def _run(mode, start_date, end_date):
"""
"""
spark = setup_spark({})
daily_update = DailyUpdate(spark, start_date, end_date, mode)
daily_update.process()
spark.stop()
def _run(mode, load_date):
"""
"""
spark = setup_spark({})
short_close = shortCloseTradeReturn(spark)
if mode == "init":
short_close.init_data(load_date)
else:
short_close.daily_compute(load_date)
def _run(mode, load_date):
"""
"""
spark = setup_spark({})
long_close = longCloseTradeReturn(spark)
if mode == "init":
long_close.init_data(load_date)
else:
long_close.daily_compute(load_date)
def _run(mode, load_date):
"""
"""
spark = setup_spark({})
daily_update = closeTradeReturnUpdate(spark)
if mode == "init":
daily_update.init_data(load_date)
else:
daily_update.update(load_date)
def process(self):
if self.mode == 'init':
# raise Exception("this module has not init function")
for (clazz, type, desc) in TASK:
print '-' * 50
print u'task[{}],enddate[{}]'.format(clazz, self.enddate)
eval(clazz)(self.sparkSession).init_data()
print '-' * 50
else:
for (clazz, type, desc) in TASK:
# 对于类型4的任务，重新启动sparkSession
if type == 4:
self.sparkSession.stop()
self.sparkSession = setup_spark({})
else:
pass
try:
if type == 3:
for compute in [90, 180, 360]:
oldtime = datetime.datetime.now()
startdate = get_natural_date(
self.enddate,
-compute + 1).strftime("%Y-%m-%d")
print u'>>task[{}],startdate[{}],enddate[{}] period[{}]'.\
format(clazz, startdate, self.enddate, compute)
eval(clazz)(self.sparkSession).daily_compute(
startdate, self.enddate, compute)
newtime = datetime.datetime.now()
print u'>>task[{}] success! spend {} seconds'.\
format(clazz, (newtime - oldtime).seconds)
elif type == 2:
oldtime = datetime.datetime.now()
startdate = self.enddate[0:7] + '-01'
print u'>>task[{}],startdate[{}],enddate[{}]'.format(
clazz, startdate, self.enddate)
eval(clazz)(self.sparkSession).daily_compute(
startdate, self.enddate)
newtime = datetime.datetime.now()
print u'>>task[{}] success! spend {} seconds'.\
format(clazz, (newtime - oldtime).seconds)
elif type == 1:
oldtime = datetime.datetime.now()
startdate = self.enddate
print u'>>task[{}],startdate[{}],enddate[{}]'.format(
clazz, startdate, self.enddate)
eval(clazz)(self.sparkSession).daily_compute(
startdate, self.enddate)
newtime = datetime.datetime.now()
print u'>>task[{}] success! spend {} seconds'.\
format(clazz, (newtime - oldtime).seconds)
except Exception as e:
print e.message
raise e
def _run(mode, load_date):
"""
"""
print '-' * 50
oldtime = datetime.datetime.now()
print 'prepare start'
spark = setup_spark({})
daily_update = closeTradeReturnUpdate(spark)
newtime = datetime.datetime.now()
print 'prepare end spend {} seconds'.format((newtime - oldtime).seconds)
print '>>task cctr, startdate[%s]' % load_date
oldtime = datetime.datetime.now()
if mode == "init":
daily_update.init_data(load_date)
else:
daily_update.update(load_date)
newtime = datetime.datetime.now()
print u'>>task success! spend {} seconds'.format(
(newtime - oldtime).seconds)
print '-' * 50
def test_evl(self):
spark = setup_spark({})
eval('{0}({1}).{2}({3},{4})'.format("StockCashFlowMerge", spark,
"daily_compute", "2017-03-14",
"2017-03-15"))
def _init_run(load_date):
"""
"""
spark = setup_spark({})
long_clsoe = longCloseReturn(spark)
cash_in_out.init_data(load_date)