def test(batch, params, i, remote, debug):
logging.info("--------------")
logging.info("episode %s" % i)
result = {'success' : False, 'error' : 'none'}
try:
portfolio = cache.get(params['portfolioKey'], remote)
episodes = params['episodes']
testParams = params['test']
wParams = params['w']
logging.debug("testParams : %s" % testParams)
fromDate = episodes['test'][i][0]
toDate = episodes['test'][i][1]
logging.info("fromDate, toDate : %s, %s" % (date.to_yyyymmdd(fromDate), date.to_yyyymmdd(toDate)))
nFromDate = episodes['train'][i][0]
nToDate = episodes['train'][i][1]
logging.info("nFromDate, nToDate : %s, %s" % (date.to_yyyymmdd(nFromDate), date.to_yyyymmdd(nToDate)))
portfolio.instantiate(fromDate, toDate, True, nFromDate, nToDate)
objective = testParams['objective']
validateResult = cache.get("batch/%s/validate/%s" % (batch, i), remote)
validateWinner_ = validateResult['winner_']
numValidateWinners = len(validateWinner_)
logging.info("candidates : %s" % numValidateWinners)
F__Total = np.zeros([portfolio.tMax, portfolio.iMax])
S_ = []
provenance_ = []
for validateWinner in validateWinner_:
W_ = validateWinner['W_']
provenance = validateWinner['provenance']
F__ = w.run_W(portfolio, W_, wParams)
S = obj.score(objective, portfolio, F__)
logging.info("S : %s" % S)
S_.append(S)
provenance_.append(provenance)
F__Total += F__
F__Blend = np.zeros([portfolio.tMax, portfolio.iMax])
SBlend = 0
if (numValidateWinners > 0):
F__Blend = F__Total / numValidateWinners
SBlend = obj.score(objective, portfolio, F__Blend)
logging.info("SBlend : %s" % SBlend)
#points = [util.Point(validateWinner['W_'].tolist()) for validateWinner in validateWinner_]
#k = max(2, int(0.25 * len(points)))
#cutoff = 0.5
#clusters = util.kmeans(points, k, cutoff)
#
#validateWinner_ = [{'W_' : np.array(c.centroid.coords, dtype=float), 'provenance' : -1} for c in clusters]
header_0 = ["t", "date"]
header_1 = ["F[%s]" % k for k in range(portfolio.iMax)]
header_2 = ["r[%s]" % k for k in range(portfolio.iMax)]
header__ = [header_0 + header_1 + header_2]
body__ = [[j] + [date.to_yyyymmdd(portfolio.date_[j])] + list(F__Blend[j]) + list(portfolio.r__[j]) for j in range(portfolio.tMax)]
excel__ = header__ + body__
excelStr = "\n".join([",".join(str(val) for val in row) for row in excel__])
#excelStr = "\n".join(str(excel__))
result = {'success' : True, 'error' : 'none', 'S_' : S_, 'SBlend' : SBlend, 'provenance' : provenance_, 'F__Blend' : F__Blend, 'excel' : excelStr}
except (KeyboardInterrupt):
raise
except Exception:
result['error'] = sys.exc_info()[0]
logging.info("error %s", result['error'])
cache.put("batch/%s/test/%s" % (batch, i), result, remote)
logging.info("--------------")
return result
def train(batch, params, i, j, remote, debug):
logging.info("--------------")
logging.info("episode %s" % i)
logging.info("iter %s" % j)
result = {'success' : False, 'error' : 'none'}
try:
portfolio = cache.get(params['portfolioKey'], remote)
episodes = params['episodes']
trainParams = params['train']
logging.debug("trainParams : %s" % trainParams)
wParams = params['w']
fromDate = episodes['train'][i][0]
toDate = episodes['train'][i][1]
logging.info("fromDate, toDate : %s, %s" % (date.to_yyyymmdd(fromDate), date.to_yyyymmdd(toDate)))
nFromDate = episodes['train'][i][0]
nToDate = episodes['train'][i][1]
logging.info("nFromDate, nToDate : %s, %s" % (date.to_yyyymmdd(nFromDate), date.to_yyyymmdd(nToDate)))
portfolio.instantiate(fromDate, toDate, True, nFromDate, nToDate)
iters = trainParams['iters']
draws = trainParams['draws']
epochs = trainParams['epochs']
alpha = trainParams['alpha']
objective = trainParams['objective']
threshold = trainParams['threshold']
winner_ = []
loser_ = []
for k in range(draws):
try:
logging.info("draw %s" % k)
W_ = w.init(portfolio.jLen, wParams)
for e in range(epochs + 1):
if (e > 0):
W_ = moo.run_epoch(portfolio, W_, alpha, wParams)
F__ = w.run_W(portfolio, W_, wParams)
logging.debug(F__)
logging.debug(portfolio.x___)
S = obj.score(objective, portfolio, F__)
if (e == 0):
logging.info("SInit : %s", S)
if (S >= threshold):
break
logging.info("SFinal : %s", S)
outcome = {'W_' : W_, 'S' : S, 'provenance' : i}
if (debug):
outcome.update({'F__' : F__})
if (S >= threshold):
winner_.append(outcome)
else:
loser_.append(outcome)
except (KeyboardInterrupt):
raise
except:
result['error'] = sys.exc_info()[0]
logging.info("error %s", result['error'])
logging.info("winners : %s" % len(winner_))
logging.info("losers : %s" % len(loser_))
result = {'success' : True, 'error' : 'none', 'winner_' : winner_, 'loser_' : loser_}
except (KeyboardInterrupt):
raise
except:
result['error'] = sys.exc_info()[0]
logging.info("error %s", result['error'])
cache.put("batch/%s/train/%s.%s" % (batch, i, j), result, remote)
logging.info("--------------")
return result
def validate(batch, params, i, remote, debug):
logging.info("--------------")
logging.info("episode %s" % i)
result = {'success' : False, 'error' : 'none'}
try:
portfolio = cache.get(params['portfolioKey'], remote)
episodes = params['episodes']
validateParams = params['validate']
logging.debug("validateParams : %s" % validateParams)
trainParams = params['train']
wParams = params['w']
fromDate = episodes['validate'][i][0]
toDate = episodes['validate'][i][1]
logging.info("fromDate, toDate : %s, %s" % (date.to_yyyymmdd(fromDate), date.to_yyyymmdd(toDate)))
nFromDate = episodes['train'][i][0]
nToDate = episodes['train'][i][1]
logging.info("nFromDate, nToDate : %s, %s" % (date.to_yyyymmdd(nFromDate), date.to_yyyymmdd(nToDate)))
portfolio.instantiate(fromDate, toDate, True, nFromDate, nToDate)
numTrainIters = trainParams['iters']
accumulate = validateParams['accumulate']
objective = validateParams['objective']
threshold = validateParams['threshold']
trainLibrary = []
winner_ = []
loser_ = []
numCandidates = 0
iTrainFrom = 0 if accumulate else i
iTrainTo = i + 1
for iTrain in range(iTrainFrom, iTrainTo):
for j in range(numTrainIters):
logging.info("train %s.%s : " % (iTrain, j))
trainResult = cache.get("batch/%s/train/%s.%s" % (batch, iTrain, j), remote)
trainWinner_ = trainResult['winner_']
numCandidates += len(trainWinner_)
for trainWinner in trainWinner_:
W_ = trainWinner['W_']
F__ = w.run_W(portfolio, W_, wParams)
S = obj.score(objective, portfolio, F__)
outcome = {'W_' : W_, 'S' : S, 'provenance' : iTrain}
if (debug):
outcome.update({'F__' : F__})
if (S >= threshold):
winner_.append(outcome)
else:
loser_.append(outcome)
logging.info("candidates : %s" % numCandidates)
logging.info("winners : %s" % len(winner_))
logging.info("losers : %s" % len(loser_))
result = {'success' : True, 'error' : 'none', 'winner_' : winner_, 'loser_' : loser_}
except (KeyboardInterrupt):
raise
except:
result['error'] = sys.exc_info()[0]
logging.info("error %s", result['error'])
cache.put("batch/%s/validate/%s" % (batch, i), result, remote)
logging.info("--------------")
return result