def doMatch(self, img_rgb):
img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY)
templates = [[
'{}'.format(i),
cv2.imread('./templates/{}.png'.format(i), 0)
] for i in range(10)]
res = MTM.matchTemplates(templates,
img_gray,
method=cv2.TM_CCOEFF_NORMED,
N_object=4,
score_threshold=0.3,
maxOverlap=0.4,
searchBox=None)
print(res)
img_rgb = MTM.drawBoxesOnRGB(img_rgb,
res,
boxThickness=2,
boxColor=(255, 255, 00),
showLabel=True,
labelColor=(255, 255, 0),
labelScale=0.5)
num = []
for i in range(len(res)):
num.append([res['BBox'][i][0], res['TemplateName'][i]])
num.sort(key=self.sortFirst)
return img_rgb, num
def processIndividual(q):
while not exitFlag:
queueLock.acquire()
if not workQueue.empty():
data = q.get()
individualId = data[0]
date = data[1]
startTime = data[2]
endTime = data[3]
mtmObject = MTM()
rewardMatrixObject = RewardMatrix()
qMatrixObject = QMatrix()
dbObject = DBUtils()
dbObject.dbConnect()
print('Calculating mtm for all trades of this individual')
mtmObject.calculateMTM(individualId, gv.aggregationUnit, date,
startTime, date, endTime, dbObject)
print('Calculating reward matrix for these individuals')
rewardMatrix = rewardMatrixObject.computeRM(
individualId, date, startTime, date, endTime, dbObject)
print('Calculating q matrix for these individuals')
qMatrixObject.calculateQMatrix(rewardMatrix, individualId,
dbObject)
queueLock.release()
#dbObject.dbClose()
else:
queueLock.release()
def detectOffset():
offsetTemplate = cv2.imread("../Header Texts/F.jpg")
template = [("F", offsetTemplate)]
img = imageGrab(845, 750, 75, 150, 0, 0)
if img is None:
return (None, None)
img_cv = cv2.cvtColor(numpy.asarray(img), cv2.COLOR_RGB2BGR)
hits = MTM.matchTemplates(template,
img_cv,
method=cv2.TM_CCOEFF_NORMED,
N_object=float("inf"),
score_threshold=0.9,
maxOverlap=0,
searchBox=None)
if len(hits['TemplateName']) != 1:
return -1, -1
else:
x, y, _, _ = hits['BBox'].iloc[0]
print(hits['BBox'].iloc[0])
x_offset = x - 16
y_offset = y - 22
print(x_offset, y_offset)
return x_offset, y_offset
def calc_boxes(template, threshold, img):
try:
hits = MTM.matchTemplates([("staff", template)], img, method=cv.TM_CCOEFF_NORMED,
N_object=float("inf"), score_threshold=threshold, maxOverlap=0.2, searchBox=None)
return hits["BBox"]
except KeyError as ke:
print(str(ke))
return []
def freeReroll(self):
gameScreen = imageGrab()
crop = gameScreen.crop((880, 140) + (975, 185))
img_cv = cv2.cvtColor(numpy.asarray(crop), cv2.COLOR_RGB2BGR)
hits = MTM.matchTemplates(self.freeRerollIcon,
img_cv,
method=cv2.TM_CCOEFF_NORMED,
N_object=float("inf"),
score_threshold=0.9,
maxOverlap=0,
searchBox=None)
if len(hits['TemplateName']) > 0:
return True
return False
def findItemListOffset(self):
gameScreen = imageGrab()
imageToSearch = gameScreen.crop((940, 170) + (1010, 400))
img_cv = cv2.cvtColor(numpy.asarray(imageToSearch), cv2.COLOR_RGB2BGR)
hits = MTM.matchTemplates(self.itemPlacementTemplate,
img_cv,
method=cv2.TM_CCOEFF_NORMED,
N_object=float("inf"),
score_threshold=0.55,
maxOverlap=0,
searchBox=None)
if len(hits['TemplateName']) > 0:
_, y_offset, _, _ = hits['BBox'].iloc[0]
return y_offset
return None
def shopOpen(self, imageCrop=None):
if imageCrop is None:
gameScreen = imageGrab()
crop = gameScreen.crop((885, 20) + (925, 110))
else:
crop = imageCrop.crop((885, 20, 925, 110))
img_cv = cv2.cvtColor(numpy.asarray(crop), cv2.COLOR_RGB2BGR)
hits = MTM.matchTemplates(self.storeIconTemplates,
img_cv,
method=cv2.TM_CCOEFF_NORMED,
N_object=1,
score_threshold=0.9,
maxOverlap=0,
searchBox=None)
if "open" in hits['TemplateName'].iloc[0]:
return True
return False
def detectUnderlord(self, img):
img_cv = cv2.cvtColor(numpy.asarray(img), cv2.COLOR_RGB2BGR)
# Look for matches with over 90% confidence
# Note MTM converts BGR images to grayscale by taking an avg across 3 channels
hits = MTM.matchTemplates(self.underlordTemplates,
img_cv,
method=cv2.TM_CCOEFF_NORMED,
N_object=float("inf"),
score_threshold=0.85,
maxOverlap=0,
searchBox=None)
# print(hits)
if len(hits['TemplateName']) > 0:
underlordName = hits['TemplateName'].iloc[0]
return underlordName
return None
def detectItem(self, img):
# Convert from PIL image type to cv2
# PIL image store in rgb format, non array
img_cv = cv2.cvtColor(numpy.asarray(img), cv2.COLOR_RGB2BGR)
# Look for matches with over 90% confidence
# Note MTM converts BGR images to grayscale by taking an avg across 3 channels
hits = MTM.matchTemplates(self.itemTemplates,
img_cv,
method=cv2.TM_CCOEFF_NORMED,
N_object=float("inf"),
score_threshold=0.45,
maxOverlap=0,
searchBox=None)
# print(hits)
if len(hits['TemplateName']) > 0:
itemName = hits['TemplateName'].iloc[0]
return itemName
return None
def countHUD(self, img, templates):
# Convert from PIL image type to cv2
# PIL image store in rgb format, non array
img_cv = cv2.cvtColor(numpy.asarray(img), cv2.COLOR_RGB2BGR)
# Look for matches with over 90% confidence
# Note MTM converts BGR images to grayscale by taking an avg across 3 channels
hits = MTM.matchTemplates(templates,
img_cv,
method=cv2.TM_CCOEFF_NORMED,
N_object=float("inf"),
score_threshold=0.9,
maxOverlap=0,
searchBox=None)
# print(hits)
if len(hits['TemplateName']) == 1:
# If only one match is found, single digit is present
itemCount = int(hits['TemplateName'].iloc[0])
elif len(hits['TemplateName']) == 0:
itemCount = -1
else:
# For two matches, look at their location in the image
# to determine the tens and ones digit
x1, _, _, _ = hits['BBox'].iloc[0]
x2, _, _, _ = hits['BBox'].iloc[1]
if x1 <= x2:
tensDigit = int(hits['TemplateName'].iloc[0])
onesDigit = int(hits['TemplateName'].iloc[1])
else:
tensDigit = int(hits['TemplateName'].iloc[1])
onesDigit = int(hits['TemplateName'].iloc[0])
itemCount = tensDigit * 10 + onesDigit
return itemCount
from skimage.data import coins
import matplotlib.pyplot as plt
import MTM, cv2
import numpy as np
print(MTM.__version__)
#%% Get image and templates by cropping
image = coins()
smallCoin = image[37:37 + 38, 80:80 + 41]
bigCoin = image[14:14 + 59, 302:302 + 65]
#%% Perform matching
tableHit = MTM.matchTemplates([('small', smallCoin), ('big', bigCoin)],
image,
score_threshold=0.3,
method=cv2.TM_CCOEFF_NORMED,
maxOverlap=0) # Correlation-score
#tableHit = MTM.matchTemplates([('small', smallCoin), ('big', bigCoin)], image, score_threshold=0.4, method=cv2.TM_SQDIFF_NORMED, maxOverlap=0) # Difference-score
print("Found {} coins".format(len(tableHit)))
print(tableHit)
#%% Display matches
Overlay = MTM.drawBoxesOnRGB(image, tableHit, showLabel=True)
plt.figure()
plt.imshow(Overlay)
#%% Use GluonCV for display
import gluoncv as gcv
__author__ = 'Ciddhi'
from datetime import timedelta, datetime
from Reallocation import *
from MTM import *
from QMatrix import *
from PerformanceMeasures import *
if __name__ == "__main__":
dbObject = DBUtils()
reallocationObject = Reallocation()
mtmObject = MTM()
rewardMatrixObject = RewardMatrix()
qMatrixObject = QMatrix()
performanceObject = PerformanceMeasures()
dbObject.dbConnect()
print('Started at : ' + str(datetime.now()))
dbObject.dbQuery("DELETE FROM asset_allocation_table")
dbObject.dbQuery("DELETE FROM asset_daily_allocation_table")
dbObject.dbQuery("DELETE FROM mtm_table")
dbObject.dbQuery("DELETE FROM tradesheet_data_table")
dbObject.dbQuery("DELETE FROM reallocation_table")
dbObject.dbQuery("DELETE FROM q_matrix_table")
date = datetime(2012, 6, 26).date()
periodEndDate = datetime(2012, 6, 28).date()
print('date : ' + str(date))
startTime = timedelta(hours=9, minutes=15)
from Reallocation import *
from RewardMatrix import *
from Training import *
from Live import *
from MTM import *
from Ranking import *
from QMatrix import *
from PerformanceMeasures import *
from Plots import *
import calendar
if __name__ == "__main__":
dbObject = DBUtils()
rankingObject = Ranking()
mtmObject = MTM()
rewardMatrixObject = RewardMatrix()
qMatrixObject = QMatrix()
trainingObject = Training()
liveObject = Live()
reallocationObject = Reallocation()
plotObject = Plots()
performanceObject = PerformanceMeasures()
dbObject.dbConnect()
'''
dbObject.dbQuery("DELETE FROM asset_allocation_table")
dbObject.dbQuery("DELETE FROM asset_daily_allocation_table")
dbObject.dbQuery("DELETE FROM mtm_table")
dbObject.dbQuery("DELETE FROM tradesheet_data_table")
def countEXP(self, img):
# Convert img from PIL to numpy
img_cv = cv2.cvtColor(numpy.asarray(img), cv2.COLOR_RGB2BGR)
# xp in underlords has the following format ##/##
# we find the position of / to distinguish curr and pool xp
slashdf = MTM.matchTemplates(self.expSlash,
img_cv,
method=cv2.TM_CCOEFF_NORMED,
N_object=float("inf"),
score_threshold=0.9,
maxOverlap=0,
searchBox=None)
# 1 = shop open , xp visible, any other cases are errors
if len(slashdf) == 1:
hits = MTM.matchTemplates(self.expTemplates,
img_cv,
method=cv2.TM_CCOEFF_NORMED,
N_object=float("inf"),
score_threshold=0.9,
maxOverlap=0,
searchBox=None)
slashX, _, _, _ = slashdf['BBox'].iloc[0]
currExpArr = []
poolExpArr = []
# Construct two ordered lists to keep track of the digits found
for index, row in hits.iterrows():
digitX, _, _, _ = row['BBox']
if digitX < slashX:
insertPos = binarySearch(currExpArr, digitX)
if insertPos < 0:
insertPos = -insertPos - 1
currExpArr.insert(insertPos, (digitX, row['TemplateName']))
else:
insertPos = binarySearch(poolExpArr, digitX)
if insertPos < 0:
insertPos = -insertPos - 1
poolExpArr.insert(insertPos, (digitX, row['TemplateName']))
currExp = 0
poolExp = 0
placeFactor = 1
# binary search is set up currently to sort smallest to highest
# the digit with the smallest x position is a higher value
# we reverse the list to account for this and start at ones column
for digit in reversed(currExpArr):
currExp = placeFactor * int(digit[1]) + currExp
placeFactor *= 10
placeFactor = 1
for digit in reversed(poolExpArr):
poolExp = placeFactor * int(digit[1]) + poolExp
placeFactor *= 10
return currExp, poolExp
else:
return -1, -1
def feedback(self, alpha, gamma, beta, individualFactor, zeroRange,
greedyLevel, workId):
logging.basicConfig(filename=gv.logFileName + str(workId) + '.log',
level=logging.INFO,
format='%(asctime)s %(message)s')
print(str(datetime.now()) + " Starting Q Learning for : ")
print("alpha = " + str(alpha))
print("gamma = " + str(gamma))
print("beta = " + str(beta))
print("individual factor = " + str(individualFactor))
print("zero range = " + str(zeroRange))
print("greedy level = " + str(greedyLevel))
logging.info("Starting Q Learning for : ")
logging.info("alpha = " + str(alpha))
logging.info("gamma = " + str(gamma))
logging.info("beta = " + str(beta))
logging.info("individual factor = " + str(individualFactor))
logging.info("zero range = " + str(zeroRange))
logging.info("greedy level = " + str(greedyLevel))
logging.info("\n")
setupObject = Setup(alpha, gamma, beta, individualFactor, zeroRange,
greedyLevel)
[
variableString, latestIndividualTable, trainingTradesheetTable,
trainingAssetTable, qMatrixTable, reallocationTable, assetTable,
dailyAssetTable, newTradesheetTable
] = setupObject.createQLearningTables()
dbObject = DBUtils(alpha, gamma, beta, individualFactor, zeroRange,
greedyLevel, latestIndividualTable,
trainingTradesheetTable, trainingAssetTable,
qMatrixTable, reallocationTable, assetTable,
dailyAssetTable, newTradesheetTable)
mtmObject = MTM()
rewardMatrixObject = RewardMatrix(alpha)
qMatrixObject = QMatrix(gamma, greedyLevel, beta)
trainingObject = Training()
liveObject = Live()
reallocationObject = Reallocation()
performanceObject = PerformanceMeasures()
performanceOutfileName = gv.performanceOutfileNameBase + variableString + '.csv'
performanceMonthlyOutfileName = gv.performanceMonthlyOutfileNameBase + variableString + '.csv'
dbObject.dbConnect()
dbObject.dbQuery("DELETE FROM " + assetTable)
dbObject.dbQuery("DELETE FROM " + dailyAssetTable)
dbObject.dbQuery("DELETE FROM " + newTradesheetTable)
dbObject.dbQuery("DELETE FROM " + reallocationTable)
dbObject.dbQuery("DELETE FROM " + qMatrixTable)
dbObject.dbQuery("DELETE FROM " + trainingAssetTable)
dbObject.dbQuery("DELETE FROM " + trainingTradesheetTable)
dbObject.dbQuery("DELETE FROM " + latestIndividualTable)
rankingStartDate = gv.startDate
rankingEndDate = rankingStartDate + timedelta(days=gv.rankingDays)
trainingStartDate = rankingEndDate + timedelta(days=1)
trainingEndDate = trainingStartDate + timedelta(
days=gv.initializationDays)
liveStartDate = trainingEndDate + timedelta(days=1)
testingStartDate = liveStartDate
liveEndDate = liveStartDate + timedelta(days=gv.liveDays)
testingEndDate = liveEndDate
periodEndDate = gv.endDate
startTime = timedelta(hours=9, minutes=15)
walkforward = 1
dbObject.resetAssetAllocation(liveStartDate, startTime)
done = False
while (not done):
dbObject.resetAssetTraining()
trainingObject.train(trainingStartDate, trainingEndDate,
walkforward, dbObject, mtmObject,
rewardMatrixObject, qMatrixObject)
dbObject.resetLatestIndividualsWalkForward()
liveObject.live(liveStartDate, liveEndDate, walkforward, dbObject,
mtmObject, rewardMatrixObject, qMatrixObject,
reallocationObject)
walkforward += 1
if liveEndDate >= periodEndDate:
done = True
else:
dbObject.updateQMatrixTableWalkForward()
dbObject.updateAssetWalkForward()
trainingEndDate = liveEndDate
trainingStartDate = trainingEndDate - timedelta(
days=gv.initializationDays)
liveStartDate = liveEndDate + timedelta(days=1)
liveEndDate = liveStartDate + timedelta(days=gv.liveDays)
if liveEndDate > periodEndDate:
liveEndDate = periodEndDate
with open(performanceOutfileName, 'w') as fp:
w = csv.writer(fp)
w.writerow([
"original performance", "number of trades",
"q learning performance", "number of trades"
])
[
performanceRef, tradesRef
] = performanceObject.CalculateReferenceTradesheetPerformanceMeasures(
testingStartDate, periodEndDate, dbObject)
[performance, trades
] = performanceObject.CalculateTradesheetPerformanceMeasures(
testingStartDate, periodEndDate, dbObject)
[
performanceTraining, tradesTraining
] = performanceObject.CalculateTrainingTradesheetPerformanceMeasures(
testingStartDate, periodEndDate, dbObject)
w.writerow([
performanceRef, tradesRef, performance, trades,
performanceTraining, tradesTraining
])
done = False
with open(performanceMonthlyOutfileName, 'w') as fp:
w = csv.writer(fp)
w.writerow([
"original performance", "number of trades",
"q learning performance", "number of trades",
"q learning training performance", "number of trades"
])
#w.writerow(["q learning performance", "number of trades"])
while not done:
[
performanceRef, tradesRef
] = performanceObject.CalculateReferenceTradesheetPerformanceMeasures(
testingStartDate, testingEndDate, dbObject)
[performance, trades
] = performanceObject.CalculateTradesheetPerformanceMeasures(
testingStartDate, testingEndDate, dbObject)
[
performanceTraining, tradesTraining
] = performanceObject.CalculateTrainingTradesheetPerformanceMeasures(
testingStartDate, testingEndDate, dbObject)
w.writerow([
performanceRef, tradesRef, performance, trades,
performanceTraining, tradesTraining
])
#w.writerow([performance, trades])
if testingEndDate >= periodEndDate:
done = True
else:
testingStartDate = testingEndDate + timedelta(days=1)
testingEndDate = testingStartDate + timedelta(
days=gv.liveDays)
if testingEndDate > periodEndDate:
testingEndDate = periodEndDate
dbObject.dbClose()
