def get_production_plan():
"""
Endpoint of the REST API that accepts a POST with a payload as we can
find in the 'example_payloads' directory.
Returns a JSON following the same structure as in 'example_response.json'
with the optimal 'p' power for each powerplant to obtain the value of 'load'.
"""
# Get informations from the payload
load = request.json["load"]
PowerPlant.wind_percent = request.json["fuels"]["wind(%)"]
prices_per_MWh = {
"gasfired": request.json["fuels"]["gas(euro/MWh)"],
"turbojet": request.json["fuels"]["kerosine(euro/MWh)"],
"windturbine": 0
}
# Instanciate powerplant objects
powerplants = []
for powerplant in request.json["powerplants"]:
pp = PowerPlant.create_pp(**powerplant)
pp.compute_cost_per_MWh(prices_per_MWh)
powerplants.append(pp)
# Calculate the unit-commitment
algo = Algo(powerplants, load)
unit_commitment = algo.calc_unit_commitment()
if unit_commitment:
return jsonify(unit_commitment)
return "WARNING: There is no possible configuration for the requested load"
def play(data,thief,police,goals):
#init graph
graph = Graph(screen,data,thief,police,goals)
#init algo
algo = Algo(data[2])
#main loop
running = True
#limit for num of turns
limit=20
#gameover var
gameover = False
while running:
screen.fill(BACKGROUND)
for event in pygame.event.get():
if event.type == pygame.MOUSEBUTTONDOWN:
#player select where to move for thief
if not gameover:
turn=graph.choose(event.pos)
if turn:
#police move base on algo here
graph.police_move(algo.move(graph.thief,graph.police))
limit-=1
elif event.type == pygame.QUIT:
running = False
#check who win
result = graph.checkWin()
if result ==1: #player win
misc.message_to_screen('Player wins!',(255,0,0),0,0,'large')
m= misc.button("Restart",850,320,150,150,(55,255,255),(0,255,0))
gameover = True
if m:
return
elif result ==2: #police win
misc.message_to_screen('Player Lose...',(0,255,0),0,0,'large')
m1= misc.button("Restart",850,320,150,150,(55,255,255),(0,255,0))
gameover = True
if m1:
return
elif limit==0:
misc.message_to_screen('Its a draw!',(0,0,255),0,0,'large')
m2= misc.button("Restart",850,320,150,150,(55,255,255),(0,255,0))
gameover = True
if m2:
return
else:
#draw game boundary
pygame.draw.rect(screen,(0,0,0),(50,50,700,700),1)
#display graph
graph.display()
#display turns left
misc.message_to_screen('Turns left: '+str(limit),(0,0,255),310,-200,'medium')
pygame.display.update()
def __init__(self):
pygame.init()
super().__init__()
self.win_w, self.win_h = 1600, 900
self.win = pygame.display.set_mode((self.win_w, self.win_h),
pygame.RESIZABLE)
self.algo = Algo(self.win)
self.update_window_size(0) ##
## Movement flags
self.moving = False
self.draw_outer_circle = False
def __init__(self, port):
self.socket = zmq.Context().socket(zmq.REP)
self.socket.bind("tcp://127.0.0.1:" + str(port))
self.algo = Algo()
self.stockHandler = self.algo.sdc
self.sip = self.algo.sip
self.should_shutdown = False
def generateDataset(self):
# 生成训练集和测试集
for line in self.__rawData:
split = line.split(',')
userId = split[0]
itemId = split[1]
self.__dataSet.setdefault(userId, set())
self.__dataSet[userId].add(itemId)
for userId, itemIds in self.__dataSet.items():
testSetItems, trainSetItems = SplitData.randomSplitList(
list(itemIds), 0.2)
self.__trainSet[userId] = set(trainSetItems)
self.__testSet[userId] = set(testSetItems)
# 在训练集中,生成用户的正样本和负样本
# 正样本为用户已经产生行为的物品
# 负样本为在整个物品列表中随机挑选用户没有产生过的物品,保证正负样本数量相同
itemsPool = [] # 所有物品,流行度高的出现的次数多
for user, items in self.__trainSet.items():
itemsPool.extend(items)
# 带正负样本的训练集
for user, items in self.__trainSet.items():
newItems = {}
for item in items:
newItems[item] = 1
# 从itemsPool中随机挑选len(items)个负样本
negativeItems = Algo.randomSelect(itemsPool, items, len(items))
for negativeItem in negativeItems:
newItems[negativeItem] = 0
self.__trainSet2[user] = newItems
class Generator:
def __init__(self, grammar: Grammar):
self.grammar = grammar
self.algo = Algo()
self.algo.fit(grammar)
def _recursive_generator(self, curr_word, word_len, words: list):
if self.algo.predict(curr_word):
words.append(curr_word)
for c in self.grammar.alphabet:
if len(curr_word) < word_len:
self._recursive_generator(curr_word + c, word_len, words)
def brute_force_generator(self, max_length=5):
words = []
self._recursive_generator('', max_length, words)
return words
def maintain(self):
algo = Algo(self)
while True:
if not self.requests.empty() and self.count < self.requests_hold:
self.count += 1
request = self.requests.get()
self.remaining_requests.append(request)
else:
break
print ('requests = ', self.remaining_requests)
robots_states, goods_states = algo.BFS(self.remaining_requests)
to_json_file(robots_states, r'../results/robots_states.json')
to_json_file(goods_states, r'../results/goods_states.json')
#These features (multithread support) will be developed in the future
'''
while True:
if not self.requests.empty():
if self.count < self.requests_hold: #Need synchronized
self.count += 1
request = self.requests.get()
#Multi-thread o day?
bfs_thread = Thread(target = algo.BFS, args = (request))
bfs_thread.start()
self.count -= done_requests
else:
#Sleep somethings
time.sleep(0.1)
else:
#Sleep somethings
break
time.sleep(0.05)
'''
def train(self,
pw,
params,
policy,
critic,
policy_loss_file,
critic_loss_file,
study_name,
beta=0) -> None:
"""
The main function for training and evaluating a policy
Repeats training and evaluation params.nb_cycles times
Stores the value and policy losses at each cycle
When the reward is greater than the best reward so far, saves the corresponding policy
:param pw: a policy wrapper, used to save the best policy into a file
:param params: the hyper-parameters of the run, specified in arguments.py or in the command line
:param policy: the trained policy
:param critic: the corresponding critic (not always used)
:param policy_loss_file: the file to record successive policy loss values
:param critic_loss_file: the file to record successive critic loss values
:param study_name: the name of the studied gradient algorithm
:param beta: a specific parameter for beta-parametrized values
:return: nothing
"""
for cycle in range(params.nb_cycles):
batch = self.make_monte_carlo_batch(params.nb_trajs, params.render,
policy)
# Update the policy
batch2 = batch.copy_batch()
algo = Algo(study_name, params.critic_estim_method, policy, critic,
params.gamma, beta, params.nstep)
algo.prepare_batch(batch)
policy_loss = batch.train_policy_td(policy)
# Update the critic
assert params.critic_update_method in [
'batch', 'dataset'
], 'unsupported critic update method'
if params.critic_update_method == "dataset":
critic_loss = algo.train_critic_from_dataset(batch2, params)
elif params.critic_update_method == "batch":
critic_loss = algo.train_critic_from_batch(batch2)
critic_loss_file.write(str(cycle) + " " + str(critic_loss) + "\n")
policy_loss_file.write(str(cycle) + " " + str(policy_loss) + "\n")
# policy evaluation part
total_reward = self.evaluate_episode(policy,
params.deterministic_eval)
# plot_trajectory(batch2, self.env, cycle+1)
# save best reward agent (no need for averaging if the policy is deterministic)
if self.best_reward < total_reward:
self.best_reward = total_reward
pw.save(self.best_reward)
def backtest_odds(self, start_date, end_date):
#breaks up date
temp = start_date.split("-")
month = int(temp[0])
day = int(temp[1])
year = int(temp[2])
cur_date = start_date
content = []
#this is actually a do-while loop
while True:
games = self.universal.get_games(cur_date)
print("date: " + cur_date)
for game in games:
print(" Game: " + str(game))
# - Strategy 0.0: Bet on algo's projected winner, no matter the odds.
# - Strategy 0.1: Bet on oddsmaker's projected winner, no matter the odds.
# All below strategies incorporate placing a bet if the algorithm projects a team to win more often than the oddsmaker projects
# - Strategy 1: Default strategy.
# - Strategy 2: Placing a bet if that team is also the algo's favorite.
# - Strategy 3: Placing a bet if that team is the algo's favorite, and the oddsmaker's underdog.
# - Strategy 4: Placing a bet if the difference between the algorithm's projected odds and the oddsmaker's odds is also >= 45
to_save = []
strat00 = {"total_bet": 0, "total_win": 0}
strat01 = {"total_bet": 0, "total_win": 0}
strat1 = {"total_bet": 0, "total_win": 0}
strat2 = {"total_bet": 0, "total_win": 0}
strat3 = {"total_bet": 0, "total_win": 0}
strat4 = {"total_bet": 0, "total_win": 0}
for x in range(0, len(games)):
#team might not exist anymore, so don't count that game
if len(games[x]['team1']) != 0 and len(games[x]['team2']) != 0:
data1 = self.universal.load_data(games[x]['team1'],
games[x]['date'])
data2 = self.universal.load_data(games[x]['team2'],
games[x]['date'])
returned1 = self.odds_calculator.analyze2(
games[x]['team1'], games[x]['team2'], data1, "away")
returned2 = self.odds_calculator.analyze2(
games[x]['team2'], games[x]['team1'], data2, "home")
# print(returned1)
# print()
# print(returned2)
# print()
algo = Algo(self.league)
algo_data = algo.calculate_V2(games[x]['date'], returned1,
returned2)
total = algo_data['total']
# to_return={}
# to_return['record_points']= odds['records']
# to_return['home_away_points']= odds['home_away']
# to_return['home_away_10_games_points']= odds['home_away_10_games']
# to_return['last_10_games_points']= odds['last_10_games']
# to_return['avg_points']= odds['avg_points']
# to_return['avg_points_10_games']= odds['avg_points_10_games']
# # to_return['win_streak']= win_streak
# to_return['win_streak_home_away']= odds['win_streak_home_away']
# to_return['total']= self.universal.convert_number(average)
odds = abs(total)
favorable_odds = round((100 / (100 - abs(odds)) - 1) * 100)
underdog_odds = round((100 / (100 - abs(odds)) - 1) * 100)
# print(str(year)+" | "+str(games[x]['team1'])+" | "+str(games[x]['team2'])+" | "+str(games[x]['game_scores']))
oddsportal_odds = self.universal.get_odds_game(
year, games[x]['team1'], games[x]['team2'],
games[x]['game_scores'])
if oddsportal_odds[0] != 0:
to_add = []
#date
to_add.append(cur_date)
#away
to_add.append(games[x]['team1'][1])
#home
to_add.append(games[x]['team2'][1])
#Algo Proj
to_add.append(str(total) + "%")
#Away proj
away_proj = 0
if total < 0:
away_proj = favorable_odds
else:
away_proj = underdog_odds * -1
to_add.append(away_proj)
#Home proj
home_proj = 0
if total > 0:
home_proj = favorable_odds
else:
home_proj = underdog_odds * -1
to_add.append(home_proj)
#Away odds
to_add.append(oddsportal_odds[0])
#Home odds
to_add.append(oddsportal_odds[1])
#Diff Away
away_diff = 0
if abs(away_proj - oddsportal_odds[0]) > 200:
away_diff = abs(away_proj -
oddsportal_odds[0]) - 200
else:
away_diff = abs(away_proj - oddsportal_odds[0])
to_add.append(away_diff)
#Diff Home
home_diff = 0
if abs(home_proj - oddsportal_odds[1]) > 200:
home_diff = abs(home_proj -
oddsportal_odds[1]) - 200
else:
home_diff = abs(home_proj - oddsportal_odds[1])
to_add.append(home_diff)
## Strategy 0.0 ##
if away_proj < 0:
#Bet
to_add.append("$100")
strat00['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[0] > 0):
to_win = 100 * (oddsportal_odds[0] / 100)
else:
to_win = 100 / (oddsportal_odds[0] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][0] > games[x][
'game_scores'][1]:
to_add.append("$" + str(100 + to_win))
strat00['total_win'] += (100 + to_win)
else:
to_add.append("$0")
else:
#Bet
to_add.append("$100")
strat00['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[1] > 0):
to_win = 100 * (oddsportal_odds[1] / 100)
else:
to_win = 100 / (oddsportal_odds[1] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][1] > games[x][
'game_scores'][0]:
to_add.append("$" + str(100 + to_win))
strat00['total_win'] += (100 + to_win)
else:
to_add.append("$0")
## Strategy 0.1 ##
if oddsportal_odds[0] < 0 and oddsportal_odds[
0] < oddsportal_odds[1]:
#Bet
to_add.append("$100")
strat01['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[0] > 0):
to_win = 100 * (oddsportal_odds[0] / 100)
else:
to_win = 100 / (oddsportal_odds[0] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][0] > games[x][
'game_scores'][1]:
to_add.append("$" + str(100 + to_win))
strat01['total_win'] += (100 + to_win)
else:
to_add.append("$0")
else:
#Bet
to_add.append("$100")
strat01['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[1] > 0):
to_win = 100 * (oddsportal_odds[1] / 100)
else:
to_win = 100 / (oddsportal_odds[1] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][1] > games[x][
'game_scores'][0]:
to_add.append("$" + str(100 + to_win))
strat01['total_win'] += (100 + to_win)
else:
to_add.append("$0")
## Strategy 1 ##
if oddsportal_odds[0] > away_proj:
#Bet
to_add.append("$100")
strat1['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[0] > 0):
to_win = 100 * (oddsportal_odds[0] / 100)
else:
to_win = 100 / (oddsportal_odds[0] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][0] > games[x][
'game_scores'][1]:
to_add.append("$" + str(100 + to_win))
strat1['total_win'] += (100 + to_win)
else:
to_add.append("$0")
elif oddsportal_odds[1] > home_proj:
#Bet
to_add.append("$100")
strat1['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[1] > 0):
to_win = 100 * (oddsportal_odds[1] / 100)
else:
to_win = 100 / (oddsportal_odds[1] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][1] > games[x][
'game_scores'][0]:
to_add.append("$" + str(100 + to_win))
strat1['total_win'] += (100 + to_win)
else:
to_add.append("$0")
else:
to_add.append("")
to_add.append("")
to_add.append("")
## Strategy 2 ##
if oddsportal_odds[0] > away_proj and away_proj < 0:
#Bet
to_add.append("$100")
strat2['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[0] > 0):
to_win = 100 * (oddsportal_odds[0] / 100)
else:
to_win = 100 / (oddsportal_odds[0] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][0] > games[x][
'game_scores'][1]:
to_add.append("$" + str(100 + to_win))
strat2['total_win'] += (100 + to_win)
else:
to_add.append("$0")
elif oddsportal_odds[1] > home_proj and home_proj < 0:
#Bet
to_add.append("$100")
strat2['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[1] > 0):
to_win = 100 * (oddsportal_odds[1] / 100)
else:
to_win = 100 / (oddsportal_odds[1] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][1] > games[x][
'game_scores'][0]:
to_add.append("$" + str(100 + to_win))
strat2['total_win'] += (100 + to_win)
else:
to_add.append("$0")
else:
to_add.append("")
to_add.append("")
to_add.append("")
## Strategy 3 ##
if oddsportal_odds[
0] > away_proj and away_proj < 0 and oddsportal_odds[
0] > 0:
#Bet
to_add.append("$100")
strat3['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[0] > 0):
to_win = 100 * (oddsportal_odds[0] / 100)
else:
to_win = 100 / (oddsportal_odds[0] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][0] > games[x][
'game_scores'][1]:
to_add.append("$" + str(100 + to_win))
strat3['total_win'] += (100 + to_win)
else:
to_add.append("$0")
elif oddsportal_odds[
1] > home_proj and home_proj < 0 and oddsportal_odds[
1] > 0:
#Bet
to_add.append("$100")
strat3['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[1] > 0):
to_win = 100 * (oddsportal_odds[1] / 100)
else:
to_win = 100 / (oddsportal_odds[1] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][1] > games[x][
'game_scores'][0]:
to_add.append("$" + str(100 + to_win))
strat3['total_win'] += (100 + to_win)
else:
to_add.append("$0")
else:
to_add.append("")
to_add.append("")
to_add.append("")
## Strategy 4 ##
if self.league == "mlb":
diff_amount = 45
elif self.league == "nba":
diff_amount = 100
if oddsportal_odds[
0] > away_proj and away_diff >= diff_amount:
#Bet
to_add.append("$100")
strat4['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[0] > 0):
to_win = 100 * (oddsportal_odds[0] / 100)
else:
to_win = 100 / (oddsportal_odds[0] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][0] > games[x][
'game_scores'][1]:
to_add.append("$" + str(100 + to_win))
strat4['total_win'] += (100 + to_win)
else:
to_add.append("$0")
elif oddsportal_odds[
1] > home_proj and home_diff >= diff_amount:
#Bet
to_add.append("$100")
strat4['total_bet'] += 100
#To Win
to_win = 0
if (oddsportal_odds[1] > 0):
to_win = 100 * (oddsportal_odds[1] / 100)
else:
to_win = 100 / (oddsportal_odds[1] * -1 / 100)
to_add.append("$" + str(to_win))
#Won
if games[x]['game_scores'][1] > games[x][
'game_scores'][0]:
to_add.append("$" + str(100 + to_win))
strat4['total_win'] += (100 + to_win)
else:
to_add.append("$0")
else:
to_add.append("")
to_add.append("")
to_add.append("")
else:
to_add = []
# #appends winning team
# if games[x]['game_scores'][0]>games[x]['game_scores'][1]:
# to_add.append(games[x]['team1'][0])
# else:
# to_add.append(games[x]['team2'][0])
# #appends score
# score=str(games[x]['game_scores'][0])+"-"+str(games[x]['game_scores'][1])
# to_add.append(score)
if len(to_add) != 0:
to_save.append(to_add)
# to_save.append(["Date", "Away", "Home", "Algo proj", "Away proj", "Home proj", "Away odds", "Home odds", "Diff away", "Diff home", "Bet", "To win", "Won"])
#only saves day's games if there were actually games on that day
if len(to_save) > 2:
#summary
strat00_profit = strat00['total_win'] - strat00['total_bet']
strat00_perc = strat00_profit / strat00['total_bet'] * 100
strat01_profit = strat01['total_win'] - strat01['total_bet']
strat01_perc = strat01_profit / strat01['total_bet'] * 100
strat1_profit = strat1['total_win'] - strat1['total_bet']
strat1_perc = strat1_profit / strat1['total_bet'] * 100
strat2_profit = strat2['total_win'] - strat2['total_bet']
if (strat2['total_bet'] > 0):
strat2_perc = strat2_profit / strat2['total_bet'] * 100
else:
strat2_perc = 0
strat3_profit = strat3['total_win'] - strat3['total_bet']
if (strat3['total_bet'] > 0):
strat3_perc = strat3_profit / strat3['total_bet'] * 100
else:
strat3_perc = 0
strat4_profit = strat4['total_win'] - strat4['total_bet']
if (strat4['total_bet'] > 0):
strat4_perc = strat4_profit / strat4['total_bet'] * 100
else:
strat4_perc = 0
#initializes with buffer columns
summary = ["", "", "", "", "", "", "", "", "", ""]
summary.append("$" + str(strat00['total_bet']))
summary.append("$" + str(strat00_profit))
summary.append(str(strat00_perc) + "%")
summary.append("$" + str(strat01['total_bet']))
summary.append("$" + str(strat01_profit))
summary.append(str(strat01_perc) + "%")
summary.append("$" + str(strat1['total_bet']))
summary.append("$" + str(strat1_profit))
summary.append(str(strat1_perc) + "%")
summary.append("$" + str(strat2['total_bet']))
summary.append("$" + str(strat2_profit))
summary.append(str(strat2_perc) + "%")
summary.append("$" + str(strat3['total_bet']))
summary.append("$" + str(strat3_profit))
summary.append(str(strat3_perc) + "%")
summary.append("$" + str(strat4['total_bet']))
summary.append("$" + str(strat4_profit))
summary.append(str(strat4_perc) + "%")
to_save.append(summary)
#space between data
to_save.append([
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
"", "", "", "", "", "", "", "", "", ""
])
content.append(to_save)
#breaks loop to act like do-while
if cur_date == end_date:
break
day += 1
if day > 31:
month += 1
day = 1
#doesn't increment year since the season's year doesn't change
if month > 12:
month = 1
day = 1
#increments season at the end of the season to sometime in the middle
if self.league == "nba":
# if "4-1-" in cur_date:
# year+=1
# month=2
# day=1
if "4-1-" in cur_date:
year += 1
month = 1
day = 15
elif self.league == "nhl":
if "4-1-" in cur_date:
year += 1
month = 2
day = 1
elif self.league == "mlb":
if "10-1-" in cur_date:
year += 1
month = 7
day = 1
cur_date = str(month) + "-" + str(day) + "-" + str(year)
# #has most recent games first
# content.reverse()
to_save = []
to_save.append([
"Date", "Away", "Home", "Algo proj", "Away proj", "Home proj",
"Away odds", "Home odds", "Diff away", "Diff home", "Bet",
"To win", "Won", "Bet", "To win", "Won", "Bet", "To win", "Won",
"Bet", "To win", "Won"
])
for x in range(0, len(content)):
for y in range(0, len(content[x])):
to_save.append(content[x][y])
if start_date != end_date:
self.universal.save_to_csv(
"./" + str(self.league) + "/analyze/" + str(self.league) +
"_Algo_V2_" + str(start_date) + "_" + str(end_date) +
"_backtest_odds.csv", to_save)
else:
self.universal.save_to_csv(
"./" + str(self.league) + "/analyze/" + str(self.league) +
"_Algo_V2_" + str(end_date) + "_backtest_odds.csv", to_save)
def __init__(self):
Algo.__init__(self)
pygame.init()
self.screen = pygame.display.set_mode((900, 900))
def team_comparison(self, algo_version, team1, team2, date, cur_year):
self.algo = Algo(self.league)
self.espn_scraper.update_data(team1, cur_year)
self.espn_scraper.update_data(team2, cur_year)
data1 = self.universal.load_data(team1, date, cur_year)
data2 = self.universal.load_data(team2, date, cur_year)
returned1 = self.analyze2(team1, team2, data1, "away")
returned2 = self.analyze2(team2, team1, data2, "home")
# print(str(team1)+" | "+str(team2))
# print(returned1)
# print(returned2)
# print()
if algo_version == "Algo_V1":
algo_data = self.algo.calculate(date, returned1, returned2)
elif algo_version == "Algo_V2":
algo_data = self.algo.calculate_V2(date, returned1, returned2)
record_points = algo_data['record_points']
home_away_points = algo_data['home_away_points']
home_away_10_games_points = algo_data['home_away_10_games_points']
last_10_games_points = algo_data['last_10_games_points']
avg_points = algo_data['avg_points']
avg_points_10_games = algo_data['avg_points_10_games']
# win_streak_10_games= algo_data['win_streak_10_games']
if self.league == "nhl":
win_streak_home_away = algo_data['win_streak_home_away']
total = algo_data['total']
to_output = []
to_output.append("")
to_output.append("Date: " + str(date))
to_output.append("Away: " + str(team1[1]) + " | Home: " +
str(team2[1]))
if algo_version == "Algo_V1":
win_streak = algo_data['win_streak']
win_streak_home_away = algo_data['win_streak_home_away']
if self.league == "nba":
to_output.append("Seasonal Record: " +
str(record_points * 10) + "/10 = " +
str(record_points))
to_output.append("Home Away: " +
str(home_away_points * 10) + "/10 = " +
str(home_away_points))
to_output.append("Home away 10: " +
str(home_away_10_games_points * 5) + "/5 = " +
str(home_away_10_games_points))
to_output.append("Last 10 games: " +
str(last_10_games_points * 5) + "/5 = " +
str(last_10_games_points))
to_output.append("Avg points: " +
str(avg_points * 8) + "/8 = " +
str(avg_points))
to_output.append("Avg points 10: " +
str(avg_points_10_games * 8) + "/8 = " +
str(avg_points_10_games))
to_output.append("Win streak: " +
str(win_streak * 3) + "/3 = " +
str(win_streak))
to_output.append("Win streak home away: " +
str(win_streak_home_away * 3) + "/3 = " +
str(win_streak_home_away))
else:
to_output.append("Seasonal Record: " +
str(record_points * 5) + "/5 = " +
str(record_points))
to_output.append("Home Away: " +
str(home_away_points * 5) + "/5 = " +
str(home_away_points))
to_output.append("Home away 10: " +
str(home_away_10_games_points * 5) + "/5 = " +
str(home_away_10_games_points))
to_output.append("Last 10 games: " +
str(last_10_games_points * 5) + "/5 = " +
str(last_10_games_points))
to_output.append("Avg points: " +
str(avg_points / 2) + "*2 = " +
str(avg_points))
to_output.append("Avg points 10: " +
str(avg_points_10_games / 2) + "*2 = " +
str(avg_points_10_games))
to_output.append("Win streak: " +
str(win_streak * 3) + "/3 = " +
str(win_streak))
to_output.append("Win streak home away: " +
str(win_streak_home_away * 3) + "/3 = " +
str(win_streak_home_away))
to_output.append("--------")
to_output.append("Total: " + str(total))
to_output.append("--------")
elif algo_version == "Algo_V2":
to_output.append("Seasonal Record: " + str(record_points) +
"%")
to_output.append("Home Away: " + str(home_away_points) +
"%")
to_output.append("Home away 10: " +
str(home_away_10_games_points) + "%")
to_output.append("Last 10 games: " +
str(last_10_games_points) + "%")
to_output.append("Avg points: " + str(avg_points) + "%")
to_output.append("Avg points 10: " +
str(avg_points_10_games) + "%")
# to_output.append("Win streak: "+str(win_streak)+"%")
if self.league == "nhl":
to_output.append("Win streak home away: " +
str(win_streak_home_away) + "%")
to_output.append("--------")
to_output.append("Total: " + str(total) + "%")
to_output.append("--------")
#chance of favorable team winning
if algo_version == "Algo_V1":
winning_odds = self.get_odds(total)
elif algo_version == "Algo_V2":
winning_odds = abs(total)
to_output.append("Perc chance to win: " + str(winning_odds) + "%")
favorable_odds = (100 / (100 - winning_odds) - 1) * 100
underdog_odds = (100 / (100 - winning_odds) - 1) * 100
to_output.append("Favorable team odds: -" + str(favorable_odds))
to_output.append("Underdog team odds: +" + str(underdog_odds))
return to_output
class Odds_Calculator:
opener = None
scraper = None
algo = None
universal = None
user_agents = []
#can be nba, nhl, nfl, mlb
league = "nba"
num_periods = {'nba': 4, 'nhl': 3, 'nfl': 4, 'mlb': 9}
def __init__(self, league):
self.league = league.lower()
self.universal = Universal_Functions(self.league)
self.espn_scraper = ESPN_Scraper(self.league)
#analyzes a single team
def single_team_analysis(self, team):
cur_year = input("Current season year: ")
self.espn_scraper.update_data(team, cur_year)
data = self.universal.load_data(team, "", cur_year)
self.analyze(team, data, cur_year)
#analyzes 2 teams and compares to determine which has best chance of winning
def team_comparison(self, algo_version, team1, team2, date, cur_year):
self.algo = Algo(self.league)
self.espn_scraper.update_data(team1, cur_year)
self.espn_scraper.update_data(team2, cur_year)
data1 = self.universal.load_data(team1, date, cur_year)
data2 = self.universal.load_data(team2, date, cur_year)
returned1 = self.analyze2(team1, team2, data1, "away")
returned2 = self.analyze2(team2, team1, data2, "home")
# print(str(team1)+" | "+str(team2))
# print(returned1)
# print(returned2)
# print()
if algo_version == "Algo_V1":
algo_data = self.algo.calculate(date, returned1, returned2)
elif algo_version == "Algo_V2":
algo_data = self.algo.calculate_V2(date, returned1, returned2)
record_points = algo_data['record_points']
home_away_points = algo_data['home_away_points']
home_away_10_games_points = algo_data['home_away_10_games_points']
last_10_games_points = algo_data['last_10_games_points']
avg_points = algo_data['avg_points']
avg_points_10_games = algo_data['avg_points_10_games']
# win_streak_10_games= algo_data['win_streak_10_games']
if self.league == "nhl":
win_streak_home_away = algo_data['win_streak_home_away']
total = algo_data['total']
to_output = []
to_output.append("")
to_output.append("Date: " + str(date))
to_output.append("Away: " + str(team1[1]) + " | Home: " +
str(team2[1]))
if algo_version == "Algo_V1":
win_streak = algo_data['win_streak']
win_streak_home_away = algo_data['win_streak_home_away']
if self.league == "nba":
to_output.append("Seasonal Record: " +
str(record_points * 10) + "/10 = " +
str(record_points))
to_output.append("Home Away: " +
str(home_away_points * 10) + "/10 = " +
str(home_away_points))
to_output.append("Home away 10: " +
str(home_away_10_games_points * 5) + "/5 = " +
str(home_away_10_games_points))
to_output.append("Last 10 games: " +
str(last_10_games_points * 5) + "/5 = " +
str(last_10_games_points))
to_output.append("Avg points: " +
str(avg_points * 8) + "/8 = " +
str(avg_points))
to_output.append("Avg points 10: " +
str(avg_points_10_games * 8) + "/8 = " +
str(avg_points_10_games))
to_output.append("Win streak: " +
str(win_streak * 3) + "/3 = " +
str(win_streak))
to_output.append("Win streak home away: " +
str(win_streak_home_away * 3) + "/3 = " +
str(win_streak_home_away))
else:
to_output.append("Seasonal Record: " +
str(record_points * 5) + "/5 = " +
str(record_points))
to_output.append("Home Away: " +
str(home_away_points * 5) + "/5 = " +
str(home_away_points))
to_output.append("Home away 10: " +
str(home_away_10_games_points * 5) + "/5 = " +
str(home_away_10_games_points))
to_output.append("Last 10 games: " +
str(last_10_games_points * 5) + "/5 = " +
str(last_10_games_points))
to_output.append("Avg points: " +
str(avg_points / 2) + "*2 = " +
str(avg_points))
to_output.append("Avg points 10: " +
str(avg_points_10_games / 2) + "*2 = " +
str(avg_points_10_games))
to_output.append("Win streak: " +
str(win_streak * 3) + "/3 = " +
str(win_streak))
to_output.append("Win streak home away: " +
str(win_streak_home_away * 3) + "/3 = " +
str(win_streak_home_away))
to_output.append("--------")
to_output.append("Total: " + str(total))
to_output.append("--------")
elif algo_version == "Algo_V2":
to_output.append("Seasonal Record: " + str(record_points) +
"%")
to_output.append("Home Away: " + str(home_away_points) +
"%")
to_output.append("Home away 10: " +
str(home_away_10_games_points) + "%")
to_output.append("Last 10 games: " +
str(last_10_games_points) + "%")
to_output.append("Avg points: " + str(avg_points) + "%")
to_output.append("Avg points 10: " +
str(avg_points_10_games) + "%")
# to_output.append("Win streak: "+str(win_streak)+"%")
if self.league == "nhl":
to_output.append("Win streak home away: " +
str(win_streak_home_away) + "%")
to_output.append("--------")
to_output.append("Total: " + str(total) + "%")
to_output.append("--------")
#chance of favorable team winning
if algo_version == "Algo_V1":
winning_odds = self.get_odds(total)
elif algo_version == "Algo_V2":
winning_odds = abs(total)
to_output.append("Perc chance to win: " + str(winning_odds) + "%")
favorable_odds = (100 / (100 - winning_odds) - 1) * 100
underdog_odds = (100 / (100 - winning_odds) - 1) * 100
to_output.append("Favorable team odds: -" + str(favorable_odds))
to_output.append("Underdog team odds: +" + str(underdog_odds))
return to_output
#gets odds of winning for algo_V1
def get_odds(self, total_points):
#puts total points at a max of 27
max_points = 27
if abs(total_points) > max_points:
total_points = max_points
x = abs(total_points) / max_points * 10
#2D polynomial that follows the percentage chance of winning per level of ranking 1-10
if self.league == "nba":
y = -0.23 * (x**2) + 7.25 * x + 47.9
else:
y = -0.23 * (x**2) + 7.25 * x + 47.9
if y < 50:
y = 50
return y
#analyzes current team
def analyze(self, team, data, end_year):
if os.path.isdir("./" + str(self.league) +
"/analyze/single_analysis/" + str(team[1])) == False:
os.mkdir("./" + str(self.league) + "/analyze/single_analysis/" +
str(team[1]))
home_away = input("Are they home or away: ").lower()
other_team = input("Playing against (letter abbreviation): ")
returned = self.analyze2(team, other_team, data, home_away)
self.save_analysis(team, data, returned, home_away)
returned['output'] = self.get_output_analysis("", team, returned,
home_away)
more_output = self.analyze_wins_ranked_teams(team, data, end_year)
# more_output=[]
for line in more_output:
returned['output'].append(line)
self.universal.save_to_txt(
"./" + str(self.league) + "/analyze/single_analysis/" +
str(team[1]) + "/" + str(team[1]) + "_analysis.txt",
returned['output'])
#analyzes whatever team needed for self.analyze()
def analyze2(self, team, other_team, data, home_away):
print("Analyzing " + str(team))
to_return = {}
season_record = self.get_seasonal_records(data)
# print("Season record: "+str(season_record))
# input("waiting...")
#seasonal win-loss ratio
to_return['seasonal_records'] = self.get_seasonal_records(data)
#average point stats
to_return['avg_game_points'] = self.get_avg_points(data)
#stats in home vs away games
to_return['home_away_record'] = self.get_home_away_record(data)
#seasonal win-loss ratio
to_return['current_win_ratio'] = self.get_current_win_ratio(data)
#last 10 games win ratio
to_return['10_game_win_ratio'] = self.analyze_10_games_win_ratio(data)
#winning or losing streaks against specified team
#definition only accepts "lal" and not ["lal", "los-angeles-lakers"], so check
if isinstance(other_team, list):
to_return[
'win_loss_streaks_against'] = self.get_win_streaks_against(
other_team[0], data)
else:
to_return[
'win_loss_streaks_against'] = self.get_win_streaks_against(
other_team, data)
return to_return
def save_analysis(self, team, data, returned, home_away):
#seasonal win-loss ratio
records = returned['seasonal_records']
to_save = []
for x in range(0, len(records)):
to_save.append(
["1-1-" + str(data[x]['year']), records[x][0] - records[x][1]])
path = "./" + str(self.league) + "/analyze/single_analysis/" + str(
team[1]) + "/" + str(team[1]) + "_seasonal_records.csv"
self.universal.save_to_csv(path, to_save)
print("Saved to " + str(path))
#average point stats
avg_points = returned['avg_game_points']
to_save = []
for x in range(0, len(avg_points['avg_game_points'])):
to_add = []
to_add.append("1-1-" + str(data[x]['year']))
to_add.append(avg_points['avg_game_points'][x])
to_add.append(avg_points['avg_other_game_points'][x])
to_add.append(avg_points['avg_game_points'][x] +
avg_points['avg_other_game_points'][x])
for y in range(0, len(avg_points['avg_quarter_points'][x])):
to_add.append(avg_points['avg_quarter_points'][x][y])
to_save.append(to_add)
path = "./" + str(self.league) + "/analyze/single_analysis/" + str(
team[1]) + "/" + str(team[1]) + "_avg_game_points.csv"
self.universal.save_to_csv(path, to_save)
print("Saved to " + str(path))
#stats in home vs away games
home_away_records = returned['home_away_record']
to_save = []
for x in range(0, len(home_away_records['home_record'])):
to_add = []
to_add.append("1-1-" + str(data[x]['year']))
to_add.append(home_away_records['home_record'][x][0])
to_add.append(home_away_records['home_record'][x][1])
to_save.append(to_add)
to_save.append(["", "", ""])
to_save.append(["", "", ""])
to_save.append(["", "", ""])
for x in range(0, len(home_away_records['away_record'])):
to_add = []
to_add.append("1-1-" + str(data[x]['year']))
to_add.append(home_away_records['away_record'][x][0])
to_add.append(home_away_records['away_record'][x][1])
to_save.append(to_add)
path = "./" + str(self.league) + "/analyze/single_analysis/" + str(
team[1]) + "/" + str(team[1]) + "_home_away_record.csv"
self.universal.save_to_csv(path, to_save)
print("Saved to " + str(path))
#seasonal win-loss ratio
win_loss = returned['current_win_ratio']
path = "./" + str(self.league) + "/analyze/single_analysis/" + str(
team[1]) + "/" + str(team[1]) + "_current_win_ratio.csv"
self.universal.save_to_csv(path, win_loss)
print(path)
#last 10 games win ratio
last_10_games = returned['10_game_win_ratio']
to_save = []
to_save.append(["Year", "win-loss", "num wins", "num games"])
for x in range(0, len(last_10_games)):
for y in range(-10, 11, 2):
to_add = []
#only has year at beginning of listing
if y == -10:
to_add.append(data[x]['year'])
else:
to_add.append("")
# to_add.append(str(y))
temp = {
'-10': '"0-10"',
'-8': '"1-9"',
'-6': '"2-8"',
'-4': '"3-7"',
'-2': '"4-6"',
'0': '"5-5"',
'2': '"6-4"',
'4': '"7-3"',
'6': '"8-2"',
'8': '"9-1"',
'10': '"10-0"'
}
#turns -4 into "3-7"
to_add.append(temp[str(y)])
to_add.append(last_10_games[x][str(y)][0])
to_add.append(last_10_games[x][str(y)][1])
#gets win percentage
if last_10_games[x][str(y)][1] != 0:
to_add.append("=C" + str(len(to_save) + 1) + "/D" +
str(len(to_save) + 1) + "*100")
else:
to_add.append(0)
to_save.append(to_add)
to_save.append(["", "", "", ""])
path = "./" + str(self.league) + "/analyze/single_analysis/" + str(
team[1]) + "/" + str(team[1]) + "_10_game_win_ratio.csv"
self.universal.save_to_csv(path, to_save)
print(path)
#winning or losing streaks against specified team
to_save = []
wins_against = returned['win_loss_streaks_against']
to_save.append(["Losing streak", wins_against['games_since_last_win']])
to_save.append(
["Winning streak", wins_against['games_since_last_loss']])
if home_away == "away":
to_save.append([
"Losing streak away", wins_against['games_since_last_win_away']
])
to_save.append([
"Winning streak away",
wins_against['games_since_last_loss_away']
])
elif home_away == "home":
to_save.append([
"Losing streak home", wins_against['games_since_last_win_home']
])
to_save.append([
"Winning streak home",
wins_against['games_since_last_loss_home']
])
path = "./" + str(self.league) + "/analyze/single_analysis/" + str(
team[1]) + "/" + str(team[1]) + "_win_loss_streaks_against.csv"
self.universal.save_to_csv(path, to_save)
print(path)
def get_output_analysis(self, indent, team, returned, home_away):
records = returned['seasonal_records']
avg_points = returned['avg_game_points']
home_away_records = returned['home_away_record']
win_loss = returned['current_win_ratio']
last_10_games = returned['10_game_win_ratio']
wins_against = returned['win_loss_streaks_against']
#### output ####
to_output = []
to_output.append("")
to_output.append("")
to_output.append(indent + team[1])
if (records[-1][0] - records[-1][1]) > (records[-2][0] -
records[-2][1]):
temp = "uptrend"
else:
temp = "downtrend"
to_output.append(indent + "Season: " +
str(records[-1][0] - records[-1][1]) + " on " +
str(temp))
if home_away == "away":
to_output.append(indent + "Home-Away: " +
str(home_away_records['away_record'][-1][0]) +
"-" +
str(home_away_records['away_record'][-1][1]) +
" away")
to_output.append(indent + " Last 10 away games: " +
str(home_away_records['away_10_games'][-1][0]) +
"-" +
str(home_away_records['away_10_games'][-1][1]))
elif home_away == "home":
to_output.append(indent + "Home-Away: " +
str(home_away_records['home_record'][-1][0]) +
"-" +
str(home_away_records['home_record'][-1][1]) +
" home")
to_output.append(indent + " Last 10 home games: " +
str(home_away_records['home_10_games'][-1][0]) +
"-" +
str(home_away_records['home_10_games'][-1][1]))
win_10_games = 0
for x in range(len(win_loss) - 1, len(win_loss) - 11, -1):
win_10_games += win_loss[x][2]
temp = {
'-10': '0-10',
'-8': '1-9',
'-6': '2-8',
'-4': '3-7',
'-2': '4-6',
'0': '5-5',
'2': '6-4',
'4': '7-3',
'6': '8-2',
'8': '9-1',
'10': '10-0'
}
to_output.append(indent + "10 Games: " + temp[str(win_10_games)])
won = last_10_games[-1][str(win_10_games)][0]
num_games = last_10_games[-1][str(win_10_games)][1]
if num_games != 0:
to_output.append(indent + " " + str(won) + " won out of " +
str(num_games) + " games | " +
str(won / num_games * 100) + "%")
else:
to_output.append(indent + " " + str(won) + " won out of " +
str(num_games) + " games | N/A%")
to_output.append(indent + "Avg points: " +
str(avg_points['avg_game_points'][-1]) + " - " +
str(avg_points['avg_other_game_points'][-1]))
to_output.append(indent + " Last 10 games: " +
str(avg_points['avg_10_games'][-1]) + " - " +
str(avg_points['avg_other_10_games'][-1]))
#on winning streak
if wins_against['games_since_last_loss'] > 0:
to_output.append(indent + "Winning streak against " +
str(wins_against['other_team']) + ": " +
str(wins_against['games_since_last_loss']))
to_output.append(indent + " Winning streak " + home_away + ": " +
str(wins_against['games_since_last_loss_' +
str(home_away)]))
elif wins_against['games_since_last_win'] > 0:
to_output.append(indent + "Losing streak against " +
str(wins_against['other_team']) + ": " +
str(wins_against['games_since_last_win']))
to_output.append(indent + " Losing streak " + home_away + ": " +
str(wins_against['games_since_last_win_' +
str(home_away)]))
return to_output
#analyzes number of wins against teams of certain rankings. Like # wins against even teams (23-25 to 27-25) or against good teams (30-15) or bad teams (15-30)... etc
def analyze_wins_ranked_teams(self, team, data, end_year):
total_output = []
for x in range(
len(data[-1]['other_team']) - 1,
len(data[-1]['other_team']) - 11, -1):
other_team = []
other_team.append(data[-1]['other_team'][x])
other_team.append("")
date = data[-1]['dates'][x]
# print("Date: "+str(date))
home_away = data[-1]['home_away'][x]
if home_away == "home":
other_home_away = "away"
elif home_away == "away":
other_home_away = "home"
temp = []
temp.append(date)
temp.append(other_team)
# temp.append()
league_teams = self.universal.load_league_teams()
#gets "los-angeles-lakers" if given "lal"
for y in range(0, len(league_teams)):
name = league_teams[y]
if name[0] == other_team[0]:
other_team[1] = name[1]
indent = " "
cur_data = self.universal.load_data(team, date, end_year)
print(cur_data[-1]['other_team'][-1])
returned = self.analyze2(team, other_team[0], cur_data,
data[-1]['home_away'][x])
output = self.get_output_analysis(indent, team, returned,
data[-1]['home_away'][x])
for line in output:
total_output.append(line)
other_data = self.universal.load_data(other_team, date, end_year)
print(
str(other_data[-1]['other_team'][-1]) + " | " + str(date) +
" | " + str(other_data[-1]['dates'][-5]))
returned = self.analyze2(other_team, team[0], other_data,
other_home_away)
output = self.get_output_analysis(indent, other_team, returned,
other_home_away)
print()
for line in output:
print(line)
total_output.append(line)
total_output.append("")
#adds winner and scores
cur_team_score = data[-1]['game_scores'][x][0]
other_team_score = data[-1]['game_scores'][x][1]
if cur_team_score > other_team_score:
total_output.append(indent + "Winner: " + team[1] + " | " +
str(cur_team_score) + "-" +
str(other_team_score))
else:
total_output.append(indent + "Winner: " + other_team[1] +
" | " + str(other_team_score) + "-" +
str(cur_team_score))
total_output.append(indent +
"----------------------------------------")
print()
return total_output
#returns wins/loss streaks against other_team
def get_win_streaks_against(self, other_team, original_data):
to_return = {}
to_return['other_team'] = other_team
to_return['games_since_last_win'] = 0
to_return['games_since_last_loss'] = 0
to_return['games_since_last_win_away'] = 0
to_return['games_since_last_win_home'] = 0
to_return['games_since_last_loss_away'] = 0
to_return['games_since_last_loss_home'] = 0
for x in range(0, len(original_data)):
data = original_data[x]
year = data['year']
for y in range(0, len(data['other_team'])):
if data['other_team'][y] == other_team:
# if x==len(original_data)-1:
# print(str(year)+" | "+str(other_team)+" | "+str(data['game_scores'][y][0])+"-"+str(data['game_scores'][y][1]))
#if won
if data['game_scores'][y][0] > data['game_scores'][y][1]:
to_return['games_since_last_win'] = 0
to_return['games_since_last_loss'] += 1
if data['home_away'][y] == "away":
to_return['games_since_last_win_away'] = 0
to_return['games_since_last_loss_away'] += 1
else:
to_return['games_since_last_win_home'] = 0
to_return['games_since_last_loss_home'] += 1
#if lost
else:
to_return['games_since_last_win'] += 1
to_return['games_since_last_loss'] = 0
if data['home_away'][y] == "away":
to_return['games_since_last_win_away'] += 1
to_return['games_since_last_loss_away'] = 0
else:
to_return['games_since_last_win_home'] += 1
to_return['games_since_last_loss_home'] = 0
return to_return
# #gets percentage of games won if ahead after 1st quarter, 2nd quarter, etc.
# def get_perc_win_quarters_ahead(self, data):
# #gets total goals for and goals against
# def get_goals_for_against(self, data):
#determines whether teams win or lose more often if they have a good or bad last 10 games
def analyze_10_games_win_ratio(self, original_data):
to_return = []
for x in range(0, len(original_data)):
data = original_data[x]
year = data['year']
#win_data['4'] will hold data for last 10 games with ratio 7-3
#increments by 2 since subtracting losses from wins of last 10 games will never have odd number
win_data = {}
for y in range(-10, 11, 2):
win_data[str(y)] = [0, 0]
last_10_record = []
for y in range(0, len(data['other_team'])):
#only gets win ratio if 10 records present
if len(last_10_record) == 10:
temp = sum(last_10_record)
#adding 1 or -1 is same as subtracting num losses from num wins
if data['game_scores'][y][0] > data['game_scores'][y][1]:
#only counts this win if 10 records already present
if len(last_10_record) == 10:
win_data[str(sum(last_10_record))][0] += 1
win_data[str(sum(last_10_record))][1] += 1
last_10_record.append(1)
else:
if len(last_10_record) == 10:
win_data[str(sum(last_10_record))][1] += 1
last_10_record.append(-1)
if len(last_10_record) > 10:
last_10_record.pop(0)
to_return.append(win_data)
return to_return
#gets win-loss ratio during each game during the current season
def get_current_win_ratio(self, original_data):
data = original_data[-1]
to_return = []
cur_score = 0
for x in range(0, len(data['game_scores'])):
to_add = []
to_add.append(data['game_scores'][x][0])
to_add.append(data['game_scores'][x][1])
# print(data['other_team'][x]+" | "+str(to_add))
if data['game_scores'][x][0] > data['game_scores'][x][1]:
temp = 1
else:
temp = -1
to_add.append(temp)
cur_score += temp
to_add.append(cur_score)
to_return.append(to_add)
return to_return
#gets wins-losses while at home or away
def get_home_away_record(self, original_data):
to_return = {}
to_return['home_record'] = []
to_return['away_record'] = []
to_return['home_10_games'] = []
to_return['away_10_games'] = []
for x in range(0, len(original_data)):
data = original_data[x]
home_away = data['home_away']
game_scores = data['game_scores']
home_record = []
away_record = []
for y in range(0, len(home_away)):
if home_away[y] == "home":
if game_scores[y][0] > game_scores[y][1]:
home_record.append(1)
else:
home_record.append(-1)
elif home_away[y] == "away":
if game_scores[y][0] > game_scores[y][1]:
away_record.append(1)
else:
away_record.append(-1)
to_return['home_record'].append(
[home_record.count(1),
home_record.count(-1)])
to_return['away_record'].append(
[away_record.count(1),
away_record.count(-1)])
#gets stats on last 10 games
home_10_games = [
home_record[-10:].count(1), home_record[-10:].count(-1)
]
away_10_games = [
away_record[-10:].count(1), away_record[-10:].count(-1)
]
to_return['home_10_games'].append(home_10_games)
to_return['away_10_games'].append(away_10_games)
return to_return
#calculates a bunch of average points stats
def get_avg_points(self, original_data):
to_return = {}
avg_game_points = []
avg_other_game_points = []
avg_10_games = []
avg_other_10_games = []
avg_quarters = []
for x in range(0, len(original_data)):
data = original_data[x]
if len(data['other_team']) != 0:
# print("Year: "+str(original_data[x]['year']))
#gets avg_game_points
total_points = 0
other_total_points = 0
for y in range(0, len(data['other_team'])):
total_points += data['game_scores'][y][0]
other_total_points += data['game_scores'][y][1]
average = total_points / len(data['other_team'])
average_other = other_total_points / len(data['other_team'])
avg_game_points.append(self.universal.convert_number(average))
avg_other_game_points.append(
self.universal.convert_number(average_other))
#gets average points for last 10 games
total_points = 0
other_total_points = 0
for y in range(
len(data['other_team']) - 1,
len(data['other_team']) - 11, -1):
total_points += data['game_scores'][y][0]
other_total_points += data['game_scores'][y][1]
average = total_points / 10
avg_10_games.append(self.universal.convert_number(average))
average = other_total_points / 10
avg_other_10_games.append(
self.universal.convert_number(average))
#gets avg_game_points
num_periods = self.num_periods[self.league]
total_quarters = [0] * num_periods * 2
for y in range(0, len(data['other_team'])):
# print(data['period_scores'][y])
# print("Num periods: "+str(num_periods))
#adds current team's 4 quarters
try:
for z in range(0, num_periods):
total_quarters[z] += int(
data['period_scores'][y][0][z])
except Exception as error:
pass
#adds other team's 4 quarters
try:
for z in range(0, len(data['period_scores'][y][1])):
total_quarters[z + num_periods] += int(
data['period_scores'][y][1][z])
except Exception as error:
pass
#gets average quarter scores
for y in range(0, len(total_quarters)):
total_quarters[y] = total_quarters[y] / len(
data['other_team'])
avg_quarters.append(total_quarters)
to_return['avg_game_points'] = avg_game_points
to_return['avg_other_game_points'] = avg_other_game_points
to_return['avg_10_games'] = avg_10_games
to_return['avg_other_10_games'] = avg_other_10_games
to_return['avg_quarter_points'] = avg_quarters
return to_return
#gets records like 2016: 49-20 for all seasons
def get_seasonal_records(self, original_data):
records = []
for x in range(0, len(original_data)):
data = original_data[x]
num_wins = 0
for y in range(0, len(data['other_team'])):
if data['game_scores'][y][0] > data['game_scores'][y][1]:
num_wins += 1
# record=num_wins-len(data['game_scores'])-num_wins
record = [num_wins, len(data['game_scores']) - num_wins]
records.append(record)
return records
def startAnalysis(self):
# Use filename from attackEdit instead of inputEdit if possible
if len(self.attackList) > 0:
infile = str(self.attackEdit.text())
else:
infile = str(self.inputEdit.text())
outfile = str(self.resultsEdit.text())
granularity = 1
trainingWin = 24
forecastingInterval = 1
print ("\nStarting analysis on %s with settings %d %d %d..."
% (infile, granularity, trainingWin, forecastingInterval))
# Get list of features (first columns is time)
infile = open(infile, 'rb')
reader = csv.reader(infile)
columns = reader.next()[1:]
print "The following features were found:", columns
# Algorithm settings
algo = Algo(granularity, trainingWin, forecastingInterval, len(columns)-1)
algo.setEMAParameter(alpha=self.emaSpin.value())
algo.setSeverityParameters(w=self.severitySpinW.value(),
L=self.severitySpinL.value())
y_time = ['Timestamp']
y_target = ['Target']
y_predict = ['Prediction']
anomalies = ['Anomaly']
detected = set()
ground_truth = set()
first = True
print "Beginning analysis..."
loadingWin = LoadingWindow()
self.mainWidget.setEnabled(False)
count = 0
for line in reader:
# Read new data from file
cur_time = float(line[0])
new_data = np.asarray(line[1:], np.float)
target, prediction = algo.run(new_data) # Magic!
if prediction != None:
y_time.append(cur_time)
y_target.append(target)
y_predict.append(float(prediction))
if algo.checkSeverity(target, float(prediction)):
detected.add(cur_time)
anomalies.append(1)
else:
anomalies.append(0)
cur_datetime = dt.datetime.fromtimestamp(cur_time)
for attack in self.attackList:
if(cur_datetime >= attack.start and cur_datetime < attack.end):
ground_truth.add(cur_time)
break
if (count % 60) == 0:
#print "Trying time: ", cur_time
QtGui.QApplication.processEvents()
count += 1
# Close the input file and save results
infile.close()
writeResults(outfile, (y_time, y_target, y_predict, anomalies))
f1_scores(detected, ground_truth)
print_stats(y_target[1:], y_predict[1:]) #Remove header
print "Ending analysis. See %s for results." % outfile
self.mainWidget.setEnabled(True)
loadingWin.close()
Photo(i, photo_params[2:], photo_params[0]))
def write_output_file(self, slides, score):
"""
Writes the output file ./outputs/<file_prefix>_<score>_<timestamp>.out
:param slides: Slides list
:param score: The score after the simulation
"""
output_file_path = "outputs/" + self.file_prefix + "_" + str(
score) + "_" + str(time.time()) + ".out"
file = open(output_file_path, "w")
print("write_output_file : " + output_file_path)
file.write(str(len(slides)) + "\n")
for slide in slides:
file.write(str(slide.photo1.id))
if slide.photo2 is not None:
file.write(" " + str(slide.photo2.id))
file.write("\n")
file.close()
# initialize Main instance
main = Main(sys.argv[1])
# initialize Algo instance
Algo(main)
def predict(grammar_file: str, word: str):
word = word.strip().replace(' ', '')
grammar = Grammar.from_file(grammar_file)
algo = Algo()
algo.fit(grammar)
return algo.predict(word)
#GUI Version of TIC-TAC-TOE using kivy
import kivy
from kivy.app import App
from kivy.uix.screenmanager import ScreenManager, Screen
from kivy.uix.gridlayout import GridLayout
from kivy.uix.label import Label
from kivy.uix.button import Button
from kivy.uix.textinput import TextInput
from algo import Algo
my_algo = Algo()
AI_mode = False
player_name = ['X', 'O']
player_sign = ['X', 'O']
#Login Page
class Login(GridLayout):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.cols = 2
self.add_widget(Label(font_size=40, text="Tic Tac Toe"))
self.add_widget(Label(text="Subham_Games", font_size=20))
self.add_widget(Label(text="Player X:", font_size=20))
self.player_x = TextInput(multiline=False, font_size=20)
self.add_widget(self.player_x)
self.add_widget(Label(text="Player O:", font_size=20))
self.player_o = TextInput(text="Computer",
multiline=False,
font_size=20)
def train(self, pw,params, policy, critic, policy_loss_file, critic_loss_file, study_name, beta=0, is_cem=False):
all_weights=np.zeros((int(params.nb_cycles+1),policy.get_weights_dim(False)))
all_rewards=np.zeros(params.nb_cycles+1)
best_reward=-np.inf
best_weights=np.zeros(policy.get_weights_dim(False))
all_pops=np.zeros((params.nb_cycles,params.population,policy.get_weights_dim(False)))
all_pops_scores= np.zeros((params.nb_cycles,params.population))
# is_kept = np.zeros((params.nb_cycles,params.population))
list_elite_index=np.zeros((params.nb_cycles,int(params.elites_frac * params.population)))
fixed=params.fix_layers
idx_best=0
if is_cem == False:
if fixed:
print(fixed)
fc1_w, fc1_b, fc2_w, fc2_b = policy.get_weights_pg()
# print(fc1_w)
# print(policy.test())
if is_cem == True:
all_weights=np.zeros((int(params.nb_cycles+1),policy.get_weights_dim(fixed)))
best_weights=np.zeros(policy.get_weights_dim(fixed))
#random init of the neural network.
#so far, all the layers are initialized with the same gaussian.
init_weights = np.array(params.sigma*np.random.randn(policy.get_weights_dim(False)))
#print(np.shape(init_weights))
#start_weights=np.array(3*np.random.randn(policy.get_weights_dim(False)))
policy.set_weights(init_weights, False)
print(fixed)
#print(params.fix_layers)
#print(policy.get_weights_dim(params.fix_layers))
study = params.study_name
noise=np.diag(np.ones(policy.get_weights_dim(fixed))*params.sigma)
#print(np.shape(noise))
#var=np.cov(init_weights[:,-policy.get_weights_dim(fixed):],rowvar=False) + noise
#mu=init_weights[:,-policy.get_weights_dim(fixed):].mean(axis=0)
var=np.diag(np.ones(policy.get_weights_dim(fixed))*np.var(init_weights))+noise
print(np.shape(var))
mu=init_weights[-policy.get_weights_dim(fixed):]
all_weights[0]=mu
all_rewards[0]=self.evaluate_episode(policy, params.deterministic_eval)
print(np.shape(mu))
rng = np.random.default_rng()
#we can draw the last layer from a different gaussian
#mu=params.sigma_bis*np.random.randn(policy.get_weights_dim(params.fix_layers))
for cycle in range(params.nb_cycles):
if is_cem == True:
rewards = np.zeros(params.population)
weights=rng.multivariate_normal(mu, var, params.population)
for p in range(params.population):
policy.set_weights(weights[p], fixed)
batch=self.make_monte_carlo_batch(params.nb_trajs_cem, params.render, policy, True)
rewards[p] = batch.train_policy_cem(policy, params.bests_frac)
all_pops[cycle,p]=weights[p]
all_pops_scores[cycle,p]=rewards[p]
elites_nb = int(params.elites_frac * params.population)
elites_idxs = rewards.argsort()[-elites_nb:]
list_elite_index[cycle]=elites_idxs
# for i in elites_idxs:
# is_kept[cycle][i]=1
elites_weights = [weights[i] for i in elites_idxs]
#update the best weights
mu = np.array(elites_weights).mean(axis=0)
var = np.cov(elites_weights,rowvar=False)+noise
#print(best_weights)
# policy evaluation part
policy.set_weights(mu, fixed)
total_reward = self.evaluate_episode(policy, params.deterministic_eval)
if total_reward>best_reward:
best_weights=mu
best_reward=total_reward
idx_best=cycle
all_rewards[cycle+1]=total_reward
# if total_reward>np.min(top_ten_scores):
# temp_min=np.argmin(top_ten_scores)
# top_ten_scores[temp_min]=total_reward
# top_ten_policies[temp_min]=mu
# Update the file for the plot
# reward_file = policy_loss_file
# reward_file.write(str(cycle) + " " + str(total_reward) + "\n")
# if (cycle+1)%3==0:
# all_weights[int((cycle+1)/3)-1]=mu
all_weights[cycle+1]=mu
elif is_cem == False:
batch = self.make_monte_carlo_batch(params.nb_trajs_pg, params.render, policy)
# Update the policy
batch2 = batch.copy_batch()
algo = Algo(study_name, params.critic_estim_method, policy, critic, params.gamma, beta, params.nstep)
algo.prepare_batch(batch)
policy_loss = batch.train_policy_td(policy)
# if (cycle+1)%3==0:
# all_weights[int((cycle+1)/3)-1]=policy.get_weights_as_numpy()
all_weights[cycle]=policy.get_weights_as_numpy()
#print(policy_loss)
# Update the critic
assert params.critic_update_method in ['batch', 'dataset'], 'unsupported critic update method'
if params.critic_update_method == "dataset":
critic_loss = algo.train_critic_from_dataset(batch2, params)
elif params.critic_update_method == "batch":
critic_loss = algo.train_critic_from_batch(batch2)
critic_loss_file.write(str(cycle) + " " + str(critic_loss) + "\n")
policy_loss_file.write(str(cycle) + " " + str(policy_loss) + "\n")
plot_trajectory(batch2, self.env, cycle+1)
# policy evaluation part
if fixed:
policy.set_weights_pg(fc1_w, fc1_b, fc2_w, fc2_b)
total_reward = self.evaluate_episode(policy, params.deterministic_eval)
all_rewards[cycle]=total_reward
if total_reward>best_reward:
best_weights=policy.get_weights_as_numpy()
best_reward=total_reward
idx_best=cycle
print(total_reward)
# X_embedded = TSNE(n_components=2).fit_transform(all_cem_weights)
# # print(np.shape(X_embedded))
# # print(X_embedded)
# plt.scatter(*zip(*X_embedded))
# return all_weights,best_weights,all_rewards,idx_best
return all_weights,all_rewards,all_pops,all_pops_scores,list_elite_index
def main(argv):
#===== Initialization =====#
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('settings_file', type=str)
parser.add_argument('-s', '--sound', action='store_true', help="use sound as a feature in analysis")
parser.add_argument('-b', '--backup', action='store_true', help="start training on backup data")
parser.add_argument('-t', '--time_allign', action='store_true', help="collect data at times which are multiples of the granularity")
args = parser.parse_args(argv[1:])
# Initialize Zway server
port = 8083
host = '172.31.16.102'
zserver = zway.Server(host, port)
# Read settings from settings file
try:
settings_dict = settings.load(args.settings_file)
except Exception as error:
print "Error reading settings file.", error
print " "
exit(1)
# Initialize Algo class
granularity = int(settings_dict['granularity'])
training_window = int(settings_dict['training_window'])
training_interval = int(settings_dict['training_interval'])
ema_alpha = float(settings_dict['ema_alpha'])
severity_omega = float(settings_dict['severity_omega'])
severity_lambda = float(settings_dict['severity_lambda'])
auto_regression = int(settings_dict['auto_regression'])
num_features = len(zserver.device_IDs())
print "Num features: ", num_features
algo = Algo(granularity, training_window, training_interval, num_features)
algo.setSeverityParameters(severity_omega, severity_lambda)
algo.setEMAParameter(ema_alpha)
# Timing procedure
granularity = settings_dict['granularity'] * 60
granularity = 1
goal_time = time.time()
if args.time_allign:
goal_time = int(goal_time) + granularity - (int(time.time()) % granularity)
#===== Analysis =====#
while(True):
# Timing procedures
while goal_time > time.time():
time.sleep(0.2)
goal_time = goal_time + granularity
# Data collection
features = collect_features(zserver)
print list(features)
# Data analysis
target, pred = algo.run(features)
if (pred != None):
print target, pred, algo.checkSeverity(target, pred)
print "theta", algo.w_opt
else:
print target, pred
def train_pg(self,
pw,
params,
policy,
critic,
policy_loss_file,
critic_loss_file,
study_name,
beta=0) -> None:
"""
The main function for training and evaluating a policy
Repeats training and evaluation params.nb_cycles times
Stores the value and policy losses at each cycle
When the reward is greater than the best reward so far, saves the corresponding policy
:param pw: a policy wrapper, used to save the best policy into a file
:param params: the hyper-parameters of the run, specified in arguments.py or in the command line
:param policy: the trained policy
:param policy_loss_file: the file to record successive policy loss values
:return: nothing
"""
# Initialize variables
self.list_weights = []
self.best_weights = np.zeros(policy.get_weights_dim())
self.list_rewards = np.zeros((int(params.nb_cycles)))
self.best_reward = -1e38
self.best_weights_idx = 0
total_reward = self.best_reward
self.list_weights.append(policy.get_weights())
if params.start_from_policy:
starting_weights = get_starting_weights(pw)
policy.set_weights(starting_weights)
print("Shape of weights vector is: ", np.shape(self.best_weights))
initial_score = self.evaluate_episode(policy,
params.deterministic_eval,
params)
total_reward = initial_score
pw.save(cycle=0, score=initial_score)
self.env.write_reward(cycle=0, reward=initial_score)
with SlowBar('Performing a repetition of PG',
max=params.nb_cycles - 1) as bar:
for cycle in range(1, params.nb_cycles):
batch = self.make_monte_carlo_batch(params.nb_trajs,
params.render, policy)
if params.reinforce:
batch.sum_rewards()
policy_loss = batch.train_policy_td(policy)
# self.env.write_gradients(gradient_angles,cycle)
policy_loss_file.write(
str(cycle) + " " + str(policy_loss) + "\n")
batch = self.make_monte_carlo_batch(
params.nb_trajs, params.render, policy)
else:
# Update the policy
batch2 = batch.copy_batch()
algo = Algo(params.study_name, params.critic_estim_method,
policy, critic, params.gamma, beta,
params.nstep)
algo.prepare_batch(batch)
policy_loss = batch.train_policy_td(policy)
# Update the critic
assert params.critic_update_method in [
'batch', 'dataset'
], 'unsupported critic update method'
if params.critic_update_method == "dataset":
critic_loss = algo.train_critic_from_dataset(
batch2, params)
elif params.critic_update_method == "batch":
critic_loss = algo.train_critic_from_batch(batch2)
critic_loss_file.write(
str(cycle) + " " + str(critic_loss) + "\n")
policy_loss_file.write(
str(cycle) + " " + str(policy_loss) + "\n")
plot_trajectory(batch2, self.env, cycle + 1)
# add the new weights to the list of weights
self.list_weights.append(policy.get_weights())
distance = np.linalg.norm(self.list_weights[-1] -
self.list_weights[-2])
self.env.write_distances(cycle, distance)
self.write_angles_global(cycle)
# policy evaluation part
if (cycle % params.eval_freq) == 0:
total_reward = self.evaluate_episode(
policy, params.deterministic_eval, params)
# wrote and store reward
self.env.write_reward(cycle, total_reward)
self.list_rewards[cycle] = total_reward
# plot_trajectory(batch2, self.env, cycle+1)
# save best reward agent (no need for averaging if the policy is deterministic)
if self.best_reward < total_reward:
self.best_reward = total_reward
self.best_weights = self.list_weights[-1]
self.best_weights_idx = cycle
# Save the best policy obtained
if (cycle % params.save_freq) == 0:
pw.save(cycle=cycle, score=total_reward)
bar.next()
# pw.rename_best(method="PG",best_cycle=self.best_weights_idx,best_score=self.best_reward)
print("Best reward: ", self.best_reward)
print("Best reward iter: ", self.best_weights_idx)
def backtest_csv_output(self, start_date, end_date):
#breaks up date
temp = start_date.split("-")
month = int(temp[0])
day = int(temp[1])
year = int(temp[2])
cur_date = start_date
content = []
#this is actually a do-while loop
while True:
games = self.universal.get_games(cur_date)
print("date: " + cur_date)
for game in games:
print(" Game: " + str(game))
to_save = []
to_save.append([
cur_date, "Away", "Home", "Algo points", "Proj winner",
"Winner", "", "", "Score"
])
for x in range(0, len(games)):
#team might not exist anymore, so don't count that game
if len(games[x]['team1']) != 0 and len(games[x]['team2']) != 0:
data1 = self.universal.load_data(games[x]['team1'],
games[x]['date'])
data2 = self.universal.load_data(games[x]['team2'],
games[x]['date'])
# print("Teams: "+str(games[x]['team1'])+" | "+str(games[x]['team2']))
# print("date: "+str(games[x]['date']))
# print(data1)
# print(data1[0]['dates'][0])
# print(data1[0]['dates'][-1])
returned1 = self.odds_calculator.analyze2(
games[x]['team1'], games[x]['team2'], data1, "away")
returned2 = self.odds_calculator.analyze2(
games[x]['team2'], games[x]['team1'], data2, "home")
# print(returned1)
# print()
# print(returned2)
# print()
algo = Algo(self.league)
if self.algo_version == "Algo_V1":
algo_data = algo.calculate(games[x]['date'], returned1,
returned2)
elif self.algo_version == "Algo_V2":
algo_data = algo.calculate_V2(games[x]['date'],
returned1, returned2)
total = algo_data['total']
to_add = []
to_add.append("")
to_add.append(games[x]['team1'][0])
to_add.append(games[x]['team2'][0])
to_add.append(total)
if self.algo_version == "Algo_V1":
#categorizes odds (points)
levels = []
levels.append([0, 3])
levels.append([3, 6])
levels.append([6, 9])
levels.append([9, 12])
levels.append([12, 15])
levels.append([15, 18])
levels.append([18, 21])
levels.append([21, 24])
levels.append([24, 27])
levels.append([27, 100])
elif self.algo_version == "Algo_V2":
#categorizes odds (percentage)
levels = []
levels.append([50, 55])
levels.append([55, 60])
levels.append([60, 65])
levels.append([65, 70])
levels.append([70, 75])
levels.append([75, 80])
levels.append([80, 85])
levels.append([85, 90])
levels.append([90, 95])
levels.append([95, 100])
level = 0
for y in range(0, len(levels)):
if (total >= levels[y][0] and total < levels[y][1]
) or (total * -1 >= levels[y][0]
and total * -1 < levels[y][1]):
level = y + 1
#appends projected team
if total > 0:
to_add.append(games[x]['team1'][0])
elif total <= 0:
to_add.append(games[x]['team2'][0])
#appends winning team
if games[x]['game_scores'][0] > games[x]['game_scores'][1]:
to_add.append(games[x]['team1'][0])
else:
to_add.append(games[x]['team2'][0])
#appends score
score = str(games[x]['game_scores'][0]) + "-" + str(
games[x]['game_scores'][1])
to_add.append(score)
# #appends algo data
# if to_add[-2]==to_add[-3]:
# to_add.append("")
# if self.algo_version=="Algo_V1":
# to_add.append(str(level))
# elif self.algo_version=="Algo_V2":
# to_add.append(total)
# elif to_add[-3]!="":
# if self.algo_version=="Algo_V1":
# to_add.append(str(level))
# elif self.algo_version=="Algo_V2":
# to_add.append(total)
# to_add.append("")
# else:
# to_add.append("")
# to_add.append("")
#appends algo data
if to_add[-2] == to_add[-3]:
to_add.append("")
to_add.append(str(level))
elif to_add[-3] != "":
to_add.append(str(level))
to_add.append("")
else:
to_add.append("")
to_add.append("")
#appends betting odds
if self.algo_version == "Algo_V1":
odds_calculator = Odds_Calculator(self.league)
odds = odds_calculator.get_odds(total)
elif self.algo_version == "Algo_V2":
odds = abs(total)
favorable_odds = (100 / (100 - abs(odds)) - 1) * 100
underdog_odds = (100 / (100 - abs(odds)) - 1) * 100
if total > 0:
to_add.append("-" + str(favorable_odds))
to_add.append("+" + str(underdog_odds))
else:
to_add.append("+" + str(underdog_odds))
to_add.append("-" + str(favorable_odds))
to_save.append(to_add)
#space between data
to_save.append(["", "", "", "", "", "", "", "", ""])
#only saves day's games if there were actually games on that day
if len(to_save) > 2:
content.append(to_save)
#breaks loop to act like do-while
if cur_date == end_date:
break
day += 1
if day > 31:
month += 1
day = 1
#doesn't increment year since the season's year doesn't change
if month > 12:
month = 1
day = 1
#increments season at the end of the season to sometime in the middle
if self.league == "nba":
if "4-1-" in cur_date:
year += 1
month = 2
day = 1
elif self.league == "nhl":
if "4-1-" in cur_date:
year += 1
month = 2
day = 1
elif self.league == "mlb":
if "10-1-" in cur_date:
year += 1
month = 7
day = 1
cur_date = str(month) + "-" + str(day) + "-" + str(year)
#has most recent games first
content.reverse()
to_save = []
for x in range(0, len(content)):
for y in range(0, len(content[x])):
to_save.append(content[x][y])
if start_date != end_date:
self.universal.save_to_csv(
"./" + str(self.league) + "/analyze/" + str(self.league) +
"_" + str(self.algo_version) + "_" + str(start_date) + "_" +
str(end_date) + "_analysis.csv", to_save)
else:
self.universal.save_to_csv(
"./" + str(self.league) + "/analyze/" + str(self.league) +
"_" + str(self.algo_version) + "_" + str(end_date) +
"_analysis.csv", to_save)
def backtest_algo2(self, number, dates, algo):
#level number corresponds to an index
wins = [0] * 11
losses = [0] * 11
#creates processes
num_processes = 16
start = int(number * (len(dates) / num_processes))
end = int((number + 1) * (len(dates) / num_processes))
print()
print(number)
print(start)
print(end)
print()
#this is actually a do-while loop
for x in range(start, end):
print("At " + str(dates[x]) + " END: " + str(dates[end - 1]))
games = self.universal.get_games(dates[x])
for y in range(0, len(games)):
#team might not exist anymore, so don't count that game
if len(games[y]['team1']) != 0 and len(games[y]['team2']) != 0:
data1 = self.universal.load_data(games[y]['team1'],
games[y]['date'])
data2 = self.universal.load_data(games[y]['team2'],
games[y]['date'])
returned1 = self.odds_calculator.analyze2(
games[y]['team1'], games[y]['team2'], data1, "away")
returned2 = self.odds_calculator.analyze2(
games[y]['team2'], games[y]['team1'], data2, "home")
algorithm = Algo(self.league)
#sets algo to backtest
algorithm.algorithm[self.league] = algo
if self.algo_version == "Algo_V1":
algo_data = algorithm.calculate(
games[y]['date'], returned1, returned2)
elif self.algo_version == "Algo_V2":
algo_data = algorithm.calculate_V2(
games[y]['date'], returned1, returned2)
total = algo_data['total']
if self.algo_version == "Algo_V1":
#categorizes odds
levels = []
levels.append([0, 3])
levels.append([3, 6])
levels.append([6, 9])
levels.append([9, 12])
levels.append([12, 15])
levels.append([15, 18])
levels.append([18, 21])
levels.append([21, 24])
levels.append([24, 27])
levels.append([27, 100])
elif self.algo_version == "Algo_V2":
#categorizes odds
levels = []
levels.append([50, 55])
levels.append([55, 60])
levels.append([60, 65])
levels.append([65, 70])
levels.append([70, 75])
levels.append([75, 80])
levels.append([80, 85])
levels.append([85, 90])
levels.append([90, 95])
levels.append([95, 100])
level = 0
for z in range(0, len(levels)):
if (total >= levels[z][0] and total < levels[z][1]
) or (total * -1 >= levels[z][0]
and total * -1 < levels[z][1]):
level = z + 1
# #0 is team1, and 1 is team2
# projected_team=0
# if self.league=="nba":
# #if team1 is projected to win
# if total>0:
# projected_team=0
# #go with home team
# elif total<=0:
# projected_team=1
# else:
# #if team1 is projected to win
# if total>0:
# projected_team=0
# #go with home team
# elif total<=0:
# projected_team=1
#0 is team1, and 1 is team2
#if team1 is projected to win
if total > 0:
projected_team = 0
#go with home team
elif total <= 0:
projected_team = 1
#0 is team1, and 1 is team2
winning_team = 0
if games[y]['game_scores'][0] > games[y]['game_scores'][1]:
winning_team = 0
else:
winning_team = 1
#if algo was right
if projected_team == winning_team:
wins[level] += 1
else:
losses[level] += 1
temp = []
for x in range(0, len(wins)):
temp.append([losses[x], wins[x]])
algo_list = str(algo).replace("[", "").replace("]",
"").replace(" ", "")
path = "./" + str(self.league) + "/analyze/backtests/" + str(
algo_list) + "_temp" + str(number) + ".csv"
self.universal.save_to_csv(path, temp)
def __init__(self, grammar: Grammar):
self.grammar = grammar
self.algo = Algo()
self.algo.fit(grammar)
class Main(PygameBasics):
def __init__(self):
pygame.init()
super().__init__()
self.win_w, self.win_h = 1600, 900
self.win = pygame.display.set_mode((self.win_w, self.win_h),
pygame.RESIZABLE)
self.algo = Algo(self.win)
self.update_window_size(0) ##
## Movement flags
self.moving = False
self.draw_outer_circle = False
""" EVENTS """
def left_click_events(self):
pygame.mouse.get_rel()
self.moving = True
def right_click_events(self):
pass
def mouse_button_up_events(self):
self.moving = False
def keydown_events(self, event):
if event.key == pygame.K_a:
self.algo.update_k(-2)
if event.key == pygame.K_d:
self.algo.update_k(2)
if event.key == pygame.K_SPACE:
self.draw_outer_circle = not self.draw_outer_circle
if event.key == pygame.K_q:
pygame.quit(), quit()
def keyup_events(self, event):
pass
def update_window_size(self, event):
""" called from pygameBasicselif event.type == pygame.VIDEORESIZE: """
if event:
self.win = pygame.display.set_mode((event.w, event.h),
pygame.RESIZABLE)
## When user resizes the window, adjust the w and h values everything else is anchored to
self.win_w = self.win.get_width()
self.win_h = self.win.get_height()
self.algo.update_pixel_anchors(self.win_w, self.win_h)
self.algo.configure()
self.algo.configure_draw()
""" UPDATES """
def updates(self):
self.update_window_size(False)
self.algo.update(self.moving, self.draw_outer_circle)
self.draw()
def draw(self):
self.win.fill(self.set.white)
self.draw_page_border()
self.algo.draw_things(self.win_w, self.win_h)
pygame.display.update()
""" MAIN """
def main(self):
while True:
self.win.fill(self.set.background)
self.set.clock.tick(self.set.FPS)
self.events()
self.updates()
def main(argv):
# Retreive settings from JSON settings file
with open(SMART_DRIVER) as driver:
jsonDataFile = json.load(driver)
granularity = int(jsonDataFile['granularity'])
training_window = int(jsonDataFile['windowSize'])
forecasting_interval = int(jsonDataFile['forecastingInterval'])
print ("\nStarting analysis on database with settings %d %d %d..."
% (granularity, training_window, forecasting_interval))
granularity_in_seconds = granularity * 60
# Initialize database
database = Database(DB_CONFIG)
# Get the list of feature numbers
id_list = getListIDs(jsonDataFile["idSelection"])
id_list = list(set(id_list)) # Remove duplicates
id_list.sort()
# Determine the range of times to pull data from
# If the user specified a timeframe, use that
if(int(jsonDataFile["specifyTime"])):
start_time = dt.datetime.strptime(jsonDataFile["beginTime"], DATE_FORMAT)
end_time = dt.datetime.strptime(jsonDataFile["endTime"], DATE_FORMAT)
# Otherwise, find the largest timeframe for which each feature has data
else:
start_time, end_time = getStartEndTimes(id_list)
print "Start, end: ", start_time, end_time
# Get the list of column headers for the features
columns = []
for id in id_list:
columns.append(jsonDataFile['data'][id-1]['columnName'])
columns.append(jsonDataFile['totalConsum'])
#print "The following features were found:", columns
# Algorithm settings
algo = Algo(granularity, training_window, forecasting_interval, len(columns)-1)
# Output lists
y_time = ['Timestamp']
y_target = ['Target']
y_predict = ['Prediction']
anomalies = ['Anomaly']
count = 0
# EWMA additions
# alpha is adjustable on a scale of (0, 1]
# The smaller value of alpha, the more averaging takes place
# A value of 1.0 means no averaging happens
#alpha = float(raw_input('Enter Value of alpha:'))
algo.setEMAParameter(alpha=1.0)
#algo.setEMAParameter(alpha=0.7)
#Recomended Severity Parameters from Paper
#algo.setSeverityParameters(w=0.53, L=3.714) # Most sensitive
#algo.setSeverityParameters(w=0.84, L=3.719) # Medium sensitive
#algo.setSeverityParameters(w=1.00, L=3.719) # Least sensitive
algo.setSeverityParameters(w=1, L=3.719) # Custom senstivity
detected = set()
ground_truth = set()
#==================== ANALYSIS ====================#
print "Beginning analysis..."
while start_time < end_time:
# FOR SMART* ONLY
# Some of the data seems bad on the 31st - too many NULLS
if (start_time > dt.datetime(2012, 5, 30) and
start_time < dt.datetime(2012, 6, 1)):
start_time = dt.datetime(2012, 6, 1)
if(count % 240 == 0):
print "trying time: %s " % start_time
count += 1
#Execute the query:
stop_time = start_time + dt.timedelta(0, granularity_in_seconds)
new_data = database.get_avg_data(start_time, stop_time, columns)
new_data = np.asarray([max(0, data) for data in new_data]) # remove 'nan' and negative
target, prediction = algo.run(new_data) # Magic!
if prediction != None:
y_time.append(start_time)
y_target.append(target)
y_predict.append(float(prediction))
if algo.checkSeverity(target, float(prediction)):
detected.add(start_time)
anomalies.append(1)
else:
anomalies.append(0)
start_time = stop_time #Increment and loop
#==================== RESULTS ====================#
# Save data for later graphing
results = y_time, y_target, y_predict, anomalies
writeResults(outfile, results)
f1_scores(detected, ground_truth)
print_stats(y_target[1:], y_predict[1:]) #Remove header
print "Ending analysis. See %s for results." % outfile
return results
"""
self._event_engine.unregister(algo)
if algo.config.launch_type == 'unactive':
self._event_manage.remove_Config(algo.config)
def deploy_algo(self):
pass
if __name__ == '__main__':
from BaseConfig import BaseConfig
from BaseAlgo import BaseAlgo
from algo import Algo
mainengin = MainEngine()
mainengin.start()
algo = Algo(BaseAlgo, BaseConfig)
mainengin.add_algo(algo)
mainengin.create_algo_instance('test_event', {
'id': 5,
'hello': 'world',
'haha': 'heihei'
})
for i in xrange(100):
mainengin.create_algo_instance('test_event', {
'id': 5,
i * 5: i,
i * 2: 'saa'
})
# mainengin.stop()
def main(argv):
# Time the analysis:
tic = time.time()
# Get commandline arguments
try:
infile = argv[1]
granularity = int(argv[2])
training_window = int(argv[3])
forecasting_interval = int(argv[4])
except Exception:
raise RuntimeError("usage: python "
+ argv[0]
+ " <infile>"
+ " <granularity>"
+ " <training_window>"
+ " <forecasting_interval>")
print ("\nStarting analysis on %s with settings %d %d %d..."
% (infile, granularity, training_window, forecasting_interval))
# Generate outfile name based on infile
outfile = infile.rstrip('.csv') + '.results.csv'
# Separate feature timestamps, feature data, and power data
dataframe = pd.read_csv(infile)
timestamps = dataframe.ix[:, 0].values
targets = dataframe.ix[:, -1].values
features_df = dataframe.ix[:, 1:-1]
# Filter features with little or no variance
print "\nRemoving features due to low variance:"
for column in features_df.columns:
values = features_df[column].values
if (values.max() == values.min()):
features_df = features_df.drop(column, 1)
print column
#dataframe = dataframe.drop('Audio Sensor', 1)
#dataframe = dataframe.drop('Luminescence_11 (Lux)', 1)
features_df = features_df[['Audio Sensor',
'Temperature_10 (C)']]
print "\nThe following features will be used: "
for column_name in features_df.columns:
print column_name
features = features_df.values
features = scaleFeatures(features)
num_features = features.shape[1]
# Algorithm settings
algo = Algo(granularity, training_window, forecasting_interval, num_features)
# Output lists
y_time = ['Timestamp']
y_target = ['Target']
y_predict = ['Prediction']
anomalies = ['Anomaly']
print "Algorithm settings:"
# EWMA additions
# alpha is adjustable on a scale of (0, 1]
# The smaller value of alpha, the more averaging takes place
# A value of 1.0 means no averaging happens
#alpha = float(raw_input('Enter Value of alpha:'))
algo.setEMAParameter(alpha=1.0)
#algo.setEMAParameter(alpha=0.75)
#algo.setEMAParameter(alpha=0.5)
#algo.setEMAParameter(alpha=0.25)
# Recomended Severity Parameters from Paper
#algo.setSeverityParameters(w=0.53, L=3.714) # Most sensitive
#algo.setSeverityParameters(w=0.84, L=3.719) # Medium sensitive
#algo.setSeverityParameters(w=1.00, L=3.719) # Least sensitive
algo.setSeverityParameters(w=1, L=2) # Custom senstivity
# More settings
algo.setMaxPrediction(500)
#USED For F1 Calculation
detected = set()
ground_truth = set()
#==================== ANALYSIS ====================#
print "\nBeginning analysis..."
count = 0
for row in features:
# Update the user with the current timestamp
cur_time = timestamps[count]
if (count % 360) == 0:
cur_dt = dt.datetime.fromtimestamp(cur_time)
print "Trying time %s" % cur_dt.strftime(DATE_FORMAT)
row = np.append(row, targets[count])
target, prediction = algo.run(row) # Magic!
# If there is a prediction, check for anomalies
if prediction != None:
y_time.append(cur_time)
y_target.append(target)
y_predict.append(float(prediction))
if algo.checkSeverity(target, float(prediction)):
detected.add(cur_time)
anomalies.append(1)
else:
anomalies.append(0)
# NOTE: If you plan on injecting attacks, you can add the
# timestamps here to create a "ground truth" set, which can then
# be used to calculate the accuracy, precision and F1 scores.
#if cur_time >= 1338696000 and cur_time <= 1338699600:
# ground_truth.add(cur_time)
#elif cur_time >= 1338955200 and cur_time <= 1338958800:
# ground_truth.add(cur_time)
#elif cur_time >= 1339128000 and cur_time <= 1339131600:
# ground_truth.add(cur_time)
# Loop back to the beginning
count += 1
#==================== RESULTS ====================#
# Save data for later graphing
results = y_time, y_target, y_predict, anomalies
writeResults(outfile, results)
#f1_scores(detected, ground_truth) #Uncomment to show F1 scores
print_stats(y_target[1:], y_predict[1:]) #Remove header
print "Run time: ", time.time() - tic
print "Ending analysis. See %s for results." % outfile
return results
