def handleDetail(code,date,parser , conf):
url=getUrl(num = code,date = date,conf=conf)
fetchData=Fetcher.fetch(url)
parseData=parser.parse(fetchData)
path = Dumper.getPath(code = code , date = date ,dataType = "detail" ,conf = conf)
Dumper.dump(path = path,data = parseData)
return True
def handle(self,code,date):
url=self.getUrl(code)
rawData = Fetcher.fetch(url)
parseData = self.parser.parse(rawData)
path = Dumper.getPath(code = code , date = date ,dataType = "instant" ,conf = self.conf)
Dumper.dump(data = parseData , path = path)
return True
def showRoom(year, month, date, room):
if (not re.match('^[a-zA-Z]{1,2}\\d+[a-zA-Z]?$', room)):
return jsonify(errMsg='Invalid room'), 500
try:
scheduleList = Cache.readCache(year, month, date)
except IOError:
html = Fetcher.fetch_html(int(year), int(month), int(date),
int("0830"), int("2200"))
scheduleList = Fetcher.parseHTML2List(html)
Cache.saveCache(year, month, date, scheduleList)
response = Scheduler.selectRoom(scheduleList, room)
res = jsonify(response)
res.mimetype = 'application/json'
res.headers['Access-Control-Allow-Origin'] = '*'
return res
def searchRoom(year, month, date, start_time, end_time):
# Create the response
try:
scheduleList = Cache.readCache(year, month, date)
except IOError:
html = Fetcher.fetch_html(int(year), int(month), int(date),
int("0830"), int("2200"))
scheduleList = Fetcher.parseHTML2List(html)
Cache.saveCache(year, month, date, scheduleList)
response = Scheduler.filterTable(scheduleList, int(start_time),
int(end_time))
# Write the response
res = jsonify(response)
res.mimetype = 'application/json'
res.headers['Access-Control-Allow-Origin'] = '*'
return res
def download(self):
"""
>>> info=Info(conf)
>>> info.download()
"""
source = self.conf.get("SOURCE_NAME",None) or SOURCE_NAME
parser = Parser.ParserFactory(source)
res=[]
for name in self.source:
url=self.source.get(name)
rawData = Fetcher.fetch(url)
parseData = parser.parse(rawData)
if parseData:
res=res+parseData
path = os.path.join(self.conf["INFO_DATA_PATH"],"info.json")
logging.debug("download info list:" + str(res))
Dumper.dump(parseData,path) #获得信息列表
codes=list(set( #并对代码去重 \
map(lambda item:item['code'],res) #获得代码列表 \
))
codes.sort() #排序
path = os.path.join(self.conf["INFO_DATA_PATH"],self.conf.get("CODE_LIST_FILE_NAME",CODE_LIST_FILE_NAME))
logging.debug("download code list:"+str(codes))
Dumper.dump(codes,path)
def get_interest(uid, username, passwd):
'''
uid:用户id,unicode
得到uid用户所发微博中的前20高频词
'''
with codecs.open('stop_words.txt', 'r', 'utf-8') as fr:
stop_words = [line.strip() for line in fr]
stop_words.append(' ')
stop_words.append('\n')
stop_words.append('\t')
myFetcher = Fetcher(username, passwd)
myFetcher.login()
follows_list = myFetcher.get_user_follows(uid)
follows_list.append(uid)
fans_list = myFetcher.get_user_fans(uid)
print len(follows_list)
print len(fans_list)
follows_keywords = get_keywords(myFetcher, follows_list)
follows_interest = {}
for word in follows_keywords:
follows_interest[word[0]] = word[1]
fans_keywords = get_keywords(myFetcher, fans_list)
fans_interest = {}
for word in fans_keywords:
fans_interest[word[0]] = word[1]
user_weibos = myFetcher.get_user_weibos(uid)
weibos = ".".join(user_weibos)
content_interest = {} #从用户发布的微博信息中提取的兴趣关键词
words = jieba.cut(weibos)
filtered_words = [
word for word in words if word not in stop_words and len(word) > 1
]
all_words_count = float(len(filtered_words))
counter = Counter(filtered_words)
key_words = counter.most_common(30)
outputs = []
for item in key_words:
if isinstance(item[0], unicode):
k_word = item[0].decode('utf-8')
weight = float(item[1]) / all_words_count
else:
k_word = item[0]
weight = float(item[1]) / all_words_count
outputs.append("%s\t%f\n" % (k_word, weight))
content_interest[k_word] = weight
#对两类兴趣词的权重进行归一化
max_weight_content = max(content_interest.values())
max_weight_follows = max(follows_interest.values())
max_weight_fans = max(fans_interest.values())
for word1, weight1 in content_interest.iteritems():
weight1 /= max_weight_content
for word2, weight2 in follows_interest.iteritems():
weight2 /= max_weight_follows
for word3, weight3 in fans_interest.iteritems():
weight3 /= max_weight_fans
interest_words = {}
all_words = follows_interest.keys() + content_interest.keys(
) + fans_interest.keys()
for word in all_words:
content_weight = content_interest.get(word, 0)
follows_weight = follows_interest.get(word, 0)
fans_weight = fans_interest.get(word, 0)
all_weight = 0.5 * follows_weight + 0.4 * content_weight + 0.1 * fans_weight
interest_words[word] = all_weight
sorted_words = sorted(interest_words.iteritems(),
key=lambda (k, v): v,
reverse=True)
for item in sorted_words[:30]:
print item[0].encode('utf-8', ''), item[1]
def main():
#Matches I want to check today, in the format:
# countryName - String, leagueName - String, howmany - Integer
todaysMatches = [
#["Norway", "Eliteserien", 6],
#["Hungary", "NB II", 2],
#["Brazil", "Serie A", 2],
#["Chile", "Primera Division", 2],
#["Brazil", "Serie B", 7],
["Ecuador", "Serie A", 1],
#["England", "National League - South", 2],
#["Denmark", "Superligaen", 3],
#["Poland", "I Liga", 1],
["Norway", "OBOS-ligaen", 2],
#["Denmark", "Viasat Divisionen", 2],
#["Iran", "Azadegan League", 9],
#["Qatar", "2nd Division League", 4],
#["Saudi-Arabia", "Division 1", 5],
#["England", "National League - North", 4],
#["Tunisia", "Ligue Professionnelle 1", 3],
#["India", "I-League", 2],
#["Morocco", "Botola Pro", 2],
#["Algeria", "Ligue 1", 5],
#["Algeria", "Ligue 2", 7],
#["Hungary", "NB I", 2],
["United-Arab-Emirates", "Arabian Gulf League", 3],
#["Poland", "Ekstraklasa", 1],
#["Iran", "Persian Gulf Cup", 1],
#["South-Africa", "Premier Soccer League", 3],
#["Qatar", "Stars League", 2],
#["Saudi-Arabia", "Pro League", 3],
#["Switzerland", "Challenge League", 2],
#["Spain", "Segunda B - Group 1", 6],
#["Spain", "Segunda B - Group 2", 5],
#["Spain", "Segunda B - Group 3", 6],
#["Spain", "Segunda B - Group 4", 9],
#["Israel", "Liga Leumit", 5],
#["Italy", "Serie C", 1],
#["France", "Ligue 2", 1],
#["France", "Ligue 1", 2],
#["France", "National", 4],
#["Switzerland", "Super League", 2],
#["Northern-Ireland", "Championship", 2],
#["Northern-Ireland", "Premiership", 4],
#["Scotland", "League One", 5],
#["Scotland", "League Two", 5],
#["Portugal", "Liga de Honra", 1],
#["Portugal", "Primeira Liga", 1],
#["Egypt", "Premier League", 1],
#["Cyprus", "1. Division", 2],
#["Germany", "Bundesliga 2", 1],
#["Germany", "Bundesliga 1", 1],
#["Germany", "Liga 3", 1],
#["Spain", "Primera Division", 1],
#["Spain", "Segunda Division", 1],
#["Egypt", "Premier League", 1],
#["Belgium", "Jupiler Pro League", 1],
#["Israel", "Ligat ha'Al", 1],
#["Netherlands", "Eerste Divisie", 1],
#["Netherlands", "Eredivisie", 1],
#["Scotland", "Championship", 1],
#["Scotland", "Premiership", 5],
#["Italy", "Serie A", 1],
#["Italy", "Serie B", 1],
#["Greece", "Super League", 4],
#["India", "Indian Super League", 1],
#["England", "Championship", 6],
#["England", "League One", 4],
#["England", "League Two", 2],
#["England", "National League", 8],
#["England", "Premier League", 1],
#["Oman", "Professional League", 2],
#["Turkey", "Super Lig", 2],
["Turkey", "TFF 1. Lig", 4],
#["Turkey", "TFF 2. Lig", 15],
#["Australia", "A-League", 1],
]
bettingLabels = []
bettingLabelsBTTS = []
bettingLabelsDraw = []
bettingLabelsDC = []
#Should tell me which matches were skipped - No odds were available on the market for it
skippedMatches = []
for match in todaysMatches:
#time.sleep(2)
upcomingLeagueFixtures = Fetch.getUpcomingFixturesForLeague(match[0], match[1], match[2])
for fixture in upcomingLeagueFixtures:
allLabels = Predict.predictOne(match[0], match[1], fixture[0], fixture[1])
bettingLabel = allLabels[0]
bettingLabelBTTS = allLabels[1]
bettingLabelDraw = allLabels[2]
bettingLabelDC = allLabels[3]
marketOdds, marketOddsBTTS = Fetch.getScoringOdds(fixture[2])
homeDCOdds, awayDCOdds = Fetch.getDCOdds(fixture[2])
#print("GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG")
#print(marketOdds)
bettingLabelDC.insert(4, homeDCOdds)
bettingLabelDC.insert(7, awayDCOdds)
#if (homeDCOdds == 0.0 or awayDCOdds == 0.0):
#bettingLabelsDC.append(bettingLabelDC)
#continue
#-------------------------Double chance insertions here
print("{} |||||||| {}".format(homeDCOdds, bettingLabelDC[3]))
print("{} |||||||| {}".format(awayDCOdds, bettingLabelDC[6]))
bettingLabelDC.insert(5, float(homeDCOdds) - float(bettingLabelDC[3]))
bettingLabelDC.insert(8, float(awayDCOdds) - float(bettingLabelDC[6]))
bettingLabelsDC.append(bettingLabelDC)
#if not marketOdds, will detect if list is empty! If not available, then dont make a label for that fixture
#if not marketOdds:
#skippedMatches.append("{} vs {} Was skipped(No market odds)".format(fixture[0], fixture[1]))
#continue
#bettingLabel.insert(4, marketOdds[0])
#bettingLabel.insert(5, float(marketOdds[0]) - bettingLabel[3])
#Inserting bet amount here based on kelly criterion formula
#betAmount = Kelly.kellyCriterion(bankroll, float(marketOdds[0]), Utils.fromOddsToDecimalProb(bettingLabel[3]), 0.125)
#bettingLabel.insert(6, betAmount)
#print(marketOdds)
#if (len(marketOdds) != 0):
#bettingLabel.insert(5, marketOdds[1])
#else:
#bettingLabel.insert(5, 0)
#bettingLabel.insert(9, float(marketOdds[1]) - bettingLabel[7])
#bettingLabel.insert(6, marketOdds[3])
#if (len(marketOdds) != 0):
#bettingLabel.insert(5, marketOdds[3])
#else:
#bettingLabel.insert(5, 0)
#bettingLabel.insert(9, float(marketOdds[2]) - bettingLabel[7])
#betAmount = Kelly.kellyCriterion(bankroll, float(marketOdds[2]), Utils.fromOddsToDecimalProb(bettingLabel[7]), 0.125)
#bettingLabel.insert(10, betAmount)
#bettingLabel.insert(15, marketOdds[3])
#bettingLabel.insert(16, float(marketOdds[3]) - bettingLabel[14])
#bettingLabelBTTS.insert(4, marketOddsBTTS[0])
#bettingLabelBTTS.insert(5, float(marketOddsBTTS[0]) - bettingLabelBTTS[3])
#bettingLabelBTTS.insert(7, marketOddsBTTS[1])
#bettingLabelBTTS.insert(8, float(marketOddsBTTS[1]) - bettingLabelBTTS[6])
bettingLabelsBTTS.append(bettingLabelBTTS)
bettingLabels.append(bettingLabel)
bettingLabelsDraw.append(bettingLabelDraw)
#time.sleep(0.1)
#print(bettingLabelsDC)
#Printing first results:
Terminal.display(bettingLabels, bettingLabelHeaders)
Terminal.display(bettingLabelsBTTS, bettingLabelBTTSHeaders)
Terminal.display(bettingLabelsDraw, bettingLabelDrawHeaders)
Terminal.display(bettingLabelsDC, bettingLabelDCHeaders)
#Gathering absolutely all labels generated to sed to Terminal Module
labels = [bettingLabels, bettingLabelsBTTS, bettingLabelsDraw, bettingLabelsDC]
Terminal.getAllLabels(labels)
labelHeaders = [bettingLabelHeaders, bettingLabelBTTSHeaders, bettingLabelDrawHeaders, bettingLabelDCHeaders]
Terminal.getAllLabelsHeaders(labelHeaders)
print(skippedMatches)
Terminal.appLoop()
import matplotlib
import cv2
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
import numpy as np
import argparse
import pickle
import os
import math
import tensorflow as tf
import Fetcher
import Model
import Config
from tqdm.autonotebook import tqdm
fetcher = Fetcher.DataFetcher();
#fetcher.processAndSave();
#fetcher.loadGestures(gestures= ["1","2"]);
fetcher.load();
model = Model.Net(numBatchTrain = 0);
model.network.summary();
model.load_checkpoint();
numSamples = 10;
for i in range(numSamples):
image, labels = fetcher.getRandomValidation();
def get_interest(uid,username,passwd):
'''
uid:用户id,unicode
得到uid用户所发微博中的前20高频词
'''
with codecs.open('stop_words.txt','r','utf-8') as fr:
stop_words = [line.strip() for line in fr]
stop_words.append(' ')
stop_words.append('\n')
stop_words.append('\t')
myFetcher = Fetcher(username,passwd)
myFetcher.login()
follows_list = myFetcher.get_user_follows(uid)
follows_list.append(uid)
fans_list = myFetcher.get_user_fans(uid)
print len(follows_list)
print len(fans_list)
follows_keywords = get_keywords(myFetcher,follows_list)
follows_interest = {}
for word in follows_keywords:
follows_interest[word[0]] = word[1]
fans_keywords = get_keywords(myFetcher,fans_list)
fans_interest = {}
for word in fans_keywords:
fans_interest[word[0]] = word[1]
user_weibos = myFetcher.get_user_weibos(uid)
weibos = ".".join(user_weibos)
content_interest = {}#从用户发布的微博信息中提取的兴趣关键词
words = jieba.cut(weibos)
filtered_words = [word for word in words if word not in stop_words and len(word) > 1]
all_words_count = float(len(filtered_words))
counter = Counter(filtered_words)
key_words = counter.most_common(30)
outputs = []
for item in key_words:
if isinstance(item[0],unicode):
k_word = item[0].decode('utf-8')
weight = float(item[1])/all_words_count
else:
k_word = item[0]
weight = float(item[1])/all_words_count
outputs.append("%s\t%f\n"%(k_word,weight))
content_interest[k_word] = weight
#对两类兴趣词的权重进行归一化
max_weight_content = max(content_interest.values())
max_weight_follows = max(follows_interest.values())
max_weight_fans = max(fans_interest.values())
for word1,weight1 in content_interest.iteritems():
weight1 /= max_weight_content
for word2,weight2 in follows_interest.iteritems():
weight2 /= max_weight_follows
for word3,weight3 in fans_interest.iteritems():
weight3 /= max_weight_fans
interest_words = {}
all_words = follows_interest.keys() + content_interest.keys() + fans_interest.keys()
for word in all_words:
content_weight = content_interest.get(word,0)
follows_weight = follows_interest.get(word,0)
fans_weight = fans_interest.get(word,0)
all_weight = 0.5*follows_weight + 0.4*content_weight + 0.1*fans_weight
interest_words[word] = all_weight
sorted_words = sorted(interest_words.iteritems(),key=lambda (k,v):v,reverse=True)
for item in sorted_words[:30]:
print item[0].encode('utf-8',''),item[1]
def fetch(self):
"""
从队列中获取地址并查询内容
:return: None
"""
log.info("Running thread %s" % threading.current_thread().name)
while True:
try:
cur_url, cur_depth = self._url_queue.get(timeout=1)
print self._url_queue.unfinished_tasks
cur_url = cur_url.strip()
except Queue.Empty as e:
log.warn(e)
continue
fetch_tool = Fetcher.Fetcher(cur_url, self._output_dir,
self._timeout, self._cookie)
if self.pro_match.match(cur_url):
# 产品详情页
self.save_product_info_url(cur_url)
content = fetch_tool.read_content()
info = fetch_tool.get_product_info(content)
product_info = []
[product_info.append(item) for item in info]
if self._lock.acquire():
self._url_visited.append(cur_url)
self._lock.release()
self.save_product_info(product_info)
elif self.fig_pattern.match(cur_url):
# 产品大类
content = fetch_tool.read_content()
if content is None or len(content) == 0:
continue
# 获取大类下的子类页面
sub_urls = fetch_tool.get_sub_urls(content)
if sub_urls is None:
continue
for item in sub_urls:
if self._lock.acquire(): # lock _url_visited, check
if item in self._url_visited:
continue
self._lock.release()
try:
if self._lock.acquire(): # lock _url_visited, add
self._url_visited.append(item)
self._lock.release()
self._url_queue.put([item, cur_depth + 1], timeout=1)
except Queue.Full as e:
log.warn(e)
break
else:
# 子类页面， 获取产品详情地址
content = fetch_tool.read_content()
product_urls = fetch_tool.get_product_url(content)
if product_urls is None:
continue
for item in product_urls:
if self._lock.acquire(): # lock _url_visited, check
if item in self._url_visited:
self._lock.release()
continue
try:
self._url_visited.append(item)
self._url_queue.put([item, cur_depth + 1],
timeout=1)
except Queue.Full as e:
log.warn(e)
break
finally:
self._lock.release()
self._url_queue.task_done()
def predictOne(countryName, leagueName, homeTeam, awayTeam):
"""
Predicts The chance of homeTeam and awayTeam to score 1 goal in the first half.
Arguments:
countryName: String
leagueName: String
homeTeam: String
awayTeam: String
Returns: list labels, other lists of bettinglabels.
"""
#Represent all labels, will contain other lists(f.ex bettingLabelDraw)
labels = []
homeTeamFixtures = Fetch.getPastFixtures(countryName, leagueName, homeTeam)
awayTeamFixtures = Fetch.getPastFixtures(countryName, leagueName, awayTeam)
homeDCChance, awayDCChance = calculateWinDrawChance(
homeTeam, awayTeam, homeTeamFixtures, awayTeamFixtures)
d = drawPredict2(homeDCChance, awayDCChance)
if (d is not 1.0):
bettingLabelDraw = [leagueName, homeTeam, awayTeam, d]
#if (awayDCChance < 0.001):
#awayDCChance = 0.001
#if (homeDCChance < 0.001):
#homeDCChance = 0.001
#print(homeTeamFixtures)
homeScoring, homeConceding = calculateScoringChance(
homeTeamFixtures, "home", homeTeam)
awayScoring, awayConceding = calculateScoringChance(
awayTeamFixtures, "away", awayTeam)
#print(homeTeamFixtures)
#print("homescoring: {}".format(homeScoring))
#Home team odds of scoring in decimal
homeDOddsOfScoring = (homeScoring + awayConceding) / 2.0
awayDOddsOfScoring = (awayScoring + homeConceding) / 2.0
#Home team Percentage odds of scoring at least 1 goal
homePOddsOfScoring = Utils.fromDecimalProbToPercentage(homeDOddsOfScoring)
awayPOddsOfScoring = Utils.fromDecimalProbToPercentage(awayDOddsOfScoring)
awayPOddsOfCleanSheet = round(100 - homePOddsOfScoring)
homePOddsOfCleanSheet = round(100 - awayPOddsOfScoring)
#print( "PPPPPP" + str(awayPOddsOfCleanSheet) )
#These 2 if statements are simply to prevent ZeroDivisionError (Very rare that these 2 variables are 0)
if (awayPOddsOfCleanSheet == 0):
awayPOddsOfCleanSheet = 1
if (homePOddsOfCleanSheet == 0):
homePOddsOfCleanSheet = 1
#Converting to european betting odds
homeBettingOddsOfScoring = round(
Utils.fromPercentageToOdds(homePOddsOfScoring), 3)
awayBettingOddsOfScoring = round(
Utils.fromPercentageToOdds(awayPOddsOfScoring), 3)
homeBettingOddsOfCleanSheet = round(
Utils.fromPercentageToOdds(homePOddsOfCleanSheet), 3)
awayBettingOddsOfCleanSheet = round(
Utils.fromPercentageToOdds(awayPOddsOfCleanSheet), 3)
#noDraw, yesDraw = drawPredict(homeDOddsOfScoring, awayDOddsOfScoring)
#bettingLabelDraw = [leagueName, homeTeam, awayTeam, homePOddsOfScoring, awayPOddsOfScoring, noDraw, yesDraw, homePOddsOfScoring - awayPOddsOfScoring]
bttsYes, bttsNo = bttsPredict(homeDOddsOfScoring, awayDOddsOfScoring)
bettingLabelBTTS = [leagueName, homeTeam, awayTeam, bttsYes, bttsNo]
bettingLabelDC = [
leagueName, homeTeam, awayTeam,
round(Utils.fromDecimalProbToOdds(homeDCChance), 2),
round(Utils.fromDecimalProbToOdds(awayDCChance), 2)
]
#Change 3. feb: Taking out all under-bets, only want over0.5, betting label too big...
bettingLabel = [
leagueName, homeTeam, awayTeam, homeBettingOddsOfScoring,
awayBettingOddsOfCleanSheet, awayBettingOddsOfScoring,
homeBettingOddsOfCleanSheet
]
print(
"{} to score against opponent is: {}% Betting odds: {} - Under 0.5: {}"
.format(homeTeam, homePOddsOfScoring, homeBettingOddsOfScoring,
awayBettingOddsOfCleanSheet))
print(
"{} to score against opponent is: {}% Betting odds: {} - Under 0.5: {}"
.format(awayTeam, awayPOddsOfScoring, awayBettingOddsOfScoring,
homeBettingOddsOfCleanSheet))
labels.append(bettingLabel)
labels.append(bettingLabelBTTS)
labels.append(bettingLabelDraw)
labels.append(bettingLabelDC)
return labels
from Models import *
import tensorflow as tf
import Config
import pickle
import Fetcher
import cv2
import math
from matplotlib import pyplot as pp
import pydot
from tqdm.autonotebook import tqdm
tf.config.optimizer.set_jit(True)
data = Fetcher.DataFetcher(imagePath="Dataset/fundus",
heightmapPath="Dataset/heightmap")
data.load()
batch_count_train = int(np.ceil(data.trainDataSize / Config.BATCH_SIZE))
batch_count_valid = int(np.ceil(data.validDataSize / Config.BATCH_SIZE))
batch_count_test = int(np.ceil(data.testDataSize / Config.BATCH_SIZE))
gen_to_disc_ratio = 3
net = FundusNet(numBatchTrain=int(batch_count_train) * gen_to_disc_ratio)
net.generator.summary()
net.generator.load_weights(filepath="ckpt_weights1\\weights", by_name=True)
startEpoch = net.load_checkpoint()
# Define loss and accuracy per epoch
all_loss = np.zeros(batch_count_train, dtype='float32')
