def handleDetail(code,date,parser , conf): url=getUrl(num = code,date = date,conf=conf) fetchData=Fetcher.fetch(url) parseData=parser.parse(fetchData) path = Dumper.getPath(code = code , date = date ,dataType = "detail" ,conf = conf) Dumper.dump(path = path,data = parseData) return True
def handle(self,code,date): url=self.getUrl(code) rawData = Fetcher.fetch(url) parseData = self.parser.parse(rawData) path = Dumper.getPath(code = code , date = date ,dataType = "instant" ,conf = self.conf) Dumper.dump(data = parseData , path = path) return True
def showRoom(year, month, date, room): if (not re.match('^[a-zA-Z]{1,2}\\d+[a-zA-Z]?$', room)): return jsonify(errMsg='Invalid room'), 500 try: scheduleList = Cache.readCache(year, month, date) except IOError: html = Fetcher.fetch_html(int(year), int(month), int(date), int("0830"), int("2200")) scheduleList = Fetcher.parseHTML2List(html) Cache.saveCache(year, month, date, scheduleList) response = Scheduler.selectRoom(scheduleList, room) res = jsonify(response) res.mimetype = 'application/json' res.headers['Access-Control-Allow-Origin'] = '*' return res
def searchRoom(year, month, date, start_time, end_time): # Create the response try: scheduleList = Cache.readCache(year, month, date) except IOError: html = Fetcher.fetch_html(int(year), int(month), int(date), int("0830"), int("2200")) scheduleList = Fetcher.parseHTML2List(html) Cache.saveCache(year, month, date, scheduleList) response = Scheduler.filterTable(scheduleList, int(start_time), int(end_time)) # Write the response res = jsonify(response) res.mimetype = 'application/json' res.headers['Access-Control-Allow-Origin'] = '*' return res
def download(self): """ >>> info=Info(conf) >>> info.download() """ source = self.conf.get("SOURCE_NAME",None) or SOURCE_NAME parser = Parser.ParserFactory(source) res=[] for name in self.source: url=self.source.get(name) rawData = Fetcher.fetch(url) parseData = parser.parse(rawData) if parseData: res=res+parseData path = os.path.join(self.conf["INFO_DATA_PATH"],"info.json") logging.debug("download info list:" + str(res)) Dumper.dump(parseData,path) #获得信息列表 codes=list(set( #并对代码去重 \ map(lambda item:item['code'],res) #获得代码列表 \ )) codes.sort() #排序 path = os.path.join(self.conf["INFO_DATA_PATH"],self.conf.get("CODE_LIST_FILE_NAME",CODE_LIST_FILE_NAME)) logging.debug("download code list:"+str(codes)) Dumper.dump(codes,path)
def get_interest(uid, username, passwd): ''' uid:用户id,unicode 得到uid用户所发微博中的前20高频词 ''' with codecs.open('stop_words.txt', 'r', 'utf-8') as fr: stop_words = [line.strip() for line in fr] stop_words.append(' ') stop_words.append('\n') stop_words.append('\t') myFetcher = Fetcher(username, passwd) myFetcher.login() follows_list = myFetcher.get_user_follows(uid) follows_list.append(uid) fans_list = myFetcher.get_user_fans(uid) print len(follows_list) print len(fans_list) follows_keywords = get_keywords(myFetcher, follows_list) follows_interest = {} for word in follows_keywords: follows_interest[word[0]] = word[1] fans_keywords = get_keywords(myFetcher, fans_list) fans_interest = {} for word in fans_keywords: fans_interest[word[0]] = word[1] user_weibos = myFetcher.get_user_weibos(uid) weibos = ".".join(user_weibos) content_interest = {} #从用户发布的微博信息中提取的兴趣关键词 words = jieba.cut(weibos) filtered_words = [ word for word in words if word not in stop_words and len(word) > 1 ] all_words_count = float(len(filtered_words)) counter = Counter(filtered_words) key_words = counter.most_common(30) outputs = [] for item in key_words: if isinstance(item[0], unicode): k_word = item[0].decode('utf-8') weight = float(item[1]) / all_words_count else: k_word = item[0] weight = float(item[1]) / all_words_count outputs.append("%s\t%f\n" % (k_word, weight)) content_interest[k_word] = weight #对两类兴趣词的权重进行归一化 max_weight_content = max(content_interest.values()) max_weight_follows = max(follows_interest.values()) max_weight_fans = max(fans_interest.values()) for word1, weight1 in content_interest.iteritems(): weight1 /= max_weight_content for word2, weight2 in follows_interest.iteritems(): weight2 /= max_weight_follows for word3, weight3 in fans_interest.iteritems(): weight3 /= max_weight_fans interest_words = {} all_words = follows_interest.keys() + content_interest.keys( ) + fans_interest.keys() for word in all_words: content_weight = content_interest.get(word, 0) follows_weight = follows_interest.get(word, 0) fans_weight = fans_interest.get(word, 0) all_weight = 0.5 * follows_weight + 0.4 * content_weight + 0.1 * fans_weight interest_words[word] = all_weight sorted_words = sorted(interest_words.iteritems(), key=lambda (k, v): v, reverse=True) for item in sorted_words[:30]: print item[0].encode('utf-8', ''), item[1]
def main(): #Matches I want to check today, in the format: # countryName - String, leagueName - String, howmany - Integer todaysMatches = [ #["Norway", "Eliteserien", 6], #["Hungary", "NB II", 2], #["Brazil", "Serie A", 2], #["Chile", "Primera Division", 2], #["Brazil", "Serie B", 7], ["Ecuador", "Serie A", 1], #["England", "National League - South", 2], #["Denmark", "Superligaen", 3], #["Poland", "I Liga", 1], ["Norway", "OBOS-ligaen", 2], #["Denmark", "Viasat Divisionen", 2], #["Iran", "Azadegan League", 9], #["Qatar", "2nd Division League", 4], #["Saudi-Arabia", "Division 1", 5], #["England", "National League - North", 4], #["Tunisia", "Ligue Professionnelle 1", 3], #["India", "I-League", 2], #["Morocco", "Botola Pro", 2], #["Algeria", "Ligue 1", 5], #["Algeria", "Ligue 2", 7], #["Hungary", "NB I", 2], ["United-Arab-Emirates", "Arabian Gulf League", 3], #["Poland", "Ekstraklasa", 1], #["Iran", "Persian Gulf Cup", 1], #["South-Africa", "Premier Soccer League", 3], #["Qatar", "Stars League", 2], #["Saudi-Arabia", "Pro League", 3], #["Switzerland", "Challenge League", 2], #["Spain", "Segunda B - Group 1", 6], #["Spain", "Segunda B - Group 2", 5], #["Spain", "Segunda B - Group 3", 6], #["Spain", "Segunda B - Group 4", 9], #["Israel", "Liga Leumit", 5], #["Italy", "Serie C", 1], #["France", "Ligue 2", 1], #["France", "Ligue 1", 2], #["France", "National", 4], #["Switzerland", "Super League", 2], #["Northern-Ireland", "Championship", 2], #["Northern-Ireland", "Premiership", 4], #["Scotland", "League One", 5], #["Scotland", "League Two", 5], #["Portugal", "Liga de Honra", 1], #["Portugal", "Primeira Liga", 1], #["Egypt", "Premier League", 1], #["Cyprus", "1. Division", 2], #["Germany", "Bundesliga 2", 1], #["Germany", "Bundesliga 1", 1], #["Germany", "Liga 3", 1], #["Spain", "Primera Division", 1], #["Spain", "Segunda Division", 1], #["Egypt", "Premier League", 1], #["Belgium", "Jupiler Pro League", 1], #["Israel", "Ligat ha'Al", 1], #["Netherlands", "Eerste Divisie", 1], #["Netherlands", "Eredivisie", 1], #["Scotland", "Championship", 1], #["Scotland", "Premiership", 5], #["Italy", "Serie A", 1], #["Italy", "Serie B", 1], #["Greece", "Super League", 4], #["India", "Indian Super League", 1], #["England", "Championship", 6], #["England", "League One", 4], #["England", "League Two", 2], #["England", "National League", 8], #["England", "Premier League", 1], #["Oman", "Professional League", 2], #["Turkey", "Super Lig", 2], ["Turkey", "TFF 1. Lig", 4], #["Turkey", "TFF 2. Lig", 15], #["Australia", "A-League", 1], ] bettingLabels = [] bettingLabelsBTTS = [] bettingLabelsDraw = [] bettingLabelsDC = [] #Should tell me which matches were skipped - No odds were available on the market for it skippedMatches = [] for match in todaysMatches: #time.sleep(2) upcomingLeagueFixtures = Fetch.getUpcomingFixturesForLeague(match[0], match[1], match[2]) for fixture in upcomingLeagueFixtures: allLabels = Predict.predictOne(match[0], match[1], fixture[0], fixture[1]) bettingLabel = allLabels[0] bettingLabelBTTS = allLabels[1] bettingLabelDraw = allLabels[2] bettingLabelDC = allLabels[3] marketOdds, marketOddsBTTS = Fetch.getScoringOdds(fixture[2]) homeDCOdds, awayDCOdds = Fetch.getDCOdds(fixture[2]) #print("GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG") #print(marketOdds) bettingLabelDC.insert(4, homeDCOdds) bettingLabelDC.insert(7, awayDCOdds) #if (homeDCOdds == 0.0 or awayDCOdds == 0.0): #bettingLabelsDC.append(bettingLabelDC) #continue #-------------------------Double chance insertions here print("{} |||||||| {}".format(homeDCOdds, bettingLabelDC[3])) print("{} |||||||| {}".format(awayDCOdds, bettingLabelDC[6])) bettingLabelDC.insert(5, float(homeDCOdds) - float(bettingLabelDC[3])) bettingLabelDC.insert(8, float(awayDCOdds) - float(bettingLabelDC[6])) bettingLabelsDC.append(bettingLabelDC) #if not marketOdds, will detect if list is empty! If not available, then dont make a label for that fixture #if not marketOdds: #skippedMatches.append("{} vs {} Was skipped(No market odds)".format(fixture[0], fixture[1])) #continue #bettingLabel.insert(4, marketOdds[0]) #bettingLabel.insert(5, float(marketOdds[0]) - bettingLabel[3]) #Inserting bet amount here based on kelly criterion formula #betAmount = Kelly.kellyCriterion(bankroll, float(marketOdds[0]), Utils.fromOddsToDecimalProb(bettingLabel[3]), 0.125) #bettingLabel.insert(6, betAmount) #print(marketOdds) #if (len(marketOdds) != 0): #bettingLabel.insert(5, marketOdds[1]) #else: #bettingLabel.insert(5, 0) #bettingLabel.insert(9, float(marketOdds[1]) - bettingLabel[7]) #bettingLabel.insert(6, marketOdds[3]) #if (len(marketOdds) != 0): #bettingLabel.insert(5, marketOdds[3]) #else: #bettingLabel.insert(5, 0) #bettingLabel.insert(9, float(marketOdds[2]) - bettingLabel[7]) #betAmount = Kelly.kellyCriterion(bankroll, float(marketOdds[2]), Utils.fromOddsToDecimalProb(bettingLabel[7]), 0.125) #bettingLabel.insert(10, betAmount) #bettingLabel.insert(15, marketOdds[3]) #bettingLabel.insert(16, float(marketOdds[3]) - bettingLabel[14]) #bettingLabelBTTS.insert(4, marketOddsBTTS[0]) #bettingLabelBTTS.insert(5, float(marketOddsBTTS[0]) - bettingLabelBTTS[3]) #bettingLabelBTTS.insert(7, marketOddsBTTS[1]) #bettingLabelBTTS.insert(8, float(marketOddsBTTS[1]) - bettingLabelBTTS[6]) bettingLabelsBTTS.append(bettingLabelBTTS) bettingLabels.append(bettingLabel) bettingLabelsDraw.append(bettingLabelDraw) #time.sleep(0.1) #print(bettingLabelsDC) #Printing first results: Terminal.display(bettingLabels, bettingLabelHeaders) Terminal.display(bettingLabelsBTTS, bettingLabelBTTSHeaders) Terminal.display(bettingLabelsDraw, bettingLabelDrawHeaders) Terminal.display(bettingLabelsDC, bettingLabelDCHeaders) #Gathering absolutely all labels generated to sed to Terminal Module labels = [bettingLabels, bettingLabelsBTTS, bettingLabelsDraw, bettingLabelsDC] Terminal.getAllLabels(labels) labelHeaders = [bettingLabelHeaders, bettingLabelBTTSHeaders, bettingLabelDrawHeaders, bettingLabelDCHeaders] Terminal.getAllLabelsHeaders(labelHeaders) print(skippedMatches) Terminal.appLoop()
import matplotlib import cv2 from tensorflow.keras.preprocessing.image import ImageDataGenerator import matplotlib.pyplot as plt import numpy as np import argparse import pickle import os import math import tensorflow as tf import Fetcher import Model import Config from tqdm.autonotebook import tqdm fetcher = Fetcher.DataFetcher(); #fetcher.processAndSave(); #fetcher.loadGestures(gestures= ["1","2"]); fetcher.load(); model = Model.Net(numBatchTrain = 0); model.network.summary(); model.load_checkpoint(); numSamples = 10; for i in range(numSamples): image, labels = fetcher.getRandomValidation();
def get_interest(uid,username,passwd): ''' uid:用户id,unicode 得到uid用户所发微博中的前20高频词 ''' with codecs.open('stop_words.txt','r','utf-8') as fr: stop_words = [line.strip() for line in fr] stop_words.append(' ') stop_words.append('\n') stop_words.append('\t') myFetcher = Fetcher(username,passwd) myFetcher.login() follows_list = myFetcher.get_user_follows(uid) follows_list.append(uid) fans_list = myFetcher.get_user_fans(uid) print len(follows_list) print len(fans_list) follows_keywords = get_keywords(myFetcher,follows_list) follows_interest = {} for word in follows_keywords: follows_interest[word[0]] = word[1] fans_keywords = get_keywords(myFetcher,fans_list) fans_interest = {} for word in fans_keywords: fans_interest[word[0]] = word[1] user_weibos = myFetcher.get_user_weibos(uid) weibos = ".".join(user_weibos) content_interest = {}#从用户发布的微博信息中提取的兴趣关键词 words = jieba.cut(weibos) filtered_words = [word for word in words if word not in stop_words and len(word) > 1] all_words_count = float(len(filtered_words)) counter = Counter(filtered_words) key_words = counter.most_common(30) outputs = [] for item in key_words: if isinstance(item[0],unicode): k_word = item[0].decode('utf-8') weight = float(item[1])/all_words_count else: k_word = item[0] weight = float(item[1])/all_words_count outputs.append("%s\t%f\n"%(k_word,weight)) content_interest[k_word] = weight #对两类兴趣词的权重进行归一化 max_weight_content = max(content_interest.values()) max_weight_follows = max(follows_interest.values()) max_weight_fans = max(fans_interest.values()) for word1,weight1 in content_interest.iteritems(): weight1 /= max_weight_content for word2,weight2 in follows_interest.iteritems(): weight2 /= max_weight_follows for word3,weight3 in fans_interest.iteritems(): weight3 /= max_weight_fans interest_words = {} all_words = follows_interest.keys() + content_interest.keys() + fans_interest.keys() for word in all_words: content_weight = content_interest.get(word,0) follows_weight = follows_interest.get(word,0) fans_weight = fans_interest.get(word,0) all_weight = 0.5*follows_weight + 0.4*content_weight + 0.1*fans_weight interest_words[word] = all_weight sorted_words = sorted(interest_words.iteritems(),key=lambda (k,v):v,reverse=True) for item in sorted_words[:30]: print item[0].encode('utf-8',''),item[1]
def fetch(self): """ 从队列中获取地址并查询内容 :return: None """ log.info("Running thread %s" % threading.current_thread().name) while True: try: cur_url, cur_depth = self._url_queue.get(timeout=1) print self._url_queue.unfinished_tasks cur_url = cur_url.strip() except Queue.Empty as e: log.warn(e) continue fetch_tool = Fetcher.Fetcher(cur_url, self._output_dir, self._timeout, self._cookie) if self.pro_match.match(cur_url): # 产品详情页 self.save_product_info_url(cur_url) content = fetch_tool.read_content() info = fetch_tool.get_product_info(content) product_info = [] [product_info.append(item) for item in info] if self._lock.acquire(): self._url_visited.append(cur_url) self._lock.release() self.save_product_info(product_info) elif self.fig_pattern.match(cur_url): # 产品大类 content = fetch_tool.read_content() if content is None or len(content) == 0: continue # 获取大类下的子类页面 sub_urls = fetch_tool.get_sub_urls(content) if sub_urls is None: continue for item in sub_urls: if self._lock.acquire(): # lock _url_visited, check if item in self._url_visited: continue self._lock.release() try: if self._lock.acquire(): # lock _url_visited, add self._url_visited.append(item) self._lock.release() self._url_queue.put([item, cur_depth + 1], timeout=1) except Queue.Full as e: log.warn(e) break else: # 子类页面, 获取产品详情地址 content = fetch_tool.read_content() product_urls = fetch_tool.get_product_url(content) if product_urls is None: continue for item in product_urls: if self._lock.acquire(): # lock _url_visited, check if item in self._url_visited: self._lock.release() continue try: self._url_visited.append(item) self._url_queue.put([item, cur_depth + 1], timeout=1) except Queue.Full as e: log.warn(e) break finally: self._lock.release() self._url_queue.task_done()
def predictOne(countryName, leagueName, homeTeam, awayTeam): """ Predicts The chance of homeTeam and awayTeam to score 1 goal in the first half. Arguments: countryName: String leagueName: String homeTeam: String awayTeam: String Returns: list labels, other lists of bettinglabels. """ #Represent all labels, will contain other lists(f.ex bettingLabelDraw) labels = [] homeTeamFixtures = Fetch.getPastFixtures(countryName, leagueName, homeTeam) awayTeamFixtures = Fetch.getPastFixtures(countryName, leagueName, awayTeam) homeDCChance, awayDCChance = calculateWinDrawChance( homeTeam, awayTeam, homeTeamFixtures, awayTeamFixtures) d = drawPredict2(homeDCChance, awayDCChance) if (d is not 1.0): bettingLabelDraw = [leagueName, homeTeam, awayTeam, d] #if (awayDCChance < 0.001): #awayDCChance = 0.001 #if (homeDCChance < 0.001): #homeDCChance = 0.001 #print(homeTeamFixtures) homeScoring, homeConceding = calculateScoringChance( homeTeamFixtures, "home", homeTeam) awayScoring, awayConceding = calculateScoringChance( awayTeamFixtures, "away", awayTeam) #print(homeTeamFixtures) #print("homescoring: {}".format(homeScoring)) #Home team odds of scoring in decimal homeDOddsOfScoring = (homeScoring + awayConceding) / 2.0 awayDOddsOfScoring = (awayScoring + homeConceding) / 2.0 #Home team Percentage odds of scoring at least 1 goal homePOddsOfScoring = Utils.fromDecimalProbToPercentage(homeDOddsOfScoring) awayPOddsOfScoring = Utils.fromDecimalProbToPercentage(awayDOddsOfScoring) awayPOddsOfCleanSheet = round(100 - homePOddsOfScoring) homePOddsOfCleanSheet = round(100 - awayPOddsOfScoring) #print( "PPPPPP" + str(awayPOddsOfCleanSheet) ) #These 2 if statements are simply to prevent ZeroDivisionError (Very rare that these 2 variables are 0) if (awayPOddsOfCleanSheet == 0): awayPOddsOfCleanSheet = 1 if (homePOddsOfCleanSheet == 0): homePOddsOfCleanSheet = 1 #Converting to european betting odds homeBettingOddsOfScoring = round( Utils.fromPercentageToOdds(homePOddsOfScoring), 3) awayBettingOddsOfScoring = round( Utils.fromPercentageToOdds(awayPOddsOfScoring), 3) homeBettingOddsOfCleanSheet = round( Utils.fromPercentageToOdds(homePOddsOfCleanSheet), 3) awayBettingOddsOfCleanSheet = round( Utils.fromPercentageToOdds(awayPOddsOfCleanSheet), 3) #noDraw, yesDraw = drawPredict(homeDOddsOfScoring, awayDOddsOfScoring) #bettingLabelDraw = [leagueName, homeTeam, awayTeam, homePOddsOfScoring, awayPOddsOfScoring, noDraw, yesDraw, homePOddsOfScoring - awayPOddsOfScoring] bttsYes, bttsNo = bttsPredict(homeDOddsOfScoring, awayDOddsOfScoring) bettingLabelBTTS = [leagueName, homeTeam, awayTeam, bttsYes, bttsNo] bettingLabelDC = [ leagueName, homeTeam, awayTeam, round(Utils.fromDecimalProbToOdds(homeDCChance), 2), round(Utils.fromDecimalProbToOdds(awayDCChance), 2) ] #Change 3. feb: Taking out all under-bets, only want over0.5, betting label too big... bettingLabel = [ leagueName, homeTeam, awayTeam, homeBettingOddsOfScoring, awayBettingOddsOfCleanSheet, awayBettingOddsOfScoring, homeBettingOddsOfCleanSheet ] print( "{} to score against opponent is: {}% Betting odds: {} - Under 0.5: {}" .format(homeTeam, homePOddsOfScoring, homeBettingOddsOfScoring, awayBettingOddsOfCleanSheet)) print( "{} to score against opponent is: {}% Betting odds: {} - Under 0.5: {}" .format(awayTeam, awayPOddsOfScoring, awayBettingOddsOfScoring, homeBettingOddsOfCleanSheet)) labels.append(bettingLabel) labels.append(bettingLabelBTTS) labels.append(bettingLabelDraw) labels.append(bettingLabelDC) return labels
from Models import * import tensorflow as tf import Config import pickle import Fetcher import cv2 import math from matplotlib import pyplot as pp import pydot from tqdm.autonotebook import tqdm tf.config.optimizer.set_jit(True) data = Fetcher.DataFetcher(imagePath="Dataset/fundus", heightmapPath="Dataset/heightmap") data.load() batch_count_train = int(np.ceil(data.trainDataSize / Config.BATCH_SIZE)) batch_count_valid = int(np.ceil(data.validDataSize / Config.BATCH_SIZE)) batch_count_test = int(np.ceil(data.testDataSize / Config.BATCH_SIZE)) gen_to_disc_ratio = 3 net = FundusNet(numBatchTrain=int(batch_count_train) * gen_to_disc_ratio) net.generator.summary() net.generator.load_weights(filepath="ckpt_weights1\\weights", by_name=True) startEpoch = net.load_checkpoint() # Define loss and accuracy per epoch all_loss = np.zeros(batch_count_train, dtype='float32')