def runFetchALL(bsid):
allgames = dao.selectAll("select id from games where fetchTimes<=1", None)
for i in range(len(allgames)):
pirod = None
if bsid > 0:
pirod = dao.selectAll(
"select years,bsid,lv,lv1 from game_pirod where game_id=%s and bsid>=%s order by bsid asc",
(allgames[i][0], bsid))
else:
pirod = dao.selectAll(
"select years,bsid,lv,lv1 from game_pirod where game_id=%s order by bsid asc",
(allgames[i][0], ))
for j in range(len(pirod)):
year = pirod[j][0]
game_id = pirod[j][1]
lv = pirod[j][2]
lv1 = pirod[j][3]
domine = None
if lv == 1 and lv1 == 0:
domine = 'League'
elif lv == 1 and lv1 == 1:
domine = 'SubLeague'
elif lv == 2 and (lv1 == 0 or lv1 == 1):
domine = 'CupMatch'
print(game_id, domine, year)
fetchGames.fetchGames(game_id, domine, year)
time.sleep(1)
dao.insert("update games set fetchTimes=fetchTimes+1 where id=%s",
(allgames[i][0], ))
def run():
while True:
print "开始处理任务"
task = dao.select(state=0)
cv.acquire()
while task == None:
cond_wait(cv)
cv.release()
ret = dao.update(state=1, update_time=now(), id=task.id)
if ret == 0:
print "任务已经被处理，直接跳出循环"
continue
page = http_crawler(task.link,task.type)
if task.type == 0:
print "处理列表任务...."
for item in page:
prefix = "http://yue.ifeng.com/news/detail_"
link = prefix + item[0]
new_task = build_task(link)
dao.insert(new_task)
cond_signal(cv)
dao.update(state=2, update_time=now(), id=task.id)
if task.type == 1:
file_name = task.link.split("/")[-1]
print "保存页面....",task.link,file_name
save_page(page,file_name)
ret = dao.update(state=2, update_time=now(), id=task.id)
print "任务完成"
def check_valid_faces(predictions, frame):
global valid_faces
tmp_valid_faces = valid_faces.copy()
for face in tmp_valid_faces:
if (datetime.datetime.now() - face["timeout"]).total_seconds() >= 10:
print("remove valid face: " + face["name"])
valid_faces.remove(face)
# todo
# mqtt send name
dao.insert("INSERT INTO public.history (pinyin, image) VALUES ('" +
face["name"] + "', '/images/" + face["name"] + ".jpg')")
# cv2.imwrite("./images/valid/" + face["name"] + ".jpg", face["frame"])
for name, _ in predictions:
if name == "unknown":
return
has_valid_face = False
tmp_valid_faces = valid_faces.copy()
for face in tmp_valid_faces:
if face["name"] == name:
has_valid_face = True
break
if not has_valid_face and len(valid_faces) < 100:
valid_faces.append({
"name": name,
"timeout": datetime.datetime.now()
})
print(valid_faces)
def runFetchPankou():
sql = "select g.bisai_id from game_data g left join pankou p on p.bisaiId=g.bisai_id where g.error=0 GROUP BY g.bisai_id HAVING count(p.id)=0 or count(p.id)=1 "
allgames = dao.selectAll(sql, None)
for i in range(len(allgames)):
id = allgames[i][0]
try:
fechPankou.fetchPankou(id)
except:
dao.insert("Update game_data set error=1 where bisai_id=%s",
(id, ))
def cadastrar(self):
try:
data = UserView.register_user()
if data is None:
return
nome, idade, telefone = data
id = len(self.__usuarios)
user = User(id, nome, idade, telefone)
self.__usuarios.append(user)
table, dict = user.toDict()
insert(table, dict)
except:
GeneralView.message('Não foi possível cadastrar o usuário, tente novamente!')
def preserve(self, content):
'''
保存数据到mysql数据库
:param content: 要保存的内容
:return:
'''
# dao.create_table(keyword[self.now_topic])
# if self.now_page == 1:
# dao.create_table(keywordDb[self.topic])
for cont in range(0, len(content)):
user = eval(content[cont][0])
dao.insert(keywordDb[self.topic], user, content[cont][1])
cont += 1
def fetchPankou(bisaiId):
dao.delete("delete from pankou where bisaiId=%s", (bisaiId, ))
print(bisaiId)
url = 'http://vip.win0168.com/AsianOdds_n.aspx?id=' + str(bisaiId)
html = request.sendRequest(url, "vip.win0168.com")
document = pq(html)
# print(document)
print(str(bisaiId) + "开始")
pankouLine = document("table").eq(1).find("tr")
bocaiGs = None
for i in range(len(pankouLine)):
if i <= 1 or i > (len(pankouLine) - 3):
continue
hang = pq(pankouLine[i]).find("td")
if pq(hang[0]).text() != '':
bocaiGs = pq(hang[0]).text()
pankouName = None
if pq(hang[1]).text() == '':
pankouName = "盘口1"
else:
pankouName = pq(hang[1]).text()
first_zhudui = pq(hang[2]).html()
first_pankou = pq(hang[3]).html()
first_cidui = pq(hang[4]).html()
finally_zhudui = pq(hang[5]).html()
finally_pankou = pq(hang[6]).html()
finally_cidui = pq(hang[7]).html()
count = dao.selectOne(
"select count(id) from bocaiGs where bocaiGsName=%s", (bocaiGs, ))
if count[0] <= 0:
dao.insert("insert into bocaiGs(bocaiGsName)"
" VALUES (%s) ", (bocaiGs, ))
first_pankou_alias = None
if first_pankou != None:
first_pankou_alias = dao.selectOne(
"select alias from pankouLabel where pankou=%s limit 1",
(first_pankou, ))
first_pankou_alias = first_pankou_alias[0]
finally_pankou_alias = None
if finally_pankou != None:
finally_pankou_alias = dao.selectOne(
"select alias from pankouLabel where pankou=%s limit 1",
(finally_pankou, ))
finally_pankou_alias = finally_pankou_alias[0]
data = (bocaiGs, pankouName, first_zhudui, first_pankou, first_cidui,
finally_zhudui, finally_pankou, finally_cidui, bisaiId,
first_pankou_alias, finally_pankou_alias)
dao.insert(
"insert into pankou(bocaiGs,pankouName,first_zhudui,first_pankou,first_cidui,finally_zhudui,finally_pankou,finally_cidui,bisaiId,first_pankou_alias,finally_pankou_alias)"
" VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ", data)
print(str(bisaiId) + "结束")
def fechCountry():
url = "http://info.win0168.com/jsData/infoHeader.js"
html_doc = request.sendRequest(url)
print(html_doc)
html_doc = html_doc.replace(' ', '')
data = html_doc.split(";")
for index in range(len(data)):
if (index == 0):
continue
item = data[index]
itemdata = item.split("=")
if (len(itemdata) != 2):
continue
val = eval(itemdata[1])
country = val[1]
image = val[2]
idx = val[3]
count = dao.selectOne("select count(id) from games where id=%s",
(index, ))
if count[0] == 0:
data1 = (index, idx, country, image)
dao.insert(
"INSERT INTO games (id,idx,country,image) VALUES (%s,%s,%s,%s)",
data1)
period = val[4]
for y in range(len(period)):
valdetail = period[y]
arr = valdetail.split(",")
id = arr[0]
name = arr[1]
lv = arr[2]
lv1 = arr[3]
for j in range(len(arr) - 4):
years = arr[j + 4]
count = dao.selectOne(
"select count(id) from game_pirod where years=%s and bsid=%s",
(years, id))
if count[0] == 0:
data2 = (name, lv, lv1, years, id, index)
dao.insert(
"INSERT INTO game_pirod (name,lv,lv1,years,bsid,game_id) VALUES (%s,%s,%s,%s,%s,%s)",
data2)
print(val[1] + '-' + val[2] + '-' + val[3] + '-')
return
def create_redirect_set():
real_url = request.form['real_url']
fake_url = request.form['fake_url']
if real_url is None or real_url == '' or \
fake_url is None or fake_url == '':
return "Bad request", 400
id = dao.insert(real_url, fake_url)
return '%s' % (id)
def check_invalid_faces(predictions, frame):
global invalid_faces
global invalid_faces_timeout
tmp_invalid_faces = invalid_faces.copy()
for face in tmp_invalid_faces:
if (datetime.datetime.now() - face["timeout"]).total_seconds() >= 10:
print("remove invalid face: " + face["name"])
invalid_faces.remove(face)
# todo
dao.insert(
"INSERT INTO public.history (pinyin, image) VALUES ('unknown', '/images/invalid/"
+ face["name"] + ".jpg')")
cv2.imwrite("./images/invalid/" + face["name"] + ".jpg",
face["frame"])
if (datetime.datetime.now() - invalid_faces_timeout).total_seconds() <= 3:
print("invalid_faces_timeout return")
return
for name, _ in predictions:
if name != "unknown":
return
invalid_faces_timeout = datetime.datetime.now()
if len(invalid_faces) < 10:
invalid_username = "******" + str(
datetime.datetime.now().timestamp())
invalid_faces.append({
"name": invalid_username,
"frame": frame,
"timeout": datetime.datetime.now()
})
print(invalid_faces)
if os.path.exists(ROOT_DIR + '/../db/{}.db'.format(test_movie_db)):
os.remove(ROOT_DIR + '/../db/{}.db'.format(test_movie_db))
if os.path.exists(ROOT_DIR + '/../db/{}.db'.format(test_user_db)):
os.remove(ROOT_DIR + '/../db/{}.db'.format(test_user_db))
deleteTestDb()
print("Checking creating test user db...")
assert (dao.create("CREATE TABLE IF NOT EXISTS test(id int, name varchar(64))",
test_user_db) == True)
assert (len(dao.select("SELECT * FROM test", test_user_db, False)) == 0)
print("Checking insert test user db...")
assert (dao.insert("INSERT INTO test(id, name) VALUES(?,?)", [123, "taro"],
test_user_db) == True)
assert (len(dao.select("SELECT * FROM test", test_user_db, False)) == 1)
print("Checking update test user db...")
assert (dao.update("UPDATE test SET name=? WHERE id=?", ["goro", 123],
test_user_db) == True)
assert (len(dao.select("SELECT * FROM test", test_user_db, False)) == 1)
assert (dao.select("SELECT * FROM test", test_user_db, False)[0][1] == "goro")
print("Checking delete a column test test db...")
assert (dao.delete("DELETE FROM test WHERE id=?", [123], test_user_db) == True)
assert (len(dao.select("SELECT * FROM test", test_user_db, False)) == 0)
print("Checking insert user provisionally into test user db...")
assert (dao.insertProvisionalUser2Db("name_test_1", "*****@*****.**",
"123abc456def",
def store(url):
print url
task = Task(id=None, priority=0, type=1, state=0, link=url, avaliable_time=now(), start_time=None, end_time=None)
dao.insert(task)
def preserve(self, content):
for cont in range(0, len(content[0])):
dao.insert(keywordDb[self.topic], content[0][cont],
content[1][cont])
cont += 1
def store(url):
print url
task = Task(id=None,priority=0,type=1,state=0,link=url,\
avaliable_time=now(),start_time=None,end_time=None)
dao.insert(task)
def fetchGames(id, domain, years):
id = str(id)
url = 'http://info.win0168.com/cn/' + domain + '/' + years + '/' + id + '.html'
html_doc = request.sendRequest(url)
document = pq(html_doc)
dataUrl = None
version = time.strftime("%Y%m%d%H", time.localtime())
if domain == 'League' or domain == 'SubLeague':
script = document.find("script")
# print(script)
text = None
for y in range(len(script)):
value = pq(script[y]).text()
if value.find("var SubSclassID") != -1:
text = value
break
SubSclassID = text.split(";")[6].split("=")[1].strip()
print(SubSclassID)
if (SubSclassID == '0'):
dataUrl = "http://info.win0168.com/jsData/matchResult/" + years + "/s" + id + ".js?version=" + version
else:
dataUrl = "http://info.win0168.com/jsData/matchResult/" + years + "/s" + id + "_" + SubSclassID + ".js?version=" + version
elif domain == 'CupMatch':
dataUrl = "http://info.win0168.com/jsData/matchResult/" + years + "/c" + id + ".js?version=" + version
print(dataUrl)
data = request.sendRequest(dataUrl)
teamStr = re.findall(r"var arrTeam =(.+?);", data)
if len(teamStr) == 0:
return
#球队
print(teamStr[0])
teams = eval(teamStr[0])
for l in range(len(teams)):
count = dao.selectOne(
"select count(id) from team where team_id=%s and game_id=%s",
(teams[l][0], id))
if count[0] > 0:
continue
else:
teamData = None
if len(teams[l]) == 5:
teamData = (teams[l][0], teams[l][1], teams[l][2], id)
dao.insert(
"INSERT INTO team (team_id,name,alias,game_id) VALUES (%s,%s,%s,%s)",
teamData)
elif len(teams[l]) == 6:
teamData = (teams[l][0], teams[l][1], teams[l][2], teams[l][5],
id)
dao.insert(
"INSERT INTO team (team_id,name,alias,image,game_id) VALUES (%s,%s,%s,%s,%s)",
teamData)
elif len(teams[l]) > 6:
teamData = (teams[l][0], teams[l][1], teams[l][2], teams[l][5],
teams[l][6], id)
dao.insert(
"INSERT INTO team (team_id,name,alias,image,mark,game_id) VALUES (%s,%s,%s,%s,%s,%s)",
teamData)
#比赛数据
gameDataStr = None
if domain == 'CupMatch':
gameDataStr = re.findall(r"jh\[\"G.+\"\] =(.+?)\];\s", data)
else:
gameDataStr = re.findall(r"jh\[\"R_.+\"\] =(.+?)\];\s", data)
lianSai = dao.selectOne(
"select name,game_id from game_pirod where bsid=%s limit 1", (id, ))
liansai = lianSai[0]
country_id = lianSai[1]
print(gameDataStr)
for j in range(len(gameDataStr)):
gameDataStr[j] = gameDataStr[j] + ']'
try:
per = eval(gameDataStr[j])
for i in range(len(per)):
data = per[i]
if '<br/>' in data[3]:
date = data[3].split('<br/>')
data[3] = years + '-' + date[0] + ' ' + date[1]
gameData = (data[0], data[1], data[2], data[3], data[4],
data[5], data[6], data[7], data[8], data[9],
data[10], data[11], data[12], data[13], j + 1,
years, liansai, country_id)
print(gameData)
count = dao.selectOne(
"select count(id) from game_data where bisai_id=%s and game_id=%s",
(data[0], id))
if count[0] > 0:
updateData = (data[6], data[7], data[8], data[9], data[10],
data[11], data[12], data[13], years, liansai,
country_id, data[0], id)
dao.insert(
"UPDATE game_data set full_score=%s,half_score=%s,first_sort=%s,second_sort=%s,full_concede=%s "
",half_concede=%s,full_bigsmall=%s,half_bigsmall=%s,years=%s,liansai=%s,country_id=%s where bisai_id=%s and game_id=%s",
updateData)
else:
dao.insert(
"INSERT INTO game_data (bisai_id,game_id,mark,bs_time,first_team_id,second_team_id,full_score, "
"half_score,first_sort,second_sort,full_concede,half_concede,full_bigsmall,half_bigsmall,turn,years,liansai,country_id) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%,%)",
gameData)
except Exception:
dataStr = ''
if gameDataStr[j].endswith("]]]"):
gamePer = re.findall(r"\[{1,2}(.+?)\]\]", gameDataStr[j])
for j in range(len(gamePer)):
dataStr = dataStr + gamePer[j] + ']'
else:
dataStr = gameDataStr[j]
per = re.findall(r"\[{1,2}(.+?)\]{1,2}", dataStr)
for i in range(len(per)):
data = per[i].replace("\'", "").split(",")
if len(data) < 13:
continue
if '<br/>' in data[3]:
date = data[3].split('<br/>')
data[3] = years.split(
'-')[0] + '-' + date[0] + ' ' + date[1]
gameData = (data[0], data[1], data[2], data[3], data[4],
data[5], data[6], data[7], data[8], data[9],
data[10], data[11], data[12], data[13], j + 1,
years, liansai, country_id)
print(gameData)
count = dao.selectOne(
"select count(id) from game_data where bisai_id=%s and game_id=%s",
(data[0], id))
if count[0] > 0:
updateData = (data[6], data[7], data[8], data[9], data[10],
data[11], data[12], data[13], years, liansai,
country_id, data[0], id)
dao.insert(
"UPDATE game_data set full_score=%s,half_score=%s,first_sort=%s,second_sort=%s,full_concede=%s "
",half_concede=%s,full_bigsmall=%s,half_bigsmall=%s,years=%s,liansai=%s,country_id=%s where bisai_id=%s and game_id=%s",
updateData)
else:
dao.insert(
"INSERT INTO game_data (bisai_id,game_id,mark,bs_time,first_team_id,second_team_id,full_score, "
"half_score,first_sort,second_sort,full_concede,half_concede,full_bigsmall,half_bigsmall,turn,years,liansai,country_id) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
gameData)
