def runFetchALL(bsid): allgames = dao.selectAll("select id from games where fetchTimes<=1", None) for i in range(len(allgames)): pirod = None if bsid > 0: pirod = dao.selectAll( "select years,bsid,lv,lv1 from game_pirod where game_id=%s and bsid>=%s order by bsid asc", (allgames[i][0], bsid)) else: pirod = dao.selectAll( "select years,bsid,lv,lv1 from game_pirod where game_id=%s order by bsid asc", (allgames[i][0], )) for j in range(len(pirod)): year = pirod[j][0] game_id = pirod[j][1] lv = pirod[j][2] lv1 = pirod[j][3] domine = None if lv == 1 and lv1 == 0: domine = 'League' elif lv == 1 and lv1 == 1: domine = 'SubLeague' elif lv == 2 and (lv1 == 0 or lv1 == 1): domine = 'CupMatch' print(game_id, domine, year) fetchGames.fetchGames(game_id, domine, year) time.sleep(1) dao.insert("update games set fetchTimes=fetchTimes+1 where id=%s", (allgames[i][0], ))
def run(): while True: print "开始处理任务" task = dao.select(state=0) cv.acquire() while task == None: cond_wait(cv) cv.release() ret = dao.update(state=1, update_time=now(), id=task.id) if ret == 0: print "任务已经被处理,直接跳出循环" continue page = http_crawler(task.link,task.type) if task.type == 0: print "处理列表任务...." for item in page: prefix = "http://yue.ifeng.com/news/detail_" link = prefix + item[0] new_task = build_task(link) dao.insert(new_task) cond_signal(cv) dao.update(state=2, update_time=now(), id=task.id) if task.type == 1: file_name = task.link.split("/")[-1] print "保存页面....",task.link,file_name save_page(page,file_name) ret = dao.update(state=2, update_time=now(), id=task.id) print "任务完成"
def check_valid_faces(predictions, frame): global valid_faces tmp_valid_faces = valid_faces.copy() for face in tmp_valid_faces: if (datetime.datetime.now() - face["timeout"]).total_seconds() >= 10: print("remove valid face: " + face["name"]) valid_faces.remove(face) # todo # mqtt send name dao.insert("INSERT INTO public.history (pinyin, image) VALUES ('" + face["name"] + "', '/images/" + face["name"] + ".jpg')") # cv2.imwrite("./images/valid/" + face["name"] + ".jpg", face["frame"]) for name, _ in predictions: if name == "unknown": return has_valid_face = False tmp_valid_faces = valid_faces.copy() for face in tmp_valid_faces: if face["name"] == name: has_valid_face = True break if not has_valid_face and len(valid_faces) < 100: valid_faces.append({ "name": name, "timeout": datetime.datetime.now() }) print(valid_faces)
def runFetchPankou(): sql = "select g.bisai_id from game_data g left join pankou p on p.bisaiId=g.bisai_id where g.error=0 GROUP BY g.bisai_id HAVING count(p.id)=0 or count(p.id)=1 " allgames = dao.selectAll(sql, None) for i in range(len(allgames)): id = allgames[i][0] try: fechPankou.fetchPankou(id) except: dao.insert("Update game_data set error=1 where bisai_id=%s", (id, ))
def cadastrar(self): try: data = UserView.register_user() if data is None: return nome, idade, telefone = data id = len(self.__usuarios) user = User(id, nome, idade, telefone) self.__usuarios.append(user) table, dict = user.toDict() insert(table, dict) except: GeneralView.message('Não foi possível cadastrar o usuário, tente novamente!')
def preserve(self, content): ''' 保存数据到mysql数据库 :param content: 要保存的内容 :return: ''' # dao.create_table(keyword[self.now_topic]) # if self.now_page == 1: # dao.create_table(keywordDb[self.topic]) for cont in range(0, len(content)): user = eval(content[cont][0]) dao.insert(keywordDb[self.topic], user, content[cont][1]) cont += 1
def fetchPankou(bisaiId): dao.delete("delete from pankou where bisaiId=%s", (bisaiId, )) print(bisaiId) url = 'http://vip.win0168.com/AsianOdds_n.aspx?id=' + str(bisaiId) html = request.sendRequest(url, "vip.win0168.com") document = pq(html) # print(document) print(str(bisaiId) + "开始") pankouLine = document("table").eq(1).find("tr") bocaiGs = None for i in range(len(pankouLine)): if i <= 1 or i > (len(pankouLine) - 3): continue hang = pq(pankouLine[i]).find("td") if pq(hang[0]).text() != '': bocaiGs = pq(hang[0]).text() pankouName = None if pq(hang[1]).text() == '': pankouName = "盘口1" else: pankouName = pq(hang[1]).text() first_zhudui = pq(hang[2]).html() first_pankou = pq(hang[3]).html() first_cidui = pq(hang[4]).html() finally_zhudui = pq(hang[5]).html() finally_pankou = pq(hang[6]).html() finally_cidui = pq(hang[7]).html() count = dao.selectOne( "select count(id) from bocaiGs where bocaiGsName=%s", (bocaiGs, )) if count[0] <= 0: dao.insert("insert into bocaiGs(bocaiGsName)" " VALUES (%s) ", (bocaiGs, )) first_pankou_alias = None if first_pankou != None: first_pankou_alias = dao.selectOne( "select alias from pankouLabel where pankou=%s limit 1", (first_pankou, )) first_pankou_alias = first_pankou_alias[0] finally_pankou_alias = None if finally_pankou != None: finally_pankou_alias = dao.selectOne( "select alias from pankouLabel where pankou=%s limit 1", (finally_pankou, )) finally_pankou_alias = finally_pankou_alias[0] data = (bocaiGs, pankouName, first_zhudui, first_pankou, first_cidui, finally_zhudui, finally_pankou, finally_cidui, bisaiId, first_pankou_alias, finally_pankou_alias) dao.insert( "insert into pankou(bocaiGs,pankouName,first_zhudui,first_pankou,first_cidui,finally_zhudui,finally_pankou,finally_cidui,bisaiId,first_pankou_alias,finally_pankou_alias)" " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ", data) print(str(bisaiId) + "结束")
def fechCountry(): url = "http://info.win0168.com/jsData/infoHeader.js" html_doc = request.sendRequest(url) print(html_doc) html_doc = html_doc.replace(' ', '') data = html_doc.split(";") for index in range(len(data)): if (index == 0): continue item = data[index] itemdata = item.split("=") if (len(itemdata) != 2): continue val = eval(itemdata[1]) country = val[1] image = val[2] idx = val[3] count = dao.selectOne("select count(id) from games where id=%s", (index, )) if count[0] == 0: data1 = (index, idx, country, image) dao.insert( "INSERT INTO games (id,idx,country,image) VALUES (%s,%s,%s,%s)", data1) period = val[4] for y in range(len(period)): valdetail = period[y] arr = valdetail.split(",") id = arr[0] name = arr[1] lv = arr[2] lv1 = arr[3] for j in range(len(arr) - 4): years = arr[j + 4] count = dao.selectOne( "select count(id) from game_pirod where years=%s and bsid=%s", (years, id)) if count[0] == 0: data2 = (name, lv, lv1, years, id, index) dao.insert( "INSERT INTO game_pirod (name,lv,lv1,years,bsid,game_id) VALUES (%s,%s,%s,%s,%s,%s)", data2) print(val[1] + '-' + val[2] + '-' + val[3] + '-') return
def create_redirect_set(): real_url = request.form['real_url'] fake_url = request.form['fake_url'] if real_url is None or real_url == '' or \ fake_url is None or fake_url == '': return "Bad request", 400 id = dao.insert(real_url, fake_url) return '%s' % (id)
def check_invalid_faces(predictions, frame): global invalid_faces global invalid_faces_timeout tmp_invalid_faces = invalid_faces.copy() for face in tmp_invalid_faces: if (datetime.datetime.now() - face["timeout"]).total_seconds() >= 10: print("remove invalid face: " + face["name"]) invalid_faces.remove(face) # todo dao.insert( "INSERT INTO public.history (pinyin, image) VALUES ('unknown', '/images/invalid/" + face["name"] + ".jpg')") cv2.imwrite("./images/invalid/" + face["name"] + ".jpg", face["frame"]) if (datetime.datetime.now() - invalid_faces_timeout).total_seconds() <= 3: print("invalid_faces_timeout return") return for name, _ in predictions: if name != "unknown": return invalid_faces_timeout = datetime.datetime.now() if len(invalid_faces) < 10: invalid_username = "******" + str( datetime.datetime.now().timestamp()) invalid_faces.append({ "name": invalid_username, "frame": frame, "timeout": datetime.datetime.now() }) print(invalid_faces)
if os.path.exists(ROOT_DIR + '/../db/{}.db'.format(test_movie_db)): os.remove(ROOT_DIR + '/../db/{}.db'.format(test_movie_db)) if os.path.exists(ROOT_DIR + '/../db/{}.db'.format(test_user_db)): os.remove(ROOT_DIR + '/../db/{}.db'.format(test_user_db)) deleteTestDb() print("Checking creating test user db...") assert (dao.create("CREATE TABLE IF NOT EXISTS test(id int, name varchar(64))", test_user_db) == True) assert (len(dao.select("SELECT * FROM test", test_user_db, False)) == 0) print("Checking insert test user db...") assert (dao.insert("INSERT INTO test(id, name) VALUES(?,?)", [123, "taro"], test_user_db) == True) assert (len(dao.select("SELECT * FROM test", test_user_db, False)) == 1) print("Checking update test user db...") assert (dao.update("UPDATE test SET name=? WHERE id=?", ["goro", 123], test_user_db) == True) assert (len(dao.select("SELECT * FROM test", test_user_db, False)) == 1) assert (dao.select("SELECT * FROM test", test_user_db, False)[0][1] == "goro") print("Checking delete a column test test db...") assert (dao.delete("DELETE FROM test WHERE id=?", [123], test_user_db) == True) assert (len(dao.select("SELECT * FROM test", test_user_db, False)) == 0) print("Checking insert user provisionally into test user db...") assert (dao.insertProvisionalUser2Db("name_test_1", "*****@*****.**", "123abc456def",
def store(url): print url task = Task(id=None, priority=0, type=1, state=0, link=url, avaliable_time=now(), start_time=None, end_time=None) dao.insert(task)
def preserve(self, content): for cont in range(0, len(content[0])): dao.insert(keywordDb[self.topic], content[0][cont], content[1][cont]) cont += 1
def store(url): print url task = Task(id=None,priority=0,type=1,state=0,link=url,\ avaliable_time=now(),start_time=None,end_time=None) dao.insert(task)
def fetchGames(id, domain, years): id = str(id) url = 'http://info.win0168.com/cn/' + domain + '/' + years + '/' + id + '.html' html_doc = request.sendRequest(url) document = pq(html_doc) dataUrl = None version = time.strftime("%Y%m%d%H", time.localtime()) if domain == 'League' or domain == 'SubLeague': script = document.find("script") # print(script) text = None for y in range(len(script)): value = pq(script[y]).text() if value.find("var SubSclassID") != -1: text = value break SubSclassID = text.split(";")[6].split("=")[1].strip() print(SubSclassID) if (SubSclassID == '0'): dataUrl = "http://info.win0168.com/jsData/matchResult/" + years + "/s" + id + ".js?version=" + version else: dataUrl = "http://info.win0168.com/jsData/matchResult/" + years + "/s" + id + "_" + SubSclassID + ".js?version=" + version elif domain == 'CupMatch': dataUrl = "http://info.win0168.com/jsData/matchResult/" + years + "/c" + id + ".js?version=" + version print(dataUrl) data = request.sendRequest(dataUrl) teamStr = re.findall(r"var arrTeam =(.+?);", data) if len(teamStr) == 0: return #球队 print(teamStr[0]) teams = eval(teamStr[0]) for l in range(len(teams)): count = dao.selectOne( "select count(id) from team where team_id=%s and game_id=%s", (teams[l][0], id)) if count[0] > 0: continue else: teamData = None if len(teams[l]) == 5: teamData = (teams[l][0], teams[l][1], teams[l][2], id) dao.insert( "INSERT INTO team (team_id,name,alias,game_id) VALUES (%s,%s,%s,%s)", teamData) elif len(teams[l]) == 6: teamData = (teams[l][0], teams[l][1], teams[l][2], teams[l][5], id) dao.insert( "INSERT INTO team (team_id,name,alias,image,game_id) VALUES (%s,%s,%s,%s,%s)", teamData) elif len(teams[l]) > 6: teamData = (teams[l][0], teams[l][1], teams[l][2], teams[l][5], teams[l][6], id) dao.insert( "INSERT INTO team (team_id,name,alias,image,mark,game_id) VALUES (%s,%s,%s,%s,%s,%s)", teamData) #比赛数据 gameDataStr = None if domain == 'CupMatch': gameDataStr = re.findall(r"jh\[\"G.+\"\] =(.+?)\];\s", data) else: gameDataStr = re.findall(r"jh\[\"R_.+\"\] =(.+?)\];\s", data) lianSai = dao.selectOne( "select name,game_id from game_pirod where bsid=%s limit 1", (id, )) liansai = lianSai[0] country_id = lianSai[1] print(gameDataStr) for j in range(len(gameDataStr)): gameDataStr[j] = gameDataStr[j] + ']' try: per = eval(gameDataStr[j]) for i in range(len(per)): data = per[i] if '<br/>' in data[3]: date = data[3].split('<br/>') data[3] = years + '-' + date[0] + ' ' + date[1] gameData = (data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], data[8], data[9], data[10], data[11], data[12], data[13], j + 1, years, liansai, country_id) print(gameData) count = dao.selectOne( "select count(id) from game_data where bisai_id=%s and game_id=%s", (data[0], id)) if count[0] > 0: updateData = (data[6], data[7], data[8], data[9], data[10], data[11], data[12], data[13], years, liansai, country_id, data[0], id) dao.insert( "UPDATE game_data set full_score=%s,half_score=%s,first_sort=%s,second_sort=%s,full_concede=%s " ",half_concede=%s,full_bigsmall=%s,half_bigsmall=%s,years=%s,liansai=%s,country_id=%s where bisai_id=%s and game_id=%s", updateData) else: dao.insert( "INSERT INTO game_data (bisai_id,game_id,mark,bs_time,first_team_id,second_team_id,full_score, " "half_score,first_sort,second_sort,full_concede,half_concede,full_bigsmall,half_bigsmall,turn,years,liansai,country_id) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%,%)", gameData) except Exception: dataStr = '' if gameDataStr[j].endswith("]]]"): gamePer = re.findall(r"\[{1,2}(.+?)\]\]", gameDataStr[j]) for j in range(len(gamePer)): dataStr = dataStr + gamePer[j] + ']' else: dataStr = gameDataStr[j] per = re.findall(r"\[{1,2}(.+?)\]{1,2}", dataStr) for i in range(len(per)): data = per[i].replace("\'", "").split(",") if len(data) < 13: continue if '<br/>' in data[3]: date = data[3].split('<br/>') data[3] = years.split( '-')[0] + '-' + date[0] + ' ' + date[1] gameData = (data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], data[8], data[9], data[10], data[11], data[12], data[13], j + 1, years, liansai, country_id) print(gameData) count = dao.selectOne( "select count(id) from game_data where bisai_id=%s and game_id=%s", (data[0], id)) if count[0] > 0: updateData = (data[6], data[7], data[8], data[9], data[10], data[11], data[12], data[13], years, liansai, country_id, data[0], id) dao.insert( "UPDATE game_data set full_score=%s,half_score=%s,first_sort=%s,second_sort=%s,full_concede=%s " ",half_concede=%s,full_bigsmall=%s,half_bigsmall=%s,years=%s,liansai=%s,country_id=%s where bisai_id=%s and game_id=%s", updateData) else: dao.insert( "INSERT INTO game_data (bisai_id,game_id,mark,bs_time,first_team_id,second_team_id,full_score, " "half_score,first_sort,second_sort,full_concede,half_concede,full_bigsmall,half_bigsmall,turn,years,liansai,country_id) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", gameData)