def downloadAllVideo():
conn = dbConn.getConn()
global VideoCount
VideoCount = 0
routerCount = 0
cTask = conn.cursor() #大的任务表
cData = conn.cursor() #每个data_表
thread_list = []
cursor = cTask.execute("SELECT name ,router,autoDownload from rsshubtasks")
for row in cursor:
if (row[1].find("douyin") >= 0 and row[2] == 1):
routerCount = routerCount + 1
cursor2 = cData.execute(
"SELECT title,link,downloaded,timestamp from rssData where rssName='{}'"
.format(row[0]))
for row2 in cursor2:
if (row2[2] == 0):
t = threading.Thread(target=downloadOneVideo,
args=(row[0], row2[0], row2[1],
row2[3]))
t.start()
thread_list.append(t)
for t in thread_list:
t.join()
logger.info(
str(routerCount) + " routers need autoDownload " + str(VideoCount) +
" videos downloaded")
conn.close()
def downloadOneVideo(name, title, url, timestamp):
conn = dbConn.getConn()
global VideoCount
try:
os.chdir(os.path.join(os.path.abspath('..'), 'web', 'static'))
head = {
'User-Agent':
'Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25'
}
page = requests.get(url.replace("/playwm/", "/play/"), headers=head)
page.encoding = 'utf-8'
#print(page.url)
webvideopage = requests.get(page.url, headers=head)
if (os.path.exists(os.path.join("video", name)) == 0):
os.makedirs(os.path.join("video", name))
with open(
os.path.join("video", name,
title + "_" + str(timestamp) + ".mp4"),
"wb") as code:
code.write(webvideopage.content)
summary = '''<p style="display:block; text-align:center;">
<video controls="controls" width="300" height="600" src="/video/{}/{}">
</video></p>'''.format(name,
quote(title) + "_" + str(timestamp) + ".mp4")
conn.execute(
"UPDATE rssData set downloaded = 1,summary='{}' where rssName='{}' and title='{}'"
.format(summary, name, title))
conn.commit()
VideoCount = VideoCount + 1
logger.info(name + " " + url + " downloaded")
except:
pass
conn.close()
def deleteRssTask(rssName):
conn = dbConn.getConn()
conn.execute("DELETE FROM normalrsslinks where name='{}'".format(rssName))
conn.execute("DELETE FROM rsshubtasks where name='{}'".format(rssName))
conn.execute("DELETE FROM rssData where rssName='{}'".format(rssName))
conn.commit()
conn.close()
def countAllRecords():
conn = dbConn.getConn()
cursor = conn.execute("select count(rowid) from rssData")
for row in cursor:
cnt = row[0]
conn.close()
return cnt
def countOneRecords(rssName):
conn = dbConn.getConn()
cursor = conn.execute(
"select count(rowid) from rssData where rssName='{}'".format(rssName))
for row in cursor:
cnt = row[0]
conn.close()
return cnt
def getRsshubServers():
serversDict = []
conn = dbConn.getConn()
cTask = conn.cursor()
cursor = cTask.execute(
"SELECT ID,Adress,FuncNum,FirstCheck,LastCheck from rsshubServers")
for row in cursor:
d = dict_factory(cursor, row)
serversDict.append(d)
conn.close()
return serversDict
def getAllDbRssData(start=0, num=10):
rssDict = []
conn = dbConn.getConn()
c = conn.cursor()
cursor = c.execute(
"SELECT rssName,title,summary,timestamp,link,downloaded from rssData order by timestamp desc limit {},{}"
.format(start, num))
for row in cursor:
d = dict_factory(cursor, row)
rssDict.append(d)
conn.close()
return rssDict
def addServer(ID, ServerAD):
conn = dbConn.getConn()
FC = time.strftime("%Y-%m-%d", time.localtime())
LC = time.strftime("%Y-%m-%d", time.localtime())
try:
FN = getFuncNum(ServerAD)
conn.execute(
"INSERT INTO rsshubServers (ID,Adress,FuncNum,FirstCheck,LastCheck) VALUES ({},'{}',{},'{}','{}')"
.format(ID, ServerAD, FN, FC, LC))
except:
logger.error("AddFailed " + ServerAD)
conn.commit()
conn.close()
def updateNormalRssLink(url, name, round, title):
if (url == "" or name == "" or round <= 0):
return -1
conn = dbConn.getConn()
try:
linksql = "UPDATE normalrsslinks set link='{}' , round={} ,title='{}' where name='{}' ".format(
url, round, title, name)
conn.execute(linksql) #修改普通rss任务
conn.commit()
conn.close()
except:
return -1
return 0
def addNormalRssLink(url, name, round, title):
if (url == "" or name == "" or round <= 0):
return -1
conn = dbConn.getConn()
try:
linksql = "INSERT INTO normalrsslinks (link , name , round , title) VALUES ('{}','{}',{},'{}')".format(
url, name, round, title)
conn.execute(linksql) #建普通rss任务
conn.commit()
conn.close()
except:
return -1
return 0
def updateRsshubTask(router, name, round, recommendedServerID, title):
if (router == "" or name == "" or round <= 0 or recommendedServerID < 0):
return -1
conn = dbConn.getConn()
try:
linksql = "UPDATE rsshubtasks set router='{}' , round={} ,recommendedServerID={},title='{}' where name='{}'".format(
router, round, recommendedServerID, title, name)
#修改rsshub任务
conn.execute(linksql)
conn.commit()
conn.close()
except:
return -1
return 0
def addRsshubTask(router, name, round, recommendedServerID, title):
if (router == "" or name == "" or round <= 0 or recommendedServerID < 0):
return -1
conn = dbConn.getConn()
try:
linksql = "INSERT INTO rsshubtasks (router , name , round ,recommendedServerID,title) VALUES ('{}','{}',{},{},'{}')".format(
router, name, round, recommendedServerID, title)
#建rsshub任务
conn.execute(linksql)
conn.commit()
conn.close()
except:
return -1
return 0
def updateAllServers():
conn = dbConn.getConn()
c = conn.cursor()
#Ad= input()
cursor = c.execute("SELECT ID,Adress from rsshubServers")
thread_list = []
for row in cursor:
t = threading.Thread(target=updateOneServer, args=(row[0], row[1]))
t.start()
thread_list.append(t)
for t in thread_list:
t.join()
conn.close()
logCenter.destroyLogger(logger)
def getRssTaskInfo(rssName):
conn = dbConn.getConn()
cTask = conn.cursor()
cursor = cTask.execute(
"SELECT link , name , round , lastget, title,unread from normalrsslinks where name='{}'"
.format(rssName))
for row in cursor:
newDict = dict_factory(cursor, row)
newDict['isRsshub'] = 0
cursor = cTask.execute(
"SELECT router , name , round , lastget , recommendedServerID ,title ,unread from rsshubtasks where name='{}'"
.format(rssName))
for row in cursor:
newDict = dict_factory(cursor, row)
newDict['isRsshub'] = 1
conn.close()
return newDict
def updateOneServer(ID, ServerAD):
conn = dbConn.getConn()
LC = time.strftime("%Y-%m-%d", time.localtime())
try:
FN = getFuncNum(ServerAD)
if (FN == 0):
FN = getFuncNum(ServerAD)
if (FN == 0):
conn.close()
return
sqlupdate = "UPDATE rsshubServers set FuncNum={},LastCheck ='{}' where ID={}".format(
FN, LC, ID)
conn.execute(sqlupdate)
conn.commit()
logger.info("Updatedone " + ServerAD + " " + str(FN) + "个功能")
except:
logger.error("UpdateFailed " + ServerAD)
conn.close()
def getRssTasks():
normalDict = []
rsshubDict = []
conn = dbConn.getConn()
cTask = conn.cursor()
cursor = cTask.execute(
"SELECT link , name , round , lastget, title,unread from normalrsslinks where active=1"
)
for row in cursor:
d = dict_factory(cursor, row)
normalDict.append(d)
cursor = cTask.execute(
"SELECT router , name , round , lastget , recommendedServerID ,title ,unread from rsshubtasks where active=1"
)
for row in cursor:
d = dict_factory(cursor, row)
rsshubDict.append(d)
conn.close()
return (normalDict, rsshubDict)
def getOneDbRssData(name, start=0, num=10):
rssDict = []
conn = dbConn.getConn()
c = conn.cursor()
cursor = c.execute(
"SELECT rssName,title,summary,timestamp,link,downloaded from rssData where rssName='{}' order by timestamp desc limit {},{}"
.format(name, start, num))
for row in cursor:
d = dict_factory(cursor, row)
rssDict.append(d)
if (rssDict != []):
conn.execute(
"UPDATE normalrsslinks set unread=0 where name='{}'".format(
row[0]))
conn.execute("UPDATE rsshubtasks set unread=0 where name='{}'".format(
row[0]))
conn.commit()
conn.close()
return rssDict
cursor = cTask.execute("SELECT name ,router,autoDownload from rsshubtasks")
for row in cursor:
if (row[1].find("douyin") >= 0 and row[2] == 1):
routerCount = routerCount + 1
cursor2 = cData.execute(
"SELECT title,link,downloaded,timestamp from rssData where rssName='{}'"
.format(row[0]))
for row2 in cursor2:
if (row2[2] == 0):
t = threading.Thread(target=downloadOneVideo,
args=(row[0], row2[0], row2[1],
row2[3]))
t.start()
thread_list.append(t)
for t in thread_list:
t.join()
logger.info(
str(routerCount) + " routers need autoDownload " + str(VideoCount) +
" videos downloaded")
conn.close()
#以下为main开始
VideoCount = 0
logger = logCenter.getLogger("dyDownload")
conn = dbConn.getConn()
if __name__ == "__main__":
downloadAllVideo()
logCenter.destroyLogger(logger)
