def download(url, episode, path, taskid):
downloadpath = "%s/%s" % (basepath, path)
downloadname = "%s/%s/%s_%s.mp4" % (basepath, path, path, episode)
if not os.path.exists(downloadpath):
os.system('mkdir -p %s' % downloadpath)
try:
sql2 = 'update spider_taskinfo set downloadstatus=2 where id=%s' % taskid
dbquest(sql2)
d_info = urllib.urlretrieve(url, downloadname)
total = int(d_info[-1]['Content-Length'])
localsize = os.path.getsize(downloadname)
etime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
if int(localsize) >= int(total) and int(localsize) > 2500000:
sql = 'update spider_taskinfo set downloadstatus=1 etime="%s" where id=%s' % (
etime, taskid)
dbquest(sql)
return 0
else:
sql = 'update spider_taskinfo set downloadstatus=0 where id=%s' % taskid
dbquest(sql)
os.system('rm -f %s' % downloadname)
return -1
except Exception, e:
sql = 'update spider_taskinfo set downloadstatus=0 where id=%s' % taskid
dbquest(sql)
os.system('rm -f %s' % downloadname)
return -1
def _gettaskinfo(self, dirname):
sql = 'select downloadurl,episode,id from spider_taskinfo where dramaurl like "%%%s%%" and downloadstatus=0' % dirname
tasklist = dbquest(sql)
return_list = []
for task in tasklist:
return_list.append((task[0], task[1], dirname, task[2]))
return return_list
def assemble_json(locatename):
sql = 'select channelid,taskname,date_format(stime,"%%Y-%%m-%%d %%H:%%i:%%s"),date_format(etime,"%%Y-%%m-%%d %%H:%%i:%%s") from luzhi_channel_task where locatename="%s";' % locatename
dbreturn = dbquest(sql)
if dbreturn != ():
return dbreturn[0]
else:
return -1
def get_id(locatename):
sql = 'select taskid from luzhi_channel_task where locatename="%s";' % locatename
dbreturn = dbquest(sql)
if dbreturn != ():
return dbreturn[0][0]
else:
return -1
def quest_url(channelid):
sql = 'select channelurl from luzhi_channel where channelid=%s' % channelid
dbreturn = dbquest(sql)
if dbreturn != ():
return dbreturn[0][0]
else:
return -1
def _gettask(self):
sql = 'select dirpath from spider_taskmain'
dirlist = dbquest(sql)
taskmain = []
for dirname in dirlist:
pertask = self._gettaskinfo(dirname[0])
taskmain = list(set(taskmain).union(set(pertask)))
return taskmain
def update_status(locatename, status, filepath):
sql = 'update luzhi_channel_task set taskstatus="%s",filepath="%s.ts" where locatename="%s";' % (
status, filepath, locatename)
try:
dbreturn = dbquest(sql)
return 0
except Exception:
return -1
def insert_id(locatename, taskid):
sql = 'update luzhi_channel_task set taskid="%s" where locatename="%s";' % (
taskid, locatename)
try:
dbreturn = dbquest(sql)
return 0
except Exception:
return -1
def _insert_task(self, daytime, channelid, taskname, starth, startm, endh,
endm, sqlstime, sqletime):
sqlstime = "%s %s:00" % (daytime, sqlstime)
sqletime = "%s %s:00" % (daytime, sqletime)
locatename = "%s_%s%s%s" % (channelid, daytime.replace(
'-', ''), starth, startm)
stime = "%s %s:%s:00" % (daytime, starth, startm)
etime = "%s %s:%s:00" % (daytime, endh, endm)
insertsql = 'insert into luzhi_channel_task(channelid,locatename,taskname,taskid,stime,etime,taskstatus) values("%s","%s","%s","1","%s","%s","1");' % (
channelid, locatename, taskname, sqlstime, sqletime)
if not dbquest(insertsql) == -1:
os.system(
'echo "%s %s * * * %s %s/task.py %s start" >>%s' %
(startm, starth, python, basepath, locatename, self.filename))
logging.info("channel:%s insert task success,starttime %s:%s" %
(channelid, starth, startm))
os.system(
'echo "%s %s * * * %s %s/task.py %s stop" >>%s' %
(endm, endh, python, basepath, locatename, self.filename))
logging.info(
"channel:%s insert task success,locatename: %s,endtime %s:%s" %
(channelid, locatename, endh, endm))
#===============================================================
import urllib
import urllib2
import json
import httplib
from Dbquest import dbquest
def get_info(url):
try:
req = urllib.urlopen(url)
code = req.getcode()
return json.loads(req.read())
except Exception:
return -1
if __name__ == "__main__":
url = "http://admins.quliebiao.com:9022/Force/Spider"
task_list = get_info(url)
if task_list != -1:
for task in task_list:
name = task['name']
dtype = task['type']
starturl = task['drama']
dirpath = task['root_dir']
sql = 'insert into spider_taskmain(dirpath,dramaname,dramatype,starturl) values("%s","%s","%s","%s")' % (
dirpath, name, dtype, starturl)
dbquest(sql)