def download_more(self):
tasks = db.select_tasks(state=STATE_DOWNLOADING)
if len(tasks) < conf.task_queue_size:
tasks = db.select_tasks(state=STATE_WAITING)
for task in tasks:
log.debug('start to download %s' % task['id'])
self._start(task['id'])
def download_more(self):
tasks = db.select_tasks(state=STATE_DOWNLOADING)
if len(tasks) < conf.task_queue_size:
tasks = db.select_tasks(state=STATE_WAITING)
for task in tasks:
log.debug('start to download %s' % task['id'])
self._start(task['id'])
def tasks(self, options):
tasks = db.select_tasks(**options)
for task in tasks:
if task['state'] == STATE_DOWNLOADING and task['update_time'] and task['speed']:
nowt = datetime.datetime.now()
parts = task['update_time'].split('.')
dt = datetime.datetime.strptime(parts[0], "%Y-%m-%d %H:%M:%S")
update_time = dt.replace(microsecond=int(parts[1]))
interval = nowt - update_time
interval_seconds = interval.seconds + interval.microseconds*1.0/1000/1000
if interval_seconds > 2 and interval_seconds * task['speed'] > conf.buffer_size:
speed = conf.buffer_size * 1.0 / interval_seconds
task['speed'] = 0 if speed < 1024 else speed
return tasks
def tasks(self, options):
tasks = db.select_tasks(**options)
for task in tasks:
if task['state'] == STATE_DOWNLOADING and task[
'update_time'] and task['speed']:
nowt = datetime.datetime.now()
parts = task['update_time'].split('.')
dt = datetime.datetime.strptime(parts[0], "%Y-%m-%d %H:%M:%S")
update_time = dt.replace(microsecond=int(parts[1]))
interval = nowt - update_time
interval_seconds = interval.seconds + interval.microseconds * 1.0 / 1000 / 1000
if interval_seconds > 2 and interval_seconds * task[
'speed'] > conf.buffer_size:
speed = conf.buffer_size * 1.0 / interval_seconds
task['speed'] = 0 if speed < 1024 else speed
return tasks
def maxspeed(self, tid):
return 0
if not conf.total_maxspeed:
return 0
if not conf.total_max:
return 0
tasks = db.select_tasks(state="!=5")
total_speed = 0
for task in tasks:
if task['id'] == tid:
continue
speed = task['speed']
if speed:
total_speed += task['speed']
if total_speed > conf.total_max:
return 1
else:
return conf.total_max - total_speed
def resume(self, tid):
# set state to waiting and load_more()
tid = int(tid)
tasks = db.select_tasks(id=tid)
if tasks:
task = tasks[0]
url = task["url"]
output = task["output"]
state = task["state"]
thsize = task["thsize"]
maxspeed = task["maxspeed"]
headers = task["headers"]
subdir = task["subdir"]
if state not in (STATE_WAITING, STATE_PAUSED, STATE_ERROR):
return
if state != STATE_WAITING:
db.update_tasks(tid, state=STATE_WAITING)
self.download_more()
def maxspeed(self, tid):
return 0
if not conf.total_maxspeed:
return 0
if not conf.total_max:
return 0
tasks = db.select_tasks(state="!=5")
total_speed = 0
for task in tasks:
if task['id'] == tid:
continue
speed = task['speed']
if speed:
total_speed += task['speed']
if total_speed > conf.total_max:
return 1
else:
return conf.total_max - total_speed
def resume(self, tid):
# set state to waiting and load_more()
tid = int(tid)
tasks = db.select_tasks(id=tid)
if tasks:
task = tasks[0]
url = task["url"]
output = task["output"]
state = task["state"]
thsize = task["thsize"]
maxspeed = task["maxspeed"]
headers = task["headers"]
subdir = task["subdir"]
if state not in (STATE_WAITING, STATE_PAUSED, STATE_ERROR):
return
if state != STATE_WAITING:
db.update_tasks(tid, state=STATE_WAITING)
self.download_more()
with open(filename) as staticfile:
filecontent = staticfile.read()
files[filename] = filecontent
return filecontent
except IOError, e:
if e.errno == 2:
raise web.notfound()
else:
raise web.notfound()
urls = (
"/(.*)", Router
)
app = web.application(urls, globals())
log.basicConfig(level=log.DEBUG, filename='webm.log')
#log.basicConfig(level=log.DEBUG)
if __name__ == "__main__":
try:
db.select_tasks(id=1)
except:
db.reset_database()
API().download_last()
try:
app.run()
except:
print "timeout"
pass
def download_last(self):
tasks = db.select_tasks(state=STATE_DOWNLOADING)
for task in tasks:
log.debug('start to download %s' % task['id'])
self._start(task['id'])
self.download_more()
def _start(self, tid):
# start a task existed
tasks = db.select_tasks(id=tid)
if not tasks:
return
db.update_tasks(tid, state=STATE_DOWNLOADING)
#create axel task
if os.fork(): # old process
return # as 200 OK
else: # sub process
# get the options
task = tasks[0]
url = task["url"]
output = task["output"]
thsize = task["thsize"]
maxspeed = task["maxspeed"]
headers = task["headers"]
subdir = task["subdir"]
force_download = True
output_file = os.path.join(conf.incomming, output)
if os.path.exists(output_file) and not os.path.exists(output_file + '.st'):
if force_download:
os.remove(output_file)
else:
print 'file completed already, skip download'
exit()
args = [os.path.join(os.getcwd(), "axel"), "-a", "-n", str(thsize), "-s", str(maxspeed)]
for header in headers.splitlines():
args.append("-H")
args.append(header)
args.append(url)
args.append("-o")
args.append(output)
os.system("mkdir -p %s" % os.path.join(conf.incomming, subdir))
axel_process = subprocess.Popen(args, shell=False, stdout=subprocess.PIPE, cwd=conf.incomming)
last_update_time = 0
while 1:
try:
line = axel_process.stdout.readline()
if not line:
break
line = line.strip()
except:
returncode = axel_process.poll()
if returncode is not None:
# axel completed
if returncode:
db.update_tasks(tid, state=STATE_ERROR, errmsg="Error, axel exit with code: %s" % returncode)
break
this_update_time = time.time()
if line.startswith(":"):
done, total, thdone, speed, left, update_time = line[1:].split("|")
if done != total and last_update_time > 0 and this_update_time - last_update_time < 1:
continue
elif line.startswith("HTTP/1."):
db.update_tasks(tid, state=STATE_ERROR, errmsg=line)
break
else:
continue
last_update_time = this_update_time
tasks = db.select_tasks(id=tid)
if tasks:
task = tasks[0]
state = task["state"]
if state == STATE_DOWNLOADING:
try:
if done == total:
#completed
db.update_tasks(tid, state=STATE_COMPLETED, left=0)
os.system("mkdir -p %s" % os.path.join(conf.downloads, subdir))
os.rename(output_file, os.path.join(conf.downloads, subdir, output))
break
db.update_tasks(tid, speed=speed, done=done, total=total, left=left)
continue
except Exception, e:
import traceback
traceback.print_exc()
db.update_tasks(tid, state=STATE_ERROR, errmsg="Error, axel exit with code: %s" % e)
try:
axel_process.terminate()
except:
pass
else:
#paused
axel_process.terminate()
else:
#deleted
axel_process.terminate()
try:
os.remove(output_file)
except:
pass
try:
os.remove(output_file + ".st")
except:
pass
returncode = axel_process.poll()
if returncode is not None:
# axel completed
if returncode:
db.update_tasks(tid, state=STATE_ERROR, errmsg="Error, axel exit with code: %s" % returncode)
self.download_more()
sys.exit()
def download_last(self):
tasks = db.select_tasks(state=STATE_DOWNLOADING)
for task in tasks:
log.debug('start to download %s' % task['id'])
self._start(task['id'])
self.download_more()
def _start(self, tid):
# start a task existed
tasks = db.select_tasks(id=tid)
if not tasks:
return
db.update_tasks(tid, state=STATE_DOWNLOADING)
#create axel task
#print "====="
#print os.fork()
if os.fork(): # old process
return # as 200 OK
else: # sub process
# get the options
task = tasks[0]
url = task["url"]
output = task["output"]
thsize = task["thsize"]
maxspeed = task["maxspeed"]
headers = task["headers"]
#subdir = task["subdir"]
downloads = task["downloads"]
ua = task["ua"]
force_download = conf.force_download
output_file = os.path.join(downloads, output)
if os.path.exists(output_file) and not os.path.exists(output_file +
'.st'):
if force_download:
os.remove(output_file)
else:
print 'file completed already, skip download'
exit()
os.system("mkdir -p %s" % os.path.join(downloads))
args = [
os.path.join(os.getcwd(), "axel"), "-a", "-n",
str(thsize), "-s",
str(maxspeed), "-U",
str(ua)
]
for header in headers.splitlines():
args.append("-H")
args.append(header)
args.append("-o")
args.append(output_file)
args.append(url)
axel_process = subprocess.Popen(args,
shell=False,
stdout=subprocess.PIPE,
cwd=downloads)
#print(args)
last_update_time = 0
while 1:
try:
line = axel_process.stdout.readline()
if not line:
break
line = line.strip()
except:
returncode = axel_process.poll()
if returncode is not None:
# axel completed
if returncode:
db.update_tasks(
tid,
state=STATE_ERROR,
errmsg="Error, axel exit with code: %s" %
returncode)
break
this_update_time = time.time()
if line.startswith(":"):
done, total, thdone, speed, left, update_time = line[
1:].split("|")
if done != total and last_update_time > 0 and this_update_time - last_update_time < 1:
continue
elif line.startswith("HTTP/1."):
db.update_tasks(tid, state=STATE_ERROR, errmsg=line)
break
else:
continue
last_update_time = this_update_time
tasks = db.select_tasks(id=tid)
if tasks:
task = tasks[0]
state = task["state"]
if state == STATE_DOWNLOADING:
try:
if done == total:
#completed
db.update_tasks(tid,
state=STATE_COMPLETED,
left=0)
os.system("mkdir -p %s" %
os.path.join(downloads))
#os.rename(output_file, os.path.join(conf.downloads, subdir, output))
break
db.update_tasks(tid,
speed=speed,
done=done,
total=total,
left=left)
continue
except Exception, e:
import traceback
traceback.print_exc()
db.update_tasks(
tid,
state=STATE_ERROR,
errmsg="Error, axel exit with code: %s" % e)
try:
axel_process.terminate()
except:
pass
else:
#paused
axel_process.terminate()
else:
#deleted
axel_process.terminate()
try:
os.remove(output_file)
except:
pass
try:
os.remove(output_file + ".st")
except:
pass
returncode = axel_process.poll()
if returncode is not None:
# axel completed
if returncode:
db.update_tasks(tid,
state=STATE_ERROR,
errmsg="Error, axel exit with code: %s" %
returncode)
self.download_more()
sys.exit()