def main():
workers = []
try:
server = HTTPServer(('', Settings.server_port), WebServer)
print "Starting WebServer on %s" % Settings.server_port
webserver_process = Process(name='webserver',
target=server.serve_forever)
webserver_process.start()
workers.append(webserver_process)
except socket.error:
print 'WebServer port already taken : %s in use' % Settings.server_port
exit()
while True:
try:
queue_manager = QueueManager()
queue_data = queue_manager.getQueue()
for queue_task in queue_data:
if int(queue_task[Settings.QINDEX_STATUS]) < 2:
dl_process = Process(
name=queue_task[Settings.QINDEX_URL],
target=FileManager.download_file,
args=(queue_task[Settings.QINDEX_URL],
queue_task[Settings.QINDEX_FILE]))
dl_process.start()
workers.append(dl_process)
time.sleep(5)
except KeyboardInterrupt:
print " Interrupt caught : stopping program"
for worker in workers:
if worker.name.startswith('webserver'):
print 'Stopping WebServer'
server.socket.close()
else:
FileManager.sigterm_catcher(worker.name)
worker.terminate()
worker.join()
break
def sigterm_catcher(url):
queue_manager = QueueManager()
current_status = queue_manager.getStatus(url)
if current_status == 3:
print "Stopping download of : %s" % url
queue_manager.setStatus(url, 1)
queue_manager.saveQueue()
def download_file(url, localfile, resume_download=True):
try:
print "Starting download of : %s" % url
queue_manager = QueueManager()
queue_manager.setStatus(url, 2)
queue_manager.saveQueue()
c = pycurl.Curl()
c.setopt(c.URL, url)
c.setopt(c.USERAGENT, "SnakElephant 0.1 (Python)")
c.setopt(c.FOLLOWLOCATION, True)
c.setopt(c.MAXREDIRS, 5)
c.setopt(c.CONNECTTIMEOUT, 30)
c.setopt(c.AUTOREFERER, True)
c.setopt(c.SSL_VERIFYHOST, False)
c.setopt(c.SSL_VERIFYPEER, False)
if resume_download and os.path.isfile(Settings.incomplete_dir +
localfile):
start_existing = os.path.getsize(Settings.incomplete_dir +
localfile)
c.setopt(c.RESUME_FROM, start_existing)
c.setopt(c.WRITEDATA,
open(Settings.incomplete_dir + localfile, "ab"))
else:
c.setopt(c.WRITEDATA,
open(Settings.incomplete_dir + localfile, "w"))
c.perform()
FileManager.move_file(url, localfile)
except pycurl.error, err:
if c.getinfo(c.HTTP_CODE) == 416:
print "Resuming download is not supported by the remote server. Restarting normally."
FileManager.download_file(url, localfile, False)
pass
else:
queue_manager.setStatus(url, 0)
queue_manager.saveQueue()
pass
def main():
workers = [ ]
try:
server = HTTPServer(('', Settings.server_port), WebServer)
print "Starting WebServer on %s" % Settings.server_port
webserver_process = Process(name='webserver', target=server.serve_forever)
webserver_process.start()
workers.append(webserver_process)
except socket.error:
print 'WebServer port already taken : %s in use' % Settings.server_port
exit()
while True:
try:
queue_manager = QueueManager()
queue_data = queue_manager.getQueue()
for queue_task in queue_data:
if int(queue_task[Settings.QINDEX_STATUS]) < 2:
dl_process = Process(name=queue_task[Settings.QINDEX_URL], target=FileManager.download_file, args=(queue_task[Settings.QINDEX_URL], queue_task[Settings.QINDEX_FILE]))
dl_process.start()
workers.append(dl_process)
time.sleep(5)
except KeyboardInterrupt:
print " Interrupt caught : stopping program"
for worker in workers:
if worker.name.startswith('webserver'):
print 'Stopping WebServer'
server.socket.close()
else:
FileManager.sigterm_catcher(worker.name)
worker.terminate()
worker.join()
break
def move_file(url, localfile):
print "Download complete : %s" % url
shutil.move(Settings.incomplete_dir + localfile,
Settings.complete_dir + localfile)
os.chmod(Settings.complete_dir + localfile, Settings.default_chmod)
queue_manager = QueueManager()
queue_manager.setStatus(url, 3)
queue_manager.saveQueue()
def do_GET(self):
authed = False
try:
if self.headers.getheader("Authorization") == None:
self.do_AUTHHEAD()
self.wfile.write("no auth header received")
pass
elif self.headers.getheader("Authorization") == "Basic %s" % Settings.web_credentials:
authed = True
pass
else:
self.wfile.write("not authenticated")
pass
if authed == False:
return
# Verification des telechargements
if self.path.startswith("/check-dl"):
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
queue_manager = QueueManager()
queue_status = queue_manager.checkStatuses()
total_space = (
statvfs(Settings.complete_dir).f_files * statvfs(Settings.complete_dir).f_bsize / 1024 / 1024 / 1024
)
free_space = (
statvfs(Settings.complete_dir).f_bavail
* statvfs(Settings.complete_dir).f_bsize
/ 1024
/ 1024
/ 1024
)
self.wfile.write(json.dumps([[free_space, total_space], queue_status]))
return
# Ajout d'un DL
if self.path.startswith("/add-url"):
fileURL = ""
qspos = self.path.find("?")
if qspos >= 0:
qs = parse_qs(self.path[qspos + 1 :], keep_blank_values=1)
if "fileURL" in qs:
fileURL = qs["fileURL"][0]
queue_manager = QueueManager()
queue_manager.addQueue(fileURL)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
return
# Default handler
if self.path == "/":
self.path = "/manager.html"
f = open(Settings.root_path + sep + self.path)
self.send_response(200)
self.end_headers()
self.wfile.write(f.read())
f.close()
return
except IOError:
self.send_error(404, "File Not Found: %s" % self.path)
def do_GET(self):
authed = False
try:
if self.headers.getheader('Authorization') == None:
self.do_AUTHHEAD()
self.wfile.write('no auth header received')
pass
elif self.headers.getheader(
'Authorization') == 'Basic %s' % Settings.web_credentials:
authed = True
pass
else:
self.wfile.write('not authenticated')
pass
if (authed == False):
return
# Verification des telechargements
if self.path.startswith('/check-dl'):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
queue_manager = QueueManager()
queue_status = queue_manager.checkStatuses()
total_space = statvfs(Settings.complete_dir).f_files * statvfs(
Settings.complete_dir).f_bsize / 1024 / 1024 / 1024
free_space = statvfs(Settings.complete_dir).f_bavail * statvfs(
Settings.complete_dir).f_bsize / 1024 / 1024 / 1024
self.wfile.write(
json.dumps([[free_space, total_space], queue_status]))
return
# Ajout d'un DL
if self.path.startswith('/add-url'):
fileURL = ''
qspos = self.path.find('?')
if qspos >= 0:
qs = parse_qs(self.path[qspos + 1:], keep_blank_values=1)
if 'fileURL' in qs:
fileURL = qs['fileURL'][0]
queue_manager = QueueManager()
queue_manager.addQueue(fileURL)
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
return
# Default handler
if (self.path == '/'):
self.path = '/manager.html'
f = open(Settings.root_path + sep + self.path)
self.send_response(200)
self.end_headers()
self.wfile.write(f.read())
f.close()
return
except IOError:
self.send_error(404, 'File Not Found: %s' % self.path)
