def main(): workers = [] try: server = HTTPServer(('', Settings.server_port), WebServer) print "Starting WebServer on %s" % Settings.server_port webserver_process = Process(name='webserver', target=server.serve_forever) webserver_process.start() workers.append(webserver_process) except socket.error: print 'WebServer port already taken : %s in use' % Settings.server_port exit() while True: try: queue_manager = QueueManager() queue_data = queue_manager.getQueue() for queue_task in queue_data: if int(queue_task[Settings.QINDEX_STATUS]) < 2: dl_process = Process( name=queue_task[Settings.QINDEX_URL], target=FileManager.download_file, args=(queue_task[Settings.QINDEX_URL], queue_task[Settings.QINDEX_FILE])) dl_process.start() workers.append(dl_process) time.sleep(5) except KeyboardInterrupt: print " Interrupt caught : stopping program" for worker in workers: if worker.name.startswith('webserver'): print 'Stopping WebServer' server.socket.close() else: FileManager.sigterm_catcher(worker.name) worker.terminate() worker.join() break
def sigterm_catcher(url): queue_manager = QueueManager() current_status = queue_manager.getStatus(url) if current_status == 3: print "Stopping download of : %s" % url queue_manager.setStatus(url, 1) queue_manager.saveQueue()
def download_file(url, localfile, resume_download=True): try: print "Starting download of : %s" % url queue_manager = QueueManager() queue_manager.setStatus(url, 2) queue_manager.saveQueue() c = pycurl.Curl() c.setopt(c.URL, url) c.setopt(c.USERAGENT, "SnakElephant 0.1 (Python)") c.setopt(c.FOLLOWLOCATION, True) c.setopt(c.MAXREDIRS, 5) c.setopt(c.CONNECTTIMEOUT, 30) c.setopt(c.AUTOREFERER, True) c.setopt(c.SSL_VERIFYHOST, False) c.setopt(c.SSL_VERIFYPEER, False) if resume_download and os.path.isfile(Settings.incomplete_dir + localfile): start_existing = os.path.getsize(Settings.incomplete_dir + localfile) c.setopt(c.RESUME_FROM, start_existing) c.setopt(c.WRITEDATA, open(Settings.incomplete_dir + localfile, "ab")) else: c.setopt(c.WRITEDATA, open(Settings.incomplete_dir + localfile, "w")) c.perform() FileManager.move_file(url, localfile) except pycurl.error, err: if c.getinfo(c.HTTP_CODE) == 416: print "Resuming download is not supported by the remote server. Restarting normally." FileManager.download_file(url, localfile, False) pass else: queue_manager.setStatus(url, 0) queue_manager.saveQueue() pass
def main(): workers = [ ] try: server = HTTPServer(('', Settings.server_port), WebServer) print "Starting WebServer on %s" % Settings.server_port webserver_process = Process(name='webserver', target=server.serve_forever) webserver_process.start() workers.append(webserver_process) except socket.error: print 'WebServer port already taken : %s in use' % Settings.server_port exit() while True: try: queue_manager = QueueManager() queue_data = queue_manager.getQueue() for queue_task in queue_data: if int(queue_task[Settings.QINDEX_STATUS]) < 2: dl_process = Process(name=queue_task[Settings.QINDEX_URL], target=FileManager.download_file, args=(queue_task[Settings.QINDEX_URL], queue_task[Settings.QINDEX_FILE])) dl_process.start() workers.append(dl_process) time.sleep(5) except KeyboardInterrupt: print " Interrupt caught : stopping program" for worker in workers: if worker.name.startswith('webserver'): print 'Stopping WebServer' server.socket.close() else: FileManager.sigterm_catcher(worker.name) worker.terminate() worker.join() break
def move_file(url, localfile): print "Download complete : %s" % url shutil.move(Settings.incomplete_dir + localfile, Settings.complete_dir + localfile) os.chmod(Settings.complete_dir + localfile, Settings.default_chmod) queue_manager = QueueManager() queue_manager.setStatus(url, 3) queue_manager.saveQueue()
def do_GET(self): authed = False try: if self.headers.getheader("Authorization") == None: self.do_AUTHHEAD() self.wfile.write("no auth header received") pass elif self.headers.getheader("Authorization") == "Basic %s" % Settings.web_credentials: authed = True pass else: self.wfile.write("not authenticated") pass if authed == False: return # Verification des telechargements if self.path.startswith("/check-dl"): self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() queue_manager = QueueManager() queue_status = queue_manager.checkStatuses() total_space = ( statvfs(Settings.complete_dir).f_files * statvfs(Settings.complete_dir).f_bsize / 1024 / 1024 / 1024 ) free_space = ( statvfs(Settings.complete_dir).f_bavail * statvfs(Settings.complete_dir).f_bsize / 1024 / 1024 / 1024 ) self.wfile.write(json.dumps([[free_space, total_space], queue_status])) return # Ajout d'un DL if self.path.startswith("/add-url"): fileURL = "" qspos = self.path.find("?") if qspos >= 0: qs = parse_qs(self.path[qspos + 1 :], keep_blank_values=1) if "fileURL" in qs: fileURL = qs["fileURL"][0] queue_manager = QueueManager() queue_manager.addQueue(fileURL) self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() return # Default handler if self.path == "/": self.path = "/manager.html" f = open(Settings.root_path + sep + self.path) self.send_response(200) self.end_headers() self.wfile.write(f.read()) f.close() return except IOError: self.send_error(404, "File Not Found: %s" % self.path)
def do_GET(self): authed = False try: if self.headers.getheader('Authorization') == None: self.do_AUTHHEAD() self.wfile.write('no auth header received') pass elif self.headers.getheader( 'Authorization') == 'Basic %s' % Settings.web_credentials: authed = True pass else: self.wfile.write('not authenticated') pass if (authed == False): return # Verification des telechargements if self.path.startswith('/check-dl'): self.send_response(200) self.send_header('Content-type', 'text/html') self.end_headers() queue_manager = QueueManager() queue_status = queue_manager.checkStatuses() total_space = statvfs(Settings.complete_dir).f_files * statvfs( Settings.complete_dir).f_bsize / 1024 / 1024 / 1024 free_space = statvfs(Settings.complete_dir).f_bavail * statvfs( Settings.complete_dir).f_bsize / 1024 / 1024 / 1024 self.wfile.write( json.dumps([[free_space, total_space], queue_status])) return # Ajout d'un DL if self.path.startswith('/add-url'): fileURL = '' qspos = self.path.find('?') if qspos >= 0: qs = parse_qs(self.path[qspos + 1:], keep_blank_values=1) if 'fileURL' in qs: fileURL = qs['fileURL'][0] queue_manager = QueueManager() queue_manager.addQueue(fileURL) self.send_response(200) self.send_header('Content-type', 'text/html') self.end_headers() return # Default handler if (self.path == '/'): self.path = '/manager.html' f = open(Settings.root_path + sep + self.path) self.send_response(200) self.end_headers() self.wfile.write(f.read()) f.close() return except IOError: self.send_error(404, 'File Not Found: %s' % self.path)