class Control(threading.Thread): def __init__(self): threading.Thread.__init__(self) #Config.load() self.saveFileControl = SaveFileControl() self.saveFileControl.start() self.log = Log() #self.toContinue = True self.autoDownload = True self.isStopped = False self.queueingCurlObjectList = [] self.curlLock = threading.Lock() #Check and load download files list if Config.checkExistence(Config.settings.queueingListFile, TYPE_FILE) > NO_EXIST: try: queueingFile = open(Config.settings.queueingListFile, 'rb') self.downloadFileList = cPickle.load(queueingFile) queueingFile.close() except EOFError: self.downloadFileList = DownloadFileList() self.log.debug('EXCEPTION Control', EOFError) #self.log.debug('Queueing file error') self.downloadFileList.setLock() self.downloadFileList.resetStatus(STAT_S) else: self.downloadFileList = DownloadFileList() self.downloadFileList.setLock() #self.mainFrame = MainFrame(None, wx.ID_ANY, "Linux Rapidshare Grabber", (0, 0), (800, 600), self, self.downloadFileList) self.mainFrame = None #show the list in GUI self.downloadFileControlList = [] def setMainFrame(self, mainFrame): self.mainFrame = mainFrame for downloadFile in self.downloadFileList.getList(): #print downloadFile self.mainFrame.addDownloadFileToList(downloadFile, PANEL_TOP) for downloadFile in self.downloadFileList.getList(True): #print downloadFile self.mainFrame.addDownloadFileToList(downloadFile, PANEL_BOT) #Add an url to the queueing list def addURL(self, fileURL): strippedFileURL = str(fileURL).strip() if (self.isStopped) or (strippedFileURL == ''): return if Config.getLinkType(strippedFileURL) == RAPIDSHARE_LINK and strippedFileURL.endswith('.html'): strippedFileURL = strippedFileURL[0: len(strippedFileURL) - len('.html')] if not self.downloadFileList.getDownloadFileByFileURL(strippedFileURL) is None: return MSG_ADD_FILEURL_EXISTED + strippedFileURL downloadFile = DownloadFile(strippedFileURL) downloadFile.setId(Config.getId()) self.downloadFileList.addQueueingFile(downloadFile) self.mainFrame.addDownloadFileToList(downloadFile, PANEL_TOP) return MSG_ADD_FILEURL_SUCCESS + strippedFileURL def addURLById(self, id): downloadFile = self.downloadFileList.getDownloadFileById(id, False) self.addURL(downloadFile.getFileURL()) #Stop all downloads, save the queueing list then quit def exit(self): #self.toContinue = False self.isStopped = True for downloadFileControl in self.downloadFileControlList: #print 'Stopping ', downloadFileControl.getDownloadFile().getId() downloadFileControl.stop() #print 'Stopping Done' self.downloadFileList.unsetLock() #self.log.debug('downloadFileListLock', self.downloadFileList.downloadFileListLock) #self.log.debug('completedFileListLock', self.downloadFileList.completedFileListLock) self.saveQueueingFiles() #for downloadFile in self.downloadFileList.getList(): #print downloadFile Config.save() #this is dangerous!!! def saveQueueingFiles(self): queueingFile = open(Config.settings.queueingListFile, 'wb') #print dir(self.downloadFileList) cPickle.dump(self.downloadFileList, queueingFile, cPickle.HIGHEST_PROTOCOL) queueingFile.close() #add a curl to the curl list def addCurlObject(self, curlObject): #lockkkk? #print 'control addCurlObject getting lock' self.curlLock.acquire() #print 'adding ', curlObject self.queueingCurlObjectList.append(curlObject) #print 'control addCurlE_OPERATION_TIMEOUTEDObject releasing lock' self.curlLock.release() #get a curl from the list def getCurlObject(self): #print 'control getCurlObject getting lock' self.curlLock.acquire() if len(self.queueingCurlObjectList) != 0: curlObject = self.queueingCurlObjectList.pop(0) #print 'control getCurlObject releasing lock' self.curlLock.release() return curlObject else: #print 'control getCurlObject releasing lock' self.curlLock.release() return None #reset the download file def resetDownload(self, id = None): if not (id is None): found = False #self.downloadFileList.changeStatus(fileURL, STAT_S) for downloadFileControl in self.downloadFileControlList: if downloadFileControl.getDownloadFile().getId() == id: downloadFileControl.resetSettings() self.mainFrame.update(downloadFileControl.getDownloadFile(), updateType = [FILESTATUS_COL, RETRY_COL]) found = True break if not found: downloadFile = self.downloadFileList.getDownloadFileById(id) #downloadFile.setStatus(STAT_S) downloadFile.resetInfo() self.mainFrame.update(downloadFile, updateType = [FILENAME_COL, FILESTATUS_COL]) else: #self.isStopped = True for downloadFileControl in self.downloadFileControlList: #if downloadFileControl.isAlive(): #downloadFileControl.getDownloadFile().setStatus(STAT_S) downloadFileControl.resetSettings() self.mainFrame.update(downloadFileControl.getDownloadFile(), updateType = [FILESTATUS_COL]) #stop the download def stopDownload(self, id = None): if not id is None: found = False #self.downloadFileList.changeStatus(fileURL, STAT_S) for downloadFileControl in self.downloadFileControlList: if downloadFileControl.getDownloadFile().getId() == id: downloadFileControl.stop() self.mainFrame.update(downloadFileControl.getDownloadFile(), updateType = [FILESTATUS_COL]) found = True downloadFileControl.reset() break if not found: downloadFile = self.downloadFileList.getDownloadFileById(id) downloadFile.setStatus(STAT_S) self.mainFrame.update(downloadFile, updateType = [FILESTATUS_COL]) else: self.autoDownload = False for downloadFileControl in self.downloadFileControlList: #if downloadFileControl.isAlive(): #downloadFileControl.getDownloadFile().setStatus(STAT_S) downloadFileControl.stop() self.mainFrame.update(downloadFileControl.getDownloadFile(), updateType = [FILESTATUS_COL]) downloadFileControl.reset() def continueDownload(self, id = None): if not (id is None): downloadFile = self.downloadFileList.getDownloadFileById(id) if downloadFile.getStatus() != STAT_D: downloadFile.setStatus(STAT_Q) self.mainFrame.update(downloadFile, updateType = [FILESTATUS_COL]) else: self.downloadFileList.resetStatus(STAT_Q) for downloadFile in self.downloadFileList.getList(): self.mainFrame.update(downloadFile, updateType = [FILESTATUS_COL]) self.autoDownload = True def deleteDownloadTop(self, id = None): #print 'deleteDownloadTop ', id if not (id is None): found = False for downloadFileControl in self.downloadFileControlList: if downloadFileControl.getDownloadFile().getId() == id: found = True self.mainFrame.deleteDownloadFileFromList(downloadFileControl.getDownloadFile().getId(), PANEL_TOP) downloadFileControl.delete() self.downloadFileList.deleteDownloadFileFromDownloadList(downloadFileControl.getDownloadFile().getId()) if not found: self.downloadFileList.deleteDownloadFileFromDownloadList(id) self.mainFrame.deleteDownloadFileFromList(id, PANEL_TOP) else: self.isStopped = True for downloadFileControl in self.downloadFileControlList: self.mainFrame.deleteDownloadFileFromList(downloadFileControl.getDownloadFile().getId(), PANEL_TOP) downloadFileControl.delete() self.downloadFileList.deleteDownloadFileFromDownloadList(downloadFileControl.getDownloadFile().getId()) self.downloadFileList.emptyList() self.mainFrame.emptyList(PANEL_TOP) def deleteDownloadBot(self, id = None): #print 'deleteDownloadBot ', id if not (id is None): self.mainFrame.deleteDownloadFileFromList(id, PANEL_BOT) self.downloadFileList.deleteDownloadFileFromCompletedList(id) else: self.mainFrame.emptyList(PANEL_BOT) def run(self): if self.mainFrame is None: self.log.debug('No mainFrame found, cannot continue') sys.exit() multiHandler = pycurl.CurlMulti() multiHandler.handles = [] i = 0 #while self.toContinue: while not self.isStopped: #noDFile = self.downloadFileList.getNumberOfDownloadingFile() #noQFile = self.downloadFileList.getNumberOfQueueingFile() #noQueueingCurlObject = len(self.queueingCurlObjectList) maxConn = Config.settings.maxConcurrentDownload num_handles = 0 while (not self.isStopped) and (self.autoDownload) and (self.downloadFileList.getNumberOfQueueingFile() > 0): if len(self.downloadFileControlList) < maxConn: #and self.downloadFileList.getNumberOfDownloadingFile() < maxConn): downloadFile = self.downloadFileList.getQueueingFile() if not (downloadFile is None): downloadFile.resetRetry() self.log.debug('Control, Creating:', i, 'instance downloadFileControl', downloadFile.getId(), downloadFile.getFileURL()) downloadFileControl = DownloadFileControl(self, self.saveFileControl, downloadFile) downloadFile.setStatus(STAT_D) #downloadFileControl.start() downloadFileControl.run() self.downloadFileControlList.append(downloadFileControl) i += 1 else: for downloadFileControl in self.downloadFileControlList: if downloadFileControl.isDone() and (self.downloadFileList.getNumberOfQueueingFile() > 0): downloadFile = self.downloadFileList.getQueueingFile() #print 'Control downloadFile is ', downloadFile downloadFileControl.setDownloadFile(downloadFile) self.log.debug('Control, Assign downloadFile for downloadFileControl', downloadFile.getId(), downloadFile.getFileURL()) downloadFileControl.continueBuildCurl() break #while True: while not self.isStopped: curlObject = self.getCurlObject() if not (curlObject is None): multiHandler.add_handle(curlObject) else: break #while True: while not self.isStopped: ret, num_handles = multiHandler.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: #print 'Perfoming break', ret, ' ', num_handles break #while True: while not self.isStopped: num_q, ok_list, err_list = multiHandler.info_read() for c in ok_list: #c.close() multiHandler.remove_handle(c) #print 'ok_list removed handle ', c httpCode = int(c.getinfo(pycurl.HTTP_CODE)) #print 'httpCode ', httpCode for downloadFileControl in self.downloadFileControlList: #print 'Debug ', downloadFileControl self.log.debug('Control OK list checking', downloadFileControl.getDownloadFile().getId(), c.downloadFileId) if downloadFileControl.getDownloadFile().getId() == c.downloadFileId: if not (c.partNo is None): self.log.debug('Control curl object belongs to downloadPartControl, checkFinish now') downloadFileControl.checkFinish() #downloadFileControl.reset() break if httpCode == 404: self.log.debug('Control OK list 404', downloadFileControl.getDownloadFile().getId()) downloadFileControl.getDownloadFile().setStatus(STAT_E) downloadFileControl.getDownloadFile().setErrorStr('404 File not found') self.mainFrame.update(downloadFileControl.getDownloadFile()) downloadFileControl.reset() elif (not downloadFileControl.isDone()) and (not downloadFileControl.isBusy()): self.log.debug('Control OK list continueBuildCurl', downloadFileControl.getDownloadFile().getId()) downloadFileControl.continueBuildCurl() break #print 'Success: ', c.downloadFileId for c, errno, errmsg in err_list: #c.close() multiHandler.remove_handle(c) #print 'err_list removed handle ', c self.log.debug('Control ERROR list error', errno, errmsg) httpCode = int(c.getinfo(pycurl.HTTP_CODE)) #print 'err_list httpCode ', httpCode for downloadFileControl in self.downloadFileControlList: #print 'Debug ', downloadFileControl if downloadFileControl.getDownloadFile().getId() == c.downloadFileId: self.log.debug('Control ERROR list checking', downloadFileControl.getDownloadFile().getId(), c.downloadFileId) if not errno in (pycurl.E_PARTIAL_FILE, pycurl.E_WRITE_ERROR, pycurl.E_ABORTED_BY_CALLBACK): self.log.debug('Control ERROR list error unexpected', errno, errmsg) if downloadFileControl.getDownloadFile().isRetryPossible(): self.log.debug('Control ERROR list retrying') downloadFileControl.getDownloadFile().setStatus(STAT_E) downloadFileControl.getDownloadFile().setErrorStr(errmsg) self.mainFrame.update(downloadFileControl.getDownloadFile(), updateType = [FILESTATUS_COL, FILEERROR_COL]) if not (c.partNo is None): self.log.debug('Control ERROR list downloadPart failed, resetting part') downloadFileControl.resetPart(c.partNo) else: self.log.debug('Control ERROR list downloadFile failed, continueBuildCurl') downloadFileControl.continueBuildCurl(True) else: self.log.debug('Control ERROR list no more retry possible, report error') downloadFileControl.reportError(errmsg) break if httpCode == 404: self.log.debug('Control ERROR list 404') downloadFileControl.getDownloadFile().setStatus(STAT_E) downloadFileControl.getDownloadFile().setErrorStr('404 File not found') self.mainFrame.update(downloadFileControl.getDownloadFile()) downloadFileControl.reset() elif (not downloadFileControl.isDone()) and (not downloadFileControl.isBusy()) and (errno != pycurl.E_ABORTED_BY_CALLBACK): self.log.debug('Control ERROR list continueBuildCurl', downloadFileControl.getDownloadFile().getId()) downloadFileControl.continueBuildCurl() break #print 'err_list Failed: ', ' errno ', errno, ' errmsg ', errmsg #freelist.append(c) #num_processed = num_processed + len(ok_list) + len(err_list) if num_q == 0: break if num_handles == 0: #print 'No more connection, I sleep a bit' time.sleep(0.5) else: ret = multiHandler.select(1.0) if ret == -1: continue #while (True): while not self.isStopped: ret, num_handles = multiHandler.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: break for downloadFileControl in self.downloadFileControlList: #if (downloadFileControl.isAlive()): # downloadFileControl.join() downloadFileControl.kill() self.mainFrame.quitNow() self.saveFileControl.kill() self.saveFileControl.join() #print 'Control Quit' def finishFile(self, downloadFile): if self.isStopped: return self.mainFrame.deleteDownloadFileFromList(downloadFile.getId(), PANEL_TOP) self.mainFrame.addDownloadFileToList(downloadFile, PANEL_BOT) self.downloadFileList.deleteDownloadFileFromDownloadList(downloadFile.getId(), True) def report(self, downloadFile, updateType): if self.isStopped: return self.mainFrame.update(downloadFile, updateType) def printDebug(self): print '===================================' print 'Total thread: ', threading.activeCount() print 'Total downloading File: ', len(self.downloadFileList.getList()) print 'List download File: ' for downloadFile in self.downloadFileList.getList(): print 'id ', downloadFile.getId(), ' status: ', downloadFile.getStatus() print 'List completed File: ', len(self.downloadFileList.getList(True)) for downloadFile in self.downloadFileList.getList(True): print 'id ', downloadFile.getId(), ' status: ', downloadFile.getStatus() print 'List DownloadFileControl: ' for downloadFileControl in self.downloadFileControlList: downloadFile = downloadFileControl.getDownloadFile() print 'id ', downloadFile.getId(), ' busy: ', downloadFileControl.busy, ', done: ', downloadFileControl.done, ', gotJob: ', downloadFileControl.gotJob