def __init__(self):

threading.Thread.__init__(self)

#Config.load()

self.saveFileControl = SaveFileControl()

self.saveFileControl.start()

self.log = Log()

#self.toContinue = True

self.autoDownload = True

self.isStopped = False

self.queueingCurlObjectList = []

self.curlLock = threading.Lock()

#Check and load download files list

if Config.checkExistence(Config.settings.queueingListFile, TYPE_FILE) > NO_EXIST:

try:

queueingFile = open(Config.settings.queueingListFile, 'rb')

self.downloadFileList = cPickle.load(queueingFile)

queueingFile.close()

except EOFError:

self.downloadFileList = DownloadFileList()

self.log.debug('EXCEPTION Control', EOFError)

#self.log.debug('Queueing file error')

self.downloadFileList.setLock()

self.downloadFileList.resetStatus(STAT_S)

else:

self.downloadFileList = DownloadFileList()

self.downloadFileList.setLock()

#self.mainFrame = MainFrame(None, wx.ID_ANY, "Linux Rapidshare Grabber", (0, 0), (800, 600), self, self.downloadFileList)

self.mainFrame = None

#show the list in GUI

self.downloadFileControlList = []

def setMainFrame(self, mainFrame):

self.mainFrame = mainFrame

for downloadFile in self.downloadFileList.getList():

#print downloadFile

self.mainFrame.addDownloadFileToList(downloadFile, PANEL_TOP)

for downloadFile in self.downloadFileList.getList(True):

#print downloadFile

self.mainFrame.addDownloadFileToList(downloadFile, PANEL_BOT)

#Add an url to the queueing list

def addURL(self, fileURL):

strippedFileURL = str(fileURL).strip()

if (self.isStopped) or (strippedFileURL == ''):

return

if Config.getLinkType(strippedFileURL) == RAPIDSHARE_LINK and strippedFileURL.endswith('.html'):

strippedFileURL = strippedFileURL[0: len(strippedFileURL) - len('.html')]

if not self.downloadFileList.getDownloadFileByFileURL(strippedFileURL) is None:

return MSG_ADD_FILEURL_EXISTED + strippedFileURL

downloadFile = DownloadFile(strippedFileURL)

downloadFile.setId(Config.getId())

self.downloadFileList.addQueueingFile(downloadFile)

self.mainFrame.addDownloadFileToList(downloadFile, PANEL_TOP)

return MSG_ADD_FILEURL_SUCCESS + strippedFileURL

def addURLById(self, id):

downloadFile = self.downloadFileList.getDownloadFileById(id, False)

self.addURL(downloadFile.getFileURL())

#Stop all downloads, save the queueing list then quit

def exit(self):

#self.toContinue = False

self.isStopped = True

for downloadFileControl in self.downloadFileControlList:

#print 'Stopping ', downloadFileControl.getDownloadFile().getId()

downloadFileControl.stop()

#print 'Stopping Done'

self.downloadFileList.unsetLock()

#self.log.debug('downloadFileListLock', self.downloadFileList.downloadFileListLock)

#self.log.debug('completedFileListLock', self.downloadFileList.completedFileListLock)

self.saveQueueingFiles()

#for downloadFile in self.downloadFileList.getList():

#print downloadFile

Config.save()

#this is dangerous!!!

def saveQueueingFiles(self):

queueingFile = open(Config.settings.queueingListFile, 'wb')

#print dir(self.downloadFileList)

cPickle.dump(self.downloadFileList, queueingFile, cPickle.HIGHEST_PROTOCOL)

queueingFile.close()

#add a curl to the curl list

def addCurlObject(self, curlObject):

#lockkkk?

#print 'control addCurlObject getting lock'

self.curlLock.acquire()

#print 'adding ', curlObject

self.queueingCurlObjectList.append(curlObject)

#print 'control addCurlE_OPERATION_TIMEOUTEDObject releasing lock'

self.curlLock.release()

#get a curl from the list

def getCurlObject(self):

#print 'control getCurlObject getting lock'

self.curlLock.acquire()

if len(self.queueingCurlObjectList) != 0:

curlObject = self.queueingCurlObjectList.pop(0)

#print 'control getCurlObject releasing lock'

self.curlLock.release()

return curlObject

else:

#print 'control getCurlObject releasing lock'

self.curlLock.release()

return None

#reset the download file

def resetDownload(self, id = None):

if not (id is None):

found = False

#self.downloadFileList.changeStatus(fileURL, STAT_S)

for downloadFileControl in self.downloadFileControlList:

if downloadFileControl.getDownloadFile().getId() == id:

downloadFileControl.resetSettings()

self.mainFrame.update(downloadFileControl.getDownloadFile(), updateType = [FILESTATUS_COL, RETRY_COL])

found = True

break

if not found:

downloadFile = self.downloadFileList.getDownloadFileById(id)

#downloadFile.setStatus(STAT_S)

downloadFile.resetInfo()

self.mainFrame.update(downloadFile, updateType = [FILENAME_COL, FILESTATUS_COL])

else:

#self.isStopped = True

for downloadFileControl in self.downloadFileControlList:

#if downloadFileControl.isAlive():

#downloadFileControl.getDownloadFile().setStatus(STAT_S)

downloadFileControl.resetSettings()

self.mainFrame.update(downloadFileControl.getDownloadFile(), updateType = [FILESTATUS_COL])

#stop the download

def stopDownload(self, id = None):

if not id is None:

found = False

#self.downloadFileList.changeStatus(fileURL, STAT_S)

for downloadFileControl in self.downloadFileControlList:

if downloadFileControl.getDownloadFile().getId() == id:

downloadFileControl.stop()

self.mainFrame.update(downloadFileControl.getDownloadFile(), updateType = [FILESTATUS_COL])

found = True

downloadFileControl.reset()

break

if not found:

downloadFile = self.downloadFileList.getDownloadFileById(id)

downloadFile.setStatus(STAT_S)

self.mainFrame.update(downloadFile, updateType = [FILESTATUS_COL])

else:

self.autoDownload = False

for downloadFileControl in self.downloadFileControlList:

#if downloadFileControl.isAlive():

#downloadFileControl.getDownloadFile().setStatus(STAT_S)

downloadFileControl.stop()

self.mainFrame.update(downloadFileControl.getDownloadFile(), updateType = [FILESTATUS_COL])

downloadFileControl.reset()

def continueDownload(self, id = None):

if not (id is None):

downloadFile = self.downloadFileList.getDownloadFileById(id)

if downloadFile.getStatus() != STAT_D:

downloadFile.setStatus(STAT_Q)

self.mainFrame.update(downloadFile, updateType = [FILESTATUS_COL])

else:

self.downloadFileList.resetStatus(STAT_Q)

for downloadFile in self.downloadFileList.getList():

self.mainFrame.update(downloadFile, updateType = [FILESTATUS_COL])

self.autoDownload = True

def deleteDownloadTop(self, id = None):

#print 'deleteDownloadTop ', id

if not (id is None):

found = False

for downloadFileControl in self.downloadFileControlList:

if downloadFileControl.getDownloadFile().getId() == id:

found = True

self.mainFrame.deleteDownloadFileFromList(downloadFileControl.getDownloadFile().getId(), PANEL_TOP)

downloadFileControl.delete()

self.downloadFileList.deleteDownloadFileFromDownloadList(downloadFileControl.getDownloadFile().getId())

if not found:

self.downloadFileList.deleteDownloadFileFromDownloadList(id)

self.mainFrame.deleteDownloadFileFromList(id, PANEL_TOP)

else:

self.isStopped = True

for downloadFileControl in self.downloadFileControlList:

self.mainFrame.deleteDownloadFileFromList(downloadFileControl.getDownloadFile().getId(), PANEL_TOP)

downloadFileControl.delete()

self.downloadFileList.deleteDownloadFileFromDownloadList(downloadFileControl.getDownloadFile().getId())

self.downloadFileList.emptyList()

self.mainFrame.emptyList(PANEL_TOP)

def deleteDownloadBot(self, id = None):

#print 'deleteDownloadBot ', id

if not (id is None):

self.mainFrame.deleteDownloadFileFromList(id, PANEL_BOT)

self.downloadFileList.deleteDownloadFileFromCompletedList(id)

else:

self.mainFrame.emptyList(PANEL_BOT)

def run(self):

if self.mainFrame is None:

self.log.debug('No mainFrame found, cannot continue')

sys.exit()

multiHandler = pycurl.CurlMulti()

multiHandler.handles = []

i = 0

#while self.toContinue:

while not self.isStopped:

#noDFile = self.downloadFileList.getNumberOfDownloadingFile()

#noQFile = self.downloadFileList.getNumberOfQueueingFile()

#noQueueingCurlObject = len(self.queueingCurlObjectList)

maxConn = Config.settings.maxConcurrentDownload

num_handles = 0

while (not self.isStopped) and (self.autoDownload) and (self.downloadFileList.getNumberOfQueueingFile() > 0):

if len(self.downloadFileControlList) < maxConn:

#and self.downloadFileList.getNumberOfDownloadingFile() < maxConn):

downloadFile = self.downloadFileList.getQueueingFile()

if not (downloadFile is None):

downloadFile.resetRetry()

self.log.debug('Control, Creating:', i, 'instance downloadFileControl', downloadFile.getId(), downloadFile.getFileURL())

downloadFileControl = DownloadFileControl(self, self.saveFileControl, downloadFile)

downloadFile.setStatus(STAT_D)

#downloadFileControl.start()

downloadFileControl.run()

self.downloadFileControlList.append(downloadFileControl)

i += 1

else:

for downloadFileControl in self.downloadFileControlList:

if downloadFileControl.isDone() and (self.downloadFileList.getNumberOfQueueingFile() > 0):

downloadFile = self.downloadFileList.getQueueingFile()

#print 'Control downloadFile is ', downloadFile

downloadFileControl.setDownloadFile(downloadFile)

self.log.debug('Control, Assign downloadFile for downloadFileControl', downloadFile.getId(), downloadFile.getFileURL())

downloadFileControl.continueBuildCurl()

break

#while True:

while not self.isStopped:

curlObject = self.getCurlObject()

if not (curlObject is None):

multiHandler.add_handle(curlObject)

else:

break

#while True:

while not self.isStopped:

ret, num_handles = multiHandler.perform()

if ret != pycurl.E_CALL_MULTI_PERFORM:

#print 'Perfoming break', ret, ' ', num_handles

break

#while True:

while not self.isStopped:

num_q, ok_list, err_list = multiHandler.info_read()

for c in ok_list:

#c.close()

multiHandler.remove_handle(c)

#print 'ok_list removed handle ', c

httpCode = int(c.getinfo(pycurl.HTTP_CODE))

#print 'httpCode ', httpCode

for downloadFileControl in self.downloadFileControlList:

#print 'Debug ', downloadFileControl

self.log.debug('Control OK list checking', downloadFileControl.getDownloadFile().getId(), c.downloadFileId)

if downloadFileControl.getDownloadFile().getId() == c.downloadFileId:

if not (c.partNo is None):

self.log.debug('Control curl object belongs to downloadPartControl, checkFinish now')

downloadFileControl.checkFinish()

#downloadFileControl.reset()

break

if httpCode == 404:

self.log.debug('Control OK list 404', downloadFileControl.getDownloadFile().getId())

downloadFileControl.getDownloadFile().setStatus(STAT_E)

downloadFileControl.getDownloadFile().setErrorStr('404 File not found')

self.mainFrame.update(downloadFileControl.getDownloadFile())

downloadFileControl.reset()

elif (not downloadFileControl.isDone()) and (not downloadFileControl.isBusy()):

self.log.debug('Control OK list continueBuildCurl', downloadFileControl.getDownloadFile().getId())

downloadFileControl.continueBuildCurl()

break

#print 'Success: ', c.downloadFileId

for c, errno, errmsg in err_list:

#c.close()

multiHandler.remove_handle(c)

#print 'err_list removed handle ', c

self.log.debug('Control ERROR list error', errno, errmsg)

httpCode = int(c.getinfo(pycurl.HTTP_CODE))

#print 'err_list httpCode ', httpCode

for downloadFileControl in self.downloadFileControlList:

#print 'Debug ', downloadFileControl

if downloadFileControl.getDownloadFile().getId() == c.downloadFileId:

self.log.debug('Control ERROR list checking', downloadFileControl.getDownloadFile().getId(), c.downloadFileId)

if not errno in (pycurl.E_PARTIAL_FILE, pycurl.E_WRITE_ERROR, pycurl.E_ABORTED_BY_CALLBACK):

self.log.debug('Control ERROR list error unexpected', errno, errmsg)

if downloadFileControl.getDownloadFile().isRetryPossible():

self.log.debug('Control ERROR list retrying')

downloadFileControl.getDownloadFile().setStatus(STAT_E)

downloadFileControl.getDownloadFile().setErrorStr(errmsg)

self.mainFrame.update(downloadFileControl.getDownloadFile(), updateType = [FILESTATUS_COL, FILEERROR_COL])

if not (c.partNo is None):

self.log.debug('Control ERROR list downloadPart failed, resetting part')

downloadFileControl.resetPart(c.partNo)

else:

self.log.debug('Control ERROR list downloadFile failed, continueBuildCurl')

downloadFileControl.continueBuildCurl(True)

else:

self.log.debug('Control ERROR list no more retry possible, report error')

downloadFileControl.reportError(errmsg)

break

if httpCode == 404:

self.log.debug('Control ERROR list 404')

downloadFileControl.getDownloadFile().setStatus(STAT_E)

downloadFileControl.getDownloadFile().setErrorStr('404 File not found')

self.mainFrame.update(downloadFileControl.getDownloadFile())

downloadFileControl.reset()

elif (not downloadFileControl.isDone()) and (not downloadFileControl.isBusy()) and (errno != pycurl.E_ABORTED_BY_CALLBACK):

self.log.debug('Control ERROR list continueBuildCurl', downloadFileControl.getDownloadFile().getId())

downloadFileControl.continueBuildCurl()

break

#print 'err_list Failed: ', ' errno ', errno, ' errmsg ', errmsg

#freelist.append(c)

#num_processed = num_processed + len(ok_list) + len(err_list)

if num_q == 0:

break

if num_handles == 0:

#print 'No more connection, I sleep a bit'

time.sleep(0.5)

else:

ret = multiHandler.select(1.0)

if ret == -1:

continue

#while (True):

while not self.isStopped:

ret, num_handles = multiHandler.perform()

if ret != pycurl.E_CALL_MULTI_PERFORM:

break

for downloadFileControl in self.downloadFileControlList:

#if (downloadFileControl.isAlive()):

# downloadFileControl.join()

downloadFileControl.kill()

self.mainFrame.quitNow()

self.saveFileControl.kill()

self.saveFileControl.join()

#print 'Control Quit'

def finishFile(self, downloadFile):

if self.isStopped:

return

self.mainFrame.deleteDownloadFileFromList(downloadFile.getId(), PANEL_TOP)

self.mainFrame.addDownloadFileToList(downloadFile, PANEL_BOT)

self.downloadFileList.deleteDownloadFileFromDownloadList(downloadFile.getId(), True)

def report(self, downloadFile, updateType):

if self.isStopped:

return

self.mainFrame.update(downloadFile, updateType)

def printDebug(self):

print '==================================='

print 'Total thread: ', threading.activeCount()

print 'Total downloading File: ', len(self.downloadFileList.getList())

print 'List download File: '

for downloadFile in self.downloadFileList.getList():

print 'id ', downloadFile.getId(), ' status: ', downloadFile.getStatus()

print 'List completed File: ', len(self.downloadFileList.getList(True))

for downloadFile in self.downloadFileList.getList(True):

print 'id ', downloadFile.getId(), ' status: ', downloadFile.getStatus()

print 'List DownloadFileControl: '

for downloadFileControl in self.downloadFileControlList:

downloadFile = downloadFileControl.getDownloadFile()

print 'id ', downloadFile.getId(), ' busy: ', downloadFileControl.busy, ', done: ', downloadFileControl.done, ', gotJob: ', downloadFileControl.gotJob