def getQueuedTorrentsCount(self): dbObject = Databases( dbType = "SQLITE3", databaseSettings = { 'databaseLocation': self.testDatabaseFile } ) return dbObject.getQueuedTorrentsCount()
def __init__(self): # Setup the database object self.database = Databases(dbType=self.defaultDatabaseType) self.logger.info('TransmissionClient INIT') if settings['client']['type'] == 'transmission': self.logger.debug('TorrentClient Setup') self.client = Transmission.Client() else: self.logger.info('TorrentClient Not Defined') raise ValueError('Torrent client type not defined!')
def test_AddBlacklistedTorrent(self): self.removeDatabase() dbObject = Databases( dbType="SQLITE3", databaseSettings={'databaseLocation': self.testDatabaseFile}) dbObject.addBlacklistedTorrent(url='http://testurl.com/test') self.assertTrue( dbObject.torrentBlacklisted(url='http://testurl.com/test')) self.removeDatabase()
def __init__(self, settings={}): # Setup the database object self.torrentDB = Databases(flannelfox.settings['database']['defaultDatabaseEngine']) self.logger = logging.getLogger(__name__) self.logger.info("TransmissionClient INIT") if settings['type'] == "transmission": self.logger.info("TorrentClient Setup") self.client = Transmission.Client(settings=settings) else: self.logger.info("TorrentClient Not Defined") raise ValueError("Torrent client type not defined!")
class TorrentClient(): ''' This should be a generic class that torrent clients can extend from and use as a roadmap all the functions in here need to be defined in the client module in order for it to work TODO: instantiate a class of the given client type based on the config setup. We will also need to return it when this is instantiated. ''' logger = logging.getLogger(__name__) # Setup the database object database = None defaultDatabaseType = settings['database']['defaultDatabaseEngine'] client = None def __init__(self): # Setup the database object self.database = Databases(dbType=self.defaultDatabaseType) self.logger.info('TransmissionClient INIT') if settings['client']['type'] == 'transmission': self.logger.debug('TorrentClient Setup') self.client = Transmission.Client() else: self.logger.info('TorrentClient Not Defined') raise ValueError('Torrent client type not defined!') def __updateHashString(self, data=None, where=None): ''' TODO: This should be used instead of the function in the Transmission Class Updates the hash string for a torrent in the database Takes: using - A list of database properties that are used to identify the torrent to be updated update - Field to update in the database ''' self.database.updateHashString(data=data, where=where) def updateQueue(self): ''' Updates the class variable queue with the latest torrent queue info Returns: Tuple (transmissionResponseCode, httpResponseCode) ''' return self.client.updateQueue() def getQueue(self): ''' Returns: List [torrents] ''' return self.client.getQueue() def verifyTorrent(self, hashString=None): ''' Verifies a corrupted torrent Takes: hashString - Hash of the specific torrent to remove Returns: bool True is action completed ''' return self.client.verifyTorrent(hashString=hashString) def stopTorrent(self, hashString=None): ''' Stops a torrent Takes: hashString - Hash of the specific torrent to remove Returns: bool True is action completed ''' return self.client.stopTorrent(hashString=hashString) def startTorrent(self, hashString=None): ''' Starts a torrent Takes: hashString - Hash of the specific torrent to remove Returns: bool True is action completed ''' return self.client.startTorrent(hashString=hashString) def removeBadTorrent(self, hashString=None, reason='No Reason Given'): ''' Removes a torrent from both transmission and the database this should be called when there is a bad torrent. Takes: hashString - Hash of the specific torrent to remove ''' # Remove the torrent from the client self.client.removeTorrent(hashString=hashString, deleteData=True, reason=reason) # Remove the torrent from the DB self.database.deleteTorrent(hashString=hashString, reason=reason) def removeDupeTorrent(self, hashString=None, url=None): ''' Removes a duplicate torrent from transmission if it does not exist and is in the database but does in transmission. This is checked via the url and the bashString. TODO: Handle hash collision Takes: hashString - Hash of the specific torrent to remove url - Url of the torrent we are adding ''' if not self.database.torrentExists(hashString=hashString, url=url): # Remove the torrent from the client self.removeTorrent(hashString=hashString, deleteData=False, reason='Duplicate Torrent') return True else: # Remove the torrent from the DB # TODO: Perhaps this should be changed to just mark the torrent as added # or blacklisted self.database.deleteTorrent(url=url, reason='Duplicate Torrent') return True def removeTorrent(self, hashString=None, deleteData=False, reason='No Reason Given'): ''' Removes a torrent from transmission Takes: hashString - Hash of the specific torrent to remove deleteData - bool, tells if the torrent data should be removed TODO: if hashString is not specified then we should remove the torrent that has the longest time since active. Returns: bool True is action completed ''' return self.client.removeTorrent(hashString=hashString, deleteData=deleteData, reason=reason) def deleteTorrent(self, hashString=None, reason='No Reason Given'): ''' Removes a torrent from transmission and deletes the associated data Takes: hashString - Hash of the specific torrent to remove TODO: if hashString is not specified then we should remove the torrent that has the longest time since active. Returns: bool True is action completed ''' # Logging is skipped here and put into the removeTorrent function to prevent # duplicate logging return self.client.removeTorrent(hashString=hashString, deleteData=True, reason=reason) def addTorrentURL(self, url=None, destination=settings['files']['defaultTorrentLocation']): ''' Attempts to load the torrent at the given url into transmission Takes: url - url of the torrent file to be added destination - where the torrent should be saved Returns: bool True is action completed successfully ''' self.logger.info('TorrentClient adding torrent1') result, response = self.client.addTorrentURL(url=url, destination=destination) self.logger.info('T {0}'.format(result)) self.logger.info('T {0}'.format(response)) self.logger.info('TorrentClient responded with ({0}, {1})'.format( result, response)) if result == 0: # Get Current Time sinceEpoch = int(time.time()) # update hash, addedOn, added in DB self.logger.info('TorrentClient added torrent') self.__updateHashString(where={'url': url}, data={ 'hashString': response, 'addedOn': sinceEpoch, 'added': 1 }) time.sleep(10) return True elif result == 1: # Torrent is broken so lets delete it from the DB, this leaves the opportunity # for the torrent to later be added again self.logger.info('TorrentClient duplicate torrent') self.removeDupeTorrent(url=url, hashString=response) time.sleep(10) return False elif result == 2: self.logger.info('TorrentClient bad torrent, but we can retry') self.database.deleteTorrent(url=url, reason=response) return False elif result == 3: self.logger.info('TorrentClient bad torrent, so blacklist it') self.database.addBlacklistedTorrent(url=url, reason=response) self.database.deleteTorrent(url=url, reason=response) return False def getSlowestSeeds(self, num=None): ''' Look for the slowest seeding torrents, slowest first Takes: num - Int, the number of torrent objects to return ''' return self.client.getSlowestSeeds(num=num) def getDormantSeeds(self, num=None): ''' Looks for a seeding torrent with the longest time since active, returns torrents, oldest first ''' return self.client.getDormantSeeds(num=num) def getDownloading(self, num=None): ''' Returns a list of torrents that are downloading Takes: num - Int, the number of torrents to return ''' return self.client.getDownloading(num=num) def getSeeding(self, num=None): ''' Returns a list of torrents that are Seeding Takes: num - Int, the number of torrents to return ''' return self.client.getSeeding(num=num) def getFinishedSeeding(self, num=None): ''' Returns a list of torrents that are finished seeding Takes: num - Int, the number of torrents to return ''' return self.client.getFinishedSeeding(num=num) def restart(self): return self.client.restart() def start(self): return self.client.start() def stop(self): return self.client.stop()
#------------------------------------------------------------------------------- # Name: Torrent # Purpose: This module is a generic torrent module that clients should use # when trying to describe torrents. Status can be added as needed. # # #------------------------------------------------------------------------------- # -*- coding: utf-8 -*- # System Includes import time from flannelfox.settings import settings from flannelfox.databases import Databases # Setup the database object TorrentDB = Databases(settings['database']['defaultDatabaseEngine']) class Status(object): Paused = 0 QueuedForVerification = 1 Verifying = 2 QueuedForDownloading = 3 Downloading = 4 QueuedForSeeding = 5 Seeding = 6 class Torrent(object): def __init__(self, hashString=None, id=None,
def __init__(self, *args): self.database = Databases( dbType = self.defaultDatabaseType ) self.torrentClient = self.setupTorrentClient() self.torrentClient.updateQueue()
class QueueReader(): logger = logging.getLogger(__name__) # Setup the database object database = None defaultDatabaseType = settings['database']['defaultDatabaseEngine'] # Torrent client object torrentClient = None def __init__(self, *args): self.database = Databases( dbType = self.defaultDatabaseType ) self.torrentClient = self.setupTorrentClient() self.torrentClient.updateQueue() def setupTorrentClient(self): if 'client' not in settings: self.logger.warning('No client was configured to monitor!') return None # Try to create a torrent client instance try: if settings['client']['type'] == 'transmission': self.logger.debug("Creating Transmission Client"); return TorrentClient() except Exception as e: self.logger.error('Could not create torrent client: {0}'.format(e)) return None if self.torrentClient == None: self.logger.error('No client was configured to monitor!') return None def checkSubDirectoryFreeSpace(self): # Check for freespace in each directory # Collect all the active destinations destinations = [] for torrent in self.torrentClient.getQueue(): if torrent['downloadDir'] not in destinations: destinations.append(torrent['downloadDir']) # Check each destination for free space for destination in destinations: if platform.system() == 'Windows': destination = 'U:' while FreeSpace.check(destination,'M') < settings['minimumFreeSpace']: finishedTorrents = self.torrentClient.getFinishedSeeding() if len(finishedTorrents) > 0: self.logger.info('Freeing up space in destination: [{0}|{1}]'.format( destination, FreeSpace.check(destination,'M')) ) # Stop a finished torrent finishedTorrent = finishedTorrents.pop() self.torrentClient.deleteTorrent( hashString=finishedTorrent['hashString'], reason='Freespace Needed (minimumFreeSpace)' ) self.torrentClient.updateQueue() def checkMainDirectoryFreeSpace(self): # Check for used space in master dir if settings['maxUsedSpace'] > 0: while int(UsedSpace.check(settings['files']['maxUsedSpaceDir'],'G')) >= int(settings['maxUsedSpace']): finishedTorrents = self.torrentClient.getFinishedSeeding() if len(finishedTorrents) > 0: self.logger.info('Freeing up space in destination: [{0}|{1}]'.format( UsedSpace.check(settings['files']['maxUsedSpaceDir'],'G'), settings['maxUsedSpace']) ) # Stop a finished torrent finishedTorrent = finishedTorrents.pop() self.torrentClient.deleteTorrent(hashString=finishedTorrent['hashString'],reason='Freespace Needed (maxUsedSpace)') self.torrentClient.updateQueue() def checkQueueSize(self): # Ensure there are not too many torrents running while len(self.torrentClient.getQueue()) > settings['queueManagement']['maxTorrents']: finishedTorrents = self.torrentClient.getFinishedSeeding() if len(finishedTorrents) <= 0: break while len(finishedTorrents) > 0: self.logger.info('Too many torrents are running, trying to remove one {0}/{1}'.format( settings['queueManagement']['maxTorrents'], len(self.torrentClient.getQueue()) )) # Stop a finished torrent finishedTorrent = finishedTorrents.pop() self.torrentClient.deleteTorrent(hashString=finishedTorrent['hashString'],reason='Too Many Torrents Running') self.torrentClient.updateQueue() def checkFinishedTorrents(self): # Remove Finished torrents is strict queue management is enabled while settings['queueManagement']['strictQueueManagement'] and len(self.torrentClient.getFinishedSeeding()) > 0: finishedTorrents = self.torrentClient.getFinishedSeeding() if len(finishedTorrents) <= 0: break self.logger.info('Strict Queue Management is enabled, stopping {0} finished torrents.'.format(len(finishedTorrents))) for finishedTorrent in finishedTorrents: self.torrentClient.deleteTorrent(hashString=finishedTorrent['hashString'], reason='Strict Queue Management Enabled and Torrent Finished') self.torrentClient.updateQueue() def addTorrents(self): # Add torrents if there is room while ( len(self.torrentClient.getQueue()) < settings['queueManagement']['maxTorrents'] and len(self.database.getQueuedTorrents(selectors=['url', 'feedDestination'],num=1)) > 0 and len(self.torrentClient.getDownloading()) < settings['queueManagement']['maxDownloadingTorrents'] and ( int(UsedSpace.check(settings['files']['maxUsedSpaceDir'],'G')) < int(settings['maxUsedSpace']) or int(settings['maxUsedSpace']) == 0 ) ): queuedTorrents = self.database.getQueuedTorrents(selectors=['url', 'feedDestination']) self.logger.info('There are {0} queued torrents, let\'s add them'.format( len(queuedTorrents) )) # Get a new torrent newTorrent = queuedTorrents.pop() # Add new torrent # If a destination was not specified then don't pass one self.logger.info('Adding: {0}'.format(newTorrent)) if newTorrent.get('feedDestination', None) is None: self.torrentClient.addTorrentURL(newTorrent['url']) else: self.torrentClient.addTorrentURL(newTorrent['url'],newTorrent['feedDestination']) self.torrentClient.updateQueue() def addTorrentsAndRemoveFinished(self): # Remove a finished torrent if room is needed to add a torrent while ( len(self.torrentClient.getQueue()) >= settings['queueManagement']['maxTorrents'] and len(self.database.getQueuedTorrents(selectors=['url', 'feedDestination'],num=1)) > 0 and len(self.torrentClient.getDownloading()) < settings['queueManagement']['maxDownloadingTorrents'] and ( int(UsedSpace.check(settings['files']['maxUsedSpaceDir'],'G')) < int(settings['maxUsedSpace']) or int(settings['maxUsedSpace']) == 0 ) ): queuedTorrents = self.database.getQueuedTorrents(selectors=['url', 'feedDestination']) dormantSeeds = self.torrentClient.getDormantSeeds() slowSeeds = self.torrentClient.getSlowestSeeds() if len(dormantSeeds) <= 0 and len(slowSeeds) <= 0: break self.logger.info('There are {0} queued torrents, let\'s make room and add them'.format( len(queuedTorrents) )) while ( ( len(dormantSeeds) > 0 or len(slowSeeds) > 0 ) and len(queuedTorrents) > 0 ): # Try to grab an old dormant seed if len(dormantSeeds) > 0: slowestFinishedSeed = dormantSeeds.pop() # Else get a slow seed else: slowestFinishedSeed = slowSeeds.pop() # Remove slow seed if self.torrentClient.deleteTorrent(hashString=slowestFinishedSeed['hashString'], reason='Making Room For a New Torrent'): # Get a new torrent newTorrent = queuedTorrents.pop() # Add new torrent # If a destination was not specified then don't pass one if newTorrent.get('feedDestination', None) is None: self.torrentClient.addTorrentURL(newTorrent['url']) else: self.torrentClient.addTorrentURL(newTorrent['url'],newTorrent['feedDestination']) self.torrentClient.updateQueue()
class TorrentClient(object): ''' This should be a generic class that torrent clients can extend from and use as a roadmap all the functions in here need to be defined in the client module in order for it to work TODO: instantiate a class of the given client type based on the config setup. We will also need to return it when this is instantiated. ''' def __init__(self, settings={}): # Setup the database object self.torrentDB = Databases(flannelfox.settings['database']['defaultDatabaseEngine']) self.logger = logging.getLogger(__name__) self.logger.info("TransmissionClient INIT") if settings['type'] == "transmission": self.logger.info("TorrentClient Setup") self.client = Transmission.Client(settings=settings) else: self.logger.info("TorrentClient Not Defined") raise ValueError("Torrent client type not defined!") def __updateHashString(self, using=None, update=None): ''' TODO: This should be used instead of the function in the Transmission Class Updates the hash string for a torrent in the database Takes: using - A list of database properties that are used to identify the torrent to be updated update - Field to update in the database ''' self.torrentDB.updateHashString(update=update, using=using) def updateQueue(self): ''' Updates the class variable queue with the latest torrent queue info Returns: Tuple (transmissionResponseCode, httpResponseCode) ''' return self.client.updateQueue() def getQueue(self): ''' Returns: List [torrents] ''' return self.client.getQueue() def verifyTorrent(self,hashString=None): ''' Verifies a corrupted torrent Takes: hashString - Hash of the specific torrent to remove Returns: bool True is action completed ''' return self.client.verifyTorrent(hashString=hashString) def stopTorrent(self,hashString=None): ''' Stops a torrent Takes: hashString - Hash of the specific torrent to remove Returns: bool True is action completed ''' return self.client.stopTorrent(hashString=hashString) def startTorrent(self, hashString=None): ''' Starts a torrent Takes: hashString - Hash of the specific torrent to remove Returns: bool True is action completed ''' return self.client.startTorrent(hashString=hashString) def removeBadTorrent(self, hashString=None, reason='No Reason Given'): ''' Removes a torrent from both transmission and the database this should be called when there is a bad torrent. Takes: hashString - Hash of the specific torrent to remove ''' # Remove the torrent from the client self.client.removeTorrent(hashString=hashString, deleteData=True, reason=reason) # Remove the torrent from the DB self.torrentDB.deleteTorrent(hashString=hashString, reason=reason) def removeDupeTorrent(self, hashString=None, url=None): ''' Removes a duplicate torrent from transmission if it does not exist and is in the database but does in transmission. This is checked via the url and the bashString. TODO: Handle hash collision Takes: hashString - Hash of the specific torrent to remove url - Url of the torrent we are adding ''' if not self.torrentDB.torrentExists(hashString=hashString, url=url): # Remove the torrent from the client self.removeTorrent(hashString=hashString, deleteData=False, reason='Duplicate Torrent') return True else: # Remove the torrent from the DB # TODO: Perhaps this should be changed to just mark the torrent as added # or blacklisted self.torrentDB.deleteTorrent(url=url, reason='Duplicate Torrent') return True def removeTorrent(self, hashString=None, deleteData=False, reason='No Reason Given'): ''' Removes a torrent from transmission Takes: hashString - Hash of the specific torrent to remove deleteData - bool, tells if the torrent data should be removed TODO: if hashString is not specified then we should remove the torrent that has the longest time since active. Returns: bool True is action completed ''' return self.client.removeTorrent(hashString=hashString, deleteData=deleteData, reason=reason) def deleteTorrent(self, hashString=None, reason='No Reason Given'): ''' Removes a torrent from transmission and deletes the associated data Takes: hashString - Hash of the specific torrent to remove TODO: if hashString is not specified then we should remove the torrent that has the longest time since active. Returns: bool True is action completed ''' # Logging is skipped here and put into the removeTorrent function to prevent # duplicate logging return self.client.removeTorrent(hashString=hashString, deleteData=True, reason=reason) def addTorrentURL(self, url=None, destination=flannelfox.settings['files']['defaultTorrentLocation']): ''' Attempts to load the torrent at the given url into transmission Takes: url - url of the torrent file to be added destination - where the torrent should be saved Returns: bool True is action completed successfully ''' self.logger.info("TorrentClient adding torrent") result, response = self.client.addTorrentURL(url=url, destination=destination) self.logger.info("TorrentClient responded with ({0}, {1})".format(result, response)) if result == 0: # Get Current Time sinceEpoch = int(time.time()) # update hash, addedOn, added in DB self.logger.info("TorrentClient added torrent") self.__updateHashString(using={u"url":url}, update={u"hashString":response, u"addedOn":sinceEpoch, u"added":1}) time.sleep(10) return True elif result == 1: # Torrent is broken so lets delete it from the DB, this leaves the opportunity # for the torrent to later be added again self.logger.info("TorrentClient duplicate torrent") self.removeDupeTorrent(url=url, hashString=response) time.sleep(10) return False elif result == 2: self.logger.info("TorrentClient bad torrent, but we can retry") self.torrentDB.deleteTorrent(url=url, reason=response) return False elif result == 3: self.logger.info("TorrentClient bad torrent, so blacklist it") self.torrentDB.addBlacklistedTorrent(url=url, reason=response) self.torrentDB.deleteTorrent(url=url, reason=response) return False def getSlowestSeeds(self, num=None): ''' Look for the slowest seeding torrents, slowest first Takes: num - Int, the number of torrent objects to return ''' return self.client.getSlowestSeeds(num=num) def getDormantSeeds(self, num=None): ''' Looks for a seeding torrent with the longest time since active, returns torrents, oldest first ''' return self.client.getDormantSeeds(num=num) def getDownloading(self, num=None): ''' Returns a list of torrents that are downloading Takes: num - Int, the number of torrents to return ''' return self.client.getDownloading(num=num) def getSeeding(self, num=None): ''' Returns a list of torrents that are Seeding Takes: num - Int, the number of torrents to return ''' return self.client.getSeeding(num=num) def getFinishedSeeding(self, num=None): ''' Returns a list of torrents that are finished seeding Takes: num - Int, the number of torrents to return ''' return self.client.getFinishedSeeding(num=num) def restart(self): return self.client.restart() def start(self): return self.client.start() def stop(self): return self.client.stop()
def __init__(self, *args): self.elements = list(*args) self.database = Databases(dbType=self.defaultDatabaseType)
class Queue(): ''' Used to Track a list of torrents and interact with the torrent database ''' # Setup the database object database = None defaultDatabaseType = settings['database']['defaultDatabaseEngine'] def __init__(self, *args): self.elements = list(*args) self.database = Databases(dbType=self.defaultDatabaseType) def __getitem__(self, idx): # Ensure the index is in the correct range if idx < 0 or idx >= len(self.elements): raise IndexError('Index out of range') return self.elements[idx] def __setitem__(self, idx, torrent): self.elements[idx] = torrent # Ensure the value was taken if self.elements[idx] == torrent: return 0 else: return -1 def __len__(self): return len(self.elements) def __iter__(self): return iter(self.elements) def __contains__(self, element): return element in self.elements def databaseTorrentExists(self, torrent): return self.database.torrentExists(torrent=torrent) #return False def databaseTorrentBlacklisted(self, torrent): return self.database.torrentBlacklisted(torrent.get('url', '')) #return False def append(self, torrent): # Check and see if the value already exists in elements if torrent in self.elements: return -1 # Check and see if the value already exists in DB elif self.databaseTorrentExists(torrent): return -1 # Ensure it is not Blacklisted elif self.databaseTorrentBlacklisted(torrent): return -1 # Append the value to elements else: self.elements.append(torrent) # Ensure the value was taken if torrent in self.elements: return 0 else: return -1 def writeToDB(self): self.database.addTorrentsToQueue(self.elements) def __str__(self): out = '' for element in self.elements: out += u'{0}\n'.format(element) return out
def test_addTorrentsToQueue_torrentExists_deleteTorrent(self): self.removeDatabase() testTorrent = Torrents.TV( torrentTitle='some.show.s01e01.720p.junk.here', url='http://testurl.com/test') testTorrent2 = Torrents.TV( torrentTitle='some.show.s01e02.720p.junk.here', url='http://testurl.com/test2') torrentQueue = Queue() torrentQueue.append(testTorrent) dbObject = Databases( dbType="SQLITE3", databaseSettings={'databaseLocation': self.testDatabaseFile}) dbObject.addTorrentsToQueue(torrentQueue) self.assertTrue(dbObject.torrentExists(testTorrent)) dbObject.updateHashString({'hashString': 'abc123'}, {'url': 'http://testurl.com/test'}) self.assertTrue( len( dbObject.getTorrentInfo(hashString='abc123', selectors=['torrentTitle', 'url'])) == 1) torrentQueue = Queue() torrentQueue.append(testTorrent2) dbObject.addTorrentsToQueue(torrentQueue) self.assertTrue(dbObject.getQueuedTorrentsCount() == 2) self.assertTrue(len(dbObject.getQueuedTorrents()) == 2) dbObject.deleteTorrent(url='http://testurl.com/test') self.assertFalse(dbObject.torrentExists(testTorrent)) self.assertTrue(dbObject.getQueuedTorrentsCount() == 1) dbObject.deleteTorrent(url='http://testurl.com/test2') self.assertFalse(dbObject.torrentExists(testTorrent)) self.assertTrue(dbObject.getQueuedTorrentsCount() == 0) self.removeDatabase()