def __isCacheUpdateNeeded(force=False, cacheFilename=None, frequency=360):
'''
Used to determine if a cachefile needs to be updated
force: force an update
cacheFilename: The full path of the file to check
frequency: how often the file should be updated in minutes
Returns Boolean
'''
logger = logging.getLogger(__name__)
try:
# Get the modification time
lastModified = __modificationDate(cacheFilename)
if lastModified == -1:
return True
logger.debug("Checking cache: {0} {1}:{2}".format(cacheFilename, frequency, math.ceil((time.time()/60 - lastModified/60))))
difference = math.ceil((time.time()/60 - lastModified/60))
if difference >= frequency:
logger.debug("Cache update needed".format(cacheFilename) )
return True
else:
logger.debug("Cache update not needed".format(cacheFilename))
return False
except Exception as e:
logger.error("Cache update for {0} could not be preformed".format(cacheFilename))
def __init__(self, dbType):
'''
Set the database module that should be used
'''
self.logger = logging.getLogger(__name__)
if dbType == u'SQLITE3':
self.Database = ff_sqlite3.Database()
self.logger.debug("SQLite3 Database initialized")
else:
self.logger.critcal("There was an issue initializing the database. [{0}]{1}".format(dbType, e))
def __init__(self, settings={}):
# Setup the database object
self.torrentDB = Databases(flannelfox.settings['database']['defaultDatabaseEngine'])
self.logger = logging.getLogger(__name__)
self.logger.info("TransmissionClient INIT")
if settings['type'] == "transmission":
self.logger.info("TorrentClient Setup")
self.client = Transmission.Client(settings=settings)
else:
self.logger.info("TorrentClient Not Defined")
raise ValueError("Torrent client type not defined!")
def __modificationDate(filename):
'''
Checks the modification time of the file it is given
filename: The full path of the file to return the timestamp of.
Returns the timestamp in seconds since epoch
'''
logger = logging.getLogger(__name__)
try:
return int(datetime.datetime.fromtimestamp(os.path.getmtime(filename)).strftime("%s"))
except:
logger.error("There was a problem getting the timestamp for:\n{0}".format(filename))
return -1
def __init__(self):
self.logger = logging.getLogger(__name__)
dbSetup = ( "PRAGMA foreign_keys = off;"
"BEGIN TRANSACTION;"
"CREATE TABLE QueuedTorrents (comparison TEXT, hashString TEXT, feedDestination TEXT, minRatio REAL, minTime INTEGER, addedOn INTEGER, added INTEGER, queuedOn INTEGER, torrentType INTEGER, proper TEXT, source TEXT, container TEXT, codec TEXT, quality TEXT, day INTEGER, month INTEGER, year INTEGER, torrentTitle TEXT, url TEXT, title TEXT, season INTEGER, episode INTEGER, releaseType TEXT, album TEXT, artist TEXT);"
"CREATE TABLE BlacklistedTorrents (url STRING PRIMARY KEY);"
"CREATE INDEX idx_Queue ON QueuedTorrents (added COLLATE BINARY ASC, queuedOn COLLATE BINARY ASC);"
"CREATE INDEX idx_FeedDestination ON QueuedTorrents (feedDestination COLLATE BINARY ASC);"
"CREATE INDEX idx_Added ON QueuedTorrents (added COLLATE BINARY DESC);"
"CREATE INDEX idx_HashString ON QueuedTorrents (hashString COLLATE BINARY ASC);"
"CREATE INDEX idx_TorrentType ON QueuedTorrents (torrentType COLLATE BINARY ASC);"
"CREATE VIEW GenericTorrentsView AS SELECT QueuedTorrents.comparison, QueuedTorrents.hashstring, QueuedTorrents.feeddestination, QueuedTorrents.minratio, QueuedTorrents.mintime, QueuedTorrents.addedon, QueuedTorrents.added, QueuedTorrents.queuedon, QueuedTorrents.day, QueuedTorrents.month, QueuedTorrents.year, QueuedTorrents.torrenttitle, QueuedTorrents.url, QueuedTorrents.title, QueuedTorrents.season, QueuedTorrents.episode, QueuedTorrents.codec, QueuedTorrents.container, QueuedTorrents.proper, QueuedTorrents.quality, QueuedTorrents.source, QueuedTorrents.torrentType FROM QueuedTorrents WHERE torrentType = 'none';"
"CREATE VIEW QueuedTorrentsView AS SELECT QueuedTorrents.comparison, QueuedTorrents.hashstring, QueuedTorrents.feeddestination, QueuedTorrents.minratio, QueuedTorrents.mintime, QueuedTorrents.addedon, QueuedTorrents.added, QueuedTorrents.queuedon, QueuedTorrents.day, QueuedTorrents.month, QueuedTorrents.year, QueuedTorrents.torrenttitle, QueuedTorrents.url, QueuedTorrents.title, QueuedTorrents.season, QueuedTorrents.episode, QueuedTorrents.codec, QueuedTorrents.container, QueuedTorrents.proper, QueuedTorrents.quality, QueuedTorrents.source, QueuedTorrents.torrentType FROM QueuedTorrents WHERE added = 0 ORDER BY queuedOn ASC;"
"CREATE VIEW MovieTorrentsView AS SELECT QueuedTorrents.comparison, QueuedTorrents.hashstring, QueuedTorrents.feeddestination, QueuedTorrents.minratio, QueuedTorrents.mintime, QueuedTorrents.addedon, QueuedTorrents.added, QueuedTorrents.queuedon, QueuedTorrents.year, QueuedTorrents.torrenttitle, QueuedTorrents.url, QueuedTorrents.title, QueuedTorrents.codec, QueuedTorrents.container, QueuedTorrents.proper, QueuedTorrents.quality, QueuedTorrents.source, QueuedTorrents.torrentType FROM QueuedTorrents WHERE torrentType = 'movie';"
"CREATE VIEW MusicTorrentsView AS SELECT QueuedTorrents.comparison, QueuedTorrents.hashstring, QueuedTorrents.feeddestination, QueuedTorrents.minratio, QueuedTorrents.mintime, QueuedTorrents.addedon, QueuedTorrents.added, QueuedTorrents.queuedon, QueuedTorrents.year, QueuedTorrents.torrenttitle, QueuedTorrents.url, QueuedTorrents.title, QueuedTorrents.album, QueuedTorrents.artist, QueuedTorrents.codec, QueuedTorrents.releaseType, QueuedTorrents.container, QueuedTorrents.proper, QueuedTorrents.quality, QueuedTorrents.source, QueuedTorrents.torrentType FROM QueuedTorrents WHERE torrentType = 'music';"
"CREATE VIEW AddedTorrentsView AS SELECT QueuedTorrents.comparison, QueuedTorrents.hashstring, QueuedTorrents.feeddestination, QueuedTorrents.minratio, QueuedTorrents.mintime, QueuedTorrents.addedon, QueuedTorrents.added, QueuedTorrents.queuedon, QueuedTorrents.day, QueuedTorrents.month, QueuedTorrents.year, QueuedTorrents.torrenttitle, QueuedTorrents.url, QueuedTorrents.title, QueuedTorrents.season, QueuedTorrents.episode, QueuedTorrents.codec, QueuedTorrents.container, QueuedTorrents.proper, QueuedTorrents.quality, QueuedTorrents.source, QueuedTorrents.torrentType FROM QueuedTorrents WHERE added = 1 ORDER BY queuedOn ASC;"
"CREATE VIEW TVTorrentsView AS SELECT QueuedTorrents.comparison, QueuedTorrents.hashstring, QueuedTorrents.feeddestination, QueuedTorrents.minratio, QueuedTorrents.mintime, QueuedTorrents.addedon, QueuedTorrents.added, QueuedTorrents.queuedon, QueuedTorrents.day, QueuedTorrents.month, QueuedTorrents.year, QueuedTorrents.torrenttitle, QueuedTorrents.url, QueuedTorrents.title, QueuedTorrents.season, QueuedTorrents.episode, QueuedTorrents.codec, QueuedTorrents.container, QueuedTorrents.proper, QueuedTorrents.quality, QueuedTorrents.source, QueuedTorrents.torrentType FROM QueuedTorrents WHERE torrentType = 'tv';"
"COMMIT TRANSACTION;"
"PRAGMA foreign_keys = on;" )
self.__execScriptDB(dbSetup)
def __init__(
self, settings={}, host=u"localhost", port=u"9091", user=None, password=None, rpcLocation=None, https=False
):
self.logger = logging.getLogger(__name__)
self.logger.info("TransmissionClient INIT")
self.logger.debug("TransmissionClient Settings: {0}".format(settings))
self.elements = {}
self.elements["queue"] = []
self.elements["sessionId"] = None
if settings != {}:
self.elements.update(settings)
else:
self.elements["host"] = host
self.elements["port"] = port
self.elements["user"] = user
self.elements["password"] = password
self.elements["rpcLocation"] = rpcLocation
self.elements["https"] = https
self.logger.debug("TransmissionClient Settings 2: {0}".format(self.elements))
# Strip off the leading slash if it exists
self.elements["rpcLocation"] = self.elements["rpcLocation"].lstrip("/")
# tag generator the keep transmisison calls matched
self.tagGenerator = self.__generateTag()
# Build the server URI
self.elements["uri"] = u"http"
if self.elements["https"]:
self.elements["uri"] += u"s"
self.elements["uri"] += u"://{0}:{1}".format(self.elements["host"], self.elements["port"])
self.logger.debug("TransmissionClient URL: {0}".format(self.elements["uri"]))
def changeCharset(data, charset="utf-8", type="xml"):
'''
Used to change the character set of a string to the desired format
data: The text to be converted
charset: The format the text should be returned in
type: The engine to be used to convert the charset
Returns the text after converted
'''
logger = logging.getLogger(__name__)
logger.debug("Tyring to convert {0} to {1}".format(charset, type))
if charset is None:
charset = "utf-8"
try:
data = BeautifulSoup(data, type)
data = data.encode(encoding=charset, errors="xmlcharrefreplace")
except Exception as e:
logger.debug("Charset Conversion issue".format(e))
data = ""
return data
def __updateCacheFile(force=False, cacheFilename=None, data=None, frequency=360):
'''
Used to update cache files for api calls. This is needed so we do not keep
asking the api servers for the same information on a frequent basis. The
fault frequency is to ask once an hour.
force: preform the update reguardless of frequency
location: where to save the file
frequency: how often to update the file in minutes
'''
logger = logging.getLogger(__name__)
try:
if __isCacheUpdateNeeded(cacheFilename=cacheFilename, frequency=frequency):
logger.debug("Cache update for {0} needed".format(cacheFilename))
with open(cacheFilename, 'w') as cache:
cache.write(data)
else:
logger.debug("Cache update for {0} not needed".format(cacheFilename) )
except Exception as e:
logger.error("There was a problem writing a cache file {0}: {1}".format(cacheFilename, e))
# flannelfox Includes
import flannelfox
from flannelfox import logging
from flannelfox import Settings
# rssdaemon Includes
from flannelfox.torrenttools import Torrents, TorrentQueue
from flannelfox.torrenttools.Torrents import TORRENT_TYPES
# TODO: can this be moved?
httpRegex = re.compile(ur"https?://([^/]+)(?:/.*)?")
# Setup the logging agent
logger = logging.getLogger(__name__)
def __readRSSFeed(url):
response = u''
xmlData = None
httpCode = None
encoding = "utf-8"
pid = os.getpid()
try:
# initialize the responses
response = None
class lastfmApi():
logger = logging.getLogger(__name__)
def __getLibraryArtistsFeed(self, apiKey, username):
currentPage = 1
maxPages = 1
httpResponse = -1
artists = []
headers = {
'Content-Type':
'application/json',
'user-agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36'
}
params = {
'method': 'library.getArtists',
'api_key': apiKey,
'user': username,
'format': 'json',
'page': currentPage
}
while currentPage <= maxPages:
reply = None
params['page'] = currentPage
try:
r = requests.get(settings['apis']['lastfm'],
headers=headers,
params=params,
timeout=60)
httpResponse = r.status_code
self.logger.debug(
'Fetched LastFm album page {0} of {1}: [{2}]'.format(
currentPage, maxPages, httpResponse))
if httpResponse == 200:
reply = r.json()
else:
raise ValueError
except Exception:
httpResponse = -1
self.logger.error(
'There was a problem fetching a Lastfm album page\n{0}'.
format(httpResponse))
return artists
maxPages = int(reply['artists']['@attr']['totalPages'])
currentPage = currentPage + 1
artists.extend(
[artist['name'] for artist in reply['artists']['artist']])
return (httpResponse, artists)
def getLibraryArtists(self, apiKey, username):
httpResponse, artists = self.__getLibraryArtistsFeed(username=username,
apiKey=apiKey)
return artists
class QueueReader():
logger = logging.getLogger(__name__)
# Setup the database object
database = None
defaultDatabaseType = settings['database']['defaultDatabaseEngine']
# Torrent client object
torrentClient = None
def __init__(self, *args):
self.database = Databases(
dbType = self.defaultDatabaseType
)
self.torrentClient = self.setupTorrentClient()
self.torrentClient.updateQueue()
def setupTorrentClient(self):
if 'client' not in settings:
self.logger.warning('No client was configured to monitor!')
return None
# Try to create a torrent client instance
try:
if settings['client']['type'] == 'transmission':
self.logger.debug("Creating Transmission Client");
return TorrentClient()
except Exception as e:
self.logger.error('Could not create torrent client: {0}'.format(e))
return None
if self.torrentClient == None:
self.logger.error('No client was configured to monitor!')
return None
def checkSubDirectoryFreeSpace(self):
# Check for freespace in each directory
# Collect all the active destinations
destinations = []
for torrent in self.torrentClient.getQueue():
if torrent['downloadDir'] not in destinations:
destinations.append(torrent['downloadDir'])
# Check each destination for free space
for destination in destinations:
if platform.system() == 'Windows':
destination = 'U:'
while FreeSpace.check(destination,'M') < settings['minimumFreeSpace']:
finishedTorrents = self.torrentClient.getFinishedSeeding()
if len(finishedTorrents) > 0:
self.logger.info('Freeing up space in destination: [{0}|{1}]'.format(
destination,
FreeSpace.check(destination,'M'))
)
# Stop a finished torrent
finishedTorrent = finishedTorrents.pop()
self.torrentClient.deleteTorrent(
hashString=finishedTorrent['hashString'],
reason='Freespace Needed (minimumFreeSpace)'
)
self.torrentClient.updateQueue()
def checkMainDirectoryFreeSpace(self):
# Check for used space in master dir
if settings['maxUsedSpace'] > 0:
while int(UsedSpace.check(settings['files']['maxUsedSpaceDir'],'G')) >= int(settings['maxUsedSpace']):
finishedTorrents = self.torrentClient.getFinishedSeeding()
if len(finishedTorrents) > 0:
self.logger.info('Freeing up space in destination: [{0}|{1}]'.format(
UsedSpace.check(settings['files']['maxUsedSpaceDir'],'G'),
settings['maxUsedSpace'])
)
# Stop a finished torrent
finishedTorrent = finishedTorrents.pop()
self.torrentClient.deleteTorrent(hashString=finishedTorrent['hashString'],reason='Freespace Needed (maxUsedSpace)')
self.torrentClient.updateQueue()
def checkQueueSize(self):
# Ensure there are not too many torrents running
while len(self.torrentClient.getQueue()) > settings['queueManagement']['maxTorrents']:
finishedTorrents = self.torrentClient.getFinishedSeeding()
if len(finishedTorrents) <= 0:
break
while len(finishedTorrents) > 0:
self.logger.info('Too many torrents are running, trying to remove one {0}/{1}'.format(
settings['queueManagement']['maxTorrents'],
len(self.torrentClient.getQueue())
))
# Stop a finished torrent
finishedTorrent = finishedTorrents.pop()
self.torrentClient.deleteTorrent(hashString=finishedTorrent['hashString'],reason='Too Many Torrents Running')
self.torrentClient.updateQueue()
def checkFinishedTorrents(self):
# Remove Finished torrents is strict queue management is enabled
while settings['queueManagement']['strictQueueManagement'] and len(self.torrentClient.getFinishedSeeding()) > 0:
finishedTorrents = self.torrentClient.getFinishedSeeding()
if len(finishedTorrents) <= 0:
break
self.logger.info('Strict Queue Management is enabled, stopping {0} finished torrents.'.format(len(finishedTorrents)))
for finishedTorrent in finishedTorrents:
self.torrentClient.deleteTorrent(hashString=finishedTorrent['hashString'], reason='Strict Queue Management Enabled and Torrent Finished')
self.torrentClient.updateQueue()
def addTorrents(self):
# Add torrents if there is room
while ( len(self.torrentClient.getQueue()) < settings['queueManagement']['maxTorrents'] and
len(self.database.getQueuedTorrents(selectors=['url', 'feedDestination'],num=1)) > 0 and
len(self.torrentClient.getDownloading()) < settings['queueManagement']['maxDownloadingTorrents'] and
(
int(UsedSpace.check(settings['files']['maxUsedSpaceDir'],'G')) < int(settings['maxUsedSpace']) or
int(settings['maxUsedSpace']) == 0
)
):
queuedTorrents = self.database.getQueuedTorrents(selectors=['url', 'feedDestination'])
self.logger.info('There are {0} queued torrents, let\'s add them'.format(
len(queuedTorrents)
))
# Get a new torrent
newTorrent = queuedTorrents.pop()
# Add new torrent
# If a destination was not specified then don't pass one
self.logger.info('Adding: {0}'.format(newTorrent))
if newTorrent.get('feedDestination', None) is None:
self.torrentClient.addTorrentURL(newTorrent['url'])
else:
self.torrentClient.addTorrentURL(newTorrent['url'],newTorrent['feedDestination'])
self.torrentClient.updateQueue()
def addTorrentsAndRemoveFinished(self):
# Remove a finished torrent if room is needed to add a torrent
while ( len(self.torrentClient.getQueue()) >= settings['queueManagement']['maxTorrents'] and
len(self.database.getQueuedTorrents(selectors=['url', 'feedDestination'],num=1)) > 0 and
len(self.torrentClient.getDownloading()) < settings['queueManagement']['maxDownloadingTorrents'] and
(
int(UsedSpace.check(settings['files']['maxUsedSpaceDir'],'G')) < int(settings['maxUsedSpace']) or
int(settings['maxUsedSpace']) == 0
)
):
queuedTorrents = self.database.getQueuedTorrents(selectors=['url', 'feedDestination'])
dormantSeeds = self.torrentClient.getDormantSeeds()
slowSeeds = self.torrentClient.getSlowestSeeds()
if len(dormantSeeds) <= 0 and len(slowSeeds) <= 0:
break
self.logger.info('There are {0} queued torrents, let\'s make room and add them'.format(
len(queuedTorrents)
))
while ( (
len(dormantSeeds) > 0 or
len(slowSeeds) > 0
) and
len(queuedTorrents) > 0
):
# Try to grab an old dormant seed
if len(dormantSeeds) > 0:
slowestFinishedSeed = dormantSeeds.pop()
# Else get a slow seed
else:
slowestFinishedSeed = slowSeeds.pop()
# Remove slow seed
if self.torrentClient.deleteTorrent(hashString=slowestFinishedSeed['hashString'], reason='Making Room For a New Torrent'):
# Get a new torrent
newTorrent = queuedTorrents.pop()
# Add new torrent
# If a destination was not specified then don't pass one
if newTorrent.get('feedDestination', None) is None:
self.torrentClient.addTorrentURL(newTorrent['url'])
else:
self.torrentClient.addTorrentURL(newTorrent['url'],newTorrent['feedDestination'])
self.torrentClient.updateQueue()
import daemon # FreeSpace Calculator from flannelfox.ostools import FreeSpace, UsedSpace # Logging from flannelfox import logging # Flannelfox Includes from flannelfox.settings import settings from flannelfox.databases import Databases from flannelfox.torrentclients import TorrentClient from flannelfox.tools import changeCharset # Setup the logger.agent logger = logging.getLogger(__name__) class QueueReader(): logger = logging.getLogger(__name__) # Setup the database object database = None defaultDatabaseType = settings['database']['defaultDatabaseEngine'] # Torrent client object torrentClient = None def __init__(self, *args): self.database = Databases( dbType = self.defaultDatabaseType
def readTraktTvConfigs(configFolder=settings['files']['traktConfigDir']):
'''
Reads the titles and other information from a specified trakt.tv list
Content-Type:application/json
trakt-api-version:2
trakt-api-key:XXXX
'''
logger = logging.getLogger(__name__)
validTypes = ['tv', 'movie']
majorFeeds = {}
for configFileName, configFileData in common.getConfigFiles(configFolder):
logger.debug('Found {} feed(s) in {}'.format(len(configFileData), configFileName))
for feedList in configFileData:
try:
# Setup some variables for the feed
feedName = None
feedType = None
feedDestination = None
minorFeeds = []
feedFilters = []
traktListResults = []
title = None
year = None
feedFilterList = []
# Make sure our list at least has some basic parts
if feedList.get('username', None) is None:
raise ValueError('A trakttv username must be specified')
if feedList.get('api_key', None) is None:
raise ValueError('A trakttv api key must be specified')
if feedList.get('minorFeeds', None) is None:
raise ValueError('You must specify one or more minorFeeds')
# Get the feedName
feedName = feedList.get('list_name','').lower().strip()
if feedName == '':
raise ValueError('Feeds with out names are not permitted')
# Get the feedType
feedType = feedList.get('type','none').lower().strip()
# Get the feedDestination
feedDestination = feedList.get('feedDestination','').strip()
if feedDestination == '':
raise ValueError('The feed has an invalid destination value')
# TODO: Check if the location exists
cacheFileName = os.path.join(getCacheDir(),feedName)
if common.isCacheStillValid(cacheFileName=cacheFileName):
traktListResults = common.readCacheFile(cacheFileName)
logger.debug('Using cache file {} for {}'.format(cacheFileName, feedName))
else:
traktapi = trakttvApi()
traktListResults = traktapi.getPublicList(
apiKey=feedList.get('api_key'),
username=feedList.get('username'),
listname=feedName
)
logger.debug('Fetching new data {}'.format(feedName))
if not isinstance(traktListResults, list) or len(traktListResults) < 1:
traktListResults = common.readCacheFile(cacheFileName)
logger.debug('Using cache file {} for {}'.format(cacheFileName, feedName))
if not isinstance(traktListResults, list) or len(traktListResults) < 1:
raise ValueError('There was not a valid feed reply nor is there a valid cache for the feed')
else:
common.updateCacheFile(cacheFileName=cacheFileName, data=traktListResults)
logger.debug('Updating cache file {} for {}'.format(cacheFileName, feedName))
logger.debug('Found {} items from {}'.format(len(traktListResults), feedName))
# Collect the feeds
logger.debug('{} contains {} minorFeed(s)'.format(feedName, len(feedList.get('minorFeeds',[]))))
for minorFeed in feedList.get('minorFeeds',[]):
try:
url = minorFeed.get('url','').strip()
minTime = int(minorFeed.get('minTime','0').strip()) # Hours Int
minRatio = float(minorFeed.get('minRatio','0.0').strip()) # Ratio Float
comparison = minorFeed.get('comparison','or').strip() # Comparison String
minorFeeds.append({'url':url,'minTime':minTime,'minRatio':minRatio,'comparison':comparison})
except (ValueError, KeyError, TypeError) as e:
logger.warning('The feed contains an invalid minorFeed:\n{0}'.format(e))
continue
feedFilters = feedList.get('filters', [])
# Loop through each show and append a filter for it
for item in traktListResults:
try:
ruleList = []
if feedList.get('like', False):
titleMatchMethod = 'titleLike'
else:
titleMatchMethod = 'title'
if 'show' not in item and feedType == 'tv':
# This happens if you select the wrong type of media tv/movie
raise ValueError('Media type is not show, but feed type is tv {0}'.format(title))
elif 'movie' not in item and feedType == 'movie':
# This happens if you select the wrong type of media tv/movie
raise ValueError('Media type is not movie, but feed type is movie {0}'.format(title))
elif 'show' in item and feedType == 'tv':
item = item['show']
title = item['title'].lower().strip().replace(' & ', ' and ')
for ch in (':', '\\', '\'', ','):
title = title.replace(ch, '')
elif 'movie' in item and feedType == 'movie':
item = item['movie']
title = item['title'].lower().strip().replace(' & ', ' and ')
for ch in (':', '\\', '\'', ','):
title = title.replace(ch, '')
year = str(item['year']).strip()
else:
raise ValueError('Could not use the trakt feed data')
for filterItem in feedFilters:
ruleList.append({'key':titleMatchMethod, 'val':title, 'exclude':False})
if year is not None:
ruleList.append({'key':'year', 'val':year, 'exclude':False})
# Load the excludes
for exclude in filterItem.get('exclude', []):
for key, val in exclude.items():
ruleList.append({'key':key.strip(), 'val':val.strip(), 'exclude':True})
for include in filterItem.get('include', []):
for key, val in include.items():
ruleList.append({'key':key.strip(), 'val':val.strip(), 'exclude':False})
feedFilterList.append(ruleList)
except Exception as e:
logger.warning('The {file} contains an invalid rule:\n{e}\n{t}'.format(file=configFileName,e=e,t=traceback.format_exc()))
continue
# Append the Config item to the dict
majorFeeds['{}.{}'.format(configFileName,feedName)] = {
'feedName':feedName,
'feedType':feedType,
'feedDestination':feedDestination,
'minorFeeds':minorFeeds,
'feedFilters':feedFilterList
}
except Exception as e:
logger.warning('The {file} contains an invalid rule:\n{e}\n{t}'.format(file=configFileName,e=e,t=traceback.format_exc()))
continue
return majorFeeds
def readRSS(configFolder=flannelfox.settings['files']['rssConfigDir']):
'''
Read the RSSFeedConfig file
rssFilter children are stackable and help to refine the filter
An empty or non-existant rssFilters will result in all items being a
match
Takes the location of the config file as a parameter
Returns a dict of filters to match torrents with
'''
logger = logging.getLogger(__name__)
majorFeeds = {}
RSS_LISTS = []
configFiles = os.listdir(configFolder)
logger.debug("Reading Count: {0} feeds".format(len(configFiles)))
try:
for configFile in configFiles:
# Skip non-json files
if not configFile.endswith('.json'):
continue
logger.debug("Loading RSS config file: {0}".format(os.path.join(configFolder,configFile)))
# Try to read in the rss lists
try:
with open(os.path.join(configFolder,configFile)) as rssJson:
RSS_LISTS = json.load(rssJson)
except Exception as e:
logger.error("There was a problem reading the rss config file\n{0}".format(e))
continue
# Loop through the rss lists
try:
for rssList in RSS_LISTS:
rssList = rssList.get("majorFeed")
# Make sure our list at least has some basic parts
if (rssList.get("list_name", None) is None or
rssList.get("feedDestination", None) is None or
rssList.get("minorFeeds", None) is None):
continue
# Setup some variables for the feed
feedName = None
feedType = None
feedDestination = None
minorFeeds = []
feedFilters = []
httpResponse = -1
title = None
year = None
useCache = False
# Get the feedName
try:
feedName = unicode(rssList.get("list_name",u"").lower().strip())
if feedName == u"":
raise ValueError
except (ValueError, KeyError) as e:
logger.warning("Feeds with out names are not permitted")
continue
# Get the feedType
try:
feedType = unicode(rssList.get("type",u"none").lower().strip())
except (ValueError, KeyError) as e:
feedType = u"none"
continue
# Get the feedDestination
try:
feedDestination = unicode(rssList.get("feedDestination",u"").strip())
# TODO: Check if the location exists
except (ValueError, KeyError) as e:
logger.warning("The feed has an invalid destination value")
continue
# Collect the feeds
try:
if rssList.get("minorFeeds",[]) is not None and len(rssList.get("minorFeeds",[])) > 0:
for minorFeed in rssList.get("minorFeeds",[]):
url = unicode(minorFeed.get("url",u"").strip())
minTime = int(minorFeed.get("minTime",u"0").strip()) # Hours Int
minRatio = float(minorFeed.get("minRatio",u"0.0").strip()) # Ratio Float
comparison = minorFeed.get("comparison",u"or").strip() # Comparison String
minorFeeds.append({u"url":url,u"minTime":minTime,u"minRatio":minRatio,u"comparison":comparison})
except (ValueError, KeyError, TypeError) as e:
logger.warning("The feed contains an invalid minorFeed:\n{0}".format(e))
continue
# Collect the feedFilters
try:
feedFilterList = []
feedFilters = rssList.get("filters", [])
# Loop through each show and append a filter for it
for filterItem in feedFilters:
ruleList = []
# Load the excludes
for exclude in filterItem.get("exclude", []):
key, val = exclude.items()[0]
ruleList.append({u"key":key, u"val":val, u"exclude":True})
for include in filterItem.get("include", []):
key, val = include.items()[0]
ruleList.append({u"key":key, u"val":val, u"exclude":False})
feedFilterList.append(ruleList)
except Exception as e:
logger.warning("The feedFilters contains an invalid rule:\n{0}".format(e))
continue
# Append the Config item to the dict
majorFeeds[configFile+'.'+feedName] = {u"feedName":feedName,u"feedType":feedType,u"feedDestination":feedDestination,u"minorFeeds":minorFeeds,u"feedFilters":feedFilterList}
except Exception as e:
logger.error("There was a problem reading a rss list file:\n{0}".format(e))
except Exception as e:
# This should only happen if there was an issue getting files names from the directory
pass
return majorFeeds
def readGoodreads(configFolder=flannelfox.settings['files']['goodreadsConfigDir']):
'''
Read the authors a user favorites on GoodReads
Creates a cachefile for the authors and updates it when needed
Returns a list of authors we want to look for
'''
logger = logging.getLogger(__name__)
logger.debug("Reading Goodreads Feed")
majorFeeds = {}
goodreadsLists = []
try:
for configFile in os.listdir(configFolder):
# Skip non-json files
if not configFile.endswith('.json'):
continue
logger.debug("Loading Goodreads config file: {0}".format(os.path.join(configFolder,configFile)))
# Try to read in the goodreads lists
try:
with open(os.path.join(configFolder,configFile)) as goodreadsJson:
goodreadsLists = json.load(goodreadsJson)
except Exception as e:
logger.error("There was a problem reading the goodreads config file\n{0}".format(e))
continue
# Loop through the goodreads lists
try:
for goodreadsList in goodreadsLists:
# Make sure our list at least has some basic parts
if (goodreadsList.get("username", None) is None or
goodreadsList.get("api_key", None) is None or
goodreadsList.get("list_name", None) is None or
goodreadsList.get("type", None) is None or
goodreadsList.get("feedDestination", None) is None or
goodreadsList.get("minorFeeds", None) is None):
continue
# Setup some variables for the feed
feedName = None
feedType = None
feedDestination = None
minorFeeds = []
feedFilters = []
goodreadsListResults = []
httpResponse = -1
title = None
year = None
useCache = False
# Get the feedName
try:
feedName = unicode(goodreadsList.get("list_name",u"").lower().strip())
if feedName == u"":
raise ValueError
except (ValueError, KeyError) as e:
logger.warning("Feeds with out names are not permitted")
continue
cacheFileName = os.path.join(flannelfox.settings['files']['goodreadsCacheDir'],feedName+'.'+configFile)
if not os.path.exists(os.path.dirname(cacheFileName)):
try:
os.makedirs(os.path.dirname(cacheFileName))
except OSError as exc: # Guard against race condition
continue
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36",
}
params = {
'key':goodreadsList.get("api_key", None)
}
# Get the feedType
try:
feedType = unicode(goodreadsList.get("type",u"none").lower().strip())
except (ValueError, KeyError) as e:
feedType = u"none"
continue
# Get the feedDestination
try:
feedDestination = unicode(goodreadsList.get("feedDestination",u"").strip())
# TODO: Check if the location exists
except (ValueError, KeyError) as e:
logger.warning("The feed has an invalid destination value")
continue
# Collect the feeds
try:
if goodreadsList.get("minorFeeds",[]) is not None and len(goodreadsList.get("minorFeeds",[])) > 0:
for minorFeed in goodreadsList.get("minorFeeds",[]):
url = unicode(minorFeed.get("url",u"").strip())
minTime = int(minorFeed.get("minTime",u"0").strip()) # Hours Int
minRatio = float(minorFeed.get("minRatio",u"0.0").strip()) # Ratio Float
comparison = minorFeed.get("comparison",u"or").strip() # Comparison String
minorFeeds.append({u"url":url,u"minTime":minTime,u"minRatio":minRatio,u"comparison":comparison})
except (ValueError, KeyError, TypeError) as e:
logger.warning("The feed contains an invalid minorFeed:\n{0}".format(e))
continue
if not __isCacheUpdateNeeded(cacheFilename=cacheFileName):
useCache = True
if not useCache:
try:
r = requests.get("{0}/user/show/{1}.xml".format(flannelfox.settings['apis']['goodreads'], goodreadsList["username"]), headers=headers, params=params, timeout=60)
httpResponse = r.status_code
if httpResponse == 200:
try:
# Parse the RSS XML and turn it into a json list
xmlData = changeCharset(r.text, "utf-8", "xml")
xmlData = ET.fromstring(xmlData)
xmlData = xmlData.find('user')
authors = xmlData.find('favorite_authors')
goodreadsListResults = []
for author in authors.iter('author'):
try:
name = author.find('name').text
if name is not None and name != "":
name = unicode(name.strip())
name = name.replace(u" & ",u" and ")
goodreadsListResults.append(name)
else:
continue
except:
continue
except Exception as e:
logger.error("There was a problem reading the goodreads xml file:\n- {0}\n- {1}".format(e,traceback.format_exc()))
else:
logger.error("There was a problem fetching a goodreads list file: {0}".format(httpResponse))
except Exception as e:
logger.error("There was a problem fetching a goodreads list file: {0}".format(e))
goodreadsListResults = []
httpResponse = -1
logger.error("Fetching goodreads list page: [{0}]".format(httpResponse))
# If we are able to get a list then cache it
# TODO: See if Last-Modified can be added to save this step when possible
if httpResponse == 200:
__updateCacheFile(cacheFilename=cacheFileName, data=json.dumps(goodreadsListResults))
else:
useCache = True
if useCache:
try:
logger.debug("Reading cache file for [{0}]".format(cacheFileName))
with open(cacheFileName) as cache:
goodreadsListResults = json.load(cache)
except Exception as e:
logger.error("There was a problem reading a goodreads list cache file: {0}".format(e))
continue
# Collect the feedFilters
try:
feedFilterList = []
majorFeedFilters = goodreadsList.get("filters", [])
for filterItem in majorFeedFilters:
# Loop through each author and append a filter for it
for item in goodreadsListResults:
ruleList = []
if goodreadsList.get("like", False):
titleMatchMethod = u"titleLike"
else:
titleMatchMethod = u"title"
ruleList.append({u"key":titleMatchMethod, u"val":title, u"exclude":False})
# Load the excludes
for exclude in filterItem.get("exclude", []):
key, val = exclude.items()[0]
ruleList.append({u"key":key, u"val":val, u"exclude":True})
for include in filterItem.get("include", []):
key, val = include.items()[0]
ruleList.append({u"key":key, u"val":val, u"exclude":False})
feedFilterList.append(ruleList)
except Exception as e:
logger.warning("The feedFilters contains an invalid rule:\n{0}".format(e))
continue
# Append the Config item to the dict
majorFeeds[configFile+'.'+feedName] = {u"feedName":feedName,u"feedType":feedType,u"feedDestination":feedDestination,u"minorFeeds":minorFeeds,u"feedFilters":feedFilterList}
#f = open("feeds_new.json", 'w')
#json.dump(majorFeeds, f)
except Exception as e:
logger.error("There was a problem reading a goodreads list file:\n{0}".format(e))
except Exception as e:
# This should only happen if there was an issue getting files names from the directory
pass
logger.debug("=================")
logger.debug("GoodreadsFilters")
logger.debug("=================")
logger.debug(majorFeeds)
logger.debug("=================")
return majorFeeds
def readTraktTV(configFolder=flannelfox.settings['files']['traktConfigDir']):
'''
Reads the titles and other information from a specified trakt.tv list
Content-Type:application/json
trakt-api-version:2
trakt-api-key:XXXX
'''
logger = logging.getLogger(__name__)
logger.debug("Reading TraktTV Feed")
majorFeeds = {}
TRAKT_TV_LISTS = []
try:
for configFile in os.listdir(configFolder):
# Skip non-json files
if not configFile.endswith('.json'):
continue
logger.debug("Loading TraktTV config file: {0}".format(os.path.join(configFolder,configFile)))
# Try to read in the trakt lists
try:
with open(os.path.join(configFolder,configFile)) as traktJson:
TRAKT_TV_LISTS = json.load(traktJson)
except Exception as e:
logger.error("There was a problem reading the trakt config file\n{0}".format(e))
continue
# Loop through the trakt.tv lists
try:
for traktList in TRAKT_TV_LISTS:
# Make sure our list at least has some basic parts
if (traktList.get("username", None) is None or
traktList.get("api_key", None) is None or
traktList.get("list_name", None) is None or
traktList.get("type", None) is None or
traktList.get("feedDestination", None) is None or
traktList.get("minorFeeds", None) is None):
continue
# Setup some variables for the feed
feedName = None
feedType = None
feedDestination = None
minorFeeds = []
feedFilters = []
traktListResults = []
httpResponse = -1
title = None
year = None
useCache = False
# Get the feedName
try:
feedName = unicode(traktList.get("list_name",u"").lower().strip())
if feedName == u"":
raise ValueError
except (ValueError, KeyError) as e:
logger.warning("Feeds with out names are not permitted")
continue
cacheFileName = os.path.join(flannelfox.settings['files']['traktCacheDir'],feedName+'.'+configFile)
if not os.path.exists(os.path.dirname(cacheFileName)):
try:
os.makedirs(os.path.dirname(cacheFileName))
except OSError as exc: # Guard against race condition
continue
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36",
"Content-Type":"application/json",
"trakt-api-version":"2",
"trakt-api-key":traktList.get("api_key","")
}
# Get the feedType
try:
feedType = unicode(traktList.get("type",u"none").lower().strip())
except (ValueError, KeyError) as e:
feedType = u"none"
continue
# Get the feedDestination
try:
feedDestination = unicode(traktList.get("feedDestination",u"").strip())
# TODO: Check if the location exists
except (ValueError, KeyError) as e:
logger.warning("The feed has an invalid destination value")
continue
# Collect the feeds
try:
if traktList.get("minorFeeds",[]) is not None and len(traktList.get("minorFeeds",[])) > 0:
for minorFeed in traktList.get("minorFeeds",[]):
url = unicode(minorFeed.get("url",u"").strip())
minTime = int(minorFeed.get("minTime",u"0").strip()) # Hours Int
minRatio = float(minorFeed.get("minRatio",u"0.0").strip()) # Ratio Float
comparison = minorFeed.get("comparison",u"or").strip() # Comparison String
minorFeeds.append({u"url":url,u"minTime":minTime,u"minRatio":minRatio,u"comparison":comparison})
except (ValueError, KeyError, TypeError) as e:
logger.warning("The feed contains an invalid minorFeed:\n{0}".format(e))
continue
if not __isCacheUpdateNeeded(cacheFilename=cacheFileName):
useCache = True
if not useCache:
try:
r = requests.get("{0}/users/{1}/lists/{2}/items".format(flannelfox.settings['apis']['trakt'], traktList["username"], traktList["list_name"]), headers=headers, timeout=60)
httpResponse = r.status_code
if httpResponse == 200:
traktListResults = r.json()
else:
logger.error("There was a problem fetching a trakt list file: {0}".format(httpResponse))
except Exception as e:
logger.error("There was a problem fetching a trakt list file: {0}".format(e))
traktListResults = []
httpResponse = -1
logger.error("Fetching trakt list page: [{0}]".format(httpResponse))
# If we are able to get a list then cache it
# TODO: See if Last-Modified can be added to save this step when possible
if httpResponse == 200:
__updateCacheFile(cacheFilename=cacheFileName, data=json.dumps(traktListResults))
else:
useCache = True
if useCache:
try:
logger.debug("Reading cache file for [{0}]".format(cacheFileName))
with open(cacheFileName) as cache:
traktListResults = json.load(cache)
except Exception as e:
logger.error("There was a problem reading a trakt list cache file: {0}".format(e))
continue
# Collect the feedFilters
try:
feedFilterList = []
majorFeedFilters = traktList.get("filters", [])
for filterItem in majorFeedFilters:
# Loop through each show and append a filter for it
for item in traktListResults:
ruleList = []
if traktList.get("like", False):
titleMatchMethod = u"titleLike"
else:
titleMatchMethod = u"title"
# Make sure we have some shows to fetch
# TODO: make this use the type field in the json file to determine if it should be show or movie
if "show" not in item and feedType == "tv":
# This happens if you select the wrong type of media tv/movie
continue;
elif "movie" not in item and feedType == "movie":
# This happens if you select the wrong type of media tv/movie
continue;
elif "show" in item and feedType == "tv":
item = item["show"]
title = item["title"].lower().strip().replace(u" & ", u" and ")
elif "movie" in item and feedType == "movie":
item = item["movie"]
title = item["title"].lower().strip().replace(u" & ", u" and ")
year = item["year"]
else:
continue
ruleList.append({u"key":titleMatchMethod, u"val":title, u"exclude":False})
if year is not None:
ruleList.append({u"key":"year", u"val":year, u"exclude":False})
# Load the excludes
for exclude in filterItem.get("exclude", []):
key, val = exclude.items()[0]
ruleList.append({u"key":key, u"val":val, u"exclude":True})
for include in filterItem.get("include", []):
key, val = include.items()[0]
ruleList.append({u"key":key, u"val":val, u"exclude":False})
feedFilterList.append(ruleList)
except Exception as e:
logger.warning("The feedFilters contains an invalid rule:\n{0}".format(e))
continue
# Append the Config item to the dict
majorFeeds[configFile+'.'+feedName] = {u"feedName":feedName,u"feedType":feedType,u"feedDestination":feedDestination,u"minorFeeds":minorFeeds,u"feedFilters":feedFilterList}
#f = open("feeds_new.json", 'w')
#json.dump(majorFeeds, f)
except Exception as e:
logger.error("There was a problem reading a trakt list file:\n{0}".format(e))
except Exception as e:
# This should only happen if there was an issue getting files names from the directory
pass
logger.debug("=================")
logger.debug("TraktMajorFilters")
logger.debug("=================")
logger.debug(majorFeeds)
logger.debug("=================")
return majorFeeds
def readLastfmArtists(configFolder=flannelfox.settings['files']['lastfmConfigDir']):
'''
Read the artists from a users lastfm library.
Creates a cachefile for the artists and updates it when needed
Returns a list of artists we want to look for
'''
logger = logging.getLogger(__name__)
majorFeeds = {}
try:
for configFile in os.listdir(configFolder):
# Skip non-json files
if not configFile.endswith('.json'):
continue
logger.debug("Loading LastFM config file: {0}".format(os.path.join(configFolder,configFile)))
# Try to read in the lastfm lists
try:
with open(os.path.join(configFolder,configFile)) as lastfmJson:
lastfmAritstsLists = json.load(lastfmJson)
except Exception as e:
logger.error("There was a problem reading the lastfm config file\n{0}".format(e))
continue
try:
for artistsList in lastfmAritstsLists:
# Make sure our list at least has some basic parts
if (artistsList.get("username", None) is None or
artistsList.get("api_key", None) is None or
artistsList.get("list_name", None) is None or
artistsList.get("type", None) is None or
artistsList.get("feedDestination", None) is None or
artistsList.get("minorFeeds", None) is None):
continue
headers = {
"Content-Type":"application/json",
"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36"
}
currentPage = 1
maxPages = 2
replies = []
artists = []
feedName = None
feedType = None
feedDestination = None
minorFeeds = []
feedFilters = []
httpResponse = -1
useCache = False
cacheFileName = "{0}/{1}".format(flannelfox.settings['files']['lastfmCacheDir'],artistsList.get("list_name")+'.'+configFile)
if not os.path.exists(os.path.dirname(cacheFileName)):
try:
os.makedirs(os.path.dirname(cacheFileName))
except OSError as exc: # Guard against race condition
continue
# Get the feedName
try:
feedName = unicode(artistsList.get("list_name",u"").lower().strip())
if feedName == u"":
raise ValueError
except (ValueError, KeyError) as e:
logger.warning("Feeds with out names are not permitted")
continue
# Get the feedType
try:
feedType = unicode(artistsList.get("type",u"none").lower().strip())
except (ValueError, KeyError) as e:
feedType = u"none"
continue
# Get the feedDestination
try:
feedDestination = unicode(artistsList.get("feedDestination",u"").strip())
# TODO: Check if the location exists
except (ValueError, KeyError) as e:
logger.warning("The feed has an invalid destination value")
continue
# Collect the feeds
try:
if artistsList.get("minorFeeds",[]) is not None and len(artistsList.get("minorFeeds",[])) > 0:
for minorFeed in artistsList.get("minorFeeds",[]):
url = unicode(minorFeed.get("url",u"").strip())
minTime = int(minorFeed.get("minTime",u"0").strip()) # Hours Int
minRatio = float(minorFeed.get("minRatio",u"0.0").strip()) # Ratio Float
comparison = minorFeed.get("comparison",u"or").strip() # Comparison String
minorFeeds.append({u"url":url,u"minTime":minTime,u"minRatio":minRatio,u"comparison":comparison})
except (ValueError, KeyError, TypeError) as e:
logger.warning("The feed contains an invalid minorFeed:\n{0}".format(e))
continue
if not __isCacheUpdateNeeded(cacheFilename=cacheFileName):
useCache = True
if not useCache:
while currentPage <= maxPages:
reply = None
call = {}
call.update({"method":"library.getArtists"})
call.update({"api_key":artistsList.get("api_key")})
call.update({"user":artistsList.get("username")})
call.update({"format":"json"})
call.update({"page":currentPage})
try:
# TODO: Covert this to a pool
r = requests.get(flannelfox.settings['apis']['lastfm'], headers=headers, params=call, timeout=60)
httpResponse = r.status_code
except Exception as e:
httpResponse = -1
logger.error("There was a problem fetching a Lastfm album page\n{0}".format(httpResponse))
if httpResponse == 200:
reply = r.json()
else:
logger.error("There was a problem fetching a Lastfm album page\n{0}".format(httpResponse))
replies = []
break # TODO: Replace this with an exception
maxPages = int(reply["artists"]["@attr"]["totalPages"])
replies.extend(reply["artists"]["artist"][:])
logger.error("Fetching Lastfm album page {0} of {1}: [{2}]".format(currentPage, maxPages, httpResponse))
currentPage = currentPage + 1
for artist in replies:
artists.append(artist["name"])
# If we are able to get a list then cache it
# TODO: See if Last-Modified can be added to save this step when possible
if httpResponse == 200:
__updateCacheFile(cacheFilename=cacheFileName, data=json.dumps(artists))
else:
useCache = True
if useCache:
try:
logger.debug("Reading cache file for [{0}]".format(cacheFileName))
with open(cacheFileName) as cache:
artists = json.load(cache)
except Exception as e:
logger.error("There was a problem reading a lastfm list cache file: {0}".format(e))
continue
# Collect the feedFilters
try:
feedFilterList = []
majorFeedFilters = artistsList.get("filters", [])
for filterItem in majorFeedFilters:
# Loop through each show and append a filter for it
for artist in artists:
ruleList = []
# Clean the artist name
artist = artist.lower().strip().replace(u" & ", u" and ")
ruleList.append({u"key":"artist", u"val":artist, u"exclude":False})
# Load the excludes
for exclude in filterItem.get("exclude", []):
key, val = exclude.items()[0]
ruleList.append({u"key":key, u"val":val, u"exclude":True})
for include in filterItem.get("include", []):
key, val = include.items()[0]
ruleList.append({u"key":key, u"val":val, u"exclude":False})
feedFilterList.append(ruleList)
except Exception as e:
logger.warning("The feedFilters contains an invalid rule:\n{0}".format(e))
continue
# Append the Config item to the dict
majorFeeds[feedName] = {u"feedName":feedName,u"feedType":feedType,u"feedDestination":feedDestination,u"minorFeeds":minorFeeds,u"feedFilters":feedFilterList}
except Exception as e:
logger.error("There was a problem reading a lastfm artists list file:\n{0}".format(e))
httpResponse = -1
artists = []
except Exception as e:
# This should only happen if there was an issue getting files names from the directory
pass
return majorFeeds
