def forcePostProcess():
if not headphones.DOWNLOAD_DIR:
logger.error(
'No DOWNLOAD_DIR has been set. Set "Music Download Directory:" to your SAB download directory on the settings page.'
)
return
else:
download_dir = headphones.DOWNLOAD_DIR.encode("utf-8")
logger.info("Checking to see if there are any folders to process in download_dir: %s" % download_dir)
# Get a list of folders in the download_dir
folders = [d for d in os.listdir(download_dir) if os.path.isdir(os.path.join(download_dir, d))]
if len(folders):
logger.info("Found %i folders to process" % len(folders))
pass
else:
logger.info("Found no folders to process in: %s" % download_dir)
return
# Parse the folder names to get artist album info
for folder in folders:
albumpath = os.path.join(download_dir, folder)
folder = unicode(folder, headphones.SYS_ENCODING, errors="replace")
logger.info("Processing: %s" % folder)
try:
name, album, year = helpers.extract_data(folder)
except:
logger.info("Couldn't parse " + folder + " into any valid format.")
continue
if name and album and year:
myDB = db.DBConnection()
release = myDB.action(
"SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?",
[name, album],
).fetchone()
if release:
logger.info(
"Found a match in the database: %s - %s. Verifying to make sure it is the correct album"
% (release["ArtistName"], release["AlbumTitle"])
)
verify(release["AlbumID"], albumpath)
else:
logger.info("Querying MusicBrainz for the release group id for: %s - %s" % (name, album))
from headphones import mb
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album))
except:
logger.error("Can not get release information for this album")
continue
if rgid:
verify(rgid, albumpath)
else:
logger.info("No match found on MusicBrainz for: %s - %s" % (name, album))
def sendNZB(nzb):
params = {}
if headphones.SAB_USERNAME:
params['ma_username'] = headphones.SAB_USERNAME
if headphones.SAB_PASSWORD:
params['ma_password'] = headphones.SAB_PASSWORD
if headphones.SAB_APIKEY:
params['apikey'] = headphones.SAB_APIKEY
if headphones.SAB_CATEGORY:
params['cat'] = headphones.SAB_CATEGORY
# if it's a normal result we just pass SAB the URL
if nzb.resultType == "nzb":
# for newzbin results send the ID to sab specifically
if nzb.provider.getID() == 'newzbin':
id = nzb.provider.getIDFromURL(nzb.url)
if not id:
logger.info("Unable to send NZB to sab, can't find ID in URL "+str(nzb.url))
return False
params['mode'] = 'addid'
params['name'] = id
else:
params['mode'] = 'addurl'
params['name'] = nzb.url
# if we get a raw data result we want to upload it to SAB
elif nzb.resultType == "nzbdata":
# Sanitize the file a bit, since we can only use ascii chars with MultiPartPostHandler
nzbdata = helpers.latinToAscii(nzb.extraInfo[0])
params['mode'] = 'addfile'
multiPartParams = {"nzbfile": (helpers.latinToAscii(nzb.name)+".nzb", nzbdata)}
if not headphones.SAB_HOST.startswith('http'):
headphones.SAB_HOST = 'http://' + headphones.SAB_HOST
if headphones.SAB_HOST.endswith('/'):
headphones.SAB_HOST = headphones.SAB_HOST[0:len(headphones.SAB_HOST)-1]
url = headphones.SAB_HOST + "/" + "api?" + urllib.urlencode(params)
try:
if nzb.resultType == "nzb":
f = urllib.urlopen(url)
elif nzb.resultType == "nzbdata":
cookies = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies),
MultipartPostHandler.MultipartPostHandler)
req = urllib2.Request(url,
multiPartParams,
headers={'User-Agent': USER_AGENT})
f = opener.open(req)
except (EOFError, IOError), e:
logger.error(u"Unable to connect to SAB with URL: %s" % url)
return False
def forcePostProcess():
download_dirs = []
if headphones.DOWNLOAD_DIR:
download_dirs.append(headphones.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING))
if headphones.DOWNLOAD_TORRENT_DIR:
download_dirs.append(headphones.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING))
logger.info('Checking to see if there are any folders to process in download_dir(s): %s' % str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Get a list of folders in the download_dir
folders = []
for download_dir in download_dirs:
for folder in os.listdir(download_dir):
path_to_folder = os.path.join(download_dir, folder)
if os.path.isdir(path_to_folder):
folders.append(path_to_folder)
if len(folders):
logger.info('Found %i folders to process' % len(folders))
else:
logger.info('Found no folders to process in: %s' % str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Parse the folder names to get artist album info
for folder in folders:
folder_basename = os.path.basename(folder).decode(headphones.SYS_ENCODING, 'replace')
logger.info('Processing: %s' % folder_basename)
try:
name, album, year = helpers.extract_data(folder_basename)
except:
logger.info("Couldn't parse " + folder_basename + " into any valid format.")
continue
if name and album and year:
myDB = db.DBConnection()
release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', [name, album]).fetchone()
if release:
logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album' % (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], folder)
else:
logger.info('Querying MusicBrainz for the release group id for: %s - %s' % (name, album))
from headphones import mb
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album))
except:
logger.error('Can not get release information for this album')
continue
if rgid:
verify(rgid, folder)
else:
logger.info('No match found on MusicBrainz for: %s - %s' % (name, album))
def sendNZB(nzb):
params = {}
# if it's a normal result we just pass SAB the URL
if nzb.resultType == "nzb":
# for newzbin results send the ID to sab specifically
if nzb.provider.getID() == 'newzbin':
id = nzb.provider.getIDFromURL(nzb.url)
if not id:
logger.info(
"Unable to send NZB to sab, can't find ID in URL " +
str(nzb.url))
return False
params['mode'] = 'addid'
params['name'] = id
else:
params['mode'] = 'addurl'
params['name'] = nzb.url
# if we get a raw data result we want to upload it to SAB
elif nzb.resultType == "nzbdata":
# Sanitize the file a bit, since we can only use ascii chars with MultiPartPostHandler
nzbdata = helpers.latinToAscii(nzb.extraInfo[0])
params['mode'] = 'addfile'
files = {"nzbfile": (helpers.latinToAscii(nzb.name) + ".nzb", nzbdata)}
headers = {'User-Agent': USER_AGENT}
logger.info("Attempting to connect to SABnzbd on url: %s" %
headphones.CONFIG.SAB_HOST)
if nzb.resultType == "nzb":
response = sab_api_call('send_nzb', params=params)
elif nzb.resultType == "nzbdata":
cookies = cookielib.CookieJar()
response = sab_api_call('send_nzb',
params=params,
method="post",
files=files,
cookies=cookies,
headers=headers)
if not response:
logger.info(u"No data returned from SABnzbd, NZB not sent")
return False
if response['status']:
logger.info(u"NZB sent to SABnzbd successfully")
return True
else:
logger.error(u"Error sending NZB to SABnzbd: %s" % response['error'])
return False
def sendNZB(nzb):
params = {}
if headphones.SAB_USERNAME:
params['ma_username'] = headphones.SAB_USERNAME
if headphones.SAB_PASSWORD:
params['ma_password'] = headphones.SAB_PASSWORD
if headphones.SAB_APIKEY:
params['apikey'] = headphones.SAB_APIKEY
if headphones.SAB_CATEGORY:
params['cat'] = headphones.SAB_CATEGORY
# if it's a normal result we just pass SAB the URL
if nzb.resultType == "nzb":
# for newzbin results send the ID to sab specifically
if nzb.provider.getID() == 'newzbin':
id = nzb.provider.getIDFromURL(nzb.url)
if not id:
logger.info("Unable to send NZB to sab, can't find ID in URL "+str(nzb.url))
return False
params['mode'] = 'addid'
params['name'] = id
else:
params['mode'] = 'addurl'
params['name'] = nzb.url
# if we get a raw data result we want to upload it to SAB
elif nzb.resultType == "nzbdata":
# Sanitize the file a bit, since we can only use ascii chars with MultiPartPostHandler
nzbdata = helpers.latinToAscii(nzb.extraInfo[0])
params['mode'] = 'addfile'
multiPartParams = {"nzbfile": (nzb.name+".nzb", nzbdata)}
if not headphones.SAB_HOST.startswith('http'):
headphones.SAB_HOST = 'http://' + headphones.SAB_HOST
if headphones.SAB_HOST.endswith('/'):
headphones.SAB_HOST = headphones.SAB_HOST[0:len(headphones.SAB_HOST)-1]
url = headphones.SAB_HOST + "/" + "api?" + urllib.urlencode(params)
try:
if nzb.resultType == "nzb":
f = urllib.urlopen(url)
elif nzb.resultType == "nzbdata":
cookies = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies),
MultipartPostHandler.MultipartPostHandler)
req = urllib2.Request(url,
multiPartParams,
headers={'User-Agent': USER_AGENT})
f = opener.open(req)
except (EOFError, IOError), e:
logger.error(u"Unable to connect to SAB with URL: %s" % url)
return False
def sendNZB(nzb):
params = {}
# if it's a normal result we just pass SAB the URL
if nzb.resultType == "nzb":
# for newzbin results send the ID to sab specifically
if nzb.provider.getID() == 'newzbin':
id = nzb.provider.getIDFromURL(nzb.url)
if not id:
logger.info("Unable to send NZB to sab, can't find ID in URL " + str(nzb.url))
return False
params['mode'] = 'addid'
params['name'] = id
else:
params['mode'] = 'addurl'
params['name'] = nzb.url
# if we get a raw data result we want to upload it to SAB
elif nzb.resultType == "nzbdata":
# Sanitize the file a bit, since we can only use ascii chars with MultiPartPostHandler
nzbdata = helpers.latinToAscii(nzb.extraInfo[0])
params['mode'] = 'addfile'
files = {"nzbfile": (helpers.latinToAscii(nzb.name) + ".nzb", nzbdata)}
headers = {'User-Agent': USER_AGENT}
logger.info("Attempting to connect to SABnzbd on url: %s" % headphones.CONFIG.SAB_HOST)
if nzb.resultType == "nzb":
response = sab_api_call('send_nzb', params=params)
elif nzb.resultType == "nzbdata":
cookies = cookielib.CookieJar()
response = sab_api_call('send_nzb', params=params, method="post", files=files,
cookies=cookies, headers=headers)
if not response:
logger.info(u"No data returned from SABnzbd, NZB not sent")
return False
if response['status']:
logger.info(u"NZB sent to SABnzbd successfully")
return True
else:
logger.error(u"Error sending NZB to SABnzbd: %s" % response['error'])
return False
release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone()
tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
downloaded_track_list = []
for r,d,f in os.walk(albumpath):
for files in f:
if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape", ".m4a")):
downloaded_track_list.append(os.path.join(r, files))
# test #1: metadata - usually works
for downloaded_track in downloaded_track_list:
try:
f = MediaFile(downloaded_track)
except:
continue
if helpers.latinToAscii(f.artist.lower()).encode('UTF-8') == helpers.latinToAscii(release['ArtistName'].lower()).encode('UTF-8') and helpers.latinToAscii(f.album.lower()).encode('UTF-8') == helpers.latinToAscii(release['AlbumTitle'].lower()).encode('UTF-8'):
doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
return
# test #2: filenames
for downloaded_track in downloaded_track_list:
track_name = os.path.splitext(downloaded_track)[0]
split_track_name = re.sub('[\.\-\_]', ' ', track_name).lower()
for track in tracks:
if helpers.latinToAscii(track['TrackTitle'].lower()).encode('UTF-8') in helpers.latinToAscii(split_track_name).encode('UTF-8'):
doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
return
# test #3: number of songs and duration
db_track_duration = 0
downloaded_track_duration = 0
def searchTorrent(albumid=None, new=False, losslessOnly=False):
myDB = db.DBConnection()
if albumid:
results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE AlbumID=?', [albumid])
else:
results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted" OR Status="Wanted Lossless"')
new = True
for albums in results:
albumid = albums[2]
reldate = albums[3]
try:
year = reldate[:4]
except TypeError:
year = ''
dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':''}
cleanalbum = helpers.latinToAscii(helpers.replace_all(albums[1], dic))
cleanartist = helpers.latinToAscii(helpers.replace_all(albums[0], dic))
# FLAC usually doesn't have a year for some reason so I'll leave it out
# Various Artist albums might be listed as VA, so I'll leave that out too
# Only use the year if the term could return a bunch of different albums, i.e. self-titled albums
if albums[0] in albums[1] or len(albums[0]) < 4 or len(albums[1]) < 4:
term = cleanartist + ' ' + cleanalbum + ' ' + year
elif albums[0] == 'Various Artists':
term = cleanalbum + ' ' + year
else:
term = cleanartist + ' ' + cleanalbum
# Replace bad characters in the term and unicode it
term = re.sub('[\.\-\/]', ' ', term).encode('utf-8')
artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8')
albumterm = re.sub('[\.\-\/]', ' ', cleanalbum).encode('utf-8')
logger.info("Searching torrents for %s since it was marked as wanted" % term)
resultlist = []
minimumseeders = int(headphones.NUMBEROFSEEDERS) - 1
if headphones.KAT:
provider = "Kick Ass Torrent"
providerurl = url_fix("http://www.kat.ph/search/" + term)
if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
categories = "7" #music
format = "2" #flac
maxsize = 10000000000
elif headphones.PREFERRED_QUALITY:
categories = "7" #music
format = "10" #mp3+flac
maxsize = 10000000000
else:
categories = "7" #music
format = "8" #mp3
maxsize = 300000000
params = {
"categories[0]": "music",
"field": "seeders",
"sorder": "desc",
"rss": "1"
}
searchURL = providerurl + "/?%s" % urllib.urlencode(params)
try:
data = urllib2.urlopen(searchURL, timeout=20).read()
except urllib2.URLError, e:
logger.warn('Error fetching data from %s: %s' % (provider, e))
data = False
if data:
logger.info(u'Parsing results from <a href="%s">KAT</a>' % searchURL)
d = feedparser.parse(data)
if not len(d.entries):
logger.info(u"No results found from %s for %s" % (provider, term))
pass
else:
for item in d.entries:
try:
rightformat = True
title = item.title
seeders = item.seeds
url = item.links[1]['url']
size = int(item.links[1]['length'])
try:
if format == "2":
request = urllib2.Request(url)
request.add_header('Accept-encoding', 'gzip')
request.add_header('Referer', 'http://kat.ph/')
response = urllib2.urlopen(request)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO( response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
if int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1:
rightformat = False
except Exception, e:
rightformat = False
if rightformat == True and size < maxsize and minimumseeders < int(seeders):
resultlist.append((title, size, url, provider))
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
else:
logger.info('%s is larger than the maxsize, the wrong format or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %i, Format: %s)' % (title, size, int(seeders), rightformat))
except Exception, e:
logger.error(u"An unknown error occurred in the KAT parser: %s" % e)
def searchNZB(albumid=None, new=False, losslessOnly=False):
myDB = db.DBConnection()
if albumid:
results = myDB.select(
"SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE AlbumID=?", [albumid]
)
else:
results = myDB.select(
'SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted" OR Status="Wanted Lossless"'
)
new = True
for albums in results:
albumid = albums[2]
reldate = albums[3]
try:
year = reldate[:4]
except TypeError:
year = ""
dic = {
"...": "",
" & ": " ",
" = ": " ",
"?": "",
"$": "s",
" + ": " ",
'"': "",
",": "",
"*": "",
".": "",
":": "",
}
cleanalbum = helpers.latinToAscii(helpers.replace_all(albums[1], dic))
cleanartist = helpers.latinToAscii(helpers.replace_all(albums[0], dic))
# FLAC usually doesn't have a year for some reason so I'll leave it out
# Various Artist albums might be listed as VA, so I'll leave that out too
# Only use the year if the term could return a bunch of different albums, i.e. self-titled albums
if albums[0] in albums[1] or len(albums[0]) < 4 or len(albums[1]) < 4:
term = cleanartist + " " + cleanalbum + " " + year
elif albums[0] == "Various Artists":
term = cleanalbum + " " + year
else:
term = cleanartist + " " + cleanalbum
# Replace bad characters in the term and unicode it
term = re.sub("[\.\-\/]", " ", term).encode("utf-8")
artistterm = re.sub("[\.\-\/]", " ", cleanartist).encode("utf-8")
logger.info("Searching for %s since it was marked as wanted" % term)
resultlist = []
if headphones.NZBMATRIX:
provider = "nzbmatrix"
if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
categories = "23"
maxsize = 10000000000
elif headphones.PREFERRED_QUALITY:
categories = "23,22"
maxsize = 2000000000
else:
categories = "22"
maxsize = 300000000
params = {
"page": "download",
"username": headphones.NZBMATRIX_USERNAME,
"apikey": headphones.NZBMATRIX_APIKEY,
"subcat": categories,
"maxage": headphones.USENET_RETENTION,
"english": 1,
"ssl": 1,
"scenename": 1,
"term": term,
}
searchURL = "http://rss.nzbmatrix.com/rss.php?" + urllib.urlencode(params)
logger.info(u'Parsing results from <a href="%s">NZBMatrix</a>' % searchURL)
try:
data = urllib2.urlopen(searchURL, timeout=20).read()
except urllib2.URLError, e:
logger.warn("Error fetching data from NZBMatrix: %s" % e)
data = False
if data:
d = feedparser.parse(data)
for item in d.entries:
try:
url = item.link
title = item.title
size = int(item.links[1]["length"])
if size < maxsize:
resultlist.append((title, size, url, provider))
logger.info("Found %s. Size: %s" % (title, helpers.bytes_to_mb(size)))
else:
logger.info(
"%s is larger than the maxsize for this category, skipping. (Size: %i bytes)"
% (title, size)
)
except AttributeError, e:
logger.info(u"No results found from NZBMatrix for %s" % term)
def searchNZB(albumid=None, new=False, losslessOnly=False):
myDB = db.DBConnection()
if albumid:
results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate, Type from albums WHERE AlbumID=?', [albumid])
else:
results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate, Type from albums WHERE Status="Wanted" OR Status="Wanted Lossless"')
new = True
for albums in results:
albumid = albums[2]
reldate = albums[3]
try:
year = reldate[:4]
except TypeError:
year = ''
dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':'', '.':'', ':':''}
cleanalbum = helpers.latinToAscii(helpers.replace_all(albums[1], dic))
cleanartist = helpers.latinToAscii(helpers.replace_all(albums[0], dic))
# FLAC usually doesn't have a year for some reason so I'll leave it out
# Various Artist albums might be listed as VA, so I'll leave that out too
# Only use the year if the term could return a bunch of different albums, i.e. self-titled albums
if albums[0] in albums[1] or len(albums[0]) < 4 or len(albums[1]) < 4:
term = cleanartist + ' ' + cleanalbum + ' ' + year
elif albums[0] == 'Various Artists':
term = cleanalbum + ' ' + year
else:
term = cleanartist + ' ' + cleanalbum
# Replace bad characters in the term and unicode it
term = re.sub('[\.\-\/]', ' ', term).encode('utf-8')
artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8')
logger.info("Searching for %s since it was marked as wanted" % term)
resultlist = []
if headphones.NZBMATRIX:
provider = "nzbmatrix"
if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
categories = "23"
elif headphones.PREFERRED_QUALITY:
categories = "23,22"
else:
categories = "22"
# Search Audiobooks/Singles/etc
if albums['Type'] == "Other":
categories = "49"
logger.info("Album type is audiobook/spokenword. Using audiobook category")
if albums['Type'] == "Single":
categories = "47"
logger.info("Album type is 'Single'. Using singles category")
# For some reason NZBMatrix is erroring out/timing out when the term starts with a "The" right now
# so we'll strip it out for the time being. This may get fixed on their end, it may not, but
# hopefully this will fix it for now. If you notice anything else it gets stuck on, please post it
# on Github so it can be added
if term.lower().startswith("the "):
term = term[4:]
params = { "page": "download",
"username": headphones.NZBMATRIX_USERNAME,
"apikey": headphones.NZBMATRIX_APIKEY,
"subcat": categories,
"maxage": headphones.USENET_RETENTION,
"english": 1,
"ssl": 1,
"scenename": 1,
"term": term
}
searchURL = "http://rss.nzbmatrix.com/rss.php?" + urllib.urlencode(params)
logger.info(u'Parsing results from <a href="%s">NZBMatrix</a>' % searchURL)
try:
data = urllib2.urlopen(searchURL, timeout=20).read()
except urllib2.URLError, e:
logger.warn('Error fetching data from NZBMatrix: %s' % e)
data = False
if data:
d = feedparser.parse(data)
for item in d.entries:
try:
url = item.link
title = item.title
size = int(item.links[1]['length'])
resultlist.append((title, size, url, provider))
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
except AttributeError, e:
logger.info(u"No results found from NZBMatrix for %s" % term)
def forcePostProcess():
download_dirs = []
if headphones.DOWNLOAD_DIR:
download_dirs.append(headphones.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING, 'replace'))
if headphones.DOWNLOAD_TORRENT_DIR:
download_dirs.append(headphones.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING, 'replace'))
# If DOWNLOAD_DIR and DOWNLOAD_TORRENT_DIR are the same, remove the duplicate to prevent us from trying to process the same folder twice.
download_dirs = list(set(download_dirs))
logger.info('Checking to see if there are any folders to process in download_dir(s): %s' % str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Get a list of folders in the download_dir
folders = []
for download_dir in download_dirs:
if not os.path.isdir(download_dir):
logger.warn('Directory ' + download_dir.decode(headphones.SYS_ENCODING, 'replace') + ' does not exist. Skipping')
continue
for folder in os.listdir(download_dir):
path_to_folder = os.path.join(download_dir, folder)
if os.path.isdir(path_to_folder):
folders.append(path_to_folder)
if len(folders):
logger.info('Found %i folders to process' % len(folders))
else:
logger.info('Found no folders to process in: %s' % str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Parse the folder names to get artist album info
myDB = db.DBConnection()
for folder in folders:
folder_basename = os.path.basename(folder).decode(headphones.SYS_ENCODING, 'replace')
logger.info('Processing: %s' % folder_basename)
# First try to see if there's a match in the snatched table, then we'll try to parse the foldername
# TODO: Iterate through underscores -> spaces, spaces -> dots, underscores -> dots (this might be hit or miss since it assumes
# all spaces/underscores came from sab replacing values
snatched = myDB.action('SELECT AlbumID, Title, Kind, Status from snatched WHERE FolderName LIKE ?', [folder_basename]).fetchone()
if snatched:
if headphones.KEEP_TORRENT_FILES and snatched['Kind'] == 'torrent' and snatched['Status'] == 'Processed':
logger.info(folder_basename + ' is a torrent folder being preserved for seeding and has already been processed. Skipping.')
continue
else:
logger.info('Found a match in the database: %s. Verifying to make sure it is the correct album' % snatched['Title'])
verify(snatched['AlbumID'], folder, snatched['Kind'])
continue
# Try to parse the folder name into a valid format
# TODO: Add metadata lookup
try:
name, album, year = helpers.extract_data(folder_basename)
except:
name = None
if name and album and year:
release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', [name, album]).fetchone()
if release:
logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album' % (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], folder)
else:
logger.info('Querying MusicBrainz for the release group id for: %s - %s' % (name, album))
from headphones import mb
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album))
except:
logger.error('Can not get release information for this album')
continue
if rgid:
verify(rgid, folder)
else:
logger.info('No match found on MusicBrainz for: %s - %s' % (name, album))
continue
else:
try:
possible_rgid = folder_basename[-36:]
# re pattern match: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
rgid = uuid.UUID(possible_rgid)
except:
logger.info("Couldn't parse " + folder_basename + " into any valid format. If adding albums from another source, they must be in an 'Artist - Album [Year]' format, or end with the musicbrainz release group id")
continue
if rgid:
rgid = possible_rgid
release = myDB.action('SELECT ArtistName, AlbumTitle, AlbumID from albums WHERE AlbumID=?', [rgid]).fetchone()
if release:
logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album' % (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], folder, forced=True)
else:
logger.info('Found a (possibly) valid Musicbrainz identifier in album folder name - continuing post-processing')
verify(rgid, folder, forced=True)
def searchNZB(albumid=None, new=False):
myDB = db.DBConnection()
if albumid:
results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted" AND AlbumID=?', [albumid])
else:
results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted"')
new = True
for albums in results:
albumid = albums[2]
reldate = albums[3]
try:
year = reldate[:4]
except TypeError:
year = ''
dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':''}
cleanartistalbum = helpers.latinToAscii(helpers.replace_all(albums[0]+' '+albums[1], dic))
# FLAC usually doesn't have a year for some reason so I'll leave it out:
term = re.sub('[\.\-\/]', ' ', '%s' % (cleanartistalbum)).encode('utf-8')
altterm = re.sub('[\.\-\/]', ' ', '%s %s' % (cleanartistalbum, year)).encode('utf-8')
# Only use the year if the term could return a bunch of different albums, i.e. self-titled albums
if albums[0] in albums[1] or len(albums[0]) < 4 or len(albums[1]) < 4:
term = altterm
logger.info("Searching for %s since it was marked as wanted" % term)
resultlist = []
if headphones.NZBMATRIX:
provider = "nzbmatrix"
if headphones.PREFERRED_QUALITY == 3:
categories = "23"
maxsize = 10000000000
elif headphones.PREFERRED_QUALITY:
categories = "23,22"
maxsize = 2000000000
else:
categories = "22"
maxsize = 300000000
params = { "page": "download",
"username": headphones.NZBMATRIX_USERNAME,
"apikey": headphones.NZBMATRIX_APIKEY,
"subcat": categories,
"age": headphones.USENET_RETENTION,
"english": 1,
"ssl": 1,
"scenename": 1,
"term": term
}
searchURL = "http://rss.nzbmatrix.com/rss.php?" + urllib.urlencode(params)
logger.info(u"Parsing results from "+searchURL)
try:
data = urllib2.urlopen(searchURL, timeout=20).read()
except urllib2.URLError, e:
logger.warn('Error fetching data from NZBMatrix: %s' % e)
data = False
if data:
d = feedparser.parse(data)
for item in d.entries:
try:
url = item.link
title = item.title
size = int(item.links[1]['length'])
if size < maxsize:
resultlist.append((title, size, url, provider))
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
else:
logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size))
except AttributeError, e:
logger.info(u"No results found from NZBMatrix for %s" % term)
for downloaded_track in downloaded_track_list:
try:
f = MediaFile(downloaded_track)
except Exception, e:
logger.info(
u"Exception from MediaFile for: " +
downloaded_track.decode(headphones.SYS_ENCODING, 'replace') +
u" : " + unicode(e))
continue
if not f.artist:
continue
if not f.album:
continue
metaartist = helpers.latinToAscii(f.artist.lower()).encode('UTF-8')
dbartist = helpers.latinToAscii(
release['ArtistName'].lower()).encode('UTF-8')
metaalbum = helpers.latinToAscii(f.album.lower()).encode('UTF-8')
dbalbum = helpers.latinToAscii(
release['AlbumTitle'].lower()).encode('UTF-8')
logger.debug('Matching metadata artist: %s with artist name: %s' %
(metaartist, dbartist))
logger.debug('Matching metadata album: %s with album name: %s' %
(metaalbum, dbalbum))
if metaartist == dbartist and metaalbum == dbalbum:
doPostProcessing(albumid, albumpath, release, tracks,
downloaded_track_list, Kind)
return
def searchTorrent(albumid=None, new=False, losslessOnly=False):
myDB = db.DBConnection()
if albumid:
results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE AlbumID=?', [albumid])
else:
results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted" OR Status="Wanted Lossless"')
new = True
for albums in results:
albumid = albums[2]
reldate = albums[3]
try:
year = reldate[:4]
except TypeError:
year = ''
dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':''}
cleanalbum = helpers.latinToAscii(helpers.replace_all(albums[1], dic))
cleanartist = helpers.latinToAscii(helpers.replace_all(albums[0], dic))
# FLAC usually doesn't have a year for some reason so I'll leave it out
# Various Artist albums might be listed as VA, so I'll leave that out too
# Only use the year if the term could return a bunch of different albums, i.e. self-titled albums
if albums[0] in albums[1] or len(albums[0]) < 4 or len(albums[1]) < 4:
term = cleanartist + ' ' + cleanalbum + ' ' + year
elif albums[0] == 'Various Artists':
term = cleanalbum + ' ' + year
else:
term = cleanartist + ' ' + cleanalbum
# Replace bad characters in the term and unicode it
term = re.sub('[\.\-\/]', ' ', term).encode('utf-8')
artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8')
logger.info("Searching torrents for %s since it was marked as wanted" % term)
resultlist = []
minimumseeders = int(headphones.NUMBEROFSEEDERS) - 1
if headphones.KAT:
provider = "Kick Ass Torrent"
providerurl = url_fix("http://www.kat.ph/search/" + term)
if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
categories = "7" #music
format = "2" #flac
maxsize = 10000000000
elif headphones.PREFERRED_QUALITY:
categories = "7" #music
format = "10" #mp3+flac
maxsize = 10000000000
else:
categories = "7" #music
format = "8" #mp3
maxsize = 300000000
params = {
"categories[0]": "music",
"field": "seeders",
"sorder": "desc",
"rss": "1"
}
searchURL = providerurl + "/?%s" % urllib.urlencode(params)
try:
data = urllib2.urlopen(searchURL, timeout=20).read()
except urllib2.URLError, e:
logger.warn('Error fetching data from %s: %s' % (provider, e))
data = False
if data:
d = feedparser.parse(data)
if not len(d.entries):
logger.info(u"No results found from %s for %s" % (provider, term))
pass
else:
for item in d.entries:
try:
rightformat = True
title = item.title
seeders = item.seeds
url = item.links[1]['url']
size = int(item.links[1]['length'])
try:
if format == "2":
request = urllib2.Request(url)
request.add_header('Accept-encoding', 'gzip')
response = urllib2.urlopen(request)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO( response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
if int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1:
rightformat = False
except Exception, e:
rightformat = False
if rightformat == True and size < maxsize and minimumseeders < int(seeders):
resultlist.append((title, size, url, provider))
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
else:
logger.info('%s is larger than the maxsize, the wrong format or has to little seeders for this category, skipping. (Size: %i bytes, Seeders: %i, Format: %s)' % (title, size, int(seeders), rightformat))
except Exception, e:
logger.error(u"An unknown error occured in the KAT parser: %s" % e)
def forcePostProcess():
download_dirs = []
if headphones.DOWNLOAD_DIR:
download_dirs.append(headphones.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING, 'replace'))
if headphones.DOWNLOAD_TORRENT_DIR:
download_dirs.append(headphones.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING, 'replace'))
logger.info('Checking to see if there are any folders to process in download_dir(s): %s' % str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Get a list of folders in the download_dir
folders = []
for download_dir in download_dirs:
for folder in os.listdir(download_dir):
path_to_folder = os.path.join(download_dir, folder)
if os.path.isdir(path_to_folder):
folders.append(path_to_folder)
if len(folders):
logger.info('Found %i folders to process' % len(folders))
else:
logger.info('Found no folders to process in: %s' % str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Parse the folder names to get artist album info
myDB = db.DBConnection()
for folder in folders:
folder_basename = os.path.basename(folder).decode(headphones.SYS_ENCODING, 'replace')
logger.info('Processing: %s' % folder_basename)
try:
name, album, year = helpers.extract_data(folder_basename)
except:
name = None
if name and album and year:
release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', [name, album]).fetchone()
if release:
logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album' % (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], folder)
else:
logger.info('Querying MusicBrainz for the release group id for: %s - %s' % (name, album))
from headphones import mb
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album))
except:
logger.error('Can not get release information for this album')
continue
if rgid:
verify(rgid, folder)
else:
logger.info('No match found on MusicBrainz for: %s - %s' % (name, album))
continue
else:
try:
possible_rgid = folder_basename[-36:]
# re pattern match: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
rgid = uuid.UUID(possible_rgid)
except:
logger.info("Couldn't parse " + folder_basename + " into any valid format. If adding albums from another source, they must be in an 'Artist - Album [Year]' format, or end with the musicbrainz release group id")
continue
if rgid:
rgid = possible_rgid
release = myDB.action('SELECT ArtistName, AlbumTitle, AlbumID from albums WHERE AlbumID=?', [rgid]).fetchone()
if release:
logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album' % (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], folder)
else:
logger.info('Found a (possibly) valid Musicbrainz identifier in album folder name - continuing post-processing')
verify(rgid, folder)
def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None):
if album_dir:
folders = [album_dir.encode(headphones.SYS_ENCODING, 'replace')]
else:
download_dirs = []
if dir:
download_dirs.append(dir.encode(headphones.SYS_ENCODING, 'replace'))
if headphones.DOWNLOAD_DIR and not dir:
download_dirs.append(headphones.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING, 'replace'))
if headphones.DOWNLOAD_TORRENT_DIR and not dir:
download_dirs.append(headphones.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING, 'replace'))
# If DOWNLOAD_DIR and DOWNLOAD_TORRENT_DIR are the same, remove the duplicate to prevent us from trying to process the same folder twice.
download_dirs = list(set(download_dirs))
logger.info('Checking to see if there are any folders to process in download_dir(s): %s', download_dirs)
# Get a list of folders in the download_dir
folders = []
for download_dir in download_dirs:
if not os.path.isdir(download_dir):
logger.warn('Directory %s does not exist. Skipping', download_dir)
continue
for folder in os.listdir(download_dir):
path_to_folder = os.path.join(download_dir, folder)
if os.path.isdir(path_to_folder):
subfolders = helpers.expand_subfolders(path_to_folder)
if expand_subfolders and subfolders is not None:
folders.extend(subfolders)
else:
folders.append(path_to_folder)
if len(folders):
logger.info('Found %i folders to process', len(folders))
else:
logger.info('Found no folders to process in: %s', download_dirs)
# Parse the folder names to get artist album info
myDB = db.DBConnection()
for folder in folders:
folder_basename = os.path.basename(folder).decode(headphones.SYS_ENCODING, 'replace')
logger.info('Processing: %s', folder_basename)
# Attempt 1: First try to see if there's a match in the snatched table,
# then we'll try to parse the foldername.
# TODO: Iterate through underscores -> spaces, spaces -> dots,
# underscores -> dots (this might be hit or miss since it assumes all
# spaces/underscores came from sab replacing values
logger.debug('Attempting to find album in the snatched table')
snatched = myDB.action('SELECT AlbumID, Title, Kind, Status from snatched WHERE FolderName LIKE ?', [folder_basename]).fetchone()
if snatched:
if headphones.KEEP_TORRENT_FILES and snatched['Kind'] == 'torrent' and snatched['Status'] == 'Processed':
logger.info('%s is a torrent folder being preserved for seeding and has already been processed. Skipping.', folder_basename)
continue
else:
logger.info('Found a match in the database: %s. Verifying to make sure it is the correct album', snatched['Title'])
verify(snatched['AlbumID'], folder, snatched['Kind'])
continue
# Attempt 2a: parse the folder name into a valid format
try:
logger.debug('Attempting to extract name, album and year from folder name')
name, album, year = helpers.extract_data(folder_basename)
except Exception as e:
name = album = year = None
if name and album:
release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', [name, album]).fetchone()
if release:
logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album', release['ArtistName'], release['AlbumTitle'])
verify(release['AlbumID'], folder)
continue
else:
logger.info('Querying MusicBrainz for the release group id for: %s - %s', name, album)
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album))
except:
logger.error('Can not get release information for this album')
rgid = None
if rgid:
verify(rgid, folder)
continue
else:
logger.info('No match found on MusicBrainz for: %s - %s', name, album)
# Attempt 2b: deduce meta data into a valid format
try:
logger.debug('Attempting to extract name, album and year from metadata')
name, album, year = helpers.extract_metadata(folder)
except Exception as e:
name = album = year = None
if name and album:
release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', [name, album]).fetchone()
if release:
logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album', release['ArtistName'], release['AlbumTitle'])
verify(release['AlbumID'], folder)
continue
else:
logger.info('Querying MusicBrainz for the release group id for: %s - %s', name, album)
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album))
except:
logger.error('Can not get release information for this album')
rgid = None
if rgid:
verify(rgid, folder)
continue
else:
logger.info('No match found on MusicBrainz for: %s - %s', name, album)
# Attempt 3: strip release group id from filename
try:
logger.debug('Attempting to extract release group from folder name')
possible_rgid = folder_basename[-36:]
# re pattern match: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
rgid = uuid.UUID(possible_rgid)
except:
logger.info("Couldn't parse '%s' into any valid format. If adding albums from another source, they must be in an 'Artist - Album [Year]' format, or end with the musicbrainz release group id", folder_basename)
rgid = possible_rgid = None
if rgid:
rgid = possible_rgid
release = myDB.action('SELECT ArtistName, AlbumTitle, AlbumID from albums WHERE AlbumID=?', [rgid]).fetchone()
if release:
logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album', release['ArtistName'], release['AlbumTitle'])
verify(release['AlbumID'], folder, forced=True)
continue
else:
logger.info('Found a (possibly) valid Musicbrainz identifier in album folder name - continuing post-processing')
verify(rgid, folder, forced=True)
continue
# Attempt 4: Hail mary. Just assume the folder name is the album name if it doesn't have a separator in it
logger.debug('Attempt to extract album name by assuming it is the folder name')
if '-' not in folder_basename:
release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE AlbumTitle LIKE ?', [folder_basename]).fetchone()
if release:
logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album', release['ArtistName'], release['AlbumTitle'])
verify(release['AlbumID'], folder)
continue
else:
logger.info('Querying MusicBrainz for the release group id for: %s', folder_basename)
try:
rgid = mb.findAlbumID(album=helpers.latinToAscii(folder_basename))
except:
logger.error('Can not get release information for this album')
rgid = None
if rgid:
verify(rgid, folder)
continue
else:
logger.info('No match found on MusicBrainz for: %s - %s', name, album)
def searchNZB(albumid=None, new=False):
myDB = db.DBConnection()
if albumid:
results = myDB.select(
'SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted" AND AlbumID=?',
[albumid])
else:
results = myDB.select(
'SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted"'
)
new = True
for albums in results:
albumid = albums[2]
reldate = albums[3]
try:
year = reldate[:4]
except TypeError:
year = ''
dic = {
'...': '',
' & ': ' ',
' = ': ' ',
'?': '',
'$': 's',
' + ': ' ',
'"': '',
',': '',
'*': ''
}
cleanalbum = helpers.latinToAscii(helpers.replace_all(albums[1], dic))
cleanartist = helpers.latinToAscii(helpers.replace_all(albums[0], dic))
# FLAC usually doesn't have a year for some reason so I'll leave it out
# Various Artist albums might be listed as VA, so I'll leave that out too
# Only use the year if the term could return a bunch of different albums, i.e. self-titled albums
if albums[0] in albums[1] or len(albums[0]) < 4 or len(albums[1]) < 4:
term = cleanartist + ' ' + cleanalbum + ' ' + year
elif albums[0] == 'Various Artists':
term = cleanalbum + ' ' + year
else:
term = cleanartist + ' ' + cleanalbum
# Replace bad characters in the term and unicode it
term = re.sub('[\.\-\/]', ' ', term).encode('utf-8')
artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8')
logger.info("Searching for %s since it was marked as wanted" % term)
resultlist = []
if headphones.NZBMATRIX:
provider = "nzbmatrix"
if headphones.PREFERRED_QUALITY == 3:
categories = "23"
maxsize = 10000000000
elif headphones.PREFERRED_QUALITY:
categories = "23,22"
maxsize = 2000000000
else:
categories = "22"
maxsize = 300000000
params = {
"page": "download",
"username": headphones.NZBMATRIX_USERNAME,
"apikey": headphones.NZBMATRIX_APIKEY,
"subcat": categories,
"age": headphones.USENET_RETENTION,
"english": 1,
"ssl": 1,
"scenename": 1,
"term": term
}
searchURL = "http://rss.nzbmatrix.com/rss.php?" + urllib.urlencode(
params)
logger.info(u'Parsing results from <a href="%s">NZBMatrix</a>' %
searchURL)
try:
data = urllib2.urlopen(searchURL, timeout=20).read()
except urllib2.URLError, e:
logger.warn('Error fetching data from NZBMatrix: %s' % e)
data = False
if data:
d = feedparser.parse(data)
for item in d.entries:
try:
url = item.link
title = item.title
size = int(item.links[1]['length'])
if size < maxsize:
resultlist.append((title, size, url, provider))
logger.info('Found %s. Size: %s' %
(title, helpers.bytes_to_mb(size)))
else:
logger.info(
'%s is larger than the maxsize for this category, skipping. (Size: %i bytes)'
% (title, size))
except AttributeError, e:
logger.info(u"No results found from NZBMatrix for %s" %
term)
def sendNZB(nzb):
params = {}
if headphones.CONFIG.SAB_USERNAME:
params['ma_username'] = headphones.CONFIG.SAB_USERNAME
if headphones.CONFIG.SAB_PASSWORD:
params['ma_password'] = headphones.CONFIG.SAB_PASSWORD
if headphones.CONFIG.SAB_APIKEY:
params['apikey'] = headphones.CONFIG.SAB_APIKEY
if headphones.CONFIG.SAB_CATEGORY:
params['cat'] = headphones.CONFIG.SAB_CATEGORY
# if it's a normal result we just pass SAB the URL
if nzb.resultType == "nzb":
# for newzbin results send the ID to sab specifically
if nzb.provider.getID() == 'newzbin':
id = nzb.provider.getIDFromURL(nzb.url)
if not id:
logger.info(
"Unable to send NZB to sab, can't find ID in URL " +
str(nzb.url))
return False
params['mode'] = 'addid'
params['name'] = id
else:
params['mode'] = 'addurl'
params['name'] = nzb.url
# if we get a raw data result we want to upload it to SAB
elif nzb.resultType == "nzbdata":
# Sanitize the file a bit, since we can only use ascii chars with MultiPartPostHandler
nzbdata = helpers.latinToAscii(nzb.extraInfo[0])
params['mode'] = 'addfile'
multiPartParams = {
"nzbfile": (helpers.latinToAscii(nzb.name) + ".nzb", nzbdata)
}
if not headphones.CONFIG.SAB_HOST.startswith('http'):
headphones.CONFIG.SAB_HOST = 'http://' + headphones.CONFIG.SAB_HOST
if headphones.CONFIG.SAB_HOST.endswith('/'):
headphones.CONFIG.SAB_HOST = headphones.CONFIG.SAB_HOST[
0:len(headphones.CONFIG.SAB_HOST) - 1]
url = headphones.CONFIG.SAB_HOST + "/" + "api?" + urllib.urlencode(params)
try:
if nzb.resultType == "nzb":
f = urllib.urlopen(url)
elif nzb.resultType == "nzbdata":
cookies = cookielib.CookieJar()
opener = urllib2.build_opener(
urllib2.HTTPCookieProcessor(cookies),
MultipartPostHandler.MultipartPostHandler)
req = urllib2.Request(url,
multiPartParams,
headers={'User-Agent': USER_AGENT})
f = opener.open(req)
except (EOFError, IOError) as e:
logger.error(u"Unable to connect to SAB with URL: %s" % url)
return False
except httplib.InvalidURL as e:
logger.error(u"Invalid SAB host, check your config. Current host: %s" %
headphones.CONFIG.SAB_HOST)
return False
except Exception as e:
logger.error(u"Error: " + str(e))
return False
if f is None:
logger.info(u"No data returned from SABnzbd, NZB not sent")
return False
try:
result = f.readlines()
except Exception as e:
logger.info(u"Error trying to get result from SAB, NZB not sent: ")
return False
if len(result) == 0:
logger.info(u"No data returned from SABnzbd, NZB not sent")
return False
sabText = result[0].strip()
logger.info(u"Result text from SAB: " + sabText)
if sabText == "ok":
logger.info(u"NZB sent to SAB successfully")
return True
elif sabText == "Missing authentication":
logger.info(u"Incorrect username/password sent to SAB, NZB not sent")
return False
else:
logger.info(u"Unknown failure sending NZB to sab. Return text is: " +
sabText)
return False
def sendNZB(nzb):
params = {}
if headphones.CONFIG.SAB_USERNAME:
params['ma_username'] = headphones.CONFIG.SAB_USERNAME
if headphones.CONFIG.SAB_PASSWORD:
params['ma_password'] = headphones.CONFIG.SAB_PASSWORD
if headphones.CONFIG.SAB_APIKEY:
params['apikey'] = headphones.CONFIG.SAB_APIKEY
if headphones.CONFIG.SAB_CATEGORY:
params['cat'] = headphones.CONFIG.SAB_CATEGORY
# if it's a normal result we just pass SAB the URL
if nzb.resultType == "nzb":
# for newzbin results send the ID to sab specifically
if nzb.provider.getID() == 'newzbin':
id = nzb.provider.getIDFromURL(nzb.url)
if not id:
logger.info("Unable to send NZB to sab, can't find ID in URL " + str(nzb.url))
return False
params['mode'] = 'addid'
params['name'] = id
else:
params['mode'] = 'addurl'
params['name'] = nzb.url
# if we get a raw data result we want to upload it to SAB
elif nzb.resultType == "nzbdata":
# Sanitize the file a bit, since we can only use ascii chars with MultiPartPostHandler
nzbdata = helpers.latinToAscii(nzb.extraInfo[0])
params['mode'] = 'addfile'
multiPartParams = {"nzbfile": (helpers.latinToAscii(nzb.name) + ".nzb", nzbdata)}
if not headphones.CONFIG.SAB_HOST.startswith('http'):
headphones.CONFIG.SAB_HOST = 'http://' + headphones.CONFIG.SAB_HOST
if headphones.CONFIG.SAB_HOST.endswith('/'):
headphones.CONFIG.SAB_HOST = headphones.CONFIG.SAB_HOST[0:len(headphones.CONFIG.SAB_HOST) - 1]
url = headphones.CONFIG.SAB_HOST + "/" + "api?" + urllib.urlencode(params)
try:
if nzb.resultType == "nzb":
f = urllib.urlopen(url)
elif nzb.resultType == "nzbdata":
cookies = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies),
MultipartPostHandler.MultipartPostHandler)
req = urllib2.Request(url,
multiPartParams,
headers={'User-Agent': USER_AGENT})
f = opener.open(req)
except (EOFError, IOError) as e:
logger.error(u"Unable to connect to SAB with URL: %s" % url)
return False
except httplib.InvalidURL as e:
logger.error(u"Invalid SAB host, check your config. Current host: %s" % headphones.CONFIG.SAB_HOST)
return False
except Exception as e:
logger.error(u"Error: " + str(e))
return False
if f is None:
logger.info(u"No data returned from SABnzbd, NZB not sent")
return False
try:
result = f.readlines()
except Exception as e:
logger.info(u"Error trying to get result from SAB, NZB not sent: ")
return False
if len(result) == 0:
logger.info(u"No data returned from SABnzbd, NZB not sent")
return False
sabText = result[0].strip()
logger.info(u"Result text from SAB: " + sabText)
if sabText == "ok":
logger.info(u"NZB sent to SAB successfully")
return True
elif sabText == "Missing authentication":
logger.info(u"Incorrect username/password sent to SAB, NZB not sent")
return False
else:
logger.info(u"Unknown failure sending NZB to sab. Return text is: " + sabText)
return False
def forcePostProcess():
download_dirs = []
if headphones.DOWNLOAD_DIR:
download_dirs.append(headphones.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING, "replace"))
if headphones.DOWNLOAD_TORRENT_DIR:
download_dirs.append(headphones.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING, "replace"))
# If DOWNLOAD_DIR and DOWNLOAD_TORRENT_DIR are the same, remove the duplicate to prevent us from trying to process the same folder twice.
download_dirs = list(set(download_dirs))
logger.info(
"Checking to see if there are any folders to process in download_dir(s): %s"
% str(download_dirs).decode(headphones.SYS_ENCODING, "replace")
)
# Get a list of folders in the download_dir
folders = []
for download_dir in download_dirs:
for folder in os.listdir(download_dir):
path_to_folder = os.path.join(download_dir, folder)
if os.path.isdir(path_to_folder):
folders.append(path_to_folder)
if len(folders):
logger.info("Found %i folders to process" % len(folders))
else:
logger.info(
"Found no folders to process in: %s" % str(download_dirs).decode(headphones.SYS_ENCODING, "replace")
)
# Parse the folder names to get artist album info
myDB = db.DBConnection()
for folder in folders:
folder_basename = os.path.basename(folder).decode(headphones.SYS_ENCODING, "replace")
logger.info("Processing: %s" % folder_basename)
# First try to see if there's a match in the snatched table, then we'll try to parse the foldername
snatched = myDB.action("SELECT AlbumID, Title from snatched WHERE FolderName LIKE ?", [folder_basename])
if snatched:
logger.info(
"Found a match in the database: %s. Verifying to make sure it is the correct album" % snatched["Title"]
)
verify(snatched["AlbumID"], folder)
continue
# Try to parse the folder name into a valid format
# TODO: Add metadata lookup
try:
name, album, year = helpers.extract_data(folder_basename)
except:
name = None
if name and album and year:
release = myDB.action(
"SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?",
[name, album],
).fetchone()
if release:
logger.info(
"Found a match in the database: %s - %s. Verifying to make sure it is the correct album"
% (release["ArtistName"], release["AlbumTitle"])
)
verify(release["AlbumID"], folder)
else:
logger.info("Querying MusicBrainz for the release group id for: %s - %s" % (name, album))
from headphones import mb
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album))
except:
logger.error("Can not get release information for this album")
continue
if rgid:
verify(rgid, folder)
else:
logger.info("No match found on MusicBrainz for: %s - %s" % (name, album))
continue
else:
try:
possible_rgid = folder_basename[-36:]
# re pattern match: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
rgid = uuid.UUID(possible_rgid)
except:
logger.info(
"Couldn't parse "
+ folder_basename
+ " into any valid format. If adding albums from another source, they must be in an 'Artist - Album [Year]' format, or end with the musicbrainz release group id"
)
continue
if rgid:
rgid = possible_rgid
release = myDB.action(
"SELECT ArtistName, AlbumTitle, AlbumID from albums WHERE AlbumID=?", [rgid]
).fetchone()
if release:
logger.info(
"Found a match in the database: %s - %s. Verifying to make sure it is the correct album"
% (release["ArtistName"], release["AlbumTitle"])
)
verify(release["AlbumID"], folder)
else:
logger.info(
"Found a (possibly) valid Musicbrainz identifier in album folder name - continuing post-processing"
)
verify(rgid, folder)
f = MediaFile(downloaded_track)
except Exception, e:
logger.info(
u"Exception from MediaFile for: "
+ downloaded_track.decode(headphones.SYS_ENCODING, "replace")
+ u" : "
+ unicode(e)
)
continue
if not f.artist:
continue
if not f.album:
continue
metaartist = helpers.latinToAscii(f.artist.lower()).encode("UTF-8")
dbartist = helpers.latinToAscii(release["ArtistName"].lower()).encode("UTF-8")
metaalbum = helpers.latinToAscii(f.album.lower()).encode("UTF-8")
dbalbum = helpers.latinToAscii(release["AlbumTitle"].lower()).encode("UTF-8")
logger.debug("Matching metadata artist: %s with artist name: %s" % (metaartist, dbartist))
logger.debug("Matching metadata album: %s with album name: %s" % (metaalbum, dbalbum))
if metaartist == dbartist and metaalbum == dbalbum:
doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
return
# test #2: filenames
logger.debug("Metadata check failed. Verifying filenames...")
for downloaded_track in downloaded_track_list:
track_name = os.path.splitext(downloaded_track)[0]
def forcePostProcess():
if not headphones.DOWNLOAD_DIR:
logger.error(
'No DOWNLOAD_DIR has been set. Set "Music Download Directory:" to your SAB download directory on the settings page.'
)
return
else:
processing = "nzb"
processing_next = "torrent"
while processing != "done":
if headphones.DOWNLOAD_DIR and processing == "nzb":
download_dir = headphones.DOWNLOAD_DIR.encode('utf-8')
if not headphones.DOWNLOAD_TORRENT_DIR:
processing_next = "done"
if headphones.DOWNLOAD_TORRENT_DIR and processing == "torrent":
download_dir = headphones.DOWNLOAD_TORRENT_DIR.encode('utf-8')
processing_next = "done"
if not headphones.DOWNLOAD_DIR and processing == "nzb":
download_dir = headphones.DOWNLOAD_TORRENT_DIR.encode('utf-8')
processing_next = "done"
logger.info(
'Checking to see if there are any folders to process in download_dir: %s'
% download_dir)
# Get a list of folders in the download_dir
folders = [
d for d in os.listdir(download_dir)
if os.path.isdir(os.path.join(download_dir, d))
]
if len(folders):
logger.info('Found %i folders to process' % len(folders))
pass
else:
logger.info('Found no folders to process in: %s' %
download_dir)
return
# Parse the folder names to get artist album info
for folder in folders:
albumpath = os.path.join(download_dir, folder)
folder = unicode(folder,
headphones.SYS_ENCODING,
errors='replace')
logger.info('Processing: %s' % folder)
try:
name, album, year = helpers.extract_data(folder)
except:
logger.info("Couldn't parse " + folder +
" into any valid format.")
continue
if name and album and year:
myDB = db.DBConnection()
release = myDB.action(
'SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?',
[name, album]).fetchone()
if release:
logger.info(
'Found a match in the database: %s - %s. Verifying to make sure it is the correct album'
% (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], albumpath)
else:
logger.info(
'Querying MusicBrainz for the release group id for: %s - %s'
% (name, album))
from headphones import mb
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name),
helpers.latinToAscii(album))
except:
logger.error(
'Can not get release information for this album'
)
continue
if rgid:
verify(rgid, albumpath)
else:
logger.info(
'No match found on MusicBrainz for: %s - %s' %
(name, album))
processing = processing_next
for r,d,f in os.walk(albumpath):
for files in f:
if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS):
downloaded_track_list.append(os.path.join(r, files))
# test #1: metadata - usually works
logger.debug('Verifying metadata...')
for downloaded_track in downloaded_track_list:
try:
f = MediaFile(downloaded_track)
except:
continue
metaartist = helpers.latinToAscii(f.artist.lower()).encode('UTF-8')
dbartist = helpers.latinToAscii(release['ArtistName'].lower()).encode('UTF-8')
metaalbum = helpers.latinToAscii(f.album.lower()).encode('UTF-8')
dbalbum = helpers.latinToAscii(release['AlbumTitle'].lower()).encode('UTF-8')
logger.debug('Matching metadata artist: %s with artist name: %s' % (metaartist, dbartist))
logger.debug('Matching metadata album: %s with album name: %s' % (metaalbum, dbalbum))
if metaartist == dbartist and metaalbum == dbalbum:
doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
return
# test #2: filenames
logger.debug('Metadata check failed. Verifying filenames...')
for downloaded_track in downloaded_track_list:
track_name = os.path.splitext(downloaded_track)[0]
def forcePostProcess():
download_dirs = []
if headphones.DOWNLOAD_DIR:
download_dirs.append(
headphones.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING, 'replace'))
if headphones.DOWNLOAD_TORRENT_DIR:
download_dirs.append(
headphones.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING,
'replace'))
logger.info(
'Checking to see if there are any folders to process in download_dir(s): %s'
% str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Get a list of folders in the download_dir
folders = []
for download_dir in download_dirs:
for folder in os.listdir(download_dir):
path_to_folder = os.path.join(download_dir, folder)
if os.path.isdir(path_to_folder):
folders.append(path_to_folder)
if len(folders):
logger.info('Found %i folders to process' % len(folders))
else:
logger.info(
'Found no folders to process in: %s' %
str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Parse the folder names to get artist album info
myDB = db.DBConnection()
for folder in folders:
folder_basename = os.path.basename(folder).decode(
headphones.SYS_ENCODING, 'replace')
logger.info('Processing: %s' % folder_basename)
try:
name, album, year = helpers.extract_data(folder_basename)
except:
name = None
if name and album and year:
release = myDB.action(
'SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?',
[name, album]).fetchone()
if release:
logger.info(
'Found a match in the database: %s - %s. Verifying to make sure it is the correct album'
% (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], folder)
else:
logger.info(
'Querying MusicBrainz for the release group id for: %s - %s'
% (name, album))
from headphones import mb
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name),
helpers.latinToAscii(album))
except:
logger.error(
'Can not get release information for this album')
continue
if rgid:
verify(rgid, folder)
else:
logger.info('No match found on MusicBrainz for: %s - %s' %
(name, album))
continue
else:
try:
possible_rgid = folder_basename[-36:]
# re pattern match: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
rgid = uuid.UUID(possible_rgid)
except:
logger.info(
"Couldn't parse " + folder_basename +
" into any valid format. If adding albums from another source, they must be in an 'Artist - Album [Year]' format, or end with the musicbrainz release group id"
)
continue
if rgid:
rgid = possible_rgid
release = myDB.action(
'SELECT ArtistName, AlbumTitle, AlbumID from albums WHERE AlbumID=?',
[rgid]).fetchone()
if release:
logger.info(
'Found a match in the database: %s - %s. Verifying to make sure it is the correct album'
% (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], folder)
else:
logger.info(
'Found a (possibly) valid Musicbrainz identifier in album folder name - continuing post-processing'
)
verify(rgid, folder)
# test #1: metadata - usually works
logger.debug('Verifying metadata...')
for downloaded_track in downloaded_track_list:
try:
f = MediaFile(downloaded_track)
except Exception, e:
logger.info(u"Exception from MediaFile for: " + downloaded_track.decode(headphones.SYS_ENCODING, 'replace') + u" : " + unicode(e))
continue
if not f.artist:
continue
if not f.album:
continue
metaartist = helpers.latinToAscii(f.artist.lower()).encode('UTF-8')
dbartist = helpers.latinToAscii(release['ArtistName'].lower()).encode('UTF-8')
metaalbum = helpers.latinToAscii(f.album.lower()).encode('UTF-8')
dbalbum = helpers.latinToAscii(release['AlbumTitle'].lower()).encode('UTF-8')
logger.debug('Matching metadata artist: %s with artist name: %s' % (metaartist, dbartist))
logger.debug('Matching metadata album: %s with album name: %s' % (metaalbum, dbalbum))
if metaartist == dbartist and metaalbum == dbalbum:
doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
return
# test #2: filenames
logger.debug('Metadata check failed. Verifying filenames...')
for downloaded_track in downloaded_track_list:
track_name = os.path.splitext(downloaded_track)[0]
def forcePostProcess():
download_dirs = []
if headphones.DOWNLOAD_DIR:
download_dirs.append(
headphones.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING, 'replace'))
if headphones.DOWNLOAD_TORRENT_DIR:
download_dirs.append(
headphones.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING,
'replace'))
# If DOWNLOAD_DIR and DOWNLOAD_TORRENT_DIR are the same, remove the duplicate to prevent us from trying to process the same folder twice.
download_dirs = list(set(download_dirs))
logger.info(
'Checking to see if there are any folders to process in download_dir(s): %s'
% str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Get a list of folders in the download_dir
folders = []
for download_dir in download_dirs:
for folder in os.listdir(download_dir):
path_to_folder = os.path.join(download_dir, folder)
if os.path.isdir(path_to_folder):
folders.append(path_to_folder)
if len(folders):
logger.info('Found %i folders to process' % len(folders))
else:
logger.info(
'Found no folders to process in: %s' %
str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Parse the folder names to get artist album info
myDB = db.DBConnection()
for folder in folders:
folder_basename = os.path.basename(folder).decode(
headphones.SYS_ENCODING, 'replace')
logger.info('Processing: %s' % folder_basename)
# First try to see if there's a match in the snatched table, then we'll try to parse the foldername
# TODO: Iterate through underscores -> spaces, spaces -> dots, underscores -> dots (this might be hit or miss since it assumes
# all spaces/underscores came from sab replacing values
snatched = myDB.action(
'SELECT AlbumID, Title, Kind, Status from snatched WHERE FolderName LIKE ?',
[folder_basename]).fetchone()
if snatched:
if headphones.KEEP_TORRENT_FILES and snatched[
'Kind'] == 'torrent' and snatched['Status'] == 'Processed':
logger.info(
folder_basename +
' is a torrent folder being preserved for seeding and has already been processed. Skipping.'
)
continue
else:
logger.info(
'Found a match in the database: %s. Verifying to make sure it is the correct album'
% snatched['Title'])
verify(snatched['AlbumID'], folder, snatched['Kind'])
continue
# Try to parse the folder name into a valid format
# TODO: Add metadata lookup
try:
name, album, year = helpers.extract_data(folder_basename)
except:
name = None
if name and album and year:
release = myDB.action(
'SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?',
[name, album]).fetchone()
if release:
logger.info(
'Found a match in the database: %s - %s. Verifying to make sure it is the correct album'
% (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], folder)
else:
logger.info(
'Querying MusicBrainz for the release group id for: %s - %s'
% (name, album))
from headphones import mb
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name),
helpers.latinToAscii(album))
except:
logger.error(
'Can not get release information for this album')
continue
if rgid:
verify(rgid, folder)
else:
logger.info('No match found on MusicBrainz for: %s - %s' %
(name, album))
continue
else:
try:
possible_rgid = folder_basename[-36:]
# re pattern match: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
rgid = uuid.UUID(possible_rgid)
except:
logger.info(
"Couldn't parse " + folder_basename +
" into any valid format. If adding albums from another source, they must be in an 'Artist - Album [Year]' format, or end with the musicbrainz release group id"
)
continue
if rgid:
rgid = possible_rgid
release = myDB.action(
'SELECT ArtistName, AlbumTitle, AlbumID from albums WHERE AlbumID=?',
[rgid]).fetchone()
if release:
logger.info(
'Found a match in the database: %s - %s. Verifying to make sure it is the correct album'
% (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], folder)
else:
logger.info(
'Found a (possibly) valid Musicbrainz identifier in album folder name - continuing post-processing'
)
verify(rgid, folder)
def verify(albumid, albumpath):
myDB = db.DBConnection()
release = myDB.action("SELECT * from albums WHERE AlbumID=?", [albumid]).fetchone()
tracks = myDB.select("SELECT * from tracks WHERE AlbumID=?", [albumid])
downloaded_track_list = []
for r, d, f in os.walk(albumpath):
for files in f:
if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape", ".m4a")):
downloaded_track_list.append(os.path.join(r, files))
# test #1: metadata - usually works
for downloaded_track in downloaded_track_list:
try:
f = MediaFile(downloaded_track)
except:
continue
if helpers.latinToAscii(f.artist.lower()).encode("UTF-8") == helpers.latinToAscii(
release["ArtistName"].lower()
).encode("UTF-8") and helpers.latinToAscii(f.album.lower()).encode("UTF-8") == helpers.latinToAscii(
release["AlbumTitle"].lower()
).encode(
"UTF-8"
):
doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
return
# test #2: filenames
for downloaded_track in downloaded_track_list:
track_name = os.path.splitext(downloaded_track)[0]
split_track_name = re.sub("[\.\-\_]", " ", track_name).lower()
for track in tracks:
if helpers.latinToAscii(track["TrackTitle"].lower()).encode("UTF-8") in helpers.latinToAscii(
split_track_name
).encode("UTF-8"):
doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
return
# test #3: number of songs and duration
db_track_duration = 0
downloaded_track_duration = 0
if len(tracks) == len(downloaded_track_list):
for track in tracks:
try:
db_track_duration += track["TrackDuration"] / 1000
except:
downloaded_track_duration = False
break
for downloaded_track in downloaded_track_list:
try:
f = MediaFile(downloaded_track)
downloaded_track_duration += f.length
except:
downloaded_track_duration = False
break
if downloaded_track_duration and db_track_duration:
delta = abs(downloaded_track_duration - db_track_duration)
if delta < 240:
doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
return
logger.warn("Could not identify album: %s. It may not be the intended album." % albumpath)
myDB.action('UPDATE snatched SET status = "Unprocessed" WHERE AlbumID=?', [albumid])
renameUnprocessedFolder(albumpath)
def searchNZB(albumid=None, new=False, losslessOnly=False):
myDB = db.DBConnection()
if albumid:
results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE AlbumID=?', [albumid])
else:
results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted" OR Status="Wanted Lossless"')
new = True
for albums in results:
albumid = albums[2]
reldate = albums[3]
try:
year = reldate[:4]
except TypeError:
year = ''
dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':'', '.':'', ':':''}
cleanalbum = helpers.latinToAscii(helpers.replace_all(albums[1], dic))
cleanartist = helpers.latinToAscii(helpers.replace_all(albums[0], dic))
# FLAC usually doesn't have a year for some reason so I'll leave it out
# Various Artist albums might be listed as VA, so I'll leave that out too
# Only use the year if the term could return a bunch of different albums, i.e. self-titled albums
if albums[0] in albums[1] or len(albums[0]) < 4 or len(albums[1]) < 4:
term = cleanartist + ' ' + cleanalbum + ' ' + year
elif albums[0] == 'Various Artists':
term = cleanalbum + ' ' + year
else:
term = cleanartist + ' ' + cleanalbum
# Replace bad characters in the term and unicode it
term = re.sub('[\.\-\/]', ' ', term).encode('utf-8')
artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8')
logger.info("Searching for %s since it was marked as wanted" % term)
resultlist = []
if headphones.NZBMATRIX:
provider = "nzbmatrix"
if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
categories = "23"
maxsize = 10000000000
elif headphones.PREFERRED_QUALITY:
categories = "23,22"
maxsize = 2000000000
else:
categories = "22"
maxsize = 300000000
# For some reason NZBMatrix is erroring out/timing out when the term starts with a "The" right now
# so we'll strip it out for the time being. This may get fixed on their end, it may not, but
# hopefully this will fix it for now. If you notice anything else it gets stuck on, please post it
# on Github so it can be added
if term.lower().startswith("the "):
term = term[4:]
params = { "page": "download",
"username": headphones.NZBMATRIX_USERNAME,
"apikey": headphones.NZBMATRIX_APIKEY,
"subcat": categories,
"maxage": headphones.USENET_RETENTION,
"english": 1,
"ssl": 1,
"scenename": 1,
"term": term
}
searchURL = "http://rss.nzbmatrix.com/rss.php?" + urllib.urlencode(params)
logger.info(u'Parsing results from <a href="%s">NZBMatrix</a>' % searchURL)
try:
data = urllib2.urlopen(searchURL, timeout=20).read()
except urllib2.URLError, e:
logger.warn('Error fetching data from NZBMatrix: %s' % e)
data = False
if data:
d = feedparser.parse(data)
for item in d.entries:
try:
url = item.link
title = item.title
size = int(item.links[1]['length'])
if size < maxsize:
resultlist.append((title, size, url, provider))
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
else:
logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size))
except AttributeError, e:
logger.info(u"No results found from NZBMatrix for %s" % term)
def forcePostProcess():
download_dirs = []
if headphones.DOWNLOAD_DIR:
download_dirs.append(
headphones.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING))
if headphones.DOWNLOAD_TORRENT_DIR:
download_dirs.append(
headphones.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING))
logger.info(
'Checking to see if there are any folders to process in download_dir(s): %s'
% str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Get a list of folders in the download_dir
folders = []
for download_dir in download_dirs:
for folder in os.listdir(download_dir):
path_to_folder = os.path.join(download_dir, folder)
if os.path.isdir(path_to_folder):
folders.append(path_to_folder)
if len(folders):
logger.info('Found %i folders to process' % len(folders))
else:
logger.info(
'Found no folders to process in: %s' %
str(download_dirs).decode(headphones.SYS_ENCODING, 'replace'))
# Parse the folder names to get artist album info
for folder in folders:
folder_basename = os.path.basename(folder).decode(
headphones.SYS_ENCODING, 'replace')
logger.info('Processing: %s' % folder_basename)
try:
name, album, year = helpers.extract_data(folder_basename)
except:
logger.info("Couldn't parse " + folder_basename +
" into any valid format.")
continue
if name and album and year:
myDB = db.DBConnection()
release = myDB.action(
'SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?',
[name, album]).fetchone()
if release:
logger.info(
'Found a match in the database: %s - %s. Verifying to make sure it is the correct album'
% (release['ArtistName'], release['AlbumTitle']))
verify(release['AlbumID'], folder)
else:
logger.info(
'Querying MusicBrainz for the release group id for: %s - %s'
% (name, album))
from headphones import mb
try:
rgid = mb.findAlbumID(helpers.latinToAscii(name),
helpers.latinToAscii(album))
except:
logger.error(
'Can not get release information for this album')
continue
if rgid:
verify(rgid, folder)
else:
logger.info('No match found on MusicBrainz for: %s - %s' %
(name, album))
else:
bestqual = nzblist[0]
logger.info(u"Found best result: %s (%s) - %s" % (bestqual[0], bestqual[2], helpers.bytes_to_mb(bestqual[1])))
if bestqual[3] == "newzbin":
#logger.info("Found a newzbin result")
reportid = bestqual[2]
params = urllib.urlencode({"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": reportid})
url = providerurl + "/api/dnzb/"
urllib._urlopener = NewzbinDownloader()
data = urllib.urlopen(url, data=params).read()
nzb = classes.NZBDataSearchResult()
nzb.extraInfo.append(data)
nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8').replace('/', '_'), helpers.latinToAscii(albums[1]).encode('UTF-8').replace('/', '_'), year)
nzb.name = nzb_folder_name
logger.info(u"Sending FILE to SABNZBD: " + nzb.name)
sab.sendNZB(nzb)
myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]])
myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name])
else:
downloadurl = bestqual[2]
nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8').replace('/', '_'), helpers.latinToAscii(albums[1]).encode('UTF-8').replace('/', '_'), year)
if headphones.SAB_HOST and not headphones.BLACKHOLE:
linkparams = {}
linkparams["mode"] = "addurl"
logger.info('%s has already been downloaded. Skipping.' % nzblist[0][0])
nzblist.pop(0)
else:
break
else:
logger.info('No more results found for %s' % term)
return "none"
logger.info(u"Pre-processing result")
(data, bestqual) = preprocess(nzblist)
if data and bestqual:
logger.info(u'Found best result: <a href="%s">%s</a> - %s' % (bestqual[2], bestqual[0], helpers.bytes_to_mb(bestqual[1])))
nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8').replace('/', '_'), helpers.latinToAscii(albums[1]).encode('UTF-8').replace('/', '_'), year)
if headphones.SAB_HOST and not headphones.BLACKHOLE:
nzb = classes.NZBDataSearchResult()
nzb.extraInfo.append(data)
nzb.name = nzb_folder_name
sab.sendNZB(nzb)
elif headphones.BLACKHOLE:
nzb_name = nzb_folder_name + '.nzb'
download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name)
try:
f = open(download_path, 'w')
f.write(data)
f.close()
for r,d,f in os.walk(albumpath):
for files in f:
if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
downloaded_track_list.append(os.path.join(r, files))
# test #1: metadata - usually works
logger.debug('Verifying metadata...')
for downloaded_track in downloaded_track_list:
try:
f = MediaFile(downloaded_track)
except Exception, e:
logger.info("Exception from MediaFile for: " + downloaded_track + " : " + str(e))
continue
metaartist = helpers.latinToAscii(f.artist.lower()).encode('UTF-8')
dbartist = helpers.latinToAscii(release['ArtistName'].lower()).encode('UTF-8')
metaalbum = helpers.latinToAscii(f.album.lower()).encode('UTF-8')
dbalbum = helpers.latinToAscii(release['AlbumTitle'].lower()).encode('UTF-8')
logger.debug('Matching metadata artist: %s with artist name: %s' % (metaartist, dbartist))
logger.debug('Matching metadata album: %s with album name: %s' % (metaalbum, dbalbum))
if metaartist == dbartist and metaalbum == dbalbum:
doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
return
# test #2: filenames
logger.debug('Metadata check failed. Verifying filenames...')
for downloaded_track in downloaded_track_list:
track_name = os.path.splitext(downloaded_track)[0]
break
else:
logger.info("No more results found for %s" % term)
return "none"
logger.info(u"Pre-processing result")
(data, bestqual) = preprocess(nzblist)
if data and bestqual:
logger.info(
u'Found best result: <a href="%s">%s</a> - %s'
% (bestqual[2], bestqual[0], helpers.bytes_to_mb(bestqual[1]))
)
nzb_folder_name = "%s - %s [%s]" % (
helpers.latinToAscii(albums[0]).encode("UTF-8").replace("/", "_"),
helpers.latinToAscii(albums[1]).encode("UTF-8").replace("/", "_"),
year,
)
if headphones.SAB_HOST and not headphones.BLACKHOLE:
nzb = classes.NZBDataSearchResult()
nzb.extraInfo.append(data)
nzb.name = nzb_folder_name
sab.sendNZB(nzb)
elif headphones.BLACKHOLE:
nzb_name = nzb_folder_name + ".nzb"
download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name)
try:
