def doReplace(self, string, replacements, remove_multiple = False, use_separator = True):
'''
replace confignames with the real thing
'''
replacements = replacements.copy()
if remove_multiple:
replacements['cd'] = ''
replacements['cd_nr'] = ''
replaced = toUnicode(string)
for x, r in replacements.iteritems():
if r is not None:
replaced = replaced.replace(u'<%s>' % toUnicode(x), toUnicode(r))
else:
#If information is not available, we don't want the tag in the filename
replaced = replaced.replace('<' + x + '>', '')
replaced = re.sub(r"[\x00:\*\?\"<>\|]", '', replaced)
sep = ' '
if use_separator:
sep = self.conf('separator')
return self.replaceDoubles(replaced).replace(' ', sep)
def add(self, path = '', part = 1, type_tuple = (), available = 1, properties = None):
if not properties: properties = {}
try:
db = get_session()
type_id = self.getType(type_tuple).get('id')
f = db.query(File).filter(File.path == toUnicode(path)).first()
if not f:
f = File()
db.add(f)
f.path = toUnicode(path)
f.part = part
f.available = available
f.type_id = type_id
db.commit()
file_dict = f.to_dict()
return file_dict
except:
log.error('Failed adding file: %s, %s', (path, traceback.format_exc()))
db.rollback()
finally:
db.close()
def getParams():
params = url_decode(getattr(flask.request, 'environ').get('QUERY_STRING', ''))
reg = re.compile('^[a-z0-9_\.]+$')
current = temp = {}
for param, value in sorted(params.iteritems()):
nest = re.split("([\[\]]+)", param)
if len(nest) > 1:
nested = []
for key in nest:
if reg.match(key):
nested.append(key)
current = temp
for item in nested:
if item is nested[-1]:
current[item] = toUnicode(unquote(value))
else:
try:
current[item]
except:
current[item] = {}
current = current[item]
else:
temp[param] = toUnicode(unquote(value))
return dictToList(temp)
def notify(self, message = '', data = None, listener = None):
if not data: data = {}
# Get all the device IDs linked to this user
devices = self.getDevices() or []
successful = 0
for device in devices:
response = self.request(
'pushes',
cache = False,
device_iden = device,
type = 'note',
title = self.default_title,
body = toUnicode(message)
)
if response:
successful += 1
else:
log.error('Unable to push notification to Pushbullet device with ID %s' % device)
for channel in self.getChannels():
response = self.request(
'pushes',
cache = False,
channel_tag = channel,
type = 'note',
title = self.default_title,
body = toUnicode(message)
)
return successful == len(devices)
def createNzbName(self, data, media):
tag = self.cpTag(media)
split_string = scanPassword(data.get('name'))
if split_string[1] != None:
return '%s%s{{%s}}' % (toSafeString(toUnicode(split_string[0])[:123 - len(tag) - len(split_string[1])]), tag, split_string[1])
else:
return '%s%s' % (toSafeString(toUnicode(data.get('name'))[:127 - len(tag)]), tag)
def parseMovie(self, movie):
year = str(movie.get('released', 'none'))[:4]
# Poster url
poster = self.getImage(movie, type = 'poster')
backdrop = self.getImage(movie, type = 'backdrop')
# 1900 is the same as None
if year == '1900' or year.lower() == 'none':
year = None
movie_data = {
'id': int(movie.get('id', 0)),
'titles': [toUnicode(movie.get('name'))],
'images': {
'posters': [poster],
'backdrops': [backdrop],
},
'imdb': movie.get('imdb_id'),
'year': year,
'plot': movie.get('overview', ''),
'tagline': '',
}
# Add alternative names
for alt in ['original_name', 'alternative_name']:
alt_name = toUnicode(movie.get(alt))
if alt_name and not alt_name in movie_data['titles'] and alt_name.lower() != 'none' and alt_name != None:
movie_data['titles'].append(alt_name)
return movie_data
def add(self, attrs={}, update_after=True):
db = get_session()
l = db.query(Library).filter_by(identifier=attrs.get("identifier")).first()
if not l:
status = fireEvent("status.get", "needs_update", single=True)
l = Library(
year=attrs.get("year"),
identifier=attrs.get("identifier"),
plot=toUnicode(attrs.get("plot")),
tagline=toUnicode(attrs.get("tagline")),
status_id=status.get("id"),
)
title = LibraryTitle(
title=toUnicode(attrs.get("title")), simple_title=self.simplifyTitle(attrs.get("title"))
)
l.titles.append(title)
db.add(l)
db.commit()
# Update library info
if update_after is not False:
handle = fireEventAsync if update_after is "async" else fireEvent
handle("library.update", identifier=l.identifier, default_title=attrs.get("title", ""))
return l.to_dict(self.default_dict)
def notify(self, message = '', data = None, listener = None):
if not data: data = {}
api_data = {
'user': self.conf('user_key'),
'token': self.conf('api_token'),
'message': toUnicode(message),
'priority': self.conf('priority'),
'sound': self.conf('sound'),
}
if data and getIdentifier(data):
api_data.update({
'url': toUnicode('http://www.imdb.com/title/%s/' % getIdentifier(data)),
'url_title': toUnicode('%s on IMDb' % getTitle(data)),
})
try:
data = self.urlopen('%s/%s' % (self.api_url, '1/messages.json'),
headers = {'Content-type': 'application/x-www-form-urlencoded'},
data = api_data)
log.info2('Pushover responded with: %s', data)
return True
except:
return False
def notify(self, message = '', data = None, listener = None):
if not data: data = {}
try:
message = message.strip()
long_message = ''
if listener == 'test':
long_message = 'This is a test message'
elif data.get('identifier'):
long_message = 'More movie info <a href="http://www.imdb.com/title/%s/">on IMDB</a>' % data['identifier']
data = {
'user_credentials': self.conf('token'),
'notification[title]': toUnicode('%s - %s' % (self.default_title, message)),
'notification[long_message]': toUnicode(long_message),
}
self.urlopen(self.url, data = data)
except:
log.error('Make sure the token provided is for the correct device')
return False
log.info('Boxcar notification successful.')
return True
def getParams(params):
reg = re.compile('^[a-z0-9_\.]+$')
current = temp = {}
for param, value in sorted(params.iteritems()):
nest = re.split("([\[\]]+)", param)
if len(nest) > 1:
nested = []
for key in nest:
if reg.match(key):
nested.append(key)
current = temp
for item in nested:
if item is nested[-1]:
current[item] = toUnicode(unquote(value))
else:
try:
current[item]
except:
current[item] = {}
current = current[item]
else:
temp[param] = toUnicode(unquote(value))
return dictToList(temp)
def calculate(self, nzb, movie):
''' Calculate the score of a NZB, used for sorting later '''
score = nameScore(toUnicode(nzb['name']), movie['library']['year'])
for movie_title in movie['library']['titles']:
score += nameRatioScore(toUnicode(nzb['name']), toUnicode(movie_title['title']))
score += sizeScore(nzb['size'])
# Torrents only
if nzb.get('seeds'):
try:
score += nzb.get('seeds') / 5
score += nzb.get('leechers') / 10
except:
pass
# Provider score
score += providerScore(nzb['provider'])
# Duplicates in name
score += duplicateScore(nzb['name'], movie['library']['titles'][0]['title'])
return score
def notify(self, message = '', data = {}, listener = None):
http_handler = HTTPSConnection("api.pushover.net:443")
api_data = {
'user': self.conf('user_key'),
'token': self.app_token,
'message': toUnicode(message),
'priority': self.conf('priority'),
}
if data and data.get('library'):
api_data.extend({
'url': toUnicode('http://www.imdb.com/title/%s/' % data['library']['identifier']),
'url_title': toUnicode('%s on IMDb' % getTitle(data['library'])),
})
http_handler.request('POST',
"/1/messages.json",
headers = {'Content-type': 'application/x-www-form-urlencoded'},
body = tryUrlencode(api_data)
)
response = http_handler.getresponse()
request_status = response.status
if request_status == 200:
log.info('Pushover notifications sent.')
return True
elif request_status == 401:
log.error('Pushover auth failed: %s', response.reason)
return False
else:
log.error('Pushover notification failed.')
return False
def add(self, attrs = {}, update_after = True):
db = get_session()
l = db.query(Library).filter_by(identifier = attrs.get('identifier')).first()
if not l:
status = fireEvent('status.get', 'needs_update', single = True)
l = Library(
year = attrs.get('year'),
identifier = attrs.get('identifier'),
plot = toUnicode(attrs.get('plot')),
tagline = toUnicode(attrs.get('tagline')),
status_id = status.get('id')
)
title = LibraryTitle(
title = toUnicode(attrs.get('title')),
simple_title = self.simplifyTitle(attrs.get('title')),
)
l.titles.append(title)
db.add(l)
db.commit()
# Update library info
if update_after is not False:
handle = fireEventAsync if update_after is 'async' else fireEvent
handle('library.update', identifier = l.identifier, default_title = toUnicode(attrs.get('title', '')))
library_dict = l.to_dict(self.default_dict)
return library_dict
def fill(self):
try:
db = get_session()
order = 0
for q in self.qualities:
# Create quality
qual = db.query(Quality).filter_by(identifier = q.get('identifier')).first()
if not qual:
log.info('Creating quality: %s', q.get('label'))
qual = Quality()
qual.order = order
qual.identifier = q.get('identifier')
qual.label = toUnicode(q.get('label'))
qual.size_min, qual.size_max = q.get('size')
db.add(qual)
# Create single quality profile
prof = db.query(Profile).filter(
Profile.core == True
).filter(
Profile.types.any(quality = qual)
).all()
if not prof:
log.info('Creating profile: %s', q.get('label'))
prof = Profile(
core = True,
label = toUnicode(qual.label),
order = order
)
db.add(prof)
profile_type = ProfileType(
quality = qual,
profile = prof,
finish = True,
order = 0
)
prof.types.append(profile_type)
order += 1
db.commit()
time.sleep(0.3) # Wait a moment
return True
except:
log.error('Failed: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
return False
def process_bind_param(self, value, dialect):
try:
return toUnicode(json.dumps(value, cls = SetEncoder))
except:
try:
return toUnicode(json.dumps(value, cls = SetEncoder, encoding = 'latin-1'))
except:
raise
def list(self, status = ['active'], limit_offset = None, starts_with = None, search = None):
db = get_session()
# Make a list from string
if not isinstance(status, (list, tuple)):
status = [status]
q = db.query(Movie) \
.join(Movie.library, Library.titles) \
.options(joinedload_all('releases.status')) \
.options(joinedload_all('releases.quality')) \
.options(joinedload_all('releases.files')) \
.options(joinedload_all('releases.info')) \
.options(joinedload_all('library.titles')) \
.options(joinedload_all('library.files')) \
.options(joinedload_all('status')) \
.options(joinedload_all('files')) \
.filter(LibraryTitle.default == True) \
.filter(or_(*[Movie.status.has(identifier = s) for s in status]))
filter_or = []
if starts_with:
starts_with = toUnicode(starts_with.lower())
if starts_with in ascii_lowercase:
filter_or.append(LibraryTitle.simple_title.startswith(starts_with))
else:
ignore = []
for letter in ascii_lowercase:
ignore.append(LibraryTitle.simple_title.startswith(toUnicode(letter)))
filter_or.append(not_(or_(*ignore)))
if search:
filter_or.append(LibraryTitle.simple_title.like('%%' + search + '%%'))
if filter_or:
q = q.filter(or_(*filter_or))
q = q.order_by(asc(LibraryTitle.simple_title))
if limit_offset:
splt = limit_offset.split(',')
limit = splt[0]
offset = 0 if len(splt) is 1 else splt[1]
q = q.limit(limit).offset(offset)
results = q.all()
movies = []
for movie in results:
temp = movie.to_dict(self.default_dict)
movies.append(temp)
return movies
def setProperty(self, identifier, value=""):
from couchpotato import get_db
db = get_db()
try:
p = db.get("property", identifier, with_doc=True)
p["doc"].update({"identifier": identifier, "value": toUnicode(value)})
db.update(p["doc"])
except:
db.insert({"_t": "property", "identifier": identifier, "value": toUnicode(value)})
def getNfo(self, data):
nfoxml = Element('movie')
types = ['rating', 'year', 'votes', 'rating', 'mpaa', 'originaltitle:original_title', 'outline:plot', 'premiered:released']
# Title
try:
el = SubElement(nfoxml, 'title')
el.text = toUnicode(data['library']['titles'][0]['title'])
except:
pass
# IMDB id
try:
el = SubElement(nfoxml, 'id')
el.text = toUnicode(data['library']['identifier'])
except:
pass
# Runtime
try:
runtime = SubElement(nfoxml, 'runtime')
runtime.text = '%s min' % data['library']['runtime']
except:
pass
# Other values
for type in types:
if ':' in type:
name, type = type.split(':')
else:
name = type
try:
if data['library'].get(type):
el = SubElement(nfoxml, name)
el.text = toUnicode(data['library'].get(type, ''))
except:
pass
# Genre
for genre in data['library'].get('genres', []):
genres = SubElement(nfoxml, 'genre')
genres.text = genre.get('name')
# Clean up the xml and return it
nfoxml = xml.dom.minidom.parseString(tostring(nfoxml))
xml_string = nfoxml.toprettyxml(indent = ' ')
text_re = re.compile('>\n\s+([^<>\s].*?)\n\s+</', re.DOTALL)
xml_string = text_re.sub('>\g<1></', xml_string)
return xml_string.encode('utf-8')
def notify(self, message = '', data = None, listener = None):
if not data: data = {}
# Extract all the settings from settings
from_address = self.conf('from')
to_address = self.conf('to')
ssl = self.conf('ssl')
smtp_server = self.conf('smtp_server')
smtp_user = self.conf('smtp_user')
smtp_pass = self.conf('smtp_pass')
smtp_port = self.conf('smtp_port')
starttls = self.conf('starttls')
# Make the basic message
email = MIMEText(toUnicode(message), _charset = Env.get('encoding'))
email['Subject'] = '%s: %s' % (self.default_title, toUnicode(message))
email['From'] = from_address
email['To'] = to_address
email['Date'] = formatdate(localtime = 1)
email['Message-ID'] = make_msgid()
try:
# Open the SMTP connection, via SSL if requested
log.debug("Connecting to host %s on port %s" % (smtp_server, smtp_port))
log.debug("SMTP over SSL %s", ("enabled" if ssl == 1 else "disabled"))
mailserver = smtplib.SMTP_SSL(smtp_server, smtp_port) if ssl == 1 else smtplib.SMTP(smtp_server, smtp_port)
if starttls:
log.debug("Using StartTLS to initiate the connection with the SMTP server")
mailserver.starttls()
# Say hello to the server
mailserver.ehlo()
# Check too see if an login attempt should be attempted
if len(smtp_user) > 0:
log.debug("Logging on to SMTP server using username \'%s\'%s", (smtp_user, " and a password" if len(smtp_pass) > 0 else ""))
mailserver.login(smtp_user.encode('utf-8'), smtp_pass.encode('utf-8'))
# Send the e-mail
log.debug("Sending the email")
mailserver.sendmail(from_address, splitString(to_address), email.as_string())
# Close the SMTP connection
mailserver.quit()
log.info('Email notification sent')
return True
except:
log.error('E-mail failed: %s', traceback.format_exc())
return False
def createMetaName(self, basename, name, root):
detected = detect(root)
if detected.get('encoding') == 'ISO-8859-2':
root = toUnicode(root);
log.info('encoding root: %s %s', (root, detected.get('encoding')))
detected = detect(name)
if detected.get('encoding') == 'ISO-8859-2':
name = toUnicode(name);
log.info('encoding name: %s %s', (name, detected.get('encoding')))
return os.path.join(root, basename.replace('%s', name))
def fill(self):
db = get_session()
for identifier, label in self.statuses.iteritems():
s = db.query(Status).filter_by(identifier=identifier).first()
if not s:
log.info("Creating status: %s", label)
s = Status(identifier=identifier, label=toUnicode(label))
db.add(s)
s.label = toUnicode(label)
db.commit()
def calculate(self, nzb, movie):
""" Calculate the score of a NZB, used for sorting later """
# Merge global and category
preferred_words = splitString(Env.setting('preferred_words', section = 'searcher').lower())
try: preferred_words = list(set(preferred_words + splitString(movie['category']['preferred'].lower())))
except: pass
score = nameScore(toUnicode(nzb['name']), movie['library']['year'], preferred_words)
for movie_title in movie['library']['titles']:
score += nameRatioScore(toUnicode(nzb['name']), toUnicode(movie_title['title']))
score += namePositionScore(toUnicode(nzb['name']), toUnicode(movie_title['title']))
score += sizeScore(nzb['size'])
# Torrents only
if nzb.get('seeders'):
try:
score += nzb.get('seeders') * 100 / 15
score += nzb.get('leechers') * 100 / 30
except:
pass
# Provider score
score += providerScore(nzb['provider'])
# Duplicates in name
score += duplicateScore(nzb['name'], getTitle(movie['library']))
# Merge global and category
ignored_words = splitString(Env.setting('ignored_words', section = 'searcher').lower())
try: ignored_words = list(set(ignored_words + splitString(movie['category']['ignored'].lower())))
except: pass
# Partial ignored words
score += partialIgnoredScore(nzb['name'], getTitle(movie['library']), ignored_words)
# Ignore single downloads from multipart
score += halfMultipartScore(nzb['name'])
# Extra provider specific check
extra_score = nzb.get('extra_score')
if extra_score:
score += extra_score(nzb)
# Scene / Nuke scoring
score += sceneScore(nzb['name'])
return score
def fill(self):
db = get_session();
order = 0
for q in self.qualities:
# Create quality
quality = db.query(Quality).filter_by(identifier = q.get('identifier')).first()
if not quality:
log.info('Creating quality: %s', q.get('label'))
quality = Quality()
db.add(quality)
quality.order = order
quality.identifier = q.get('identifier')
quality.label = toUnicode(q.get('label'))
quality.size_min, quality.size_max = q.get('size')
# Create single quality profile
profile = db.query(Profile).filter(
Profile.core == True
).filter(
Profile.types.any(quality = quality)
).all()
if not profile:
log.info('Creating profile: %s', q.get('label'))
profile = Profile(
core = True,
label = toUnicode(quality.label),
order = order
)
db.add(profile)
profile_type = ProfileType(
quality = quality,
profile = profile,
finish = True,
order = 0
)
profile.types.append(profile_type)
order += 1
db.commit()
#db.close()
return True
def getType(self, type):
db = get_session()
type, identifier = type
ft = db.query(FileType).filter_by(identifier=identifier).first()
if not ft:
ft = FileType(
type=toUnicode(type), identifier=identifier, name=toUnicode(identifier[0].capitalize() + identifier[1:])
)
db.add(ft)
db.commit()
return ft
def safeMessage(self, msg, replace_tuple = ()):
from couchpotato.core.helpers.encoding import ss, toUnicode
msg = ss(msg)
try:
if isinstance(replace_tuple, tuple):
msg = msg % tuple([ss(x) if not isinstance(x, (int, float)) else x for x in list(replace_tuple)])
elif isinstance(replace_tuple, dict):
msg = msg % dict((k, ss(v)) for k, v in replace_tuple.iteritems())
else:
msg = msg % ss(replace_tuple)
except Exception as e:
self.logger.error('Failed encoding stuff to log "%s": %s' % (msg, e))
self.setup()
if not self.is_develop:
for replace in self.replace_private:
msg = re.sub('(\?%s=)[^\&]+' % replace, '?%s=xxx' % replace, msg)
msg = re.sub('(&%s=)[^\&]+' % replace, '&%s=xxx' % replace, msg)
# Replace api key
try:
api_key = self.Env.setting('api_key')
if api_key:
msg = msg.replace(api_key, 'API_KEY')
except:
pass
return toUnicode(msg)
def registerStatic(self, plugin_file, add_to_head = True):
# Register plugin path
self.plugin_path = os.path.dirname(plugin_file)
static_folder = toUnicode(os.path.join(self.plugin_path, 'static'))
if not os.path.isdir(static_folder):
return
# Get plugin_name from PluginName
s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__)
class_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
# View path
path = 'static/plugin/%s/' % class_name
# Add handler to Tornado
Env.get('app').add_handlers(".*$", [(Env.get('web_base') + path + '(.*)', StaticFileHandler, {'path': static_folder})])
# Register for HTML <HEAD>
if add_to_head:
for f in glob.glob(os.path.join(self.plugin_path, 'static', '*')):
ext = getExt(f)
if ext in ['js', 'css']:
fireEvent('register_%s' % ('script' if ext in 'js' else 'style'), path + os.path.basename(f), f)
def notify(self, message = '', data = {}, listener = None):
if self.isDisabled(): return
try:
params = {
'label': self.default_title,
'msg': toUnicode(message),
}
headers = {
'Authorization': "Basic %s" % base64.encodestring('%s:%s' % (self.conf('username'), self.conf('api_key')))[:-1]
}
handle = self.urlopen(self.url, params = params, headers = headers)
result = json.loads(handle)
if result['status'] != 'success' or result['response_message'] != 'OK':
raise Exception
except:
log.error('Notification failed: %s', traceback.format_exc())
return False
log.info('Notifo notification successful.')
return True
def _searchOnTitle(self, title, movie, quality, results):
q = '%s %s' % (title, movie['library']['year'])
params = tryUrlencode({
'search': q,
'catid': ','.join([str(x) for x in self.getCatId(quality['identifier'])]),
'user': self.conf('username', default = ''),
'api': self.conf('api_key', default = ''),
})
nzbs = self.getRSSData(self.urls['search'] % params)
for nzb in nzbs:
enclosure = self.getElement(nzb, 'enclosure').attrib
nzb_id = parse_qs(urlparse(self.getTextElement(nzb, 'link')).query).get('id')[0]
results.append({
'id': nzb_id,
'name': toUnicode(self.getTextElement(nzb, 'title')),
'age': self.calculateAge(int(time.mktime(parse(self.getTextElement(nzb, 'pubDate')).timetuple()))),
'size': tryInt(enclosure['length']) / 1024 / 1024,
'url': enclosure['url'],
'detail_url': self.urls['detail_url'] % nzb_id,
'description': self.getTextElement(nzb, 'description')
})
def get(self, identifiers):
if not isinstance(identifiers, (list)):
identifiers = [identifiers]
db = get_session()
return_list = []
for identifier in identifiers:
if self.status_cached.get(identifier):
return_list.append(self.status_cached.get(identifier))
continue
s = db.query(Status).filter_by(identifier=identifier).first()
if not s:
s = Status(identifier=identifier, label=toUnicode(identifier.capitalize()))
db.add(s)
db.commit()
status_dict = s.to_dict()
self.status_cached[identifier] = status_dict
return_list.append(status_dict)
return return_list if len(identifiers) > 1 else return_list[0]
def _searchOnHost(self, host, media, quality, results):
torrents = self.getJsonData(self.buildUrl(media, host), cache_timeout = 1800)
if torrents:
try:
if torrents.get('error'):
log.error('%s: %s', (torrents.get('error'), host['host']))
elif torrents.get('results'):
for torrent in torrents.get('results', []):
results.append({
'id': torrent.get('torrent_id'),
'protocol': 'torrent' if re.match('^(http|https|ftp)://.*$', torrent.get('download_url')) else 'torrent_magnet',
'provider_extra': urlparse(host['host']).hostname or host['host'],
'name': toUnicode(torrent.get('release_name')),
'url': torrent.get('download_url'),
'detail_url': torrent.get('details_url'),
'size': torrent.get('size'),
'score': host['extra_score'],
'seeders': torrent.get('seeders'),
'leechers': torrent.get('leechers'),
'seed_ratio': host['seed_ratio'],
'seed_time': host['seed_time'],
})
except:
log.error('Failed getting results from %s: %s', (host['host'], traceback.format_exc()))
def cleanup(self):
# Wait a bit after starting before cleanup
time.sleep(3)
log.debug('Cleaning up unused files')
python_cache = Env.get('cache')._path
try:
db = get_session()
for root, dirs, walk_files in os.walk(Env.get('cache_dir')):
for filename in walk_files:
if os.path.splitext(filename)[1] in [
'.png', '.jpg', '.jpeg'
]:
file_path = os.path.join(root, filename)
f = db.query(File).filter(
File.path == toUnicode(file_path)).first()
if not f:
os.remove(file_path)
except:
log.error('Failed removing unused file: %s',
traceback.format_exc())
def getSubtitleLanguage(self, group):
detected_languages = {}
# Subliminal scanner
try:
paths = group['files']['movie']
scan_result = []
for p in paths:
if not group['is_dvd']:
video = Video.from_path(toUnicode(p))
video_result = [(video, video.scan())]
scan_result.extend(video_result)
for video, detected_subtitles in scan_result:
for s in detected_subtitles:
if s.language and s.path not in paths:
detected_languages[s.path] = [s.language]
except:
log.debug('Failed parsing subtitle languages for %s: %s',
(paths, traceback.format_exc()))
# IDX
for extra in group['files']['subtitle_extra']:
try:
if os.path.isfile(extra):
output = open(extra, 'r')
txt = output.read()
output.close()
idx_langs = re.findall('\nid: (\w+)', txt)
sub_file = '%s.sub' % os.path.splitext(extra)[0]
if len(idx_langs) > 0 and os.path.isfile(sub_file):
detected_languages[sub_file] = idx_langs
except:
log.error('Failed parsing subtitle idx for %s: %s',
(extra, traceback.format_exc()))
return detected_languages
def notify(self, message='', data={}, listener=None):
data = {
'apikey': self.conf('api_key'),
'application': self.default_title,
'description': toUnicode(message),
'priority': self.conf('priority'),
}
headers = {'Content-type': 'application/x-www-form-urlencoded'}
try:
self.urlopen(self.urls['api'],
headers=headers,
params=data,
multipart=True,
show_error=False)
log.info('Prowl notifications sent.')
return True
except:
log.error('Prowl failed: %s', traceback.format_exc())
return False
def edit(self):
params = getParams()
db = get_session()
available_status = fireEvent('status.get', 'available', single = True)
ids = [x.strip() for x in params.get('id').split(',')]
for movie_id in ids:
m = db.query(Movie).filter_by(id = movie_id).first()
if not m:
continue
m.profile_id = params.get('profile_id')
# Remove releases
for rel in m.releases:
if rel.status_id is available_status.get('id'):
db.delete(rel)
db.commit()
# Default title
if params.get('default_title'):
for title in m.library.titles:
title.default = toUnicode(params.get('default_title', '')).lower() == toUnicode(title.title).lower()
db.commit()
fireEvent('movie.restatus', m.id)
movie_dict = m.to_dict(self.default_dict)
fireEventAsync('searcher.single', movie_dict, on_complete = self.createNotifyFront(movie_id))
#db.close()
return jsonified({
'success': True,
})
def save(self):
params = getParams()
db = get_session()
p = db.query(Profile).filter_by(id = params.get('id')).first()
if not p:
p = Profile()
db.add(p)
p.label = toUnicode(params.get('label'))
p.order = params.get('order', p.order if p.order else 0)
p.core = params.get('core', False)
#delete old types
[db.delete(t) for t in p.types]
order = 0
for type in params.get('types', []):
t = ProfileType(
order = order,
finish = type.get('finish') if order > 0 else 1,
wait_for = params.get('wait_for'),
quality_id = type.get('quality_id')
)
p.types.append(t)
order += 1
db.commit()
profile_dict = p.to_dict(self.to_dict)
return jsonified({
'success': True,
'profile': profile_dict
})
def partial(self):
log_type = getParam('type', 'all')
total_lines = tryInt(getParam('lines', 30))
log_lines = []
for x in range(0, 50):
path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '')
# Check see if the log exists
if not os.path.isfile(path):
break
reversed_lines = []
f = open(path, 'r')
reversed_lines = toUnicode(f.read()).split('[0m\n')
reversed_lines.reverse()
brk = False
for line in reversed_lines:
if log_type == 'all' or '%s ' % log_type.upper() in line:
log_lines.append(line)
if len(log_lines) >= total_lines:
brk = True
break
if brk:
break
log_lines.reverse()
return jsonified({
'success': True,
'log': '[0m\n'.join(log_lines),
})
def searchSingle(self, group):
if self.isDisabled(): return
try:
available_languages = sum(group['subtitle_language'].itervalues(),
[])
downloaded = []
files = [toUnicode(x) for x in group['files']['movie']]
log.debug('Searching for subtitles for: %s', files)
for lang in self.getLanguages():
if lang not in available_languages:
download = subliminal.download_subtitles(
files,
multi=True,
force=False,
languages=[lang],
services=self.services,
cache_dir=Env.get('cache_dir'))
for subtitle in download:
downloaded.extend(download[subtitle])
for d_sub in downloaded:
log.info('Found subtitle (%s): %s',
(d_sub.language.alpha2, files))
group['files']['subtitle'].append(d_sub.path)
group['subtitle_language'][d_sub.path] = [
d_sub.language.alpha2
]
return True
except:
log.error('Failed searching for subtitle: %s',
(traceback.format_exc()))
return False
def _searchOnTitle(self, title, movie, quality, results):
q = '%s %s' % (title, movie['info']['year'])
params = tryUrlencode({
'search':
q,
'catid':
','.join([str(x) for x in self.getCatId(quality)]),
'user':
self.conf('username', default=''),
'api':
self.conf('api_key', default=''),
})
if len(self.conf('custom_tag')) > 0:
params = '%s&%s' % (params, self.conf('custom_tag'))
nzbs = self.getJsonData(self.urls['search'] % params)
if isinstance(nzbs, list):
for nzb in nzbs:
results.append({
'id':
nzb.get('nzbid'),
'name':
toUnicode(nzb.get('release')),
'age':
self.calculateAge(tryInt(nzb.get('usenetage'))),
'size':
tryInt(nzb.get('sizebytes')) / 1024 / 1024,
'url':
nzb.get('getnzb'),
'detail_url':
nzb.get('details'),
'description':
nzb.get('weblink')
})
def cleanup(self):
# Wait a bit after starting before cleanup
log.debug('Cleaning up unused files')
try:
db = get_db()
cache_dir = Env.get('cache_dir')
medias = db.all('media', with_doc = True)
files = []
for media in medias:
file_dict = media['doc'].get('files', {})
for x in file_dict.keys():
files.extend(file_dict[x])
for f in os.listdir(cache_dir):
if os.path.splitext(f)[1] in ['.png', '.jpg', '.jpeg']:
file_path = os.path.join(cache_dir, f)
if toUnicode(file_path) not in files:
os.remove(file_path)
except:
log.error('Failed removing unused file: %s', traceback.format_exc())
def notify(self, message = '', data = None, listener = None):
if not data: data = {}
data = {
'action': "send",
'uri': "",
'title': self.default_title,
'message': toUnicode(message),
'broadcast': self.conf('broadcast'),
'username': self.conf('username'),
}
headers = {
'Content-type': 'application/x-www-form-urlencoded'
}
try:
self.urlopen(self.conf('url'), headers = headers, data = data, show_error = False)
return True
except:
log.error('AndroidPN failed: %s', traceback.format_exc())
return False
def edit(self, id = '', **kwargs):
db = get_session()
available_status = fireEvent('status.get', 'available', single = True)
ids = splitString(id)
for movie_id in ids:
m = db.query(Movie).filter_by(id = movie_id).first()
if not m:
continue
m.profile_id = kwargs.get('profile_id')
# Remove releases
for rel in m.releases:
if rel.status_id is available_status.get('id'):
db.delete(rel)
db.commit()
# Default title
if kwargs.get('default_title'):
for title in m.library.titles:
title.default = toUnicode(kwargs.get('default_title', '')).lower() == toUnicode(title.title).lower()
db.commit()
fireEvent('movie.restatus', m.id)
movie_dict = m.to_dict(self.default_dict)
fireEventAsync('searcher.single', movie_dict, on_complete = self.createNotifyFront(movie_id))
db.expire_all()
return {
'success': True,
}
def notify(self, message='', data=None, listener=None):
if not data: data = {}
try:
db = get_session()
data['notification_type'] = listener if listener else 'unknown'
n = Notif(message=toUnicode(message), data=data)
db.add(n)
db.commit()
ndict = n.to_dict()
ndict['type'] = 'notification'
ndict['time'] = time.time()
self.frontend(type=listener, data=data)
return True
except:
log.error('Failed notify: %s', traceback.format_exc())
db.rollback()
finally:
db.close()
def partial(self, type='all', lines=30, **kwargs):
total_lines = tryInt(lines)
log_lines = []
for x in range(0, 50):
path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '')
# Check see if the log exists
if not os.path.isfile(path):
break
f = open(path, 'r')
reversed_lines = toUnicode(f.read()).split('[0m\n')
reversed_lines.reverse()
brk = False
for line in reversed_lines:
if type == 'all' or '%s ' % type.upper() in line:
log_lines.append(line)
if len(log_lines) >= total_lines:
brk = True
break
if brk:
break
log_lines.reverse()
return {
'success': True,
'log': '[0m\n'.join(log_lines),
}
def safeMessage(self, msg, replace_tuple=()):
from couchpotato.core.helpers.encoding import ss, toUnicode
msg = ss(msg)
try:
msg = msg % replace_tuple
except:
try:
if isinstance(replace_tuple, tuple):
msg = msg % tuple([ss(x) for x in list(replace_tuple)])
else:
msg = msg % ss(replace_tuple)
except Exception as e:
self.logger.error('Failed encoding stuff to log "%s": %s' %
(msg, e))
self.setup()
if not self.is_develop:
for replace in self.replace_private:
msg = re.sub('(\?%s=)[^\&]+' % replace, '?%s=xxx' % replace,
msg)
msg = re.sub('(&%s=)[^\&]+' % replace, '&%s=xxx' % replace,
msg)
# Replace api key
try:
api_key = self.Env.setting('api_key')
if api_key:
msg = msg.replace(api_key, 'API_KEY')
except:
pass
return toUnicode(msg)
def notify(self, message='', data=None, listener=None):
if not data: data = {}
url = self.conf('url')
if not url:
log.error('Please provide the URL')
return False
post_data = {
'type': listener,
'movie': getTitle(data)
if listener != 'test' else 'Test Movie Title (2016)',
'message': toUnicode(message)
}
try:
self.urlopen(url, data=post_data, show_error=False)
return True
except:
log.error('Webhook notification failed: %s',
traceback.format_exc())
return False
def save(self, **kwargs):
db = get_session()
c = db.query(Category).filter_by(id=kwargs.get('id')).first()
if not c:
c = Category()
db.add(c)
c.order = kwargs.get('order', c.order if c.order else 0)
c.label = toUnicode(kwargs.get('label'))
c.path = toUnicode(kwargs.get('path'))
c.ignored = toUnicode(kwargs.get('ignored'))
c.preferred = toUnicode(kwargs.get('preferred'))
c.required = toUnicode(kwargs.get('required'))
c.destination = toUnicode(kwargs.get('destination'))
db.commit()
category_dict = c.to_dict()
return {'success': True, 'category': category_dict}
def getDefaultTitle(
self,
info,
):
# Set default title
default_title = toUnicode(info.get('title'))
titles = info.get('titles', [])
counter = 0
def_title = None
for title in titles:
if (len(default_title) == 0 and counter
== 0) or len(titles) == 1 or title.lower() == toUnicode(
default_title.lower()) or (
toUnicode(default_title) == six.u('')
and toUnicode(titles[0]) == title):
def_title = toUnicode(title)
break
counter += 1
if not def_title:
def_title = toUnicode(titles[0])
return def_title or 'UNKNOWN'
def createNzbName(self, data, movie):
tag = self.cpTag(movie)
return '%s%s' % (toSafeString(
toUnicode(data.get('name'))[:127 - len(tag)]), tag)
def calculate(self, nzb, movie):
""" Calculate the score of a NZB, used for sorting later """
# Merge global and category
preferred_words = splitString(
Env.setting('preferred_words', section='searcher').lower())
try:
preferred_words = list(
set(preferred_words +
splitString(movie['category']['preferred'].lower())))
except:
pass
score = nameScore(toUnicode(nzb['name']), movie['library']['year'],
preferred_words)
for movie_title in movie['library']['titles']:
score += nameRatioScore(toUnicode(nzb['name']),
toUnicode(movie_title['title']))
score += namePositionScore(toUnicode(nzb['name']),
toUnicode(movie_title['title']))
score += sizeScore(nzb['size'])
# Torrents only
if nzb.get('seeders'):
try:
score += nzb.get('seeders') * 100 / 15
score += nzb.get('leechers') * 100 / 30
except:
pass
# Provider score
score += providerScore(nzb['provider'])
# Duplicates in name
score += duplicateScore(nzb['name'], getTitle(movie['library']))
# Merge global and category
ignored_words = splitString(
Env.setting('ignored_words', section='searcher').lower())
try:
ignored_words = list(
set(ignored_words +
splitString(movie['category']['ignored'].lower())))
except:
pass
# Partial ignored words
score += partialIgnoredScore(nzb['name'], getTitle(movie['library']),
ignored_words)
# Ignore single downloads from multipart
score += halfMultipartScore(nzb['name'])
# Extra provider specific check
extra_score = nzb.get('extra_score')
if extra_score:
score += extra_score(nzb)
# Scene / Nuke scoring
score += sceneScore(nzb['name'])
return score
def parseMovie(self, movie, extended=True):
# Do request, append other items
movie = self.request(
'movie/%s' % movie.get('id'), {
'append_to_response':
'alternative_titles' + (',images,casts' if extended else '')
})
if not movie:
return
# Images
poster = self.getImage(movie, type='poster', size='w154')
poster_original = self.getImage(movie, type='poster', size='original')
backdrop_original = self.getImage(movie,
type='backdrop',
size='original')
extra_thumbs = self.getMultImages(
movie, type='backdrops', size='original') if extended else []
images = {
'poster': [poster] if poster else [],
#'backdrop': [backdrop] if backdrop else [],
'poster_original': [poster_original] if poster_original else [],
'backdrop_original':
[backdrop_original] if backdrop_original else [],
'actors': {},
'extra_thumbs': extra_thumbs
}
# Genres
try:
genres = [genre.get('name') for genre in movie.get('genres', [])]
except:
genres = []
# 1900 is the same as None
year = str(movie.get('release_date') or '')[:4]
if not movie.get('release_date') or year == '1900' or year.lower(
) == 'none':
year = None
# Gather actors data
actors = {}
if extended:
# Full data
cast = movie.get('casts', {}).get('cast', [])
for cast_item in cast:
try:
actors[toUnicode(cast_item.get('name'))] = toUnicode(
cast_item.get('character'))
images['actors'][toUnicode(
cast_item.get('name'))] = self.getImage(
cast_item, type='profile', size='original')
except:
log.debug('Error getting cast info for %s: %s',
(cast_item, traceback.format_exc()))
movie_data = {
'type': 'movie',
'via_tmdb': True,
'tmdb_id': movie.get('id'),
'titles': [toUnicode(movie.get('title'))],
'original_title': movie.get('original_title'),
'images': images,
'imdb': movie.get('imdb_id'),
'runtime': movie.get('runtime'),
'released': str(movie.get('release_date')),
'year': tryInt(year, None),
'plot': movie.get('overview'),
'genres': genres,
'collection': getattr(movie.get('belongs_to_collection'), 'name',
None),
'actor_roles': actors
}
movie_data = dict((k, v) for k, v in movie_data.items() if v)
# Add alternative names
if movie_data['original_title'] and movie_data[
'original_title'] not in movie_data['titles']:
movie_data['titles'].append(movie_data['original_title'])
# Add alternative titles
alternate_titles = movie.get('alternative_titles',
{}).get('titles', [])
for alt in alternate_titles:
alt_name = alt.get('title')
if alt_name and alt_name not in movie_data[
'titles'] and alt_name.lower(
) != 'none' and alt_name is not None:
movie_data['titles'].append(alt_name)
return movie_data
def getUnicode(self, section, option):
value = self.p.get(section, option).decode('unicode_escape')
return toUnicode(value).strip()
def single(self, movie):
done_status = fireEvent('status.get', 'done', single=True)
if not movie['profile'] or movie['status_id'] == done_status.get('id'):
log.debug(
'Movie doesn\'t have a profile or already done, assuming in manage tab.'
)
return
db = get_session()
pre_releases = fireEvent('quality.pre_releases', single=True)
release_dates = fireEvent('library.update_release_date',
identifier=movie['library']['identifier'],
merge=True)
available_status = fireEvent('status.get', 'available', single=True)
ignored_status = fireEvent('status.get', 'ignored', single=True)
default_title = getTitle(movie['library'])
if not default_title:
log.error(
'No proper info found for movie, removing it from library to cause it from having more issues.'
)
fireEvent('movie.delete', movie['id'], single=True)
return
fireEvent('notify.frontend',
type='searcher.started.%s' % movie['id'],
data=True,
message='Searching for "%s"' % default_title)
ret = False
for quality_type in movie['profile']['types']:
if not self.couldBeReleased(quality_type['quality']['identifier'],
release_dates, pre_releases):
log.info(
'Too early to search for %s, %s',
(quality_type['quality']['identifier'], default_title))
continue
has_better_quality = 0
# See if better quality is available
for release in movie['releases']:
if release['quality']['order'] <= quality_type['quality'][
'order'] and release['status_id'] not in [
available_status.get('id'),
ignored_status.get('id')
]:
has_better_quality += 1
# Don't search for quality lower then already available.
if has_better_quality is 0:
log.info('Search for %s in %s',
(default_title, quality_type['quality']['label']))
quality = fireEvent(
'quality.single',
identifier=quality_type['quality']['identifier'],
single=True)
results = fireEvent('yarr.search', movie, quality, merge=True)
sorted_results = sorted(results,
key=lambda k: k['score'],
reverse=True)
if len(sorted_results) == 0:
log.debug(
'Nothing found for %s in %s',
(default_title, quality_type['quality']['label']))
download_preference = self.conf('preferred_method')
if download_preference != 'both':
sorted_results = sorted(
sorted_results,
key=lambda k: k['type'],
reverse=(download_preference == 'torrent'))
# Check if movie isn't deleted while searching
if not db.query(Movie).filter_by(id=movie.get('id')).first():
break
# Add them to this movie releases list
for nzb in sorted_results:
rls = db.query(Release).filter_by(
identifier=md5(nzb['url'])).first()
if not rls:
rls = Release(
identifier=md5(nzb['url']),
movie_id=movie.get('id'),
quality_id=quality_type.get('quality_id'),
status_id=available_status.get('id'))
db.add(rls)
db.commit()
else:
[db.delete(info) for info in rls.info]
db.commit()
for info in nzb:
try:
if not isinstance(
nzb[info],
(str, unicode, int, long, float)):
continue
rls_info = ReleaseInfo(identifier=info,
value=toUnicode(nzb[info]))
rls.info.append(rls_info)
db.commit()
except InterfaceError:
log.debug('Couldn\'t add %s to ReleaseInfo: %s',
(info, traceback.format_exc()))
nzb['status_id'] = rls.status_id
for nzb in sorted_results:
if nzb['status_id'] == ignored_status.get('id'):
log.info('Ignored: %s', nzb['name'])
continue
if nzb['score'] <= 0:
log.info('Ignored, score to low: %s', nzb['name'])
continue
downloaded = self.download(data=nzb, movie=movie)
if downloaded is True:
ret = True
break
elif downloaded != 'try_next':
break
else:
log.info(
'Better quality (%s) already available or snatched for %s',
(quality_type['quality']['label'], default_title))
fireEvent('movie.restatus', movie['id'])
break
# Break if CP wants to shut down
if self.shuttingDown() or ret:
break
fireEvent('notify.frontend',
type='searcher.ended.%s' % movie['id'],
data=True)
#db.close()
return ret
def getReleaseNameYear(self, release_name, file_name = None):
release_name = release_name.strip(' .-_')
# Use guessit first
guess = {}
if file_name:
try:
guessit = guess_movie_info(toUnicode(file_name))
if guessit.get('title') and guessit.get('year'):
guess = {
'name': guessit.get('title'),
'year': guessit.get('year'),
}
except:
log.debug('Could not detect via guessit "%s": %s', (file_name, traceback.format_exc()))
# Backup to simple
release_name = os.path.basename(release_name.replace('\\', '/'))
cleaned = ' '.join(re.split('\W+', simplifyString(release_name)))
cleaned = re.sub(self.clean, ' ', cleaned)
year = None
for year_str in [file_name, release_name, cleaned]:
if not year_str: continue
year = self.findYear(year_str)
if year:
break
cp_guess = {}
if year: # Split name on year
try:
movie_name = cleaned.rsplit(year, 1).pop(0).strip()
if movie_name:
cp_guess = {
'name': movie_name,
'year': int(year),
}
except:
pass
if not cp_guess: # Split name on multiple spaces
try:
movie_name = cleaned.split(' ').pop(0).strip()
cp_guess = {
'name': movie_name,
'year': int(year) if movie_name[:4] != year else 0,
}
except:
pass
if cp_guess.get('year') == guess.get('year') and len(cp_guess.get('name', '')) > len(guess.get('name', '')):
cp_guess['other'] = guess
return cp_guess
elif guess == {}:
cp_guess['other'] = guess
return cp_guess
guess['other'] = cp_guess
return guess
def download(self, data, media, manual=False):
if not data.get('protocol'):
data['protocol'] = data['type']
data['type'] = 'movie'
# Test to see if any downloaders are enabled for this type
downloader_enabled = fireEvent('download.enabled',
manual,
data,
single=True)
if downloader_enabled:
snatched_status, done_status, active_status = fireEvent(
'status.get', ['snatched', 'done', 'active'], single=True)
# Download release to temp
filedata = None
if data.get('download') and (ismethod(data.get('download'))
or isfunction(data.get('download'))):
filedata = data.get('download')(url=data.get('url'),
nzb_id=data.get('id'))
if filedata == 'try_next':
return filedata
download_result = fireEvent('download',
data=data,
movie=media,
manual=manual,
filedata=filedata,
single=True)
log.debug('Downloader result: %s', download_result)
if download_result:
try:
# Mark release as snatched
db = get_session()
rls = db.query(Relea).filter_by(
identifier=md5(data['url'])).first()
if rls:
renamer_enabled = Env.setting('enabled', 'renamer')
# Save download-id info if returned
if isinstance(download_result, dict):
for key in download_result:
rls_info = ReleaseInfo(
identifier='download_%s' % key,
value=toUnicode(download_result.get(key)))
rls.info.append(rls_info)
db.commit()
log_movie = '%s (%s) in %s' % (getTitle(
media['library']), media['library']['year'],
rls.quality.label)
snatch_message = 'Snatched "%s": %s' % (
data.get('name'), log_movie)
log.info(snatch_message)
fireEvent('%s.snatched' % data['type'],
message=snatch_message,
data=rls.to_dict())
# If renamer isn't used, mark media done
if not renamer_enabled:
try:
if media['status_id'] == active_status.get(
'id'):
for profile_type in media['profile'][
'types']:
if profile_type[
'quality_id'] == rls.quality.id and profile_type[
'finish']:
log.info(
'Renamer disabled, marking media as finished: %s',
log_movie)
# Mark release done
self.updateStatus(
rls.id, status=done_status)
# Mark media done
mdia = db.query(Media).filter_by(
id=media['id']).first()
mdia.status_id = done_status.get(
'id')
mdia.last_edit = int(time.time())
db.commit()
except:
log.error(
'Failed marking media finished, renamer disabled: %s',
traceback.format_exc())
else:
self.updateStatus(rls.id, status=snatched_status)
except:
log.error('Failed marking media finished: %s',
traceback.format_exc())
return True
log.info(
'Tried to download, but none of the "%s" downloaders are enabled or gave an error',
(data.get('protocol')))
return False
def getNfo(self, movie_info=None, data=None, i=0):
if not data: data = {}
if not movie_info: movie_info = {}
nfoxml = Element('details')
movie_root = SubElement(nfoxml, 'movie')
movie_root.attrib["isExtra"] = "false"
movie_root.attrib["isSet"] = "false"
movie_root.attrib["isTV"] = "false"
# Title
try:
el = SubElement(movie_root, 'title')
el.text = toUnicode(getTitle(data))
except:
pass
# IMDB id
try:
imdb_id = etree.SubElement(movie_root, "id")
imdb_id.attrib["moviedb"] = "imdb"
imdb_id.text = myShow['imdb_id']
except:
pass
# Runtime
try:
runtime = SubElement(movie_root, 'runtime')
runtime.text = '%s min' % movie_info.get('runtime')
except:
pass
# Other values
types = [
'year', 'mpaa', 'original_title', 'outline', 'plot', 'tagline',
'releaseDate:released'
]
for type in types:
if ':' in type:
name, type = type.split(':')
else:
name = type
try:
if movie_info.get(type):
el = SubElement(movie_root, name)
el.text = toUnicode(movie_info.get(type, ''))
except:
pass
# Rating
for rating_type in ['imdb', 'rotten', 'tmdb']:
try:
r, v = movie_info['rating'][rating_type]
rating = SubElement(movie_root, 'rating')
rating.text = str(r)
votes = SubElement(movie_root, 'votes')
votes.text = str(v)
break
except:
log.debug('Failed adding rating info from %s: %s',
(rating_type, traceback.format_exc()))
# Genre
Genres = SubElement(movie_root, "genres")
for genre in movie_info.get('genres', []):
genres = SubElement(Genres, "Genre")
genres.text = toUnicode(genre)
# Actors
Actors = SubElement(movie_root, "cast")
for actor_name in movie_info.get('actor_roles', {}):
actor = SubElement(Actors, 'actor')
actor.text = toUnicode(actor_name)
# Directors
for director_name in movie_info.get('directors', []):
director = SubElement(movie_root, 'director')
director.text = toUnicode(director_name)
# Images / Thumbnail
for image_url in movie_info['images']['poster_original']:
image = SubElement(movie_root, 'thumb')
image.text = toUnicode(image_url)
# Images / Thumbnail
for image_url in movie_info['images']['backdrop_original']:
image = SubElement(movie_root, 'fanart')
image.text = toUnicode(image_url)
# Add file metadata
# Video data
if data['meta_data'].get('video'):
vcodec = SubElement(movie_root, 'videoCodec')
vcodec.text = toUnicode(data['meta_data']['video'])
resolution = SubElement(movie_root, 'resolution')
resolution.text = str(
data['meta_data']['resolution_width']) + "x" + str(
data['meta_data']['resolution_height'])
# Audio data
if data['meta_data'].get('audio'):
acodec = SubElement(movie_root, 'audioCodec')
acodec.text = toUnicode(data['meta_data'].get('audio'))
channels = SubElement(movie_root, 'audioChannels')
channels.text = toUnicode(data['meta_data'].get('audio_channels'))
# Clean up the xml and return it
nfoxml = xml.dom.minidom.parseString(tostring(nfoxml))
xml_string = nfoxml.toprettyxml(indent=' ')
text_re = re.compile('>\n\s+([^<>\s].*?)\n\s+</', re.DOTALL)
xml_string = text_re.sub('>\g<1></', xml_string)
return xml_string.encode('utf-8')
def notify(self, message='', data=None, listener=None):
if not data: data = {}
# Extract all the settings from settings
from_address = self.conf('from')
to_address = self.conf('to')
ssl = self.conf('ssl')
smtp_server = self.conf('smtp_server')
smtp_user = self.conf('smtp_user')
smtp_pass = self.conf('smtp_pass')
smtp_port = self.conf('smtp_port')
starttls = self.conf('starttls')
# Make the basic message
message = MIMEText(toUnicode(message), _charset=Env.get('encoding'))
message['Subject'] = self.default_title
message['From'] = from_address
message['To'] = to_address
message['Date'] = formatdate(localtime=1)
message['Message-ID'] = make_msgid()
try:
# Open the SMTP connection, via SSL if requested
log.debug("Connecting to host %s on port %s" %
(smtp_server, smtp_port))
log.debug("SMTP over SSL %s",
("enabled" if ssl == 1 else "disabled"))
mailserver = smtplib.SMTP_SSL(
smtp_server, smtp_port) if ssl == 1 else smtplib.SMTP(
smtp_server, smtp_port)
if starttls:
log.debug(
"Using StartTLS to initiate the connection with the SMTP server"
)
mailserver.starttls()
# Say hello to the server
mailserver.ehlo()
# Check too see if an login attempt should be attempted
if len(smtp_user) > 0:
log.debug("Logging on to SMTP server using username \'%s\'%s",
(smtp_user,
" and a password" if len(smtp_pass) > 0 else ""))
mailserver.login(smtp_user, smtp_pass)
# Send the e-mail
log.debug("Sending the email")
mailserver.sendmail(from_address, splitString(to_address),
message.as_string())
# Close the SMTP connection
mailserver.quit()
log.info('Email notification sent')
return True
except:
log.error('E-mail failed: %s', traceback.format_exc())
return False
def _searchOnHost(self, host, media, quality, results):
query = self.buildUrl(media, host)
url = '%s%s' % (self.getUrl(host['host']), query)
nzbs = self.getRSSData(url,
cache_timeout=1800,
headers={'User-Agent': Env.getIdentifier()})
for nzb in nzbs:
date = None
spotter = None
for item in nzb:
if date and spotter:
break
if item.attrib.get('name') == 'usenetdate':
date = item.attrib.get('value')
break
# Get the name of the person who posts the spot
if item.attrib.get('name') == 'poster':
if "@spot.net" in item.attrib.get('value'):
spotter = item.attrib.get('value').split("@")[0]
continue
if not date:
date = self.getTextElement(nzb, 'pubDate')
nzb_id = self.getTextElement(nzb, 'guid').split('/')[-1:].pop()
name = self.getTextElement(nzb, 'title')
if not name:
continue
name_extra = ''
if spotter:
name_extra = spotter
description = ''
if "@spot.net" in nzb_id:
try:
# Get details for extended description to retrieve passwords
query = self.buildDetailsUrl(nzb_id, host['api_key'])
url = '%s%s' % (self.getUrl(host['host']), query)
nzb_details = self.getRSSData(
url,
cache_timeout=1800,
headers={'User-Agent': Env.getIdentifier()})[0]
description = self.getTextElement(nzb_details,
'description')
# Extract a password from the description
password = re.search(
'(?:' + self.passwords_regex +
')(?: *)(?:\:|\=)(?: *)(.*?)\<br\>|\n|$',
description,
flags=re.I).group(1)
if password:
name += ' {{%s}}' % password.strip()
except:
log.debug('Error getting details of "%s": %s',
(name, traceback.format_exc()))
results.append({
'id':
nzb_id,
'provider_extra':
urlparse(host['host']).hostname or host['host'],
'name':
toUnicode(name),
'name_extra':
name_extra,
'age':
self.calculateAge(int(time.mktime(parse(date).timetuple()))),
'size':
int(self.getElement(nzb, 'enclosure').attrib['length']) /
1024 / 1024,
'url': ((self.getUrl(host['host']) + self.urls['download']) %
tryUrlencode(nzb_id)) + self.getApiExt(host),
'detail_url': (cleanHost(host['host']) + self.urls['detail']) %
tryUrlencode(nzb_id),
'content':
self.getTextElement(nzb, 'description'),
'description':
description,
'score':
host['extra_score'],
})
def scan(self, movie_folder = None, download_info = None):
if self.isDisabled():
return
if self.renaming_started is True:
log.info('Renamer is already running, if you see this often, check the logs above for errors.')
return
# Check to see if the "to" folder is inside the "from" folder.
if movie_folder and not os.path.isdir(movie_folder) or not os.path.isdir(self.conf('from')) or not os.path.isdir(self.conf('to')):
l = log.debug if movie_folder else log.error
l('Both the "To" and "From" have to exist.')
return
elif self.conf('from') in self.conf('to'):
log.error('The "to" can\'t be inside of the "from" folder. You\'ll get an infinite loop.')
return
elif (movie_folder and movie_folder in [self.conf('to'), self.conf('from')]):
log.error('The "to" and "from" folders can\'t be inside of or the same as the provided movie folder.')
return
self.renaming_started = True
# make sure the movie folder name is included in the search
folder = None
files = []
if movie_folder:
log.info('Scanning movie folder %s...', movie_folder)
movie_folder = movie_folder.rstrip(os.path.sep)
folder = os.path.dirname(movie_folder)
# Get all files from the specified folder
try:
for root, folders, names in os.walk(movie_folder):
files.extend([os.path.join(root, name) for name in names])
except:
log.error('Failed getting files from %s: %s', (movie_folder, traceback.format_exc()))
db = get_session()
# Extend the download info with info stored in the downloaded release
download_info = self.extendDownloadInfo(download_info)
groups = fireEvent('scanner.scan', folder = folder if folder else self.conf('from'),
files = files, download_info = download_info, return_ignored = False, single = True)
destination = self.conf('to')
folder_name = self.conf('folder_name')
file_name = self.conf('file_name')
trailer_name = self.conf('trailer_name')
nfo_name = self.conf('nfo_name')
separator = self.conf('separator')
# Statusses
done_status, active_status, downloaded_status, snatched_status = \
fireEvent('status.get', ['done', 'active', 'downloaded', 'snatched'], single = True)
for group_identifier in groups:
group = groups[group_identifier]
rename_files = {}
remove_files = []
remove_releases = []
movie_title = getTitle(group['library'])
# Add _UNKNOWN_ if no library item is connected
if not group['library'] or not movie_title:
self.tagDir(group, 'unknown')
continue
# Rename the files using the library data
else:
group['library'] = fireEvent('library.update', identifier = group['library']['identifier'], single = True)
if not group['library']:
log.error('Could not rename, no library item to work with: %s', group_identifier)
continue
library = group['library']
movie_title = getTitle(library)
# Find subtitle for renaming
fireEvent('renamer.before', group)
# Remove weird chars from moviename
movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', movie_title)
# Put 'The' at the end
name_the = movie_name
if movie_name[:4].lower() == 'the ':
name_the = movie_name[4:] + ', The'
replacements = {
'ext': 'mkv',
'namethe': name_the.strip(),
'thename': movie_name.strip(),
'year': library['year'],
'first': name_the[0].upper(),
'quality': group['meta_data']['quality']['label'],
'quality_type': group['meta_data']['quality_type'],
'video': group['meta_data'].get('video'),
'audio': group['meta_data'].get('audio'),
'group': group['meta_data']['group'],
'source': group['meta_data']['source'],
'resolution_width': group['meta_data'].get('resolution_width'),
'resolution_height': group['meta_data'].get('resolution_height'),
'imdb_id': library['identifier'],
'cd': '',
'cd_nr': '',
}
for file_type in group['files']:
# Move nfo depending on settings
if file_type is 'nfo' and not self.conf('rename_nfo'):
log.debug('Skipping, renaming of %s disabled', file_type)
if self.conf('cleanup'):
for current_file in group['files'][file_type]:
remove_files.append(current_file)
continue
# Subtitle extra
if file_type is 'subtitle_extra':
continue
# Move other files
multiple = len(group['files'][file_type]) > 1 and not group['is_dvd']
cd = 1 if multiple else 0
for current_file in sorted(list(group['files'][file_type])):
current_file = toUnicode(current_file)
# Original filename
replacements['original'] = os.path.splitext(os.path.basename(current_file))[0]
replacements['original_folder'] = fireEvent('scanner.remove_cptag', group['dirname'], single = True)
# Extension
replacements['ext'] = getExt(current_file)
# cd #
replacements['cd'] = ' cd%d' % cd if multiple else ''
replacements['cd_nr'] = cd if multiple else ''
# Naming
final_folder_name = self.doReplace(folder_name, replacements).lstrip('. ')
final_file_name = self.doReplace(file_name, replacements).lstrip('. ')
replacements['filename'] = final_file_name[:-(len(getExt(final_file_name)) + 1)]
# Meta naming
if file_type is 'trailer':
final_file_name = self.doReplace(trailer_name, replacements, remove_multiple = True).lstrip('. ')
elif file_type is 'nfo':
final_file_name = self.doReplace(nfo_name, replacements, remove_multiple = True).lstrip('. ')
# Seperator replace
if separator:
final_file_name = final_file_name.replace(' ', separator)
# Move DVD files (no structure renaming)
if group['is_dvd'] and file_type is 'movie':
found = False
for top_dir in ['video_ts', 'audio_ts', 'bdmv', 'certificate']:
has_string = current_file.lower().find(os.path.sep + top_dir + os.path.sep)
if has_string >= 0:
structure_dir = current_file[has_string:].lstrip(os.path.sep)
rename_files[current_file] = os.path.join(destination, final_folder_name, structure_dir)
found = True
break
if not found:
log.error('Could not determine dvd structure for: %s', current_file)
# Do rename others
else:
if file_type is 'leftover':
if self.conf('move_leftover'):
rename_files[current_file] = os.path.join(destination, final_folder_name, os.path.basename(current_file))
elif file_type not in ['subtitle']:
rename_files[current_file] = os.path.join(destination, final_folder_name, final_file_name)
# Check for extra subtitle files
if file_type is 'subtitle':
remove_multiple = False
if len(group['files']['movie']) == 1:
remove_multiple = True
sub_langs = group['subtitle_language'].get(current_file, [])
# rename subtitles with or without language
sub_name = self.doReplace(file_name, replacements, remove_multiple = remove_multiple)
rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)
rename_extras = self.getRenameExtras(
extra_type = 'subtitle_extra',
replacements = replacements,
folder_name = folder_name,
file_name = file_name,
destination = destination,
group = group,
current_file = current_file,
remove_multiple = remove_multiple,
)
# Don't add language if multiple languages in 1 subtitle file
if len(sub_langs) == 1:
sub_name = final_file_name.replace(replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext']))
rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)
rename_files = mergeDicts(rename_files, rename_extras)
# Filename without cd etc
elif file_type is 'movie':
rename_extras = self.getRenameExtras(
extra_type = 'movie_extra',
replacements = replacements,
folder_name = folder_name,
file_name = file_name,
destination = destination,
group = group,
current_file = current_file
)
rename_files = mergeDicts(rename_files, rename_extras)
group['filename'] = self.doReplace(file_name, replacements, remove_multiple = True)[:-(len(getExt(final_file_name)) + 1)]
group['destination_dir'] = os.path.join(destination, final_folder_name)
if multiple:
cd += 1
# Before renaming, remove the lower quality files
library = db.query(Library).filter_by(identifier = group['library']['identifier']).first()
remove_leftovers = True
# Add it to the wanted list before we continue
if len(library.movies) == 0:
profile = db.query(Profile).filter_by(core = True, label = group['meta_data']['quality']['label']).first()
fireEvent('movie.add', params = {'identifier': group['library']['identifier'], 'profile_id': profile.id}, search_after = False)
db.expire_all()
library = db.query(Library).filter_by(identifier = group['library']['identifier']).first()
for movie in library.movies:
# Mark movie "done" onces it found the quality with the finish check
try:
if movie.status_id == active_status.get('id') and movie.profile:
for profile_type in movie.profile.types:
if profile_type.quality_id == group['meta_data']['quality']['id'] and profile_type.finish:
movie.status_id = done_status.get('id')
movie.last_edit = int(time.time())
db.commit()
except Exception, e:
log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc()))
# Go over current movie releases
for release in movie.releases:
# When a release already exists
if release.status_id is done_status.get('id'):
# This is where CP removes older, lesser quality releases
if release.quality.order > group['meta_data']['quality']['order']:
log.info('Removing lesser quality %s for %s.', (movie.library.titles[0].title, release.quality.label))
for current_file in release.files:
remove_files.append(current_file)
remove_releases.append(release)
# Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc
elif release.quality.order is group['meta_data']['quality']['order']:
log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (movie.library.titles[0].title, release.quality.label))
for current_file in release.files:
remove_files.append(current_file)
remove_releases.append(release)
# Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan
else:
log.info('Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label))
# Add exists tag to the .ignore file
self.tagDir(group, 'exists')
# Notify on rename fail
download_message = 'Renaming of %s (%s) canceled, exists in %s already.' % (movie.library.titles[0].title, group['meta_data']['quality']['label'], release.quality.label)
fireEvent('movie.renaming.canceled', message = download_message, data = group)
remove_leftovers = False
break
elif release.status_id is snatched_status.get('id'):
if release.quality.id is group['meta_data']['quality']['id']:
log.debug('Marking release as downloaded')
try:
release.status_id = downloaded_status.get('id')
release.last_edit = int(time.time())
except Exception, e:
log.error('Failed marking release as finished: %s %s', (e, traceback.format_exc()))
db.commit()
# Remove leftover files
if self.conf('cleanup') and not self.conf('move_leftover') and remove_leftovers and \
not (self.conf('file_action') != 'move' and self.downloadIsTorrent(download_info)):
log.debug('Removing leftover files')
for current_file in group['files']['leftover']:
remove_files.append(current_file)
elif not remove_leftovers: # Don't remove anything
break
def _searchOnTitle(self, title, movie, quality, results):
page = 0
total_pages = 1
cats = self.getCatId(quality)
while page < total_pages:
movieTitle = tryUrlencode('"%s" %s' %
(title, movie['info']['year']))
search_url = self.urls['search'] % (movieTitle, page, cats[0])
page += 1
data = self.getHTMLData(search_url)
if data:
try:
results_table = None
data_split = splitString(data, '<table')
soup = None
for x in data_split:
soup = BeautifulSoup(x)
results_table = soup.find('table',
attrs={'class': 'koptekst'})
if results_table:
break
if not results_table:
return
try:
pagelinks = soup.findAll(href=re.compile('page'))
page_numbers = [
int(
re.search('page=(?P<page_number>.+'
')', i['href']).group('page_number'))
for i in pagelinks
]
total_pages = max(page_numbers)
except:
pass
entries = results_table.find_all('tr')
for result in entries[1:]:
prelink = result.find(href=re.compile('details.php'))
link = prelink['href']
download = result.find(
'a', href=re.compile('download.php'))['href']
if link and download:
def extra_score(item):
trusted = (0, 10)[result.find(
'img', alt=re.compile('Trusted'))
is not None]
vip = (0, 20)[result.find(
'img', alt=re.compile('VIP')) is not None]
confirmed = (0, 30)[result.find(
'img', alt=re.compile('Helpers'))
is not None]
moderated = (0, 50)[result.find(
'img', alt=re.compile('Moderator'))
is not None]
return confirmed + trusted + vip + moderated
id = re.search('id=(?P<id>\d+)&', link).group('id')
url = self.urls['download'] % download
fileSize = self.parseSize(
result.select('td.rowhead')[8].text)
results.append({
'id':
id,
'name':
toUnicode(prelink.find('b').text),
'url':
url,
'detail_url':
self.urls['detail'] % link,
'size':
fileSize,
'seeders':
tryInt(result.find_all('td')[2].string),
'leechers':
tryInt(result.find_all('td')[3].string),
'extra_score':
extra_score,
'get_more_info':
self.getMoreInfo
})
except:
log.error('Failed getting results from %s: %s',
(self.getName(), traceback.format_exc()))
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
q = '"%s %s" %s' % (simplifyString(getTitle(
movie['library'])), movie['library']['year'],
quality.get('identifier'))
for ignored in Env.setting('ignored_words', 'searcher').split(','):
q = '%s -%s' % (q, ignored.strip())
params = {
'q': q,
'ig': '1',
'rpp': 200,
'st': 1,
'sp': 1,
'ns': 1,
}
cache_key = 'nzbclub.%s.%s.%s' % (movie['library']['identifier'],
quality.get('identifier'), q)
data = self.getCache(cache_key,
self.urls['search'] % tryUrlencode(params))
if data:
try:
try:
data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item')
except Exception, e:
log.debug('%s, %s', (self.getName(), e))
return results
for nzb in nzbs:
nzbclub_id = tryInt(
self.getTextElement(
nzb, "link").split('/nzb_view/')[1].split('/')[0])
enclosure = self.getElement(nzb, "enclosure").attrib
size = enclosure['length']
date = self.getTextElement(nzb, "pubDate")
def extra_check(item):
full_description = self.getCache(
'nzbclub.%s' % nzbclub_id,
item['detail_url'],
cache_timeout=25920000)
for ignored in [
'ARCHIVE inside ARCHIVE', 'Incomplete',
'repair impossible'
]:
if ignored in full_description:
log.info(
'Wrong: Seems to be passworded or corrupted files: %s',
new['name'])
return False
return True
new = {
'id':
nzbclub_id,
'type':
'nzb',
'provider':
self.getName(),
'name':
toUnicode(self.getTextElement(nzb, "title")),
'age':
self.calculateAge(
int(time.mktime(parse(date).timetuple()))),
'size':
tryInt(size) / 1024 / 1024,
'url':
enclosure['url'].replace(' ', '_'),
'download':
self.download,
'detail_url':
self.getTextElement(nzb, "link"),
'description':
'',
'get_more_info':
self.getMoreInfo,
'extra_check':
extra_check
}
is_correct_movie = fireEvent('searcher.correct_movie',
nzb=new,
movie=movie,
quality=quality,
imdb_results=False,
single=True)
if is_correct_movie:
new['score'] = fireEvent('score.calculate',
new,
movie,
single=True)
results.append(new)
self.found(new)
return results
except SyntaxError:
log.error('Failed to parse XML response from NZBClub')
