def test_pick_result(p, app_config, create_search_result, search_provider, create_tvshow, tvepisode, caplog):
caplog.set_level(logging.DEBUG, logger='medusa')
# Given
config_attrs = p.get('config', {})
for attr, value in iteritems(config_attrs):
app_config(attr, value)
series_attrs = p.get('series', {})
series = create_tvshow(**series_attrs)
provider_attrs = p.get('provider', {})
make_result = functools.partial(
create_search_result,
provider=search_provider(**provider_attrs),
series=series,
episodes=[tvepisode]
)
results = [make_result(**item) for item in p['results']]
expected = p['expected']
if isinstance(expected, int):
expected = results[expected]
# When
actual = pick_result(results)
# Then
assert expected == actual
def test_pick_result(p, app_config, create_search_result, search_provider, create_tvshow, tvepisode, caplog):
caplog.set_level(logging.DEBUG, logger='medusa')
# Given
config_attrs = p.get('config', {})
for attr, value in iteritems(config_attrs):
app_config(attr, value)
series_attrs = p.get('series', {})
series = create_tvshow(**series_attrs)
provider_attrs = p.get('provider', {})
make_result = functools.partial(
create_search_result,
provider=search_provider(**provider_attrs),
series=series,
episodes=[tvepisode]
)
results = [make_result(**item) for item in p['results']]
expected = p['expected']
if isinstance(expected, int):
expected = results[expected]
# When
actual = pick_result(results)
# Then
assert expected == actual
def _get_proper_results(self): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
"""Retrieve a list of recently aired episodes, and search for these episodes in the different providers."""
propers = {}
# For each provider get the list of propers
original_thread_name = threading.currentThread().name
providers = enabled_providers('backlog')
search_date = datetime.datetime.today() - datetime.timedelta(
days=app.PROPERS_SEARCH_DAYS)
main_db_con = db.DBConnection()
if not app.POSTPONE_IF_NO_SUBS:
# Get the recently aired (last 2 days) shows from DB
recently_aired = main_db_con.select(
'SELECT indexer, showid, season, episode, status, airdate'
' FROM tv_episodes'
' WHERE airdate >= ?'
' AND status = ?', [search_date.toordinal(), DOWNLOADED])
else:
# Get recently subtitled episodes (last 2 days) from DB
# Episode status becomes downloaded only after found subtitles
last_subtitled = search_date.strftime(History.date_format)
recently_aired = main_db_con.select(
'SELECT indexer_id AS indexer, showid, season, episode FROM history '
'WHERE date >= ? AND action = ?', [last_subtitled, SUBTITLED])
if not recently_aired:
log.info('No recently aired new episodes, nothing to search for')
return []
# Loop through the providers, and search for releases
for cur_provider in providers:
threading.currentThread().name = '{thread} :: [{provider}]'.format(
thread=original_thread_name, provider=cur_provider.name)
log.info('Searching for any new PROPER releases from {provider}',
{'provider': cur_provider.name})
try:
cur_propers = cur_provider.find_propers(recently_aired)
except AuthException as e:
log.debug('Authentication error: {error}', {'error': ex(e)})
continue
# if they haven't been added by a different provider than add the proper to the list
for proper in cur_propers:
name = self._sanitize_name(proper.name)
if name not in propers:
log.debug('Found new possible proper result: {name}',
{'name': proper.name})
propers[name] = proper
threading.currentThread().name = original_thread_name
# take the list of unique propers and get it sorted by
sorted_propers = sorted(list(itervalues(propers)),
key=operator.attrgetter('date'),
reverse=True)
final_propers = []
# Keep only items from last PROPER_SEARCH_DAYS setting in processed propers:
latest_proper = datetime.datetime.now() - datetime.timedelta(
days=app.PROPERS_SEARCH_DAYS)
self.processed_propers = [
p for p in self.processed_propers if p.get('date') >= latest_proper
]
# Get proper names from processed propers
processed_propers_names = [
proper.get('name') for proper in self.processed_propers
if proper.get('name')
]
for cur_proper in sorted_propers:
if not self.ignore_processed_propers and cur_proper.name in processed_propers_names:
log.debug(u'Proper already processed. Skipping: {proper_name}',
{'proper_name': cur_proper.name})
continue
try:
cur_proper.parse_result = NameParser().parse(cur_proper.name)
except (InvalidNameException, InvalidShowException) as error:
log.debug('{error}', {'error': error})
continue
if not cur_proper.parse_result.proper_tags:
log.info('Skipping non-proper: {name}',
{'name': cur_proper.name})
continue
if not cur_proper.series.episodes.get(cur_proper.parse_result.season_number) or \
any([ep for ep in cur_proper.parse_result.episode_numbers
if not cur_proper.series.episodes[cur_proper.parse_result.season_number].get(ep)]):
log.info('Skipping proper for wrong season/episode: {name}',
{'name': cur_proper.name})
continue
log.debug(
'Proper tags for {proper}: {tags}', {
'proper': cur_proper.name,
'tags': cur_proper.parse_result.proper_tags
})
if not cur_proper.parse_result.series_name:
log.debug('Ignoring invalid show: {name}',
{'name': cur_proper.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({
'name': cur_proper.name,
'date': cur_proper.date
})
continue
if not cur_proper.parse_result.episode_numbers:
log.debug('Ignoring full season instead of episode: {name}',
{'name': cur_proper.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({
'name': cur_proper.name,
'date': cur_proper.date
})
continue
log.debug(
'Successful match! Matched {original_name} to show {new_name}',
{
'original_name': cur_proper.parse_result.original_name,
'new_name': cur_proper.parse_result.series.name
})
# Map the indexerid in the db to the show's indexerid
cur_proper.indexerid = cur_proper.parse_result.series.indexerid
# Map the indexer in the db to the show's indexer
cur_proper.indexer = cur_proper.parse_result.series.indexer
# Map our Proper instance
cur_proper.series = cur_proper.parse_result.series
cur_proper.actual_season = cur_proper.parse_result.season_number \
if cur_proper.parse_result.season_number is not None else 1
cur_proper.actual_episodes = cur_proper.parse_result.episode_numbers
cur_proper.release_group = cur_proper.parse_result.release_group
cur_proper.version = cur_proper.parse_result.version
cur_proper.quality = cur_proper.parse_result.quality
cur_proper.content = None
cur_proper.proper_tags = cur_proper.parse_result.proper_tags
# filter release, in this case, it's just a quality gate. As we only send one result.
wanted_results = filter_results(cur_proper)
best_result = pick_result(wanted_results)
if not best_result:
log.info('Rejected proper: {name}', {'name': cur_proper.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({
'name': cur_proper.name,
'date': cur_proper.date
})
continue
# only get anime proper if it has release group and version
if best_result.series.is_anime:
if not best_result.release_group and best_result.version == -1:
log.info(
'Ignoring proper without release group and version: {name}',
{'name': best_result.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({
'name': cur_proper.name,
'date': cur_proper.date
})
continue
# check if we have the episode as DOWNLOADED
main_db_con = db.DBConnection()
sql_results = main_db_con.select(
'SELECT quality, release_name '
'FROM tv_episodes WHERE indexer = ? '
'AND showid = ? AND season = ? '
'AND episode = ? AND status = ?', [
best_result.indexer, best_result.series.indexerid,
best_result.actual_season, best_result.actual_episodes[0],
DOWNLOADED
])
if not sql_results:
log.info(
"Ignoring proper because this episode doesn't have 'DOWNLOADED' status: {name}",
{'name': best_result.name})
continue
# only keep the proper if we have already downloaded an episode with the same quality
old_quality = int(sql_results[0]['quality'])
if old_quality != best_result.quality:
log.info(
'Ignoring proper because quality is different: {name}',
{'name': best_result.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({
'name': cur_proper.name,
'date': cur_proper.date
})
continue
# only keep the proper if we have already downloaded an episode with the same codec
release_name = sql_results[0]['release_name']
if release_name:
release_name_guess = NameParser()._parse_string(release_name)
current_codec = release_name_guess.video_codec
# Ignore proper if codec differs from downloaded release codec
if all([
current_codec, best_result.parse_result.video_codec,
best_result.parse_result.video_codec != current_codec
]):
log.info(
'Ignoring proper because codec is different: {name}',
{'name': best_result.name})
if best_result.name not in processed_propers_names:
self.processed_propers.append({
'name': best_result.name,
'date': best_result.date
})
continue
streaming_service = release_name_guess.guess.get(
u'streaming_service')
# Ignore proper if streaming service differs from downloaded release streaming service
if best_result.parse_result.guess.get(
u'streaming_service') != streaming_service:
log.info(
'Ignoring proper because streaming service is different: {name}',
{'name': best_result.name})
if best_result.name not in processed_propers_names:
self.processed_propers.append({
'name': best_result.name,
'date': best_result.date
})
continue
else:
log.debug(
"Coudn't find a release name in database. Skipping codec comparison for: {name}",
{'name': best_result.name})
# check if we actually want this proper (if it's the right release group and a higher version)
if best_result.series.is_anime:
main_db_con = db.DBConnection()
sql_results = main_db_con.select(
'SELECT release_group, version '
'FROM tv_episodes WHERE indexer = ? AND showid = ? '
'AND season = ? AND episode = ?', [
best_result.indexer, best_result.series.indexerid,
best_result.actual_season,
best_result.actual_episodes[0]
])
old_version = int(sql_results[0]['version'])
old_release_group = (sql_results[0]['release_group'])
if -1 < old_version < best_result.version:
log.info(
'Found new anime version {new} to replace existing version {old}: {name}',
{
'old': old_version,
'new': best_result.version,
'name': best_result.name
})
else:
log.info(
'Ignoring proper with the same or lower version: {name}',
{'name': best_result.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({
'name': best_result.name,
'date': best_result.date
})
continue
if old_release_group != best_result.release_group:
log.info(
'Ignoring proper from release group {new} instead of current group {old}',
{
'new': best_result.release_group,
'old': old_release_group
})
if best_result.name not in processed_propers_names:
self.processed_propers.append({
'name': best_result.name,
'date': best_result.date
})
continue
# if the show is in our list and there hasn't been a proper already added for that particular episode
# then add it to our list of propers
if best_result.indexerid != -1 and (
best_result.indexerid, best_result.actual_season,
best_result.actual_episodes) not in list(
map(
operator.attrgetter('indexerid', 'actual_season',
'actual_episodes'),
final_propers)):
log.info('Found a desired proper: {name}',
{'name': best_result.name})
final_propers.append(best_result)
if best_result.name not in processed_propers_names:
self.processed_propers.append({
'name': best_result.name,
'date': best_result.date
})
return final_propers
def _get_proper_results(self): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
"""Retrieve a list of recently aired episodes, and search for these episodes in the different providers."""
propers = {}
# For each provider get the list of propers
original_thread_name = threading.currentThread().name
providers = enabled_providers('backlog')
search_date = datetime.datetime.today() - datetime.timedelta(days=app.PROPERS_SEARCH_DAYS)
main_db_con = db.DBConnection()
if not app.POSTPONE_IF_NO_SUBS:
# Get the recently aired (last 2 days) shows from DB
recently_aired = main_db_con.select(
'SELECT indexer, showid, season, episode, status, airdate'
' FROM tv_episodes'
' WHERE airdate >= ?'
' AND status = ?',
[search_date.toordinal(), DOWNLOADED]
)
else:
# Get recently subtitled episodes (last 2 days) from DB
# Episode status becomes downloaded only after found subtitles
last_subtitled = search_date.strftime(History.date_format)
recently_aired = main_db_con.select('SELECT indexer_id AS indexer, showid, season, episode FROM history '
'WHERE date >= ? AND action = ?', [last_subtitled, SUBTITLED])
if not recently_aired:
log.info('No recently aired new episodes, nothing to search for')
return []
# Loop through the providers, and search for releases
for cur_provider in providers:
threading.currentThread().name = '{thread} :: [{provider}]'.format(thread=original_thread_name,
provider=cur_provider.name)
log.info('Searching for any new PROPER releases from {provider}', {'provider': cur_provider.name})
try:
cur_propers = cur_provider.find_propers(recently_aired)
except AuthException as e:
log.debug('Authentication error: {error}', {'error': ex(e)})
continue
# if they haven't been added by a different provider than add the proper to the list
for proper in cur_propers:
name = self._sanitize_name(proper.name)
if name not in propers:
log.debug('Found new possible proper result: {name}', {'name': proper.name})
propers[name] = proper
threading.currentThread().name = original_thread_name
# take the list of unique propers and get it sorted by
sorted_propers = sorted(list(itervalues(propers)), key=operator.attrgetter('date'), reverse=True)
final_propers = []
# Keep only items from last PROPER_SEARCH_DAYS setting in processed propers:
latest_proper = datetime.datetime.now() - datetime.timedelta(days=app.PROPERS_SEARCH_DAYS)
self.processed_propers = [p for p in self.processed_propers if p.get('date') >= latest_proper]
# Get proper names from processed propers
processed_propers_names = [proper.get('name') for proper in self.processed_propers if proper.get('name')]
for cur_proper in sorted_propers:
if not self.ignore_processed_propers and cur_proper.name in processed_propers_names:
log.debug(u'Proper already processed. Skipping: {proper_name}', {'proper_name': cur_proper.name})
continue
try:
cur_proper.parse_result = NameParser().parse(cur_proper.name)
except (InvalidNameException, InvalidShowException) as error:
log.debug('{error}', {'error': error})
continue
if not cur_proper.parse_result.proper_tags:
log.info('Skipping non-proper: {name}', {'name': cur_proper.name})
continue
if not cur_proper.series.episodes.get(cur_proper.parse_result.season_number) or \
any([ep for ep in cur_proper.parse_result.episode_numbers
if not cur_proper.series.episodes[cur_proper.parse_result.season_number].get(ep)]):
log.info('Skipping proper for wrong season/episode: {name}', {'name': cur_proper.name})
continue
log.debug('Proper tags for {proper}: {tags}', {
'proper': cur_proper.name,
'tags': cur_proper.parse_result.proper_tags
})
if not cur_proper.parse_result.series_name:
log.debug('Ignoring invalid show: {name}', {'name': cur_proper.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({'name': cur_proper.name, 'date': cur_proper.date})
continue
if not cur_proper.parse_result.episode_numbers:
log.debug('Ignoring full season instead of episode: {name}', {'name': cur_proper.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({'name': cur_proper.name, 'date': cur_proper.date})
continue
log.debug('Successful match! Matched {original_name} to show {new_name}',
{'original_name': cur_proper.parse_result.original_name,
'new_name': cur_proper.parse_result.series.name
})
# Map the indexerid in the db to the show's indexerid
cur_proper.indexerid = cur_proper.parse_result.series.indexerid
# Map the indexer in the db to the show's indexer
cur_proper.indexer = cur_proper.parse_result.series.indexer
# Map our Proper instance
cur_proper.series = cur_proper.parse_result.series
cur_proper.actual_season = cur_proper.parse_result.season_number \
if cur_proper.parse_result.season_number is not None else 1
cur_proper.actual_episodes = cur_proper.parse_result.episode_numbers
cur_proper.release_group = cur_proper.parse_result.release_group
cur_proper.version = cur_proper.parse_result.version
cur_proper.quality = cur_proper.parse_result.quality
cur_proper.content = None
cur_proper.proper_tags = cur_proper.parse_result.proper_tags
# filter release, in this case, it's just a quality gate. As we only send one result.
wanted_results = filter_results(cur_proper)
best_result = pick_result(wanted_results)
if not best_result:
log.info('Rejected proper: {name}', {'name': cur_proper.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({'name': cur_proper.name, 'date': cur_proper.date})
continue
# only get anime proper if it has release group and version
if best_result.series.is_anime:
if not best_result.release_group and best_result.version == -1:
log.info('Ignoring proper without release group and version: {name}', {'name': best_result.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({'name': cur_proper.name, 'date': cur_proper.date})
continue
# check if we have the episode as DOWNLOADED
main_db_con = db.DBConnection()
sql_results = main_db_con.select('SELECT quality, release_name '
'FROM tv_episodes WHERE indexer = ? '
'AND showid = ? AND season = ? '
'AND episode = ? AND status = ?',
[best_result.indexer,
best_result.series.indexerid,
best_result.actual_season,
best_result.actual_episodes[0],
DOWNLOADED])
if not sql_results:
log.info("Ignoring proper because this episode doesn't have 'DOWNLOADED' status: {name}", {
'name': best_result.name
})
continue
# only keep the proper if we have already downloaded an episode with the same quality
old_quality = int(sql_results[0]['quality'])
if old_quality != best_result.quality:
log.info('Ignoring proper because quality is different: {name}', {'name': best_result.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({'name': cur_proper.name, 'date': cur_proper.date})
continue
# only keep the proper if we have already downloaded an episode with the same codec
release_name = sql_results[0]['release_name']
if release_name:
release_name_guess = NameParser()._parse_string(release_name)
current_codec = release_name_guess.video_codec
# Ignore proper if codec differs from downloaded release codec
if all([current_codec, best_result.parse_result.video_codec,
best_result.parse_result.video_codec != current_codec]):
log.info('Ignoring proper because codec is different: {name}', {'name': best_result.name})
if best_result.name not in processed_propers_names:
self.processed_propers.append({'name': best_result.name, 'date': best_result.date})
continue
streaming_service = release_name_guess.guess.get(u'streaming_service')
# Ignore proper if streaming service differs from downloaded release streaming service
if best_result.parse_result.guess.get(u'streaming_service') != streaming_service:
log.info('Ignoring proper because streaming service is different: {name}',
{'name': best_result.name})
if best_result.name not in processed_propers_names:
self.processed_propers.append({'name': best_result.name, 'date': best_result.date})
continue
else:
log.debug("Coudn't find a release name in database. Skipping codec comparison for: {name}", {
'name': best_result.name
})
# check if we actually want this proper (if it's the right release group and a higher version)
if best_result.series.is_anime:
main_db_con = db.DBConnection()
sql_results = main_db_con.select(
'SELECT release_group, version '
'FROM tv_episodes WHERE indexer = ? AND showid = ? '
'AND season = ? AND episode = ?',
[best_result.indexer, best_result.series.indexerid, best_result.actual_season,
best_result.actual_episodes[0]])
old_version = int(sql_results[0]['version'])
old_release_group = (sql_results[0]['release_group'])
if -1 < old_version < best_result.version:
log.info('Found new anime version {new} to replace existing version {old}: {name}',
{'old': old_version,
'new': best_result.version,
'name': best_result.name
})
else:
log.info('Ignoring proper with the same or lower version: {name}', {'name': best_result.name})
if cur_proper.name not in processed_propers_names:
self.processed_propers.append({'name': best_result.name, 'date': best_result.date})
continue
if old_release_group != best_result.release_group:
log.info('Ignoring proper from release group {new} instead of current group {old}',
{'new': best_result.release_group,
'old': old_release_group})
if best_result.name not in processed_propers_names:
self.processed_propers.append({'name': best_result.name, 'date': best_result.date})
continue
# if the show is in our list and there hasn't been a proper already added for that particular episode
# then add it to our list of propers
if best_result.indexerid != -1 and (
best_result.indexerid, best_result.actual_season, best_result.actual_episodes
) not in list(map(operator.attrgetter('indexerid', 'actual_season', 'actual_episodes'), final_propers)):
log.info('Found a desired proper: {name}', {'name': best_result.name})
final_propers.append(best_result)
if best_result.name not in processed_propers_names:
self.processed_propers.append({'name': best_result.name, 'date': best_result.date})
return final_propers