def to_dict(self, advanced=False):
"""Return the guess as a dict containing only base types, ie:
where dates, languages, countries, etc. are converted to strings.
if advanced is True, return the data as a json string containing
also the raw information of the properties."""
data = dict(self)
for prop, value in data.items():
if isinstance(value, datetime.date):
data[prop] = value.isoformat()
elif isinstance(value, (UnicodeMixin, base_text_type)):
data[prop] = u(value)
elif isinstance(value, (Language, Country)):
data[prop] = value.guessit
elif isinstance(value, list):
data[prop] = [u(x) for x in value]
if advanced:
metadata = self.metadata(prop)
prop_data = {'value': data[prop]}
if metadata.raw:
prop_data['raw'] = metadata.raw
if metadata.confidence:
prop_data['confidence'] = metadata.confidence
data[prop] = prop_data
return data
def to_dict(self, advanced=False):
"""Return the guess as a dict containing only base types, ie:
where dates, languages, countries, etc. are converted to strings.
if advanced is True, return the data as a json string containing
also the raw information of the properties."""
data = dict(self)
for prop, value in data.items():
if isinstance(value, datetime.date):
data[prop] = value.isoformat()
elif isinstance(value, (UnicodeMixin, base_text_type)):
data[prop] = u(value)
elif isinstance(value, (Language, Country)):
data[prop] = value.guessit
elif isinstance(value, list):
data[prop] = [u(x) for x in value]
if advanced:
metadata = self.metadata(prop)
prop_data = {"value": data[prop]}
if metadata.raw:
prop_data["raw"] = metadata.raw
if metadata.confidence:
prop_data["confidence"] = metadata.confidence
data[prop] = prop_data
return data
def to_dict(self):
data = dict(self)
for prop, value in data.items():
if isinstance(value, datetime.date):
data[prop] = value.isoformat()
elif isinstance(value, (Language, Country, base_text_type)):
data[prop] = u(value)
elif isinstance(value, list):
data[prop] = [u(x) for x in value]
return data
def to_dict(self):
data = dict(self)
for prop, value in data.items():
if isinstance(value, datetime.date):
data[prop] = value.isoformat()
elif isinstance(value, (Language, Country, base_text_type)):
data[prop] = u(value)
elif isinstance(value, list):
data[prop] = [u(x) for x in value]
return data
def load_file_in_same_dir(ref_file, filename):
"""Load a given file. Works even when the file is contained inside a zip."""
path = split_path(ref_file)[:-1] + [filename]
for i, p in enumerate(path):
if p.endswith('.zip'):
zfilename = os.path.join(*path[:i + 1])
zfile = zipfile.ZipFile(zfilename)
return u(zfile.read('/'.join(path[i + 1:])))
return u(io.open(os.path.join(*path), encoding='utf-8').read())
def to_dict(self, advanced=False):
data = dict(self)
for prop, value in data.items():
if isinstance(value, datetime.date):
data[prop] = value.isoformat()
elif isinstance(value, (Language, Country, base_text_type)):
data[prop] = u(value)
elif isinstance(value, list):
data[prop] = [u(x) for x in value]
if advanced:
data[prop] = {"value": data[prop], "raw": self.raw(prop), "confidence": self.confidence(prop)}
return data
def reverse(self, name):
with_country = (GuessitConverter._with_country_regexp.match(name)
or GuessitConverter._with_country_regexp2.match(name))
name = u(name.lower())
if with_country:
lang = Language.fromguessit(with_country.group(1).strip())
lang.country = babelfish.Country.fromguessit(
with_country.group(2).strip())
return lang.alpha3, lang.country.alpha2 if lang.country else None, lang.script or None
# exceptions come first, as they need to override a potential match
# with any of the other guessers
try:
return self.guessit_exceptions[name]
except KeyError:
pass
for conv in [
babelfish.Language, babelfish.Language.fromalpha3b,
babelfish.Language.fromalpha2, babelfish.Language.fromname,
babelfish.Language.fromopensubtitles
]:
try:
c = conv(name)
return c.alpha3, c.country, c.script
except (ValueError, babelfish.LanguageReverseError):
pass
raise babelfish.LanguageReverseError(name)
def guess_file(filename, info='filename', options=None, **kwargs):
options = options or {}
filename = u(filename)
if not options.get('yaml') and not options.get('show_property'):
print('For:', filename)
guess = guess_file_info(filename, info, options, **kwargs)
if not options.get('unidentified'):
try:
del guess['unidentified']
except KeyError:
pass
if options.get('show_property'):
print(guess.get(options.get('show_property'), ''))
return
if options.get('yaml'):
import yaml
for k, v in guess.items():
if isinstance(v, list) and len(v) == 1:
guess[k] = v[0]
ystr = yaml.safe_dump({filename: dict(guess)}, default_flow_style=False, allow_unicode=True)
i = 0
for yline in ystr.splitlines():
if i == 0:
print("? " + yline[:-1])
elif i == 1:
print(":" + yline[1:])
else:
print(yline)
i += 1
return
print('GuessIt found:', guess.nice_string(options.get('advanced')))
def guess_file(filename, info='filename', options=None, **kwargs):
options = options or {}
filename = u(filename)
guess = guess_file_info(filename, info, options, **kwargs)
if options.get('show_property'):
print (guess[options.get('show_property')])
return
print('For:', filename)
if options.get('yaml'):
try:
import yaml
for k, v in guess.items():
if isinstance(v, list) and len(v) == 1:
guess[k] = v[0]
ystr = yaml.safe_dump({filename: dict(guess)}, default_flow_style=False)
i = 0
for yline in ystr.splitlines():
if i == 0:
print("? " + yline[:-1])
elif i == 1:
print(":" + yline[1:])
else:
print(yline)
i = i + 1
return
except ImportError: # pragma: no cover
print('PyYAML not found. Using default output.')
print('GuessIt found:', guess.nice_string(options.get('advanced')))
def guess_file(filename, info='filename', options=None, **kwargs):
options = options or {}
filename = u(filename)
print('For:', filename)
guess = guess_file_info(filename, info, options, **kwargs)
if options.get('yaml'):
try:
import yaml
for k, v in guess.items():
if isinstance(v, list) and len(v) == 1:
guess[k] = v[0]
ystr = yaml.safe_dump({filename: dict(guess)},
default_flow_style=False)
i = 0
for yline in ystr.splitlines():
if i == 0:
print("? " + yline[:-1])
elif i == 1:
print(":" + yline[1:])
else:
print(yline)
i = i + 1
return
except ImportError: # pragma: no cover
print('PyYAML not found. Using default output.')
print('GuessIt found:', guess.nice_string(options.get('advanced')))
def guess_file(filename, info='filename', options=None, **kwargs):
options = options or {}
filename = u(filename)
if not options.get('yaml') and not options.get('show_property'):
print('For:', filename)
guess = guess_file_info(filename, info, options, **kwargs)
if not options.get('unidentified'):
try:
del guess['unidentified']
except KeyError:
pass
if options.get('show_property'):
print(guess.get(options.get('show_property'), ''))
return
if options.get('yaml'):
import yaml
for k, v in guess.items():
if isinstance(v, list) and len(v) == 1:
guess[k] = v[0]
ystr = yaml.safe_dump({filename: dict(guess)}, default_flow_style=False, allow_unicode=True)
i = 0
for yline in ystr.splitlines():
if i == 0:
print("? " + yline[:-1])
elif i == 1:
print(":" + yline[1:])
else:
print(yline)
i += 1
return
print('GuessIt found:', guess.nice_string(options.get('advanced')))
def reverse(self, name):
with_country = (GuessitConverter._with_country_regexp.match(name) or
GuessitConverter._with_country_regexp2.match(name))
name = u(name.lower())
if with_country:
lang = Language.fromguessit(with_country.group(1).strip())
lang.country = babelfish.Country.fromguessit(with_country.group(2).strip())
return (lang.alpha3, lang.country.alpha2 if lang.country else None, lang.script or None)
# exceptions come first, as they need to override a potential match
# with any of the other guessers
try:
return self.guessit_exceptions[name]
except KeyError:
pass
for conv in [babelfish.Language,
babelfish.Language.fromalpha3b,
babelfish.Language.fromalpha2,
babelfish.Language.fromname,
babelfish.Language.fromopensubtitles]:
try:
c = conv(name)
return c.alpha3, c.country, c.script
except (ValueError, babelfish.LanguageReverseError):
pass
raise babelfish.LanguageReverseError(name)
def __init__(self, filename, options=None, **kwargs):
options = dict(options or {})
for k, v in kwargs.items():
if k not in options or not options[k]:
options[
k] = v # options dict has priority over keyword arguments
self._validate_options(options)
if not PY3 and not isinstance(filename, unicode):
log.warning('Given filename to matcher is not unicode...')
filename = filename.decode('utf-8')
filename = normalize_unicode(filename)
clean_function = None
if options and options.get('clean_function'):
clean_function = options.get('clean_function')
if not hasattr(clean_function, '__call__'):
module, function = clean_function.rsplit('.')
if not module:
module = 'guessit.textutils'
clean_function = getattr(__import__(module), function)
if not clean_function:
log.error(
'Can\'t find clean function %s. Default will be used.'
% options.get('clean_function'))
clean_function = clean_default
else:
clean_function = clean_default
self.match_tree = MatchTree(filename, clean_function=clean_function)
self.options = options
self._transfo_calls = []
# sanity check: make sure we don't process a (mostly) empty string
if clean_function(filename).strip() == '':
return
from guessit.plugins import transformers
try:
mtree = self.match_tree
if 'type' in self.options:
mtree.guess.set('type', self.options['type'], confidence=0.0)
# Process
for transformer in transformers.all_transformers():
disabled = options.get('disabled_transformers')
if not disabled or not transformer.name in disabled:
self._process(transformer, False)
# Post-process
for transformer in transformers.all_transformers():
disabled = options.get('disabled_transformers')
if not disabled or not transformer.name in disabled:
self._process(transformer, True)
log.debug('Found match tree:\n%s' % u(mtree))
except TransformerException as e:
log.debug('An error has occurred in Transformer %s: %s' %
(e.transformer, e))
def to_dict(self, advanced=False):
data = dict(self)
for prop, value in data.items():
if isinstance(value, datetime.date):
data[prop] = value.isoformat()
elif isinstance(value, (Language, Country, base_text_type)):
data[prop] = u(value)
elif isinstance(value, list):
data[prop] = [u(x) for x in value]
if advanced:
data[prop] = {
"value": data[prop],
"raw": self.raw(prop),
"confidence": self.confidence(prop)
}
return data
def __init__(self, country, strict=False):
country = u(country.strip().lower())
self.alpha3 = country_to_alpha3.get(country)
if self.alpha3 is None and strict:
msg = 'The given string "%s" could not be identified as a country'
raise ValueError(msg % country)
if self.alpha3 is None:
self.alpha3 = 'unk'
def __init__(self, filename, options=None, **kwargs):
options = dict(options or {})
for k, v in kwargs.items():
if k not in options or not options[k]:
options[k] = v # options dict has priority over keyword arguments
self._validate_options(options)
if not PY3 and not isinstance(filename, unicode):
log.warning('Given filename to matcher is not unicode...')
filename = filename.decode('utf-8')
filename = normalize_unicode(filename)
clean_function = None
if options and options.get('clean_function'):
clean_function = options.get('clean_function')
if not hasattr(clean_function, '__call__'):
module, function = clean_function.rsplit('.')
if not module:
module = 'guessit.textutils'
clean_function = getattr(__import__(module), function)
if not clean_function:
log.error('Can\'t find clean function %s. Default will be used.' % options.get('clean_function'))
clean_function = clean_default
else:
clean_function = clean_default
self.match_tree = MatchTree(filename, clean_function=clean_function)
self.options = options
self._transfo_calls = []
# sanity check: make sure we don't process a (mostly) empty string
if clean_function(filename).strip() == '':
return
from guessit.plugins import transformers
try:
mtree = self.match_tree
if 'type' in self.options:
mtree.guess.set('type', self.options['type'], confidence=0.0)
# Process
for transformer in transformers.all_transformers():
disabled = options.get('disabled_transformers')
if not disabled or not transformer.name in disabled:
self._process(transformer, False)
# Post-process
for transformer in transformers.all_transformers():
disabled = options.get('disabled_transformers')
if not disabled or not transformer.name in disabled:
self._process(transformer, True)
log.debug('Found match tree:\n%s' % u(mtree))
except TransformerException as e:
log.debug('An error has occurred in Transformer %s: %s' % (e.transformer, e))
def __init__(self, language, country=None, strict=False, scheme=None):
language = u(language.strip().lower())
with_country = (Language._with_country_regexp.match(language)
or Language._with_country_regexp2.match(language))
if with_country:
self.lang = Language(with_country.group(1)).lang
self.country = Country(with_country.group(2))
return
self.lang = None
self.country = Country(country) if country else None
# first look for scheme specific languages
if scheme == 'opensubtitles':
if language == 'br':
self.lang = 'bre'
return
elif language == 'se':
self.lang = 'sme'
return
elif scheme is not None:
log.warning(
'Unrecognized scheme: "%s" - Proceeding with standard one' %
scheme)
# look for ISO language codes
if len(language) == 2:
self.lang = lng2_to_lng3.get(language)
elif len(language) == 3:
self.lang = (language if language in lng3 else
lng3term_to_lng3.get(language))
else:
self.lang = (lng_en_name_to_lng3.get(language)
or lng_fr_name_to_lng3.get(language))
# general language exceptions
if self.lang is None and language in lng_exceptions:
lang, country = lng_exceptions[language]
self.lang = Language(lang).alpha3
self.country = Country(country) if country else None
msg = 'The given string "%s" could not be identified as a language' % language
if self.lang is None and strict:
raise ValueError(msg)
if self.lang is None:
log.debug(msg)
self.lang = 'und'
def __init__(self, language, country=None, strict=False, scheme=None):
language = u(language.strip().lower())
with_country = (Language._with_country_regexp.match(language) or
Language._with_country_regexp2.match(language))
if with_country:
self.lang = Language(with_country.group(1)).lang
self.country = Country(with_country.group(2))
return
self.lang = None
self.country = Country(country) if country else None
# first look for scheme specific languages
if scheme == 'opensubtitles':
if language == 'br':
self.lang = 'bre'
return
elif language == 'se':
self.lang = 'sme'
return
elif scheme is not None:
log.warning('Unrecognized scheme: "%s" - Proceeding with standard one' % scheme)
# look for ISO language codes
if len(language) == 2:
self.lang = lng2_to_lng3.get(language)
elif len(language) == 3:
self.lang = (language
if language in lng3
else lng3term_to_lng3.get(language))
else:
self.lang = (lng_en_name_to_lng3.get(language) or
lng_fr_name_to_lng3.get(language))
# general language exceptions
if self.lang is None and language in lng_exceptions:
lang, country = lng_exceptions[language]
self.lang = Language(lang).alpha3
self.country = Country(country) if country else None
msg = 'The given string "%s" could not be identified as a language' % language
if self.lang is None and strict:
raise ValueError(msg)
if self.lang is None:
log.debug(msg)
self.lang = 'und'
def __init__(self, language, country=None, strict=False):
language = u(language.strip().lower())
country = babelfish.Country(country.upper()) if country else None
try:
self.lang = babelfish.Language.fromguessit(language)
# user given country overrides guessed one
if country:
self.lang.country = country
except babelfish.LanguageReverseError:
msg = 'The given string "%s" could not be identified as a language' % language
if strict:
raise ValueError(msg)
log.debug(msg)
self.lang = UNDETERMINED
def __init__(self, language, country=None, strict=False):
language = u(language.strip().lower())
country = babelfish.Country(country.upper()) if country else None
try:
self.lang = babelfish.Language.fromguessit(language)
# user given country overrides guessed one
if country:
self.lang.country = country
except babelfish.LanguageReverseError:
msg = 'The given string "%s" could not be identified as a language' % language
if strict:
raise ValueError(msg)
log.debug(msg)
self.lang = UNDETERMINED
def __init__(self, filename, filetype='autodetect', opts=None, transfo_opts=None):
if opts is None:
opts = []
if not isinstance(opts, list):
raise ValueError('opts must be a list of option names! Received: type=%s val=%s',
type(opts), opts)
if transfo_opts is None:
transfo_opts = {}
if not isinstance(transfo_opts, dict):
raise ValueError('transfo_opts must be a dict of { transfo_name: (args, kwargs) }. ' +
'Received: type=%s val=%s', type(transfo_opts), transfo_opts)
valid_filetypes = ('autodetect', 'subtitle', 'info', 'video',
'movie', 'moviesubtitle', 'movieinfo',
'episode', 'episodesubtitle', 'episodeinfo')
if filetype not in valid_filetypes:
raise ValueError("filetype needs to be one of %s" % valid_filetypes)
if not PY3 and not isinstance(filename, unicode):
log.warning('Given filename to matcher is not unicode...')
filename = filename.decode('utf-8')
filename = normalize_unicode(filename)
self.filename = filename
self.match_tree = MatchTree(filename)
self.filetype = filetype
self.opts = opts
self.transfo_opts = transfo_opts
self._transfo_calls = []
# sanity check: make sure we don't process a (mostly) empty string
if clean_string(filename) == '':
return
try:
mtree = self.match_tree
mtree.guess.set('type', filetype, confidence=1.0)
for transformer in transformers.extensions.objects():
self._apply_transfo(transformer)
log.debug('Found match tree:\n%s' % u(mtree))
except TransfoException as e:
log.debug('An error has occured in Transformer %s: %s' % (e.transformer, e))
def __init__(self, filename, options=None, **kwargs):
options = dict(options or {})
for k, v in kwargs.items():
if k not in options or not options[k]:
options[
k] = v # options dict has priority over keyword arguments
self._validate_options(options)
if not PY3 and not isinstance(filename, unicode):
log.warning('Given filename to matcher is not unicode...')
filename = filename.decode('utf-8')
filename = normalize_unicode(filename)
self.match_tree = MatchTree(filename)
self.options = options
self._transfo_calls = []
# sanity check: make sure we don't process a (mostly) empty string
if clean_string(filename) == '':
return
from guessit.plugins import transformers
try:
mtree = self.match_tree
if 'type' in self.options:
mtree.guess.set('type', self.options['type'], confidence=0.0)
# Process
for transformer in transformers.all_transformers():
self._process(transformer, False)
# Post-process
for transformer in transformers.all_transformers():
self._process(transformer, True)
log.debug('Found match tree:\n%s' % u(mtree))
except TransformerException as e:
log.debug('An error has occurred in Transformer %s: %s' %
(e.transformer, e))
def __init__(self, filename, options=None, **kwargs):
options = dict(options or {})
for k, v in kwargs.items():
if k not in options or not options[k]:
options[k] = v # options dict has priority over keyword arguments
self._validate_options(options)
if not PY3 and not isinstance(filename, unicode):
log.warning('Given filename to matcher is not unicode...')
filename = filename.decode('utf-8')
filename = normalize_unicode(filename)
self.match_tree = MatchTree(filename)
self.options = options
self._transfo_calls = []
# sanity check: make sure we don't process a (mostly) empty string
if clean_string(filename) == '':
return
from guessit.plugins import transformers
try:
mtree = self.match_tree
if 'type' in self.options:
mtree.guess.set('type', self.options['type'], confidence=0.0)
# Process
for transformer in transformers.all_transformers():
self._process(transformer, False)
# Post-process
for transformer in transformers.all_transformers():
self._process(transformer, True)
log.debug('Found match tree:\n%s' % u(mtree))
except TransformerException as e:
log.debug('An error has occured in Transformer %s: %s' % (e.transformer, e))
def process(mtree):
"""
try to identify the remaining unknown groups by looking at their
position relative to other known elements
"""
def found_property(node, name, value, confidence):
node.guess = Guess({name: value},
confidence=confidence)
log.debug('Found with confidence %.2f: %s' % (confidence, node.guess))
def found_title(node, confidence):
found_property(node, 'title', node.clean_value, confidence)
basename = mtree.node_at((-2,))
all_valid = lambda leaf: len(leaf.clean_value) > 0
basename_leftover = basename.unidentified_leaves(valid=all_valid)
try:
folder = mtree.node_at((-3,))
folder_leftover = folder.unidentified_leaves()
except ValueError:
folder = None
folder_leftover = []
log.debug('folder: %s' % u(folder_leftover))
log.debug('basename: %s' % u(basename_leftover))
# specific cases:
# if we find the same group both in the folder name and the filename,
# it's a good candidate for title
if (folder_leftover and basename_leftover and
folder_leftover[0].clean_value == basename_leftover[0].clean_value):
found_title(folder_leftover[0], confidence=0.8)
return
# specific cases:
# if the basename contains a number first followed by an unidentified
# group, and the folder only contains 1 unidentified one, then we have
# a series
# ex: Millenium Trilogy (2009)/(1)The Girl With The Dragon Tattoo(2009).mkv
try:
series = folder_leftover[0]
filmNumber = basename_leftover[0]
title = basename_leftover[1]
basename_leaves = basename.leaves()
num = int(filmNumber.clean_value)
log.debug('series: %s' % series.clean_value)
log.debug('title: %s' % title.clean_value)
if (series.clean_value != title.clean_value and
series.clean_value != filmNumber.clean_value and
basename_leaves.index(filmNumber) == 0 and
basename_leaves.index(title) == 1):
found_title(title, confidence=0.6)
found_property(series, 'filmSeries',
series.clean_value, confidence=0.6)
found_property(filmNumber, 'filmNumber',
num, confidence=0.6)
return
except Exception:
pass
# specific cases:
# - movies/tttttt (yyyy)/tttttt.ccc
try:
if mtree.node_at((-4, 0)).value.lower() == 'movies':
folder = mtree.node_at((-3,))
# Note:too generic, might solve all the unittests as they all
# contain 'movies' in their path
#
#if containing_folder.is_leaf() and not containing_folder.guess:
# containing_folder.guess =
# Guess({ 'title': clean_string(containing_folder.value) },
# confidence=0.7)
year_group = folder.first_leaf_containing('year')
groups_before = folder.previous_unidentified_leaves(year_group)
found_title(groups_before[0], confidence=0.8)
return
except Exception:
pass
# if we have either format or videoCodec in the folder containing the file
# or one of its parents, then we should probably look for the title in
# there rather than in the basename
try:
props = mtree.previous_leaves_containing(mtree.children[-2],
['videoCodec', 'format',
'language'])
except IndexError:
props = []
if props:
group_idx = props[0].node_idx[0]
if all(g.node_idx[0] == group_idx for g in props):
# if they're all in the same group, take leftover info from there
leftover = mtree.node_at((group_idx,)).unidentified_leaves()
if leftover:
found_title(leftover[0], confidence=0.7)
return
# look for title in basename if there are some remaining undidentified
# groups there
if basename_leftover:
title_candidate = basename_leftover[0]
# if basename is only one word and the containing folder has at least
# 3 words in it, we should take the title from the folder name
# ex: Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi
# ex: Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi <-- TODO: gets caught here?
if (title_candidate.clean_value.count(' ') == 0 and
folder_leftover and
folder_leftover[0].clean_value.count(' ') >= 2):
found_title(folder_leftover[0], confidence=0.7)
return
# if there are only many unidentified groups, take the first of which is
# not inside brackets or parentheses.
# ex: Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi
if basename_leftover[0].is_explicit():
for basename_leftover_elt in basename_leftover:
if not basename_leftover_elt.is_explicit():
found_title(basename_leftover_elt, confidence=0.8)
return
# if all else fails, take the first remaining unidentified group in the
# basename as title
found_title(title_candidate, confidence=0.6)
return
# if there are no leftover groups in the basename, look in the folder name
if folder_leftover:
found_title(folder_leftover[0], confidence=0.5)
return
# if nothing worked, look if we have a very small group at the beginning
# of the basename
basename = mtree.node_at((-2,))
basename_leftover = basename.unidentified_leaves(valid=lambda leaf: True)
if basename_leftover:
found_title(basename_leftover[0], confidence=0.4)
return
def process(self, mtree, options=None):
"""
try to identify the remaining unknown groups by looking at their
position relative to other known elements
"""
if 'title' in mtree.info:
return
path_nodes = list(filter(lambda x: x.category == 'path', mtree.nodes()))
basename = path_nodes[-2]
all_valid = lambda leaf: len(leaf.clean_value) > 0
basename_leftover = list(basename.unidentified_leaves(valid=all_valid))
try:
folder = path_nodes[-3]
folder_leftover = list(folder.unidentified_leaves())
except IndexError:
folder = None
folder_leftover = []
self.log.debug('folder: %s' % u(folder_leftover))
self.log.debug('basename: %s' % u(basename_leftover))
# specific cases:
# if we find the same group both in the folder name and the filename,
# it's a good candidate for title
if (folder_leftover and basename_leftover and
folder_leftover[0].clean_value == basename_leftover[0].clean_value and
not GuessMovieTitleFromPosition.excluded_word(folder_leftover[0])):
found_property(folder_leftover[0], 'title', confidence=0.8)
return
# specific cases:
# if the basename contains a number first followed by an unidentified
# group, and the folder only contains 1 unidentified one, then we have
# a series
# ex: Millenium Trilogy (2009)/(1)The Girl With The Dragon Tattoo(2009).mkv
if len(folder_leftover) > 0 and len(basename_leftover) > 1:
series = folder_leftover[0]
film_number = basename_leftover[0]
title = basename_leftover[1]
basename_leaves = list(basename.leaves())
num = None
try:
num = int(film_number.clean_value)
except ValueError:
pass
if num:
self.log.debug('series: %s' % series.clean_value)
self.log.debug('title: %s' % title.clean_value)
if (series.clean_value != title.clean_value and
series.clean_value != film_number.clean_value and
basename_leaves.index(film_number) == 0 and
basename_leaves.index(title) == 1 and
not GuessMovieTitleFromPosition.excluded_word(title, series)):
found_property(title, 'title', confidence=0.6)
found_property(series, 'filmSeries', confidence=0.6)
found_property(film_number, 'filmNumber', num, confidence=0.6)
return
if folder:
year_group = folder.first_leaf_containing('year')
if year_group:
groups_before = folder.previous_unidentified_leaves(year_group)
if groups_before:
try:
node = next(groups_before)
if not GuessMovieTitleFromPosition.excluded_word(node):
found_property(node, 'title', confidence=0.8)
return
except StopIteration:
pass
# if we have either format or videoCodec in the folder containing the
# file or one of its parents, then we should probably look for the title
# in there rather than in the basename
try:
props = list(mtree.previous_leaves_containing(mtree.children[-2],
['videoCodec',
'format',
'language']))
except IndexError:
props = []
if props:
group_idx = props[0].node_idx[0]
if all(g.node_idx[0] == group_idx for g in props):
# if they're all in the same group, take leftover info from there
leftover = mtree.node_at((group_idx,)).unidentified_leaves()
try:
node = next(leftover)
if not GuessMovieTitleFromPosition.excluded_word(node):
found_property(node, 'title', confidence=0.7)
return
except StopIteration:
pass
# look for title in basename if there are some remaining unidentified
# groups there
if basename_leftover:
# if basename is only one word and the containing folder has at least
# 3 words in it, we should take the title from the folder name
# ex: Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi
# ex: Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi <-- TODO: gets caught here?
if (basename_leftover[0].clean_value.count(' ') == 0 and
folder_leftover and folder_leftover[0].clean_value.count(' ') >= 2 and
not GuessMovieTitleFromPosition.excluded_word(folder_leftover[0])):
found_property(folder_leftover[0], 'title', confidence=0.7)
return
# if there are only many unidentified groups, take the first of which is
# not inside brackets or parentheses.
# ex: Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi
if basename_leftover[0].is_explicit():
for basename_leftover_elt in basename_leftover:
if not basename_leftover_elt.is_explicit() and not GuessMovieTitleFromPosition.excluded_word(basename_leftover_elt):
found_property(basename_leftover_elt, 'title', confidence=0.8)
return
# if all else fails, take the first remaining unidentified group in the
# basename as title
if not GuessMovieTitleFromPosition.excluded_word(basename_leftover[0]):
found_property(basename_leftover[0], 'title', confidence=0.6)
return
# if there are no leftover groups in the basename, look in the folder name
if folder_leftover and not GuessMovieTitleFromPosition.excluded_word(folder_leftover[0]):
found_property(folder_leftover[0], 'title', confidence=0.5)
return
# if nothing worked, look if we have a very small group at the beginning
# of the basename
basename_leftover = basename.unidentified_leaves(valid=lambda leaf: True)
try:
node = next(basename_leftover)
if not GuessMovieTitleFromPosition.excluded_word(node):
found_property(node, 'title', confidence=0.4)
return
except StopIteration:
pass
def process(self, mtree, options=None):
"""
try to identify the remaining unknown groups by looking at their
position relative to other known elements
"""
if 'title' in mtree.info:
return
path_nodes = list(filter(lambda x: x.category == 'path',
mtree.nodes()))
basename = path_nodes[-2]
all_valid = lambda leaf: len(leaf.clean_value) > 0
basename_leftover = list(basename.unidentified_leaves(valid=all_valid))
try:
folder = path_nodes[-3]
folder_leftover = list(folder.unidentified_leaves())
except IndexError:
folder = None
folder_leftover = []
self.log.debug('folder: %s' % u(folder_leftover))
self.log.debug('basename: %s' % u(basename_leftover))
# specific cases:
# if we find the same group both in the folder name and the filename,
# it's a good candidate for title
if (folder_leftover and basename_leftover
and folder_leftover[0].clean_value
== basename_leftover[0].clean_value
and not GuessMovieTitleFromPosition.excluded_word(
folder_leftover[0])):
found_property(folder_leftover[0], 'title', confidence=0.8)
return
# specific cases:
# if the basename contains a number first followed by an unidentified
# group, and the folder only contains 1 unidentified one, then we have
# a series
# ex: Millenium Trilogy (2009)/(1)The Girl With The Dragon Tattoo(2009).mkv
if len(folder_leftover) > 0 and len(basename_leftover) > 1:
series = folder_leftover[0]
film_number = basename_leftover[0]
title = basename_leftover[1]
basename_leaves = list(basename.leaves())
num = None
try:
num = int(film_number.clean_value)
except ValueError:
pass
if num:
self.log.debug('series: %s' % series.clean_value)
self.log.debug('title: %s' % title.clean_value)
if (series.clean_value != title.clean_value
and series.clean_value != film_number.clean_value
and basename_leaves.index(film_number) == 0
and basename_leaves.index(title) == 1
and not GuessMovieTitleFromPosition.excluded_word(
title, series)):
found_property(title, 'title', confidence=0.6)
found_property(series, 'filmSeries', confidence=0.6)
found_property(film_number,
'filmNumber',
num,
confidence=0.6)
return
if folder:
year_group = folder.first_leaf_containing('year')
if year_group:
groups_before = folder.previous_unidentified_leaves(year_group)
if groups_before:
try:
node = next(groups_before)
if not GuessMovieTitleFromPosition.excluded_word(node):
found_property(node, 'title', confidence=0.8)
return
except StopIteration:
pass
# if we have either format or videoCodec in the folder containing the
# file or one of its parents, then we should probably look for the title
# in there rather than in the basename
try:
props = list(
mtree.previous_leaves_containing(
mtree.children[-2], ['videoCodec', 'format', 'language']))
except IndexError:
props = []
if props:
group_idx = props[0].node_idx[0]
if all(g.node_idx[0] == group_idx for g in props):
# if they're all in the same group, take leftover info from there
leftover = mtree.node_at((group_idx, )).unidentified_leaves()
try:
node = next(leftover)
if not GuessMovieTitleFromPosition.excluded_word(node):
found_property(node, 'title', confidence=0.7)
return
except StopIteration:
pass
# look for title in basename if there are some remaining unidentified
# groups there
if basename_leftover:
# if basename is only one word and the containing folder has at least
# 3 words in it, we should take the title from the folder name
# ex: Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi
# ex: Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi <-- TODO: gets caught here?
if (basename_leftover[0].clean_value.count(' ') == 0
and folder_leftover
and folder_leftover[0].clean_value.count(' ') >= 2
and not GuessMovieTitleFromPosition.excluded_word(
folder_leftover[0])):
found_property(folder_leftover[0], 'title', confidence=0.7)
return
# if there are only many unidentified groups, take the first of which is
# not inside brackets or parentheses.
# ex: Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi
if basename_leftover[0].is_explicit():
for basename_leftover_elt in basename_leftover:
if not basename_leftover_elt.is_explicit(
) and not GuessMovieTitleFromPosition.excluded_word(
basename_leftover_elt):
found_property(basename_leftover_elt,
'title',
confidence=0.8)
return
# if all else fails, take the first remaining unidentified group in the
# basename as title
if not GuessMovieTitleFromPosition.excluded_word(
basename_leftover[0]):
found_property(basename_leftover[0], 'title', confidence=0.6)
return
# if there are no leftover groups in the basename, look in the folder name
if folder_leftover and not GuessMovieTitleFromPosition.excluded_word(
folder_leftover[0]):
found_property(folder_leftover[0], 'title', confidence=0.5)
return
# if nothing worked, look if we have a very small group at the beginning
# of the basename
basename_leftover = basename.unidentified_leaves(
valid=lambda leaf: True)
try:
node = next(basename_leftover)
if not GuessMovieTitleFromPosition.excluded_word(node):
found_property(node, 'title', confidence=0.4)
return
except StopIteration:
pass
def process(self, mtree, options=None):
"""
try to identify the remaining unknown groups by looking at their
position relative to other known elements
"""
basename = mtree.node_at((-2, ))
all_valid = lambda leaf: len(leaf.clean_value) > 0
basename_leftover = basename.unidentified_leaves(valid=all_valid)
try:
folder = mtree.node_at((-3, ))
folder_leftover = folder.unidentified_leaves()
except ValueError:
folder = None
folder_leftover = []
self.log.debug('folder: %s' % u(folder_leftover))
self.log.debug('basename: %s' % u(basename_leftover))
# specific cases:
# if we find the same group both in the folder name and the filename,
# it's a good candidate for title
if (folder_leftover and basename_leftover
and folder_leftover[0].clean_value
== basename_leftover[0].clean_value):
found_property(folder_leftover[0], 'title', confidence=0.8)
return
# specific cases:
# if the basename contains a number first followed by an unidentified
# group, and the folder only contains 1 unidentified one, then we have
# a series
# ex: Millenium Trilogy (2009)/(1)The Girl With The Dragon Tattoo(2009).mkv
try:
series = folder_leftover[0]
filmNumber = basename_leftover[0]
title = basename_leftover[1]
basename_leaves = basename.leaves()
num = int(filmNumber.clean_value)
self.log.debug('series: %s' % series.clean_value)
self.log.debug('title: %s' % title.clean_value)
if (series.clean_value != title.clean_value
and series.clean_value != filmNumber.clean_value
and basename_leaves.index(filmNumber) == 0
and basename_leaves.index(title) == 1):
found_property(title, 'title', confidence=0.6)
found_property(series, 'filmSeries', confidence=0.6)
found_property(filmNumber, 'filmNumber', num, confidence=0.6)
return
except Exception:
pass
# specific cases:
# - movies/tttttt (yyyy)/tttttt.ccc
try:
if mtree.node_at((-4, 0)).value.lower() == 'movies':
folder = mtree.node_at((-3, ))
# Note:too generic, might solve all the unittests as they all
# contain 'movies' in their path
#
# if containing_folder.is_leaf() and not containing_folder.guess:
# containing_folder.guess =
# Guess({ 'title': clean_string(containing_folder.value) },
# confidence=0.7)
year_group = folder.first_leaf_containing('year')
groups_before = folder.previous_unidentified_leaves(year_group)
found_property(groups_before[0], 'title', confidence=0.8)
return
except Exception:
pass
# if we have either format or videoCodec in the folder containing the file
# or one of its parents, then we should probably look for the title in
# there rather than in the basename
try:
props = mtree.previous_leaves_containing(
mtree.children[-2], ['videoCodec', 'format', 'language'])
except IndexError:
props = []
if props:
group_idx = props[0].node_idx[0]
if all(g.node_idx[0] == group_idx for g in props):
# if they're all in the same group, take leftover info from there
leftover = mtree.node_at((group_idx, )).unidentified_leaves()
if leftover:
found_property(leftover[0], 'title', confidence=0.7)
return
# look for title in basename if there are some remaining unidentified
# groups there
if basename_leftover:
# if basename is only one word and the containing folder has at least
# 3 words in it, we should take the title from the folder name
# ex: Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi
# ex: Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi <-- TODO: gets caught here?
if (basename_leftover[0].clean_value.count(' ') == 0
and folder_leftover
and folder_leftover[0].clean_value.count(' ') >= 2):
found_property(folder_leftover[0], 'title', confidence=0.7)
return
# if there are only many unidentified groups, take the first of which is
# not inside brackets or parentheses.
# ex: Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi
if basename_leftover[0].is_explicit():
for basename_leftover_elt in basename_leftover:
if not basename_leftover_elt.is_explicit():
found_property(basename_leftover_elt,
'title',
confidence=0.8)
return
# if all else fails, take the first remaining unidentified group in the
# basename as title
found_property(basename_leftover[0], 'title', confidence=0.6)
return
# if there are no leftover groups in the basename, look in the folder name
if folder_leftover:
found_property(folder_leftover[0], 'title', confidence=0.5)
return
# if nothing worked, look if we have a very small group at the beginning
# of the basename
basename = mtree.node_at((-2, ))
basename_leftover = basename.unidentified_leaves(
valid=lambda leaf: True)
if basename_leftover:
found_property(basename_leftover[0], 'title', confidence=0.4)
return
def __unicode__(self):
return u(self.to_dict())
def checkFields(self, groundTruth, guess_func, remove_type=True,
exclude_files=None):
total = 0
exclude_files = exclude_files or []
fails = defaultdict(list)
additionals = defaultdict(list)
for filename, required_fields in groundTruth.items():
filename = u(filename)
if filename in exclude_files:
continue
log.debug('\n' + '-' * 120)
log.info('Guessing information for file: %s' % filename)
options = required_fields.pop('options') if 'options' in required_fields else None
if options:
args = shlex.split(options)
options = get_opts().parse_args(args)
options = vars(options)
try:
found = guess_func(filename, options)
except Exception as e:
fails[filename].append("An exception has occured in %s: %s" % (filename, e))
log.exception("An exception has occured in %s: %s" % (filename, e))
continue
total = total + 1
# no need for these in the unittests
if remove_type:
try:
del found['type']
except:
pass
for prop in ('container', 'mimetype', 'unidentified'):
if prop in found:
del found[prop]
# props which are list of just 1 elem should be opened for easier writing of the tests
for prop in ('language', 'subtitleLanguage', 'other', 'episodeDetails', 'unidentified'):
value = found.get(prop, None)
if isinstance(value, list) and len(value) == 1:
found[prop] = value[0]
# look for missing properties
for prop, value in required_fields.items():
if prop not in found:
log.debug("Prop '%s' not found in: %s" % (prop, filename))
fails[filename].append("'%s' not found in: %s" % (prop, filename))
continue
# if both properties are strings, do a case-insensitive comparison
if (isinstance(value, base_text_type) and
isinstance(found[prop], base_text_type)):
if value.lower() != found[prop].lower():
log.debug("Wrong prop value [str] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
elif isinstance(value, list) and isinstance(found[prop], list):
if found[prop] and isinstance(found[prop][0], babelfish.Language):
# list of languages
s1 = set(Language.fromguessit(s) for s in value)
s2 = set(found[prop])
else:
# by default we assume list of strings and do a case-insensitive
# comparison on their elements
s1 = set(u(s).lower() for s in value)
s2 = set(u(s).lower() for s in found[prop])
if s1 != s2:
log.debug("Wrong prop value [list] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
elif isinstance(found[prop], babelfish.Language):
try:
if babelfish.Language.fromguessit(value) != found[prop]:
raise ValueError
except:
log.debug("Wrong prop value [Language] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
elif isinstance(found[prop], babelfish.Country):
try:
if babelfish.Country.fromguessit(value) != found[prop]:
raise ValueError
except:
log.debug("Wrong prop value [Country] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
# otherwise, just compare their values directly
else:
if found[prop] != value:
log.debug("Wrong prop value for '%s': expected = '%s' [%s] - received = '%s' [%s]" % (prop, u(value), type(value), u(found[prop]), type(found[prop])))
fails[filename].append("'%s': expected = '%s' [%s] - received = '%s' [%s]" % (prop, u(value), type(value), u(found[prop]), type(found[prop])))
# look for additional properties
for prop, value in found.items():
if prop not in required_fields:
log.debug("Found additional info for prop = '%s': '%s'" % (prop, u(value)))
additionals[filename].append("'%s': '%s'" % (prop, u(value)))
correct = total - len(fails)
log.info('SUMMARY: Guessed correctly %d out of %d filenames' % (correct, total))
for failed_entry, failed_properties in fails.items():
log.error('---- ' + failed_entry + ' ----')
for failed_property in failed_properties:
log.error("FAILED: " + failed_property)
for additional_entry, additional_properties in additionals.items():
log.warning('---- ' + additional_entry + ' ----')
for additional_property in additional_properties:
log.warning("ADDITIONAL: " + additional_property)
self.assertTrue(correct == total,
msg='Correct: %d < Total: %d' % (correct, total))
def __init__(self, filename, filetype='autodetect', opts=None, transfo_opts=None):
"""An iterative matcher tries to match different patterns that appear
in the filename.
The 'filetype' argument indicates which type of file you want to match.
If it is 'autodetect', the matcher will try to see whether it can guess
that the file corresponds to an episode, or otherwise will assume it is
a movie.
The recognized 'filetype' values are:
[ autodetect, subtitle, info, movie, moviesubtitle, movieinfo, episode,
episodesubtitle, episodeinfo ]
The IterativeMatcher works mainly in 2 steps:
First, it splits the filename into a match_tree, which is a tree of groups
which have a semantic meaning, such as episode number, movie title,
etc...
The match_tree created looks like the following:
0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111
0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000
0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000
__________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___
xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc
[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv
The first 3 lines indicates the group index in which a char in the
filename is located. So for instance, x264 is the group (0, 4, 1), and
it corresponds to a video codec, denoted by the letter'v' in the 4th line.
(for more info, see guess.matchtree.to_string)
Second, it tries to merge all this information into a single object
containing all the found properties, and does some (basic) conflict
resolution when they arise.
When you create the Matcher, you can pass it:
- a list 'opts' of option names, that act as global flags
- a dict 'transfo_opts' of { transfo_name: (transfo_args, transfo_kwargs) }
with which to call the transfo.process() function.
"""
valid_filetypes = ('autodetect', 'subtitle', 'info', 'video',
'movie', 'moviesubtitle', 'movieinfo',
'episode', 'episodesubtitle', 'episodeinfo')
if filetype not in valid_filetypes:
raise ValueError("filetype needs to be one of %s" % valid_filetypes)
if not PY3 and not isinstance(filename, unicode):
log.warning('Given filename to matcher is not unicode...')
filename = filename.decode('utf-8')
filename = normalize_unicode(filename)
if opts is None:
opts = []
if not isinstance(opts, list):
raise ValueError('opts must be a list of option names! Received: type=%s val=%s',
type(opts), opts)
if transfo_opts is None:
transfo_opts = {}
if not isinstance(transfo_opts, dict):
raise ValueError('transfo_opts must be a dict of { transfo_name: (args, kwargs) }. '+
'Received: type=%s val=%s', type(transfo_opts), transfo_opts)
self.match_tree = MatchTree(filename)
# sanity check: make sure we don't process a (mostly) empty string
if clean_string(filename) == '':
return
mtree = self.match_tree
mtree.guess.set('type', filetype, confidence=1.0)
def apply_transfo(transfo_name, *args, **kwargs):
transfo = __import__('guessit.transfo.' + transfo_name,
globals=globals(), locals=locals(),
fromlist=['process'], level=0)
default_args, default_kwargs = transfo_opts.get(transfo_name, ((), {}))
all_args = args or default_args
all_kwargs = dict(default_kwargs)
all_kwargs.update(kwargs) # keep all kwargs merged together
transfo.process(mtree, *all_args, **all_kwargs)
# 1- first split our path into dirs + basename + ext
apply_transfo('split_path_components')
# 2- guess the file type now (will be useful later)
apply_transfo('guess_filetype', filetype)
if mtree.guess['type'] == 'unknown':
return
# 3- split each of those into explicit groups (separated by parentheses
# or square brackets)
apply_transfo('split_explicit_groups')
# 4- try to match information for specific patterns
# NOTE: order needs to comply to the following:
# - website before language (eg: tvu.org.ru vs russian)
# - language before episodes_rexps
# - properties before language (eg: he-aac vs hebrew)
# - release_group before properties (eg: XviD-?? vs xvid)
if mtree.guess['type'] in ('episode', 'episodesubtitle', 'episodeinfo'):
strategy = [ 'guess_date', 'guess_website', 'guess_release_group',
'guess_properties', 'guess_language',
'guess_video_rexps',
'guess_episodes_rexps', 'guess_weak_episodes_rexps' ]
else:
strategy = [ 'guess_date', 'guess_website', 'guess_release_group',
'guess_properties', 'guess_language',
'guess_video_rexps' ]
if 'nolanguage' in opts:
strategy.remove('guess_language')
for name in strategy:
apply_transfo(name)
# more guessers for both movies and episodes
apply_transfo('guess_bonus_features')
apply_transfo('guess_year', skip_first_year=('skip_first_year' in opts))
if 'nocountry' not in opts:
apply_transfo('guess_country')
apply_transfo('guess_idnumber')
# split into '-' separated subgroups (with required separator chars
# around the dash)
apply_transfo('split_on_dash')
# 5- try to identify the remaining unknown groups by looking at their
# position relative to other known elements
if mtree.guess['type'] in ('episode', 'episodesubtitle', 'episodeinfo'):
apply_transfo('guess_episode_info_from_position')
else:
apply_transfo('guess_movie_title_from_position')
# 6- perform some post-processing steps
apply_transfo('post_process')
log.debug('Found match tree:\n%s' % u(mtree))
def __init__(self, filename, filetype='autodetect'):
"""An iterative matcher tries to match different patterns that appear
in the filename.
The 'filetype' argument indicates which type of file you want to match.
If it is 'autodetect', the matcher will try to see whether it can guess
that the file corresponds to an episode, or otherwise will assume it is
a movie.
The recognized 'filetype' values are:
[ autodetect, subtitle, movie, moviesubtitle, episode, episodesubtitle ]
The IterativeMatcher works mainly in 2 steps:
First, it splits the filename into a match_tree, which is a tree of groups
which have a semantic meaning, such as episode number, movie title,
etc...
The match_tree created looks like the following:
0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111
0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000
0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000
__________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___
xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc
[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv
The first 3 lines indicates the group index in which a char in the
filename is located. So for instance, x264 is the group (0, 4, 1), and
it corresponds to a video codec, denoted by the letter'v' in the 4th line.
(for more info, see guess.matchtree.to_string)
Second, it tries to merge all this information into a single object
containing all the found properties, and does some (basic) conflict
resolution when they arise.
"""
valid_filetypes = ('autodetect', 'subtitle', 'video', 'movie',
'moviesubtitle', 'episode', 'episodesubtitle')
if filetype not in valid_filetypes:
raise ValueError("filetype needs to be one of %s" %
valid_filetypes)
if not PY3 and not isinstance(filename, unicode):
log.warning('Given filename to matcher is not unicode...')
self.match_tree = MatchTree(filename)
mtree = self.match_tree
mtree.guess.set('type', filetype, confidence=1.0)
def apply_transfo(transfo_name, *args, **kwargs):
transfo = __import__('guessit.transfo.' + transfo_name,
globals=globals(),
locals=locals(),
fromlist=['process'],
level=-1)
transfo.process(mtree, *args, **kwargs)
# 1- first split our path into dirs + basename + ext
apply_transfo('split_path_components')
# 2- guess the file type now (will be useful later)
apply_transfo('guess_filetype', filetype)
if mtree.guess['type'] == 'unknown':
return
# 3- split each of those into explicit groups (separated by parentheses
# or square brackets)
apply_transfo('split_explicit_groups')
# 4- try to match information for specific patterns
# NOTE: order needs to comply to the following:
# - website before language (eg: tvu.org.ru vs russian)
# - language before episodes_rexps
# - properties before language (eg: he-aac vs hebrew)
# - release_group before properties (eg: XviD-?? vs xvid)
if mtree.guess['type'] in ('episode', 'episodesubtitle'):
strategy = [
'guess_date', 'guess_website', 'guess_release_group',
'guess_properties', 'guess_language', 'guess_video_rexps',
'guess_episodes_rexps', 'guess_weak_episodes_rexps'
]
else:
strategy = [
'guess_date', 'guess_website', 'guess_release_group',
'guess_properties', 'guess_language', 'guess_video_rexps'
]
for name in strategy:
apply_transfo(name)
# more guessers for both movies and episodes
for name in ['guess_bonus_features', 'guess_year', 'guess_country']:
apply_transfo(name)
# split into '-' separated subgroups (with required separator chars
# around the dash)
apply_transfo('split_on_dash')
# 5- try to identify the remaining unknown groups by looking at their
# position relative to other known elements
if mtree.guess['type'] in ('episode', 'episodesubtitle'):
apply_transfo('guess_episode_info_from_position')
else:
apply_transfo('guess_movie_title_from_position')
# 6- perform some post-processing steps
apply_transfo('post_process')
log.debug('Found match tree:\n%s' % u(mtree))
def detect_filename(filename, filetype, info=['filename']):
filename = u(filename)
print('For:', filename)
print('GuessIt found:',
guess_file_info(filename, filetype, info).nice_string())
def checkFields(self,
groundTruth,
guess_func,
remove_type=True,
exclude_files=None):
total = 0
exclude_files = exclude_files or []
fails = defaultdict(list)
additionals = defaultdict(list)
for filename, required_fields in groundTruth.items():
filename = u(filename)
if filename in exclude_files:
continue
log.debug('\n' + '-' * 120)
log.info('Guessing information for file: %s' % filename)
options = required_fields.pop(
'options') if 'options' in required_fields else None
if options:
args = shlex.split(options)
options, _ = option_parser.parse_args(args)
options = vars(options)
found = guess_func(filename, options)
total = total + 1
# no need for these in the unittests
if remove_type:
try:
del found['type']
except:
pass
for prop in ('container', 'mimetype'):
if prop in found:
del found[prop]
# props which are list of just 1 elem should be opened for easier writing of the tests
for prop in ('language', 'subtitleLanguage', 'other',
'episodeDetails'):
value = found.get(prop, None)
if isinstance(value, list) and len(value) == 1:
found[prop] = value[0]
# look for missing properties
for prop, value in required_fields.items():
if prop not in found:
log.debug("Prop '%s' not found in: %s" % (prop, filename))
fails[filename].append("'%s' not found in: %s" %
(prop, filename))
continue
# if both properties are strings, do a case-insensitive comparison
if (isinstance(value, base_text_type)
and isinstance(found[prop], base_text_type)):
if value.lower() != found[prop].lower():
log.debug(
"Wrong prop value [str] for '%s': expected = '%s' - received = '%s'"
% (prop, u(value), u(found[prop])))
fails[filename].append(
"'%s': expected = '%s' - received = '%s'" %
(prop, u(value), u(found[prop])))
elif isinstance(value, list) and isinstance(found[prop], list):
if found[prop] and isinstance(found[prop][0],
babelfish.Language):
# list of languages
s1 = set(Language.fromguessit(s) for s in value)
s2 = set(found[prop])
else:
# by default we assume list of strings and do a case-insensitive
# comparison on their elements
s1 = set(u(s).lower() for s in value)
s2 = set(u(s).lower() for s in found[prop])
if s1 != s2:
log.debug(
"Wrong prop value [list] for '%s': expected = '%s' - received = '%s'"
% (prop, u(value), u(found[prop])))
fails[filename].append(
"'%s': expected = '%s' - received = '%s'" %
(prop, u(value), u(found[prop])))
elif isinstance(found[prop], babelfish.Language):
try:
if babelfish.Language.fromguessit(
value) != found[prop]:
raise ValueError
except:
log.debug(
"Wrong prop value [Language] for '%s': expected = '%s' - received = '%s'"
% (prop, u(value), u(found[prop])))
fails[filename].append(
"'%s': expected = '%s' - received = '%s'" %
(prop, u(value), u(found[prop])))
elif isinstance(found[prop], babelfish.Country):
try:
if babelfish.Country.fromguessit(value) != found[prop]:
raise ValueError
except:
log.debug(
"Wrong prop value [Country] for '%s': expected = '%s' - received = '%s'"
% (prop, u(value), u(found[prop])))
fails[filename].append(
"'%s': expected = '%s' - received = '%s'" %
(prop, u(value), u(found[prop])))
# otherwise, just compare their values directly
else:
if found[prop] != value:
log.debug(
"Wrong prop value for '%s': expected = '%s' [%s] - received = '%s' [%s]"
% (prop, u(value), type(value), u(
found[prop]), type(found[prop])))
fails[filename].append(
"'%s': expected = '%s' [%s] - received = '%s' [%s]"
% (prop, u(value), type(value), u(
found[prop]), type(found[prop])))
# look for additional properties
for prop, value in found.items():
if prop not in required_fields:
log.debug("Found additional info for prop = '%s': '%s'" %
(prop, u(value)))
additionals[filename].append("'%s': '%s'" %
(prop, u(value)))
correct = total - len(fails)
log.info('SUMMARY: Guessed correctly %d out of %d filenames' %
(correct, total))
for failed_entry, failed_properties in fails.items():
log.error('---- ' + failed_entry + ' ----')
for failed_property in failed_properties:
log.error("FAILED: " + failed_property)
for additional_entry, additional_properties in additionals.items():
log.warn('---- ' + additional_entry + ' ----')
for additional_property in additional_properties:
log.warn("ADDITIONAL: " + additional_property)
self.assertTrue(correct == total,
msg='Correct: %d < Total: %d' % (correct, total))
def detect_filename(filename, filetype, info=['filename']):
filename = u(filename)
print('For:', filename)
print('GuessIt found:', guess_file_info(filename, filetype, info).nice_string())
def detect_filename(filename, filetype, info=["filename"], advanced=False):
filename = u(filename)
print("For:", filename)
print("GuessIt found:", guess_file_info(filename, filetype, info).nice_string(advanced))
def __unicode__(self):
return u(self.to_dict())
def detect_filename(filename, filetype, info=['filename'], advanced=False):
filename = u(filename)
print('For:', filename)
print('GuessIt found:',
guess_file_info(filename, filetype, info).nice_string(advanced))
def __init__(self, filename, filetype="autodetect", opts=None):
"""An iterative matcher tries to match different patterns that appear
in the filename.
The 'filetype' argument indicates which type of file you want to match.
If it is 'autodetect', the matcher will try to see whether it can guess
that the file corresponds to an episode, or otherwise will assume it is
a movie.
The recognized 'filetype' values are:
[ autodetect, subtitle, movie, moviesubtitle, episode, episodesubtitle ]
The IterativeMatcher works mainly in 2 steps:
First, it splits the filename into a match_tree, which is a tree of groups
which have a semantic meaning, such as episode number, movie title,
etc...
The match_tree created looks like the following:
0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111
0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000
0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000
__________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___
xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc
[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv
The first 3 lines indicates the group index in which a char in the
filename is located. So for instance, x264 is the group (0, 4, 1), and
it corresponds to a video codec, denoted by the letter'v' in the 4th line.
(for more info, see guess.matchtree.to_string)
Second, it tries to merge all this information into a single object
containing all the found properties, and does some (basic) conflict
resolution when they arise.
"""
valid_filetypes = ("autodetect", "subtitle", "video", "movie", "moviesubtitle", "episode", "episodesubtitle")
if filetype not in valid_filetypes:
raise ValueError("filetype needs to be one of %s" % valid_filetypes)
if not PY3 and not isinstance(filename, unicode):
log.warning("Given filename to matcher is not unicode...")
filename = filename.decode("utf-8")
filename = normalize_unicode(filename)
if opts is None:
opts = []
elif isinstance(opts, base_text_type):
opts = opts.split()
self.match_tree = MatchTree(filename)
mtree = self.match_tree
mtree.guess.set("type", filetype, confidence=1.0)
def apply_transfo(transfo_name, *args, **kwargs):
transfo = __import__(
"guessit.transfo." + transfo_name, globals=globals(), locals=locals(), fromlist=["process"], level=0
)
transfo.process(mtree, *args, **kwargs)
# 1- first split our path into dirs + basename + ext
apply_transfo("split_path_components")
# 2- guess the file type now (will be useful later)
apply_transfo("guess_filetype", filetype)
if mtree.guess["type"] == "unknown":
return
# 3- split each of those into explicit groups (separated by parentheses
# or square brackets)
apply_transfo("split_explicit_groups")
# 4- try to match information for specific patterns
# NOTE: order needs to comply to the following:
# - website before language (eg: tvu.org.ru vs russian)
# - language before episodes_rexps
# - properties before language (eg: he-aac vs hebrew)
# - release_group before properties (eg: XviD-?? vs xvid)
if mtree.guess["type"] in ("episode", "episodesubtitle"):
strategy = [
"guess_date",
"guess_website",
"guess_release_group",
"guess_properties",
"guess_language",
"guess_video_rexps",
"guess_episodes_rexps",
"guess_weak_episodes_rexps",
]
else:
strategy = [
"guess_date",
"guess_website",
"guess_release_group",
"guess_properties",
"guess_language",
"guess_video_rexps",
]
if "nolanguage" in opts:
strategy.remove("guess_language")
for name in strategy:
apply_transfo(name)
# more guessers for both movies and episodes
apply_transfo("guess_bonus_features")
apply_transfo("guess_year", skip_first_year=("skip_first_year" in opts))
if "nocountry" not in opts:
apply_transfo("guess_country")
# split into '-' separated subgroups (with required separator chars
# around the dash)
apply_transfo("split_on_dash")
# 5- try to identify the remaining unknown groups by looking at their
# position relative to other known elements
if mtree.guess["type"] in ("episode", "episodesubtitle"):
apply_transfo("guess_episode_info_from_position")
else:
apply_transfo("guess_movie_title_from_position")
# 6- perform some post-processing steps
apply_transfo("post_process")
log.debug("Found match tree:\n%s" % u(mtree))
def checkFields(self, groundTruth, guess_func, remove_type=True,
exclude_files=None):
correct, total = 0, 0
exclude_files = exclude_files or []
for filename, required_fields in groundTruth.items():
filename = u(filename)
if filename in exclude_files:
continue
log.debug('\n' + '-' * 120)
log.info('Guessing information for file: %s' % filename)
found = guess_func(filename)
total = total + 1
is_incomplete = [False]
def error(*args):
log.warning(args[0] % args[1:])
is_incomplete[0] = True
# no need for these in the unittests
if remove_type:
try:
del found['type']
except:
pass
for prop in ('container', 'mimetype'):
if prop in found:
del found[prop]
# props which are list of just 1 elem should be opened for easier writing of the tests
for prop in ('language', 'subtitleLanguage', 'other'):
value = found.get(prop, None)
if isinstance(value, list) and len(value) == 1:
found[prop] = value[0]
# look for missing properties
for prop, value in required_fields.items():
if prop not in found:
error("Prop '%s' not found in: %s", prop, filename)
continue
# if both properties are strings, do a case-insensitive comparison
if (isinstance(value, base_text_type) and
isinstance(found[prop], base_text_type)):
if value.lower() != found[prop].lower():
error("Wrong prop value [str] for '%s': expected = '%s' - received = '%s'",
prop, u(value), u(found[prop]))
# if both are lists, we assume list of strings and do a case-insensitive
# comparison on their elements
elif isinstance(value, list) and isinstance(found[prop], list):
s1 = set(u(s).lower() for s in value)
s2 = set(u(s).lower() for s in found[prop])
if s1 != s2:
error("Wrong prop value [list] for '%s': expected = '%s' - received = '%s'",
prop, u(value), u(found[prop]))
# otherwise, just compare their values directly
else:
if found[prop] != value:
error("Wrong prop value for '%s': expected = '%s' [%s] - received = '%s' [%s]",
prop, u(value), type(value), u(found[prop]), type(found[prop]))
# look for additional properties
for prop, value in found.items():
if prop not in required_fields:
log.warning("Found additional info for prop = '%s': '%s'" % (prop, u(value)))
if not is_incomplete[0]:
correct = correct + 1
log.info('SUMMARY: Guessed correctly %d out of %d filenames' % (correct, total))
self.assertTrue(correct == total,
msg='Correct: %d < Total: %d' % (correct, total))
def detect_filename(filename, filetype, info=['filename'], advanced=False):
filename = u(filename)
print('For:', filename)
print('GuessIt found:', guess_file_info(filename, filetype, info).nice_string(advanced))
def checkFields(self,
groundTruth,
guess_func,
remove_type=True,
exclude_files=None):
correct, total = 0, 0
exclude_files = exclude_files or []
for filename, required_fields in groundTruth.items():
filename = u(filename)
if filename in exclude_files:
continue
log.debug('\n' + '-' * 120)
log.info('Guessing information for file: %s' % filename)
found = guess_func(filename)
total = total + 1
is_incomplete = [False]
def error(*args):
log.warning(args[0] % args[1:])
is_incomplete[0] = True
# no need for these in the unittests
if remove_type:
try:
del found['type']
except:
pass
for prop in ('container', 'mimetype'):
if prop in found:
del found[prop]
# props which are list of just 1 elem should be opened for easier writing of the tests
for prop in ('language', 'subtitleLanguage', 'other'):
value = found.get(prop, None)
if isinstance(value, list) and len(value) == 1:
found[prop] = value[0]
# look for missing properties
for prop, value in required_fields.items():
if prop not in found:
error("Prop '%s' not found in: %s", prop, filename)
continue
# if both properties are strings, do a case-insensitive comparison
if (isinstance(value, base_text_type)
and isinstance(found[prop], base_text_type)):
if value.lower() != found[prop].lower():
error(
"Wrong prop value [str] for '%s': expected = '%s' - received = '%s'",
prop, u(value), u(found[prop]))
# if both are lists, we assume list of strings and do a case-insensitive
# comparison on their elements
elif isinstance(value, list) and isinstance(found[prop], list):
s1 = set(u(s).lower() for s in value)
s2 = set(u(s).lower() for s in found[prop])
if s1 != s2:
error(
"Wrong prop value [list] for '%s': expected = '%s' - received = '%s'",
prop, u(value), u(found[prop]))
# otherwise, just compare their values directly
else:
if found[prop] != value:
error(
"Wrong prop value for '%s': expected = '%s' [%s] - received = '%s' [%s]",
prop, u(value), type(value), u(found[prop]),
type(found[prop]))
# look for additional properties
for prop, value in found.items():
if prop not in required_fields:
log.warning("Found additional info for prop = '%s': '%s'" %
(prop, u(value)))
if not is_incomplete[0]:
correct = correct + 1
log.info('SUMMARY: Guessed correctly %d out of %d filenames' %
(correct, total))
self.assertTrue(correct == total,
msg='Correct: %d < Total: %d' % (correct, total))
def detect_filename(filename, filetype, info=["filename"]):
filename = u(filename)
print("For:", filename)
print("GuessIt found:", guess_file_info(filename, filetype, info).nice_string())