def id(self, source, key, identifier=None, resolve_mappings=True):
# Retrieve mapping from database
supported, match = self.map(
source, key,
identifier=identifier,
resolve_mappings=resolve_mappings
)
if not supported:
return False, (None, None)
if not match or not match.valid:
return True, (None, None)
# Find valid identifier
for id_service, id_key in match.identifiers.items():
if id_service == source:
continue
# Strip media from identifier key
id_service_parts = id_service.split(':', 1)
if len(id_service_parts) == 2:
id_service, _ = tuple(id_service_parts)
if id_service in ['tvdb', 'tmdb', 'tvrage']:
id_key = try_convert(id_key, int, id_key)
return True, (id_service, id_key)
log.info('[%s/%s] - Unable to find valid identifier in %r', source, key, match.identiifers)
return True, (None, None)
def process_period(self, period, group):
policy = BACKUP_RETENTION[period]
# Retrieve options
p_files = policy.get('files')
if p_files is None:
raise ValueError('Policy "%s" is missing the "files" attribute')
# Build lists of revisions, grouped by period
revisions_grouped = {}
for base_path, dirs, files in os.walk(group.path):
# Strip UNC prefix from `base_path`
if base_path.startswith('\\\\?\\'):
base_path = base_path[4:]
# Ensure directory starts with a year
rel_path = os.path.relpath(base_path, group.path)
try:
year = rel_path[:rel_path.index(os.path.sep)]
except ValueError:
year = rel_path
if len(year) != 4 or try_convert(year, int) is None:
continue
# Search for revision metadata files
for name in files:
# Ensure file name matches the policy "files" filter
if not fnmatch(name, p_files):
continue
# Build path
path = os.path.join(base_path, name)
# Match revision metadata against regex pattern
if not BACKUP_NAME_REGEX.match(name):
continue
# Load metadata from file
try:
revision = BackupRevision.load(path)
except Exception, ex:
log.warn('Unable to load revision at %r: %s', path, ex, exc_info=True)
continue
# Retrieve timestamp period
key = self.timestamp_period(period, revision.timestamp)
if key == self.timestamp_period(period, self.now):
# Backup occurred in the current period
continue
# Store details in `revisions_grouped` dictionary
if key not in revisions_grouped:
revisions_grouped[key] = []
revisions_grouped[key].append(revision)
def _parse(cls, lines):
processors = {}
extra = {}
# Parse lines into `processors` and `extra`
section = None
current = {}
for line in lines:
# Handle section break
if line == '':
# Store current attributes
if section == 'processor':
num = try_convert(current.pop('processor', None), int)
if num is None:
num = len(processors)
processors[num] = current
elif section == 'extra':
extra.update(current)
elif current:
log.debug('Discarding unknown attributes: %r', current)
# Reset state
section = None
current = {}
# Continue with next line
continue
# Parse attribute from line
parts = [part.strip() for part in line.split(':', 1)]
if len(parts) < 2:
log.debug('Unable to parse attribute from line: %r', line)
continue
# Retrieve attribute components
key, value = parts[0], parts[1]
if not key:
log.debug('Invalid key returned for line: %r', line)
continue
# Transform `key`
key = key.lower()
key = key.replace(' ', '_')
# Check for section-identifier
if not section:
if key == 'processor':
section = 'processor'
else:
section = 'extra'
# Store attribute in current dictionary
current[key] = value
# Store any leftover extra attributes
if section == 'extra' and current:
extra.update(current)
# Return result
return processors, extra
def _parse(cls, lines):
processors = {}
extra = {}
# Parse lines into `processors` and `extra`
section = None
current = {}
for line in lines:
# Handle section break
if line == '':
# Store current attributes
if section == 'processor':
num = try_convert(current.pop('processor', None), int)
if num is None:
num = len(processors)
processors[num] = current
elif section == 'extra':
extra.update(current)
elif current:
log.debug('Discarding unknown attributes: %r', current)
# Reset state
section = None
current = {}
# Continue with next line
continue
# Parse attribute from line
parts = [part.strip() for part in line.split(':', 1)]
if len(parts) < 2:
log.debug('Unable to parse attribute from line: %r', line)
continue
# Retrieve attribute components
key, value = parts[0], parts[1]
if not key:
log.debug('Invalid key returned for line: %r', line)
continue
# Transform `key`
key = key.lower()
key = key.replace(' ', '_')
# Check for section-identifier
if not section:
if key == 'processor':
section = 'processor'
else:
section = 'extra'
# Store attribute in current dictionary
current[key] = value
# Store any leftover extra attributes
if section == 'extra' and current:
extra.update(current)
# Return result
return processors, extra
def _build_request(self, match, item, episode=None):
if not match:
log.warn('Invalid value provided for "match" parameter')
return None
if not item:
log.warn('Invalid value provided for "item" parameter')
return None
# Retrieve identifier
id_service = match.identifiers.keys()[0]
id_key = try_convert(match.identifiers[id_service], int, match.identifiers[id_service])
if type(id_key) not in [int, str]:
log.info('Unsupported key: %r', id_key)
return None
# Determine media type
if isinstance(match, MovieMatch):
media = 'movie'
elif isinstance(match, EpisodeMatch):
media = 'show'
else:
log.warn('Unknown match: %r', match)
return None
# Strip media from identifier key
id_service_parts = id_service.split(':', 1)
if len(id_service_parts) == 2:
id_service, id_media = tuple(id_service_parts)
else:
id_media = None
if id_media and id_media != media:
log.warn('Identifier mismatch, [%s: %r] doesn\'t match %r', id_service, id_key, media)
return None
# Build request
request = {
media: {
'title': item.title,
'ids': {
id_service: id_key
}
}
}
if item.year:
request[media]['year'] = item.year
elif episode and episode.year:
request[media]['year'] = episode.year
else:
log.warn('Missing "year" parameter on %r', item)
# Add episode parameters
if isinstance(match, EpisodeMatch):
if match.absolute_num is not None:
log.info('Absolute mappings are not supported')
return None
if match.season_num is None or match.episode_num is None:
log.warn('Missing season or episode number in %r', match)
return None
request['episode'] = {
'season': match.season_num,
'number': match.episode_num
}
if episode:
request['episode']['title'] = episode.title
return request
def map_items(self, key, store, media=None):
# Retrieve key map
if media is not None:
keys = self.keys(media)
table = self.table(media)
if keys is None or table is None:
log.debug('[%-38s] Collection has been ignored (unknown/unsupported media)', '/'.join(key))
return
else:
keys = None
table = None
# Map each item in store
log.debug('[%-38s] Building table from collection...', '/'.join(key))
for pk, item in store.iteritems():
# Trim `pk` season/episode values
if len(pk) > 2:
pk = tuple(pk[:2])
if pk[0] not in GUID_SERVICES:
log.info('Ignoring item %r with an unknown primary agent: %r', item, pk)
continue
# Detect media type from `item`
if media is not None:
i_media = media
i_keys = keys
i_table = table
else:
i_media = self.media(item)
i_keys = self.keys(i_media)
i_table = self.table(i_media)
# Store `pk` in `keys
if i_keys is not None:
i_keys.add(pk)
# Map `item.keys` -> `pk`
for key in item.keys:
# Expand `key`
if type(key) is not tuple or len(key) != 2:
continue
service, id = key
# Check if agent is supported
if service not in GUID_SERVICES:
continue
# Cast service id to integer
if service in ['tvdb', 'tmdb', 'tvrage']:
id = try_convert(id, int, id)
# Store key in table
key = (service, id)
if key in i_table:
continue
i_table[key] = pk
# Map episodes in show
if i_media == 'episodes':
if type(item) is trakt_objects.Show:
if pk not in self.episode_keys:
self.episode_keys[pk] = set()
for identifier, _ in item.episodes():
self.episode_keys[pk].add(identifier)
elif type(item) is trakt_objects.Episode:
# TODO
pass
else:
log.debug('Unknown episode item: %r', item)
# Task checkpoint
self.task.checkpoint()
class GuidParser(object):
@classmethod
def parse(cls, guid, episode=None):
media = (GuidMatch.Media.Episode if episode else GuidMatch.Media.Movie)
# Ensure guid is valid
if not guid or not guid.valid:
return GuidMatch(media, guid, invalid=True)
# Process guid episode identifier overrides
if episode and len(episode) == 2:
season_num, episode_num = episode
if guid.season is not None:
episode = guid.season, episode_num
# Process natively supported guid services
if guid.service in GUID_SERVICES:
episodes = None
if episode and len(episode) == 2:
episodes = [episode]
return GuidMatch(media,
guid,
episodes=episodes,
supported=True,
found=True)
# Process episode
if episode:
return cls.parse_episode(guid, episode)
# Process shows + movies
supported, (service,
key) = ModuleManager['mapper'].id(guid.service,
guid.id,
resolve_mappings=False)
# Validate match
if not supported:
return GuidMatch(media, guid)
if not service or not key:
return GuidMatch(media, guid, supported=True)
# Validate identifier
if type(key) is list:
log.info('[%s/%s] - List keys are not supported', guid.service,
guid.id)
return GuidMatch(media, guid, supported=True)
if type(key) not in [int, str]:
log.info('[%s/%s] - Unsupported key: %r', guid.service, guid.id,
key)
return GuidMatch(media, guid, supported=True)
log.debug('[%s/%s] - Mapped to: %s/%s', guid.service, guid.id, service,
key)
# Return movie/show match
return GuidMatch(media,
Guid.construct(service, key, matched=True),
supported=True,
found=True)
@classmethod
def parse_episode(cls, guid, (season_num, episode_num)):
episodes = [(season_num, episode_num)]
# Map episode to a supported service (via OEM)
supported, match = ModuleManager['mapper'].map_episode(
guid, season_num, episode_num, resolve_mappings=False)
# Validate match
if not supported:
return GuidMatch(GuidMatch.Media.Episode, guid, episodes=episodes)
if not match or not match.identifiers:
log.debug('Unable to find mapping for %r S%02dE%02d', guid,
season_num, episode_num)
return GuidMatch(GuidMatch.Media.Episode,
guid,
episodes=episodes,
supported=True)
# Retrieve identifier
service = match.identifiers.keys()[0]
key = match.identifiers[service]
if type(key) is list:
log.info('[%s/%s] - List keys are not supported', guid.service,
guid.id)
return GuidMatch(GuidMatch.Media.Episode,
guid,
episodes=episodes,
supported=True)
# Cast `key` numbers to integers
key = try_convert(key, int, key)
# Validate show identifier
if type(key) not in [int, str]:
log.info('[%s/%s] - Unsupported key: %r', guid.service, guid.id,
key)
return GuidMatch(GuidMatch.Media.Episode,
guid,
episodes=episodes,
supported=True)
# Process episode matches
if isinstance(match, EpisodeMatch):
# Ensure match doesn't include an absolute number
if match.absolute_num is not None:
log.info(
'[%s/%s] - Episode mappings with absolute numbers are not supported',
guid.service, guid.id)
return GuidMatch(GuidMatch.Media.Episode,
guid,
episodes=episodes,
supported=True)
# Update `episodes` list
if match.mappings:
# Use match mappings
episodes = []
for mapping in match.mappings:
log.debug('[%s/%s] (S%02dE%02d) - Mapped to: %r',
guid.service, guid.id, season_num, episode_num,
mapping)
episodes.append((int(mapping.season), int(mapping.number)))
else:
# Use match identifier
log.debug('[%s/%s] (S%02dE%02d) - Mapped to: %r', guid.service,
guid.id, season_num, episode_num, match)
episodes = [(int(match.season_num), int(match.episode_num))]
# Return episode match
return GuidMatch(GuidMatch.Media.Episode,
Guid.construct(service, key, matched=True),
episodes=episodes,
supported=True,
found=True)
# Process movie matches
if isinstance(match, MovieMatch):
log.debug('[%s/%s] (S%02dE%02d) - Mapped to: %r', guid.service,
guid.id, season_num, episode_num, match)
# Return movie match
return GuidMatch(GuidMatch.Media.Movie,
Guid.construct(service, key, matched=True),
supported=True,
found=True)
# Unknown value for `match` returned
log.warn('Unknown match returned: %r', match)
return GuidMatch(GuidMatch.Media.Episode,
guid,
episodes=episodes,
supported=True)