def move_old_locations(destination, dryrun=False):
fetcher = ProgramFetcher(None, None, destination)
for program_info in fetcher.get_all_program_infos():
if program_info.version >= 1:
logger.info('Skipping %s', program_info)
continue
program = program_info.program
logger.info('Targeting %s', program['title'])
seasons = program_info.seasons
files_to_move = []
for season, entries in seasons.items():
season_folder = Entry.get_season_folder(destination, program,
season)
for i, target_entry in enumerate(entries.sorted()):
if target_entry.episode.number is None:
raise RuntimeError(
'You need to attempt sync for this program once '
'before running this migration.')
src_entry = copy.deepcopy(target_entry)
src_entry.episode = Episode(None)
src_entry.episode.number = i + 1
src_dest = os.path.join(
season_folder,
src_entry.get_target_basename(program, season),
)
target_dest = os.path.join(
season_folder,
target_entry.get_target_basename(program, season),
)
if src_dest != target_dest and os.path.isfile(src_dest):
files_to_move.append((src_dest, target_dest))
while files_to_move:
i = 0
src_dest, target_dest = files_to_move.pop(0)
if dryrun:
logger.info('Would move %s to %s', src_dest, target_dest)
elif os.path.isfile(target_dest):
temp_path = next(tempfile._get_candidate_names())
temp_path = os.path.join(
os.path.dirname(target_dest),
temp_path,
)
logger.info(
'%s exists. Moving temporarily to %s and postponing',
target_dest,
temp_path,
)
shutil.move(src_dest, temp_path)
files_to_move.append((temp_path, target_dest))
else:
os.makedirs(os.path.dirname(target_dest), exist_ok=True)
logger.info('Moving %s to %s', src_dest, target_dest)
shutil.move(src_dest, target_dest)
if not dryrun:
program_info.version = 1
program_info.write()
def test_entry_to_dict():
e = Entry(
'some_fn',
'some_url',
datetime.datetime(2017, 6, 14),
'some_etag',
{'id': 'some_episode'},
)
assert e.to_dict() == {
'fn': 'some_fn',
'url': 'some_url',
'date': '2017/06/14',
'etag': 'some_etag',
'episode': {
'id': 'some_episode'
},
}
def test_entry_from_dict():
data = {
'fn': 'some_fn',
'url': 'some_url',
'date': '2017/06/14',
'etag': 'some_etag',
'episode': {
'id': 'some_episode'
},
}
e = Entry.from_dict(data)
assert e.fn == data['fn']
assert e.url == data['url']
assert e.date == datetime.datetime(2017, 6, 14)
assert e.etag == data['etag']
assert e.episode.data == data['episode']
del data['episode']
e = Entry.from_dict(data)
assert e.episode.data == {}
def test_entry_equality():
assert Entry('fdsa', 'fdsa', None,
'etag') == Entry('asdf', 'asdf', datetime.datetime.min,
'etag')
assert Entry('fdsa', 'fdsa', None,
'etag') == Entry('fdsa', 'fdsa', None, 'etag')
assert Entry('fdsa', 'fdsa', None, 'etag') != Entry(
'fdsa', 'fdsa', None, 'not-same-etag')
def create_program_info(location='/tv/Program', seasons=None):
pi = ProgramInfo(location, initialize_empty=True)
pi.program = {'id': 'some-id', 'title': 'Program'}
pi.seasons = {
key: EntrySet([
Entry(
'',
'',
datetime.datetime(2020, 1, 10),
f'e{number}',
episode={'number': number},
) for number in value
])
for key, value in (seasons or {}).items()
}
pi.write()
return pi
def test_choose_best_item():
s = EntrySet()
episode_real = {
'id': 'some_id_from_ruv_api',
'number': 1,
}
episode_generated = {
'number': 2,
}
item1 = Entry('fn1', 'url1', datetime.datetime.min, 'etag1', None)
item2 = Entry('fn2', 'url2', datetime.datetime.min, 'etag2', episode_real)
assert s._choose_best_item(item1, item2).episode.to_dict() == episode_real
assert s._choose_best_item(item2, item1).episode.to_dict() == episode_real
item1.episode = Episode(episode_generated)
assert s._choose_best_item(item1, item2).episode.to_dict() == episode_real
assert s._choose_best_item(item2, item1).episode.to_dict() == episode_real
item2.episode = None
assert (s._choose_best_item(item1,
item2).episode.to_dict() == episode_generated)
assert (s._choose_best_item(item2,
item1).episode.to_dict() == episode_generated)
def get_entry(self, date, fn, episode=None):
cache_key = f'{date.strftime(DATE_FORMAT)}-{fn}'
if not self.cache.has(cache_key):
try:
r = requests.head(
URL_TEMPLATE.format(
date=date.strftime(DATE_FORMAT),
fn=fn,
openclose='opid' if self.prefer_open else 'lokad',
)
)
except Exception as e:
logger.error('Error getting entry: %s', e)
return None
logger.info(
'Checking %s - %s - %s (is_open: %s)'
% (date.strftime(DATE_FORMAT), fn, r.ok, self.prefer_open,)
)
if r.ok:
self.cache.set(
cache_key,
{
'success': True,
'url': r.url,
'etag': r.headers['ETag'],
'checked_at': datetime.datetime.now().strftime(
DATETIME_FORMAT,
),
},
)
else:
self.cache.set(
cache_key,
{
'success': False,
'status_code': r.status_code,
'checked_at': datetime.datetime.now().strftime(
DATETIME_FORMAT,
),
},
)
info = self.cache.get(cache_key)
checked_at = parse_datetime(info['checked_at'])
if info['success']:
return Entry(
fn=fn,
url=info['url'],
date=date,
etag=info['etag'],
episode=episode,
)
elif (
# Don't remove unless we last checked before the show was aired
checked_at <= (date + datetime.timedelta(1))
and
# And we haven't checked this link for over 1 hours
abs((checked_at - datetime.datetime.now()).total_seconds() / 3600)
> 1
and
# And the show should have been aired
date <= (datetime.datetime.now() + datetime.timedelta(1))
):
self.cache.remove(cache_key)
return self.get_entry(date, fn)
def seasons(self):
return {
int(key): EntrySet(Entry.from_dict(entry) for entry in entries)
for key, entries in self._data.items()
if key not in NON_SEASON_FIELDS and key.isdigit()
}
def organize(self):
# TODO: Use ProgramInfo class
logger.info(f'Organizing {self.program["title"]}')
info_fn = os.path.join(
self.destination,
self.program['title'],
)
os.makedirs(info_fn, exist_ok=True)
try:
program_info = ProgramInfo(info_fn)
except FileNotFoundError:
program_info = ProgramInfo(info_fn, initialize_empty=True)
seasons = program_info.seasons
program_info.program = self.program
# seasons = {
# 1: EntrySet({entry, entry, entry}),
# 2: EntrySet({entry, entry, entry}),
# }
# Sort episodes into seasons
for entry in sorted(self.episode_entries,
key=lambda entry: entry.date):
for season in seasons.keys():
if any(
abs((e.date - entry.date).days) < 10
for e in seasons[season]):
seasons[season].add(entry)
break
else:
int_season_numbers = [
int(season) for season in seasons
if isinstance(season, int) or season.isdigit()
]
season = max(int_season_numbers or [0]) + 1
seasons[season] = EntrySet([entry])
# Calculate target paths for entries
for season, entries in seasons.items():
season_folder = Entry.get_season_folder(self.destination,
self.program, season)
os.makedirs(season_folder, exist_ok=True)
for i, entry in enumerate(entries.sorted()):
if not entry.episode.number:
entry.episode.number = EntrySet.find_target_number(
entries, i)
basename = entry.get_target_basename(
self.program,
season,
)
target_path = os.path.join(
season_folder,
basename,
)
entry.set_target_path(target_path)
# Finally, make sure we don't have the same etag multiple times,
# prefer the first one in chronological order
found_etags = []
for season, entries in seasons.items():
for entry in [
entry for entry in entries if entry.etag in found_etags
]:
entries.remove(entry)
found_etags += [entry.etag for entry in entries]
program_info.seasons = seasons
if not settings.dryrun:
program_info.write()
missing_migrations = range(
program_info.version,
PROGRAM_INFO_VERSION,
)
for migration_entry in missing_migrations:
logger.error(
'Missing migration %d. Run `ruv-dl migrate %d`. You can '
'supply `--dryrun` (e.g. `ruv-dl --dryrun migrate ...`) to '
'see what will be done.',
migration_entry + 1,
migration_entry + 1,
)
if missing_migrations:
return []
return [
entry for entry in itertools.chain(*seasons.values())
if not entry.exists_on_disk()
]