def test_get_change_list(self):
items, last_id, _ = updater.get_updates(1, 5, TEST_URL,
'worst[ext=mp4]')
self.assertEqual(len(items), 5)
self.assertEqual(items[0]['ID'], last_id)
test_last_id = items[2]['ID']
self.assertIsNotNone(test_last_id)
items, last_id, _ = updater.get_updates(1, 5, TEST_URL,
'worst[ext=mp4]', test_last_id)
self.assertEqual(len(items), 2)
self.assertEqual(items[0]['ID'], last_id)
def test_get_updates(self):
kinds = [
updater.get_format('video', 'high'),
updater.get_format('video', 'low'),
updater.get_format('audio', 'high'),
updater.get_format('audio', 'low'),
]
for kind in kinds:
with self.subTest(kind):
feed, items, _ = updater.get_updates(1, 1, TEST_URL, kind)
self.assertIsNotNone(feed)
self.assertIsNotNone(items)
def test_last_id(self):
feed, items, last_id = updater.get_updates(1, 1, TEST_URL,
'worstaudio')
self.assertEqual(len(items), 1)
self.assertEqual(items[0]['ID'], last_id)
def test_get_title_issue33(self):
url = 'https://youtube.com/channel/UC9-y-6csu5WGm29I7JiwpnA'
items, _, _ = updater.get_updates(1, 1, url, 'best[ext=mp4]')
for item in items:
self.assertNotEqual('_', item.get('Title'))
def _update(item):
# Unpack fields
feed_id = item['id']
url = item['url']
last_id = item['last_id']
start = int(item['start'])
count = int(item['count'])
fmt = item.get('format', 'video')
quality = item.get('quality', 'high')
ytdl_fmt = updater.get_format(fmt, quality)
# Playlist need special handling
link_type = item.get('link_type')
is_playlist = link_type == 'playlist'
old_episodes = []
if is_playlist:
# Query old episodes in advance for playlist in order to compare the diff
old_episodes = _get_episodes(feed_id)
# Invoke youtube-dl and pull updates
print(
'Updating feed {} (last id: {}, start: {}, count: {}, fmt: {}, type: {})'
.format(feed_id, last_id, start, count, ytdl_fmt, link_type))
new_episodes, new_last_id, dirty = updater.get_updates(
start, count, url, ytdl_fmt, last_id, old_episodes)
if new_last_id is None:
# Sometimes youtube-dl fails to pull updates
print('! New last id is None, retrying...')
new_episodes, new_last_id, dirty = updater.get_updates(
start, count, url, ytdl_fmt, last_id, old_episodes)
if not dirty:
print('No updates found for {}'.format(feed_id))
return
else:
print('Found {} new episode(s) (new last id: {})'.format(
len(new_episodes), new_last_id))
# Get records from DynamoDB and decompress episodes
if is_playlist:
episodes = new_episodes
else:
old_episodes = _get_episodes(feed_id)
episodes = new_episodes + old_episodes # Prepand the new episodes
if is_playlist:
del episodes[count:]
# Compress episodes and submit update query
data = bytes(json.dumps(episodes), 'utf-8')
compressed = gzip.compress(data)
print('Sending new compressed data of size: {} bytes ({} episodes)'.format(
len(compressed), len(episodes)))
feeds_table.update_item(
Key={
'HashID': feed_id,
},
UpdateExpression=
'SET #episodesData = :data, #last_id = :last_id, #updated_at = :now REMOVE #episodes',
ExpressionAttributeNames={
'#episodesData': 'EpisodesData',
'#episodes': 'Episodes',
'#last_id': 'LastID',
'#updated_at': 'UpdatedAt',
},
ExpressionAttributeValues={
':now': int(datetime.datetime.utcnow().timestamp()),
':last_id': new_last_id,
':data': compressed,
},
)