def create_channel(data):
with get_db_context() as (engine, session):
# Verify that the URL/Name/Link aren't taken
try:
check_for_channel_conflicts(
session,
url=data.get('url'),
name=data['name'],
link=sanitize_link(data['name']),
directory=str(data['directory']),
)
except APIError as e:
raise ValidationError from e
channel = Channel(
name=data['name'],
url=data.get('url'),
match_regex=data.get('match_regex'),
link=sanitize_link(data['name']),
directory=str(data['directory']),
download_frequency=data.get('download_frequency', DEFAULT_DOWNLOAD_FREQUENCY),
)
session.add(channel)
session.commit()
session.flush()
session.refresh(channel)
# Save these changes to the local.yaml as well
channels = get_channels_config(session)
save_settings_config(channels)
return channel.dict()
def wrapped(*args, **kwargs):
# Dummy queue and reporter to receive messages.
q = Queue()
reporter = ProgressReporter(q, 2)
# Convert the channel/video structure to a file structure for the test.
file_structure = []
for channel, paths in structure.items():
for path in paths:
file_structure.append(f'{channel}/{path}')
file_structure.append(f'{channel}/')
with build_test_directories(file_structure) as tempdir:
args, kwargs = insert_parameter(func, 'tempdir', tempdir, args,
kwargs)
with get_db_context(commit=True) as (engine, session):
for channel in structure:
channel = Channel(directory=str(tempdir / channel),
name=channel,
link=channel)
session.add(channel)
session.flush()
session.refresh(channel)
refresh_channel_videos(channel, reporter)
return func(*args, **kwargs)
def test_get_channel_videos(self):
with get_db_context(commit=True) as (engine, session):
channel1 = Channel(name='Foo', link='foo')
channel2 = Channel(name='Bar', link='bar')
session.add(channel1)
session.add(channel2)
session.flush()
session.refresh(channel1)
session.refresh(channel2)
# Channels don't have videos yet
d = dict(channel_link=channel1.link)
request, response = api_app.test_client.post(f'/api/videos/search',
data=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 0
with get_db_context(commit=True) as (engine, session):
vid1 = Video(title='vid1',
channel_id=channel2.id,
video_path='foo')
vid2 = Video(title='vid2',
channel_id=channel1.id,
video_path='foo')
session.add(vid1)
session.add(vid2)
# Videos are gotten by their respective channels
request, response = api_app.test_client.post(f'/api/videos/search',
data=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 1
assert response.json['totals']['videos'] == 1
self.assertDictContains(
response.json['videos'][0],
dict(id=2, title='vid2', channel_id=channel1.id))
d = dict(channel_link=channel2.link)
request, response = api_app.test_client.post(f'/api/videos/search',
data=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 1
self.assertDictContains(
response.json['videos'][0],
dict(id=1, title='vid1', channel_id=channel2.id))
def test_get_channel_videos_pagination(self):
with get_db_context(commit=True) as (engine, session):
channel1 = Channel(name='Foo', link='foo')
session.add(channel1)
session.flush()
session.refresh(channel1)
for i in range(50):
session.add(
Video(title=f'Foo.Video{i}',
channel_id=channel1.id,
video_path='foo'))
channel2 = Channel(name='Bar', link='bar')
session.add(channel2)
session.flush()
session.refresh(channel2)
session.add(
Video(title='vid2', channel_id=channel2.id, video_path='foo'))
# Get first, second, third, and empty pages of videos.
tests = [
# (offset, video_count)
(0, 20),
(20, 20),
(40, 10),
(50, 0),
]
last_ids = []
for offset, video_count in tests:
d = dict(channel_link=channel1.link, order_by='id', offset=offset)
_, response = api_app.test_client.post(f'/api/videos/search',
data=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == video_count
current_ids = [i['id'] for i in response.json['videos']]
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
last_ids = current_ids
def test_get_video(self):
"""
Test that you get can information about a video. Test that video file can be gotten.
"""
def raise_unknown_file(_):
raise UnknownFile()
with get_db_context(commit=True) as (engine, session), \
mock.patch('api.videos.common.get_absolute_video_info_json', raise_unknown_file):
channel = Channel(name='Foo', link='foo')
session.add(channel)
session.flush()
session.refresh(channel)
now = datetime.utcnow()
session.add(
Video(title='vid1', channel_id=channel.id, upload_date=now))
session.add(
Video(title='vid2',
channel_id=channel.id,
upload_date=now + timedelta(seconds=1)))
# Test that a 404 is returned when no video exists
_, response = api_app.test_client.get('/api/videos/video/10')
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
assert response.json == {
'code': 1,
'api_error': 'The video could not be found.',
'message': ''
}
# Get the video info we inserted
_, response = api_app.test_client.get('/api/videos/video/1')
assert response.status_code == HTTPStatus.OK, response.json
self.assertDictContains(response.json['video'], {'title': 'vid1'})
# The next video is included.
self.assertIsNone(response.json['prev'])
self.assertDictContains(response.json['next'], {'title': 'vid2'})
def test_get_video_prev_next(self):
"""
Test that the previous and next videos will be retrieved when fetching a video.
"""
with get_db_context(commit=True) as (engine, session):
for _ in range(4):
session.add(Channel(link=str(uuid4())))
channel1, channel2, channel3, channel4 = session.query(
Channel).all()
now = datetime.utcnow()
second = timedelta(seconds=1)
# The upload_date decides the order of the prev/next videos.
session.add(
Video(title=f'vid1', channel_id=channel1.id, upload_date=now))
session.add(
Video(title=f'vid2',
channel_id=channel1.id,
upload_date=now + second))
session.add(
Video(title=f'vid3',
channel_id=channel2.id,
upload_date=now + (second * 4)))
session.add(
Video(title=f'vid4',
channel_id=channel1.id,
upload_date=now + (second * 3)))
session.add(
Video(title=f'vid5',
channel_id=channel2.id,
upload_date=now + (second * 2)))
session.add(
Video(title=f'vid6',
channel_id=channel2.id,
upload_date=now + (second * 5)))
session.add(Video(title=f'vid7', channel_id=channel1.id))
session.add(
Video(title=f'vid8',
channel_id=channel2.id,
upload_date=now + (second * 7)))
session.add(
Video(title=f'vid9',
channel_id=channel3.id,
upload_date=now + (second * 8)))
session.add(Video(title=f'vid10', channel_id=channel4.id))
session.add(Video(title=f'vid11', channel_id=channel4.id))
session.commit()
tests = [
# Channel 1's videos were inserted in upload_date order.
(1, (None, 'vid2')),
(2, ('vid1', 'vid4')),
(4,
('vid2',
None)), # 7 has no upload_date, so it doesn't come after 4.
(7, (None, None)
), # 7 has no upload_date, so we don't know the order of it.
# Channel 3 has only one video.
(9, (None, None)),
# Channel 2 was inserted out of order.
(5, (None, 'vid3')),
(3, ('vid5', 'vid6')),
(8, ('vid6', None)),
# Channel 4's videos have no upload date, so we don't know what is previous/next.
(10, (None, None)),
]
for id_, (prev_title, next_title) in tests:
video = session.query(Video).filter_by(id=id_).one()
prev_video, next_video = get_surrounding_videos(
session, id_, video.channel_id)
if prev_title is None:
self.assertIsNone(prev_video)
else:
self.assertDictContains(prev_video, {'title': prev_title})
if next_title is None:
self.assertIsNone(next_video)
else:
self.assertDictContains(next_video, {'title': next_title})
def test_refresh_videos(self):
# There should be no messages until a refresh is called.
pytest.raises(Empty, refresh_queue.get_nowait)
# Setup a fake channel directory.
with get_db_context() as (engine, session), \
tempfile.TemporaryDirectory() as channel_dir:
channel_path = pathlib.Path(channel_dir)
# Files in subdirectories should be found and handled properly.
subdir = channel_path / 'subdir'
subdir.mkdir()
# These are the types of files that will be found first.
vid1 = pathlib.Path(subdir /
'channel name_20000101_abcdefghijk_title.mp4')
vid1.touch()
vid2 = pathlib.Path(channel_path /
'channel name_20000102_bcdefghijkl_title.webm')
vid2.touch()
# This video is named the same as vid1, except for the file extension. Its possible that this was
# downloaded later, or maybe the old video format has fallen out of favor. WROLPi should ignore this
# duplicate file.
vid1_alt = pathlib.Path(
subdir / 'channel name_20000101_abcdefghijk_title.webm')
vid1_alt.touch()
# These files are associated with the video files above, and should be found "near" them.
poster1 = pathlib.Path(
subdir / 'channel name_20000101_abcdefghijk_title.jpg')
poster1.touch()
poster2 = pathlib.Path(
channel_path / 'channel name_20000102_bcdefghijkl_title.jpg')
poster2.touch()
# Create a channel, associate videos with it.
channel = Channel(directory=channel_dir, link='foo', name='foo')
session.add(channel)
session.flush()
session.refresh(channel)
video1 = upsert_video(session, vid1, channel)
video2 = upsert_video(session, vid2, channel)
session.commit()
self.assertEqual({i.video_path
for i in channel.videos},
{'subdir/' + vid1.name, vid2.name})
# Poster files were found.
self.assertEqual(video1.poster_path, 'subdir/' + poster1.name)
self.assertEqual(video2.poster_path, poster2.name)
# Add a bogus file, this should be removed during the refresh
self.assertNotIn('foo', {i.video_path for i in channel.videos})
session.add(Video(video_path='foo', channel_id=channel.id))
session.flush()
session.refresh(channel)
self.assertIn('foo', {i.video_path for i in channel.videos})
self.assertEqual(len(channel.videos), 3)
# Add a video that isn't in the DB, it should be found and any meta files associated with it
vid3 = pathlib.Path(channel_path /
'channel name_20000103_cdefghijklm_title.flv')
vid3.touch()
description3 = pathlib.Path(
channel_path /
'channel name_20000103_cdefghijklm_title.description')
description3.touch()
# An orphan meta-file should be ignored. This shouldn't show up anywhere. But, it shouldn't be deleted.
poster3 = pathlib.Path(
channel_path / 'channel name_20000104_defghijklmn_title.jpg')
poster3.touch()
# Finally, call the refresh. Again, it should remove the "foo" video, then discover this 3rd video
# file and it's description.
api_app.test_client.post('/api/videos:refresh')
# Bogus file was removed
self.assertNotIn('foo', {i.video_path for i in channel.videos})
# Final channel video list we built
expected = {
('subdir/' + vid1.name, 'subdir/' + poster1.name,
None), # in a subdirectory, no description
(vid2.name, poster2.name, None), # no description
(vid3.name, None, description3.name), # no poster
}
self.assertEqual(
{(i.video_path, i.poster_path, i.description_path)
for i in channel.videos}, expected)
assert poster3.is_file(), 'Orphan jpg file was deleted!'
# During the refresh process, messages are pushed to a queue, make sure there are messages there
messages = get_all_messages_in_queue(refresh_queue)
assert 'refresh-started' in [i.get('code') for i in messages]