def test_delete__episodes_in_another_podcast__ok(
self, client, episode_data, user, mocked_s3, dbs
):
dbs = dbs
podcast_1 = await_(Podcast.async_create(dbs, **get_podcast_data(created_by_id=user.id)))
episode_data["status"] = Episode.Status.PUBLISHED
episode_data["podcast_id"] = podcast_1.id
episode_1 = await_(Episode.async_create(dbs, **episode_data))
episode_1_1 = await_(Episode.async_create(dbs, **get_episode_data(podcast_1, "published")))
podcast_2 = await_(Podcast.async_create(dbs, **get_podcast_data()))
episode_data["status"] = Episode.Status.PUBLISHED
episode_data["podcast_id"] = podcast_2.id
# creating episode with same `source_id` in another podcast
episode_2 = await_(Episode.async_create(dbs, **episode_data))
await_(dbs.commit())
client.login(user)
url = self.url.format(id=podcast_1.id)
response = client.delete(url)
assert response.status_code == 200
assert await_(Podcast.async_get(dbs, id=podcast_1.id)) is None
assert await_(Episode.async_get(dbs, id=episode_1.id)) is None
assert await_(Podcast.async_get(dbs, id=podcast_2.id)) is not None
assert await_(Episode.async_get(dbs, id=episode_2.id)) is not None
mocked_s3.delete_files_async.assert_called_with([episode_1_1.file_name])
def test_generate__several_podcasts__ok(self, user, mocked_s3, dbs):
podcast_1 = await_(Podcast.async_create(dbs, **get_podcast_data()))
podcast_2 = await_(Podcast.async_create(dbs, **get_podcast_data()))
await_(dbs.commit())
generate_rss_task = tasks.GenerateRSSTask(db_session=dbs)
result_code = await_(generate_rss_task.run(podcast_1.id, podcast_2.id))
assert result_code == FinishCode.OK
for podcast in [podcast_1, podcast_2]:
expected_file_path = mocked_s3.tmp_upload_dir / f"{podcast.publish_id}.xml"
assert os.path.exists(
expected_file_path
), f"File {expected_file_path} didn't uploaded"
def test_filter_by_status__ok(self, client, user, episode_data,
mocked_redis, dbs):
podcast_1 = await_(
Podcast.async_create(dbs,
**get_podcast_data(created_by_id=user.id)))
podcast_2 = await_(
Podcast.async_create(dbs,
**get_podcast_data(created_by_id=user.id)))
episode_data["created_by_id"] = user.id
p1_episode_new = create_episode(dbs, episode_data, podcast_1,
STATUS.NEW, MB_1)
p1_episode_down = create_episode(dbs, episode_data, podcast_1,
STATUS.DOWNLOADING, MB_2)
p2_episode_down = create_episode(dbs, episode_data, podcast_2,
STATUS.DOWNLOADING, MB_4)
# p2_episode_new
create_episode(dbs, episode_data, podcast_2, STATUS.NEW, MB_1)
mocked_redis.async_get_many.return_value = mocked_redis.async_return({
p1_episode_new.file_name.partition(".")[0]: {
"status": EpisodeStatus.DL_PENDING,
"processed_bytes": 0,
"total_bytes": MB_1,
},
p1_episode_down.file_name.partition(".")[0]: {
"status": EpisodeStatus.DL_EPISODE_DOWNLOADING,
"processed_bytes": MB_1,
"total_bytes": MB_2,
},
p2_episode_down.file_name.partition(".")[0]: {
"status": EpisodeStatus.DL_EPISODE_DOWNLOADING,
"processed_bytes": MB_1,
"total_bytes": MB_4,
},
})
client.login(user)
response = client.get(self.url)
response_data = self.assert_ok_response(response)
assert response_data == [
_progress(podcast_2,
p2_episode_down,
current_size=MB_1,
completed=25.0),
_progress(podcast_1,
p1_episode_down,
current_size=MB_1,
completed=50.0),
]
def test_delete__same_episode_exists__ok(
self,
client,
podcast,
episode_data,
mocked_s3,
same_episode_status,
delete_called,
dbs,
):
source_id = get_video_id()
user_1 = create_user(dbs)
user_2 = create_user(dbs)
podcast_1 = await_(
Podcast.async_create(dbs,
db_commit=True,
**get_podcast_data(created_by_id=user_1.id)))
podcast_2 = await_(
Podcast.async_create(dbs,
db_commit=True,
**get_podcast_data(created_by_id=user_2.id)))
episode_data["created_by_id"] = user_1.id
_ = create_episode(dbs,
episode_data,
podcast_1,
status=same_episode_status,
source_id=source_id)
episode_data["created_by_id"] = user_2.id
episode_2 = create_episode(dbs,
episode_data,
podcast_2,
status=Episode.Status.NEW,
source_id=source_id)
url = self.url.format(id=episode_2.id)
client.login(user_2)
response = client.delete(url)
assert response.status_code == 204, f"Delete API is not available: {response.text}"
assert await_(Episode.async_get(dbs, id=episode_2.id)) is None
if delete_called:
mocked_s3.delete_files_async.assert_called_with(
[episode_2.file_name])
else:
assert not mocked_s3.delete_files_async.called
def test_get_list__filter_by_created_by__ok(self, client, dbs):
user_1 = create_user(dbs)
user_2 = create_user(dbs)
podcast_data = get_podcast_data()
podcast_data["created_by_id"] = user_1.id
await_(Podcast.async_create(dbs, db_commit=True, **podcast_data))
podcast_data = get_podcast_data()
podcast_data["created_by_id"] = user_2.id
podcast_2 = await_(Podcast.async_create(dbs, db_commit=True, **podcast_data))
client.login(user_2)
response = client.get(self.url)
response_data = self.assert_ok_response(response)
assert response_data == [_podcast(podcast_2)]
def test_create__same_episode_in_other_podcast__ok(
self, podcast, episode, user, mocked_youtube, dbs
):
mocked_youtube.extract_info.return_value = {
"id": episode.source_id,
"title": "Updated title",
"description": "Updated description",
"webpage_url": "https://new.watch.site/updated/",
"thumbnail": "https://link.to.image/updated/",
"uploader": "Updated author",
"duration": 123,
}
new_podcast = await_(Podcast.async_create(dbs, db_commit=True, **get_podcast_data()))
episode_creator = EpisodeCreator(
dbs,
podcast_id=new_podcast.id,
source_url=episode.watch_url,
user_id=user.id,
)
new_episode: Episode = await_(episode_creator.create())
assert episode is not None
assert new_episode.id != episode.id
assert new_episode.source_id == episode.source_id
assert new_episode.watch_url == "https://new.watch.site/updated/"
assert new_episode.title == "Updated title"
assert new_episode.description == "Updated description"
assert new_episode.image_url == "https://link.to.image/updated/"
assert new_episode.author == "Updated author"
assert new_episode.length == 123
def generate_rss(podcast_id: int) -> Optional[str]:
""" Allows to download and recreate specific rss (by requested podcast.publish_id) """
podcast = Podcast.get_by_id(podcast_id)
logger.info("START rss generation for %s", podcast)
src_file = podcast_utils.render_rss_to_file(podcast_id)
filename = os.path.basename(src_file)
storage = StorageS3()
result_url = storage.upload_file(src_file,
filename,
remote_path=settings.S3_BUCKET_RSS_PATH)
if not result_url:
logger.error("Couldn't upload RSS file to storage. SKIP")
exit(1)
podcast.rss_link = result_url
podcast.save()
logger.info("RSS file uploaded, podcast record updated")
if not settings.TEST_MODE:
logger.info("Removing file [%s]", src_file)
podcast_utils.delete_file(filepath=src_file)
src_file = None
logger.info("FINISH generation")
return src_file
def render_rss_to_file(podcast_id: int) -> str:
""" Generate rss for Podcast and Episodes marked as "published" """
logger.info(f"Podcast #{podcast_id}: RSS generation has been started.")
podcast = Podcast.get_by_id(podcast_id)
# noinspection PyComparisonWithNone
episodes = podcast.get_episodes(podcast.created_by).where(
Episode.status == Episode.STATUS_PUBLISHED,
Episode.published_at != None, # noqa: E711
)
context = {"episodes": episodes, "settings": settings}
with open(os.path.join(settings.TEMPLATE_PATH, "rss",
"feed_template.xml")) as fh:
template = Template(fh.read())
rss_filename = os.path.join(settings.TMP_RSS_PATH,
f"{podcast.publish_id}.xml")
logger.info(
f"Podcast #{podcast.publish_id}: Generation new file rss [{rss_filename}]"
)
with open(rss_filename, "w") as fh:
result_rss = template.render(podcast=podcast, **context)
fh.write(result_rss)
logger.info(f"Podcast #{podcast_id}: RSS generation has been finished.")
return rss_filename
def test_get_list__check_episodes_count__ok(self, client, user, loop, dbs):
dbs = dbs
podcast_1 = await_(Podcast.async_create(dbs, **get_podcast_data(created_by_id=user.id)))
create_episode(dbs, get_episode_data(), podcast_1)
create_episode(dbs, get_episode_data(), podcast_1)
podcast_2 = await_(Podcast.async_create(dbs, **get_podcast_data(created_by_id=user.id)))
create_episode(dbs, get_episode_data(), podcast_2)
client.login(user)
response = client.get(self.url)
response_data = self.assert_ok_response(response)
expected_episodes_counts = {podcast_1.id: 2, podcast_2.id: 1}
actual_episodes_counts = {
podcast["id"]: podcast["episodes_count"] for podcast in response_data
}
assert expected_episodes_counts == actual_episodes_counts
def test_generate__upload_failed__fail(self, podcast, mocked_s3, dbs):
mocked_s3.upload_file.side_effect = lambda *_, **__: ""
generate_rss_task = tasks.GenerateRSSTask(db_session=dbs)
result_code = await_(generate_rss_task.run(podcast.id))
assert result_code == FinishCode.ERROR
podcast_1 = await_(Podcast.async_get(dbs, id=podcast.id))
assert podcast_1.rss_link is None
def test_delete__ok(self, client, podcast, user, mocked_s3, dbs):
client.login(user)
url = self.url.format(id=podcast.id)
response = client.delete(url)
assert response.status_code == 200
assert await_(Podcast.async_get(dbs, id=podcast.id)) is None
mocked_s3.delete_files_async.assert_called_with(
[f"{podcast.publish_id}.xml"], remote_path=settings.S3_BUCKET_RSS_PATH
)
def test_generate__single_podcast__ok(self, user, mocked_s3, dbs):
podcast_1: Podcast = await_(
Podcast.async_create(dbs, **get_podcast_data()))
podcast_2: Podcast = await_(
Podcast.async_create(dbs, **get_podcast_data()))
episode_data = get_episode_data(podcast_1, creator=user)
episode_data["status"] = Episode.Status.NEW
episode_new = await_(Episode.async_create(dbs, **episode_data))
episode_data = get_episode_data(podcast_1, creator=user)
episode_data["status"] = Episode.Status.DOWNLOADING
episode_downloading = await_(Episode.async_create(dbs, **episode_data))
episode_data = get_episode_data(podcast_1, creator=user)
episode_data["status"] = Episode.Status.PUBLISHED
episode_data["published_at"] = datetime.now()
episode_published = await_(Episode.async_create(dbs, **episode_data))
episode_data = get_episode_data(podcast_2, creator=user)
episode_data["status"] = Episode.Status.PUBLISHED
episode_podcast_2 = await_(Episode.async_create(dbs, **episode_data))
await_(dbs.commit())
expected_file_path = mocked_s3.tmp_upload_dir / f"{podcast_1.publish_id}.xml"
generate_rss_task = tasks.GenerateRSSTask(db_session=dbs)
result_code = await_(generate_rss_task.run(podcast_1.id))
assert result_code == FinishCode.OK
assert os.path.exists(
expected_file_path), f"File {expected_file_path} didn't uploaded"
with open(expected_file_path) as file:
generated_rss_content = file.read()
assert episode_published.title in generated_rss_content
assert episode_published.description in generated_rss_content
assert episode_published.file_name in generated_rss_content
for episode in [episode_new, episode_downloading, episode_podcast_2]:
assert episode.source_id not in generated_rss_content, f"{episode} in RSS {podcast_1}"
podcast_1 = await_(Podcast.async_get(dbs, id=podcast_1.id))
assert podcast_1.rss_link == str(expected_file_path)
async def post(self, request):
cleaned_data = await self._validate(request)
podcast = await Podcast.async_create(
db_session=request.db_session,
name=cleaned_data["name"],
publish_id=Podcast.generate_publish_id(),
description=cleaned_data["description"],
created_by_id=request.user.id,
)
return self._response(podcast, status_code=status.HTTP_201_CREATED)
async def _get_object(self) -> Podcast:
try:
podcast = await self.request.app.objects.get(
Podcast.select().where(
Podcast.created_by_id == self.user.id).order_by(
Podcast.created_at.desc()))
except peewee.DoesNotExist:
podcast = await Podcast.create_first_podcast(
self.request.app.objects, self.user.id)
return podcast
def test_create__ok(self, client, user, podcast_data, dbs):
podcast_data = {
"name": podcast_data["name"],
"description": podcast_data["description"],
}
client.login(user)
response = client.post(self.url, json=podcast_data)
response_data = self.assert_ok_response(response, status_code=201)
podcast = await_(Podcast.async_get(dbs, id=response_data["id"]))
assert podcast is not None
assert response_data == _podcast(podcast)
async def test_podcasts__create__ok(client, db_objects, podcast, urls):
request_data = {"name": "test name", "description": "test description"}
response = await client.post(urls.podcasts_list,
data=request_data,
allow_redirects=False)
assert response.status == 302
created_podcast = await db_objects.get(Podcast.select().order_by(
Podcast.created_at.desc()))
assert response.headers["Location"] == f"/podcasts/{created_podcast.id}/"
assert created_podcast.name == "test name"
assert created_podcast.description == "test description"
def test_filter_by_user__ok(self, client, episode_data, mocked_redis, dbs):
user_1 = create_user(dbs)
user_2 = create_user(dbs)
podcast_1 = await_(
Podcast.async_create(dbs,
**get_podcast_data(created_by_id=user_1.id)))
podcast_2 = await_(
Podcast.async_create(dbs,
**get_podcast_data(created_by_id=user_2.id)))
ep_data_1 = get_episode_data(creator=user_1)
ep_data_2 = get_episode_data(creator=user_2)
p1_episode_down = create_episode(dbs, ep_data_1, podcast_1,
STATUS.DOWNLOADING, MB_2)
p2_episode_down = create_episode(dbs, ep_data_2, podcast_2,
STATUS.DOWNLOADING, MB_4)
await_(dbs.commit())
mocked_redis.async_get_many.return_value = mocked_redis.async_return({
p1_episode_down.file_name.partition(".")[0]: {
"status": EpisodeStatus.DL_EPISODE_DOWNLOADING,
"processed_bytes": MB_1,
"total_bytes": MB_2,
},
p2_episode_down.file_name.partition(".")[0]: {
"status": EpisodeStatus.DL_EPISODE_DOWNLOADING,
"processed_bytes": MB_1,
"total_bytes": MB_4,
},
})
client.login(user_1)
response = client.get(self.url)
response_data = self.assert_ok_response(response)
assert response_data == [
_progress(podcast_1,
p1_episode_down,
current_size=MB_1,
completed=50.0),
]
async def post(self):
cleaned_data = await self._validate()
podcast = await self.request.app.objects.create(
Podcast,
**dict(
publish_id=Podcast.generate_publish_id(),
name=cleaned_data["name"],
description=cleaned_data.get("description", ""),
created_by_id=self.user.id,
),
)
return redirect(self.request, "podcast_details", podcast_id=podcast.id)
def test_delete__episodes_deleted_too__ok(self, client, podcast, user, mocked_s3, dbs):
episode_1 = await_(Episode.async_create(dbs, **get_episode_data(podcast)))
episode_2 = await_(Episode.async_create(dbs, **get_episode_data(podcast, "published")))
await_(dbs.commit())
client.login(user)
url = self.url.format(id=podcast.id)
response = client.delete(url)
assert response.status_code == 200
assert await_(Podcast.async_get(dbs, id=podcast.id)) is None
assert await_(Episode.async_get(dbs, id=episode_1.id)) is None
assert await_(Episode.async_get(dbs, id=episode_2.id)) is None
mocked_s3.delete_files_async.assert_called_with([episode_2.file_name])
def test_sign_up__ok(self, client, user_invite, dbs):
request_data = self._sign_up_data(user_invite)
response = client.post(self.url, json=request_data)
response_data = self.assert_ok_response(response, status_code=201)
user = await_(User.async_get(dbs, email=request_data["email"]))
assert user is not None, f"User wasn't created with {request_data=}"
assert_tokens(response_data, user)
await_(dbs.refresh(user_invite))
assert user_invite.user_id == user.id
assert user_invite.is_applied
assert await_(Podcast.async_get(dbs,
created_by_id=user.id)) is not None
def test_download_episode__file_correct__ignore(
self,
episode_data,
podcast_data,
mocked_youtube,
mocked_ffmpeg,
mocked_s3,
mocked_generate_rss_task,
dbs,
):
podcast_1 = await_(Podcast.async_create(dbs, **get_podcast_data()))
podcast_2 = await_(Podcast.async_create(dbs, **get_podcast_data()))
episode_data.update({
"status": "published",
"source_id": mocked_youtube.video_id,
"watch_url": mocked_youtube.watch_url,
"file_size": 1024,
"podcast_id": podcast_1.id,
})
await_(Episode.async_create(dbs, **episode_data))
episode_data["status"] = "new"
episode_data["podcast_id"] = podcast_2.id
episode_2 = await_(Episode.async_create(dbs, **episode_data))
await_(dbs.commit())
mocked_s3.get_file_size.return_value = episode_2.file_size
result = await_(DownloadEpisodeTask(db_session=dbs).run(episode_2.id))
await_(dbs.refresh(episode_2))
mocked_generate_rss_task.run.assert_called_with(
podcast_1.id, podcast_2.id)
assert result == FinishCode.SKIP
assert not mocked_youtube.download.called
assert not mocked_ffmpeg.called
assert episode_2.status == Episode.Status.PUBLISHED
assert episode_2.published_at == episode_2.created_at
def test_create__same_episode__extract_failed__ok(
self, podcast, episode, user, mocked_youtube, dbs
):
mocked_youtube.extract_info.side_effect = ExtractorError("Something went wrong here")
new_podcast = await_(Podcast.async_create(dbs, **get_podcast_data()))
episode_creator = EpisodeCreator(
dbs,
podcast_id=new_podcast.id,
source_url=episode.watch_url,
user_id=user.id,
)
new_episode: Episode = await_(episode_creator.create())
assert episode is not None
assert new_episode.id != episode.id
assert new_episode.source_id == episode.source_id
assert new_episode.watch_url == episode.watch_url
def podcast(podcast_data, user, loop, dbs):
podcast_data["created_by_id"] = user.id
podcast = loop.run_until_complete(Podcast.async_create(
dbs, **podcast_data))
loop.run_until_complete(dbs.commit())
return podcast
def regenerate_rss():
podcasts = list(Podcast.select())
for podcast in podcasts:
generate_rss(podcast.id)
def podcast(db_objects, user, podcast_data):
with db_objects.allow_sync():
yield Podcast.create(**podcast_data)
def teardown_module(module):
print(f"module teardown {module}")
Episode.truncate_table()
Podcast.truncate_table()
User.truncate_table()
