def test_delete__episodes_in_another_podcast__ok( self, client, episode_data, user, mocked_s3, dbs ): dbs = dbs podcast_1 = await_(Podcast.async_create(dbs, **get_podcast_data(created_by_id=user.id))) episode_data["status"] = Episode.Status.PUBLISHED episode_data["podcast_id"] = podcast_1.id episode_1 = await_(Episode.async_create(dbs, **episode_data)) episode_1_1 = await_(Episode.async_create(dbs, **get_episode_data(podcast_1, "published"))) podcast_2 = await_(Podcast.async_create(dbs, **get_podcast_data())) episode_data["status"] = Episode.Status.PUBLISHED episode_data["podcast_id"] = podcast_2.id # creating episode with same `source_id` in another podcast episode_2 = await_(Episode.async_create(dbs, **episode_data)) await_(dbs.commit()) client.login(user) url = self.url.format(id=podcast_1.id) response = client.delete(url) assert response.status_code == 200 assert await_(Podcast.async_get(dbs, id=podcast_1.id)) is None assert await_(Episode.async_get(dbs, id=episode_1.id)) is None assert await_(Podcast.async_get(dbs, id=podcast_2.id)) is not None assert await_(Episode.async_get(dbs, id=episode_2.id)) is not None mocked_s3.delete_files_async.assert_called_with([episode_1_1.file_name])
def test_generate__several_podcasts__ok(self, user, mocked_s3, dbs): podcast_1 = await_(Podcast.async_create(dbs, **get_podcast_data())) podcast_2 = await_(Podcast.async_create(dbs, **get_podcast_data())) await_(dbs.commit()) generate_rss_task = tasks.GenerateRSSTask(db_session=dbs) result_code = await_(generate_rss_task.run(podcast_1.id, podcast_2.id)) assert result_code == FinishCode.OK for podcast in [podcast_1, podcast_2]: expected_file_path = mocked_s3.tmp_upload_dir / f"{podcast.publish_id}.xml" assert os.path.exists( expected_file_path ), f"File {expected_file_path} didn't uploaded"
def test_filter_by_status__ok(self, client, user, episode_data, mocked_redis, dbs): podcast_1 = await_( Podcast.async_create(dbs, **get_podcast_data(created_by_id=user.id))) podcast_2 = await_( Podcast.async_create(dbs, **get_podcast_data(created_by_id=user.id))) episode_data["created_by_id"] = user.id p1_episode_new = create_episode(dbs, episode_data, podcast_1, STATUS.NEW, MB_1) p1_episode_down = create_episode(dbs, episode_data, podcast_1, STATUS.DOWNLOADING, MB_2) p2_episode_down = create_episode(dbs, episode_data, podcast_2, STATUS.DOWNLOADING, MB_4) # p2_episode_new create_episode(dbs, episode_data, podcast_2, STATUS.NEW, MB_1) mocked_redis.async_get_many.return_value = mocked_redis.async_return({ p1_episode_new.file_name.partition(".")[0]: { "status": EpisodeStatus.DL_PENDING, "processed_bytes": 0, "total_bytes": MB_1, }, p1_episode_down.file_name.partition(".")[0]: { "status": EpisodeStatus.DL_EPISODE_DOWNLOADING, "processed_bytes": MB_1, "total_bytes": MB_2, }, p2_episode_down.file_name.partition(".")[0]: { "status": EpisodeStatus.DL_EPISODE_DOWNLOADING, "processed_bytes": MB_1, "total_bytes": MB_4, }, }) client.login(user) response = client.get(self.url) response_data = self.assert_ok_response(response) assert response_data == [ _progress(podcast_2, p2_episode_down, current_size=MB_1, completed=25.0), _progress(podcast_1, p1_episode_down, current_size=MB_1, completed=50.0), ]
def test_delete__same_episode_exists__ok( self, client, podcast, episode_data, mocked_s3, same_episode_status, delete_called, dbs, ): source_id = get_video_id() user_1 = create_user(dbs) user_2 = create_user(dbs) podcast_1 = await_( Podcast.async_create(dbs, db_commit=True, **get_podcast_data(created_by_id=user_1.id))) podcast_2 = await_( Podcast.async_create(dbs, db_commit=True, **get_podcast_data(created_by_id=user_2.id))) episode_data["created_by_id"] = user_1.id _ = create_episode(dbs, episode_data, podcast_1, status=same_episode_status, source_id=source_id) episode_data["created_by_id"] = user_2.id episode_2 = create_episode(dbs, episode_data, podcast_2, status=Episode.Status.NEW, source_id=source_id) url = self.url.format(id=episode_2.id) client.login(user_2) response = client.delete(url) assert response.status_code == 204, f"Delete API is not available: {response.text}" assert await_(Episode.async_get(dbs, id=episode_2.id)) is None if delete_called: mocked_s3.delete_files_async.assert_called_with( [episode_2.file_name]) else: assert not mocked_s3.delete_files_async.called
def test_get_list__filter_by_created_by__ok(self, client, dbs): user_1 = create_user(dbs) user_2 = create_user(dbs) podcast_data = get_podcast_data() podcast_data["created_by_id"] = user_1.id await_(Podcast.async_create(dbs, db_commit=True, **podcast_data)) podcast_data = get_podcast_data() podcast_data["created_by_id"] = user_2.id podcast_2 = await_(Podcast.async_create(dbs, db_commit=True, **podcast_data)) client.login(user_2) response = client.get(self.url) response_data = self.assert_ok_response(response) assert response_data == [_podcast(podcast_2)]
def test_create__same_episode_in_other_podcast__ok( self, podcast, episode, user, mocked_youtube, dbs ): mocked_youtube.extract_info.return_value = { "id": episode.source_id, "title": "Updated title", "description": "Updated description", "webpage_url": "https://new.watch.site/updated/", "thumbnail": "https://link.to.image/updated/", "uploader": "Updated author", "duration": 123, } new_podcast = await_(Podcast.async_create(dbs, db_commit=True, **get_podcast_data())) episode_creator = EpisodeCreator( dbs, podcast_id=new_podcast.id, source_url=episode.watch_url, user_id=user.id, ) new_episode: Episode = await_(episode_creator.create()) assert episode is not None assert new_episode.id != episode.id assert new_episode.source_id == episode.source_id assert new_episode.watch_url == "https://new.watch.site/updated/" assert new_episode.title == "Updated title" assert new_episode.description == "Updated description" assert new_episode.image_url == "https://link.to.image/updated/" assert new_episode.author == "Updated author" assert new_episode.length == 123
def generate_rss(podcast_id: int) -> Optional[str]: """ Allows to download and recreate specific rss (by requested podcast.publish_id) """ podcast = Podcast.get_by_id(podcast_id) logger.info("START rss generation for %s", podcast) src_file = podcast_utils.render_rss_to_file(podcast_id) filename = os.path.basename(src_file) storage = StorageS3() result_url = storage.upload_file(src_file, filename, remote_path=settings.S3_BUCKET_RSS_PATH) if not result_url: logger.error("Couldn't upload RSS file to storage. SKIP") exit(1) podcast.rss_link = result_url podcast.save() logger.info("RSS file uploaded, podcast record updated") if not settings.TEST_MODE: logger.info("Removing file [%s]", src_file) podcast_utils.delete_file(filepath=src_file) src_file = None logger.info("FINISH generation") return src_file
def render_rss_to_file(podcast_id: int) -> str: """ Generate rss for Podcast and Episodes marked as "published" """ logger.info(f"Podcast #{podcast_id}: RSS generation has been started.") podcast = Podcast.get_by_id(podcast_id) # noinspection PyComparisonWithNone episodes = podcast.get_episodes(podcast.created_by).where( Episode.status == Episode.STATUS_PUBLISHED, Episode.published_at != None, # noqa: E711 ) context = {"episodes": episodes, "settings": settings} with open(os.path.join(settings.TEMPLATE_PATH, "rss", "feed_template.xml")) as fh: template = Template(fh.read()) rss_filename = os.path.join(settings.TMP_RSS_PATH, f"{podcast.publish_id}.xml") logger.info( f"Podcast #{podcast.publish_id}: Generation new file rss [{rss_filename}]" ) with open(rss_filename, "w") as fh: result_rss = template.render(podcast=podcast, **context) fh.write(result_rss) logger.info(f"Podcast #{podcast_id}: RSS generation has been finished.") return rss_filename
def test_get_list__check_episodes_count__ok(self, client, user, loop, dbs): dbs = dbs podcast_1 = await_(Podcast.async_create(dbs, **get_podcast_data(created_by_id=user.id))) create_episode(dbs, get_episode_data(), podcast_1) create_episode(dbs, get_episode_data(), podcast_1) podcast_2 = await_(Podcast.async_create(dbs, **get_podcast_data(created_by_id=user.id))) create_episode(dbs, get_episode_data(), podcast_2) client.login(user) response = client.get(self.url) response_data = self.assert_ok_response(response) expected_episodes_counts = {podcast_1.id: 2, podcast_2.id: 1} actual_episodes_counts = { podcast["id"]: podcast["episodes_count"] for podcast in response_data } assert expected_episodes_counts == actual_episodes_counts
def test_generate__upload_failed__fail(self, podcast, mocked_s3, dbs): mocked_s3.upload_file.side_effect = lambda *_, **__: "" generate_rss_task = tasks.GenerateRSSTask(db_session=dbs) result_code = await_(generate_rss_task.run(podcast.id)) assert result_code == FinishCode.ERROR podcast_1 = await_(Podcast.async_get(dbs, id=podcast.id)) assert podcast_1.rss_link is None
def test_delete__ok(self, client, podcast, user, mocked_s3, dbs): client.login(user) url = self.url.format(id=podcast.id) response = client.delete(url) assert response.status_code == 200 assert await_(Podcast.async_get(dbs, id=podcast.id)) is None mocked_s3.delete_files_async.assert_called_with( [f"{podcast.publish_id}.xml"], remote_path=settings.S3_BUCKET_RSS_PATH )
def test_generate__single_podcast__ok(self, user, mocked_s3, dbs): podcast_1: Podcast = await_( Podcast.async_create(dbs, **get_podcast_data())) podcast_2: Podcast = await_( Podcast.async_create(dbs, **get_podcast_data())) episode_data = get_episode_data(podcast_1, creator=user) episode_data["status"] = Episode.Status.NEW episode_new = await_(Episode.async_create(dbs, **episode_data)) episode_data = get_episode_data(podcast_1, creator=user) episode_data["status"] = Episode.Status.DOWNLOADING episode_downloading = await_(Episode.async_create(dbs, **episode_data)) episode_data = get_episode_data(podcast_1, creator=user) episode_data["status"] = Episode.Status.PUBLISHED episode_data["published_at"] = datetime.now() episode_published = await_(Episode.async_create(dbs, **episode_data)) episode_data = get_episode_data(podcast_2, creator=user) episode_data["status"] = Episode.Status.PUBLISHED episode_podcast_2 = await_(Episode.async_create(dbs, **episode_data)) await_(dbs.commit()) expected_file_path = mocked_s3.tmp_upload_dir / f"{podcast_1.publish_id}.xml" generate_rss_task = tasks.GenerateRSSTask(db_session=dbs) result_code = await_(generate_rss_task.run(podcast_1.id)) assert result_code == FinishCode.OK assert os.path.exists( expected_file_path), f"File {expected_file_path} didn't uploaded" with open(expected_file_path) as file: generated_rss_content = file.read() assert episode_published.title in generated_rss_content assert episode_published.description in generated_rss_content assert episode_published.file_name in generated_rss_content for episode in [episode_new, episode_downloading, episode_podcast_2]: assert episode.source_id not in generated_rss_content, f"{episode} in RSS {podcast_1}" podcast_1 = await_(Podcast.async_get(dbs, id=podcast_1.id)) assert podcast_1.rss_link == str(expected_file_path)
async def post(self, request): cleaned_data = await self._validate(request) podcast = await Podcast.async_create( db_session=request.db_session, name=cleaned_data["name"], publish_id=Podcast.generate_publish_id(), description=cleaned_data["description"], created_by_id=request.user.id, ) return self._response(podcast, status_code=status.HTTP_201_CREATED)
async def _get_object(self) -> Podcast: try: podcast = await self.request.app.objects.get( Podcast.select().where( Podcast.created_by_id == self.user.id).order_by( Podcast.created_at.desc())) except peewee.DoesNotExist: podcast = await Podcast.create_first_podcast( self.request.app.objects, self.user.id) return podcast
def test_create__ok(self, client, user, podcast_data, dbs): podcast_data = { "name": podcast_data["name"], "description": podcast_data["description"], } client.login(user) response = client.post(self.url, json=podcast_data) response_data = self.assert_ok_response(response, status_code=201) podcast = await_(Podcast.async_get(dbs, id=response_data["id"])) assert podcast is not None assert response_data == _podcast(podcast)
async def test_podcasts__create__ok(client, db_objects, podcast, urls): request_data = {"name": "test name", "description": "test description"} response = await client.post(urls.podcasts_list, data=request_data, allow_redirects=False) assert response.status == 302 created_podcast = await db_objects.get(Podcast.select().order_by( Podcast.created_at.desc())) assert response.headers["Location"] == f"/podcasts/{created_podcast.id}/" assert created_podcast.name == "test name" assert created_podcast.description == "test description"
def test_filter_by_user__ok(self, client, episode_data, mocked_redis, dbs): user_1 = create_user(dbs) user_2 = create_user(dbs) podcast_1 = await_( Podcast.async_create(dbs, **get_podcast_data(created_by_id=user_1.id))) podcast_2 = await_( Podcast.async_create(dbs, **get_podcast_data(created_by_id=user_2.id))) ep_data_1 = get_episode_data(creator=user_1) ep_data_2 = get_episode_data(creator=user_2) p1_episode_down = create_episode(dbs, ep_data_1, podcast_1, STATUS.DOWNLOADING, MB_2) p2_episode_down = create_episode(dbs, ep_data_2, podcast_2, STATUS.DOWNLOADING, MB_4) await_(dbs.commit()) mocked_redis.async_get_many.return_value = mocked_redis.async_return({ p1_episode_down.file_name.partition(".")[0]: { "status": EpisodeStatus.DL_EPISODE_DOWNLOADING, "processed_bytes": MB_1, "total_bytes": MB_2, }, p2_episode_down.file_name.partition(".")[0]: { "status": EpisodeStatus.DL_EPISODE_DOWNLOADING, "processed_bytes": MB_1, "total_bytes": MB_4, }, }) client.login(user_1) response = client.get(self.url) response_data = self.assert_ok_response(response) assert response_data == [ _progress(podcast_1, p1_episode_down, current_size=MB_1, completed=50.0), ]
async def post(self): cleaned_data = await self._validate() podcast = await self.request.app.objects.create( Podcast, **dict( publish_id=Podcast.generate_publish_id(), name=cleaned_data["name"], description=cleaned_data.get("description", ""), created_by_id=self.user.id, ), ) return redirect(self.request, "podcast_details", podcast_id=podcast.id)
def test_delete__episodes_deleted_too__ok(self, client, podcast, user, mocked_s3, dbs): episode_1 = await_(Episode.async_create(dbs, **get_episode_data(podcast))) episode_2 = await_(Episode.async_create(dbs, **get_episode_data(podcast, "published"))) await_(dbs.commit()) client.login(user) url = self.url.format(id=podcast.id) response = client.delete(url) assert response.status_code == 200 assert await_(Podcast.async_get(dbs, id=podcast.id)) is None assert await_(Episode.async_get(dbs, id=episode_1.id)) is None assert await_(Episode.async_get(dbs, id=episode_2.id)) is None mocked_s3.delete_files_async.assert_called_with([episode_2.file_name])
def test_sign_up__ok(self, client, user_invite, dbs): request_data = self._sign_up_data(user_invite) response = client.post(self.url, json=request_data) response_data = self.assert_ok_response(response, status_code=201) user = await_(User.async_get(dbs, email=request_data["email"])) assert user is not None, f"User wasn't created with {request_data=}" assert_tokens(response_data, user) await_(dbs.refresh(user_invite)) assert user_invite.user_id == user.id assert user_invite.is_applied assert await_(Podcast.async_get(dbs, created_by_id=user.id)) is not None
def test_download_episode__file_correct__ignore( self, episode_data, podcast_data, mocked_youtube, mocked_ffmpeg, mocked_s3, mocked_generate_rss_task, dbs, ): podcast_1 = await_(Podcast.async_create(dbs, **get_podcast_data())) podcast_2 = await_(Podcast.async_create(dbs, **get_podcast_data())) episode_data.update({ "status": "published", "source_id": mocked_youtube.video_id, "watch_url": mocked_youtube.watch_url, "file_size": 1024, "podcast_id": podcast_1.id, }) await_(Episode.async_create(dbs, **episode_data)) episode_data["status"] = "new" episode_data["podcast_id"] = podcast_2.id episode_2 = await_(Episode.async_create(dbs, **episode_data)) await_(dbs.commit()) mocked_s3.get_file_size.return_value = episode_2.file_size result = await_(DownloadEpisodeTask(db_session=dbs).run(episode_2.id)) await_(dbs.refresh(episode_2)) mocked_generate_rss_task.run.assert_called_with( podcast_1.id, podcast_2.id) assert result == FinishCode.SKIP assert not mocked_youtube.download.called assert not mocked_ffmpeg.called assert episode_2.status == Episode.Status.PUBLISHED assert episode_2.published_at == episode_2.created_at
def test_create__same_episode__extract_failed__ok( self, podcast, episode, user, mocked_youtube, dbs ): mocked_youtube.extract_info.side_effect = ExtractorError("Something went wrong here") new_podcast = await_(Podcast.async_create(dbs, **get_podcast_data())) episode_creator = EpisodeCreator( dbs, podcast_id=new_podcast.id, source_url=episode.watch_url, user_id=user.id, ) new_episode: Episode = await_(episode_creator.create()) assert episode is not None assert new_episode.id != episode.id assert new_episode.source_id == episode.source_id assert new_episode.watch_url == episode.watch_url
def podcast(podcast_data, user, loop, dbs): podcast_data["created_by_id"] = user.id podcast = loop.run_until_complete(Podcast.async_create( dbs, **podcast_data)) loop.run_until_complete(dbs.commit()) return podcast
def regenerate_rss(): podcasts = list(Podcast.select()) for podcast in podcasts: generate_rss(podcast.id)
def podcast(db_objects, user, podcast_data): with db_objects.allow_sync(): yield Podcast.create(**podcast_data)
def teardown_module(module): print(f"module teardown {module}") Episode.truncate_table() Podcast.truncate_table() User.truncate_table()