def handle(self, *args, **options): category = 'anime' start = 0 if options.get('id'): anime_id = options.get('id')[0] anime = Work.objects.filter(category__slug='anime').get( id=anime_id) if anime.anidb_aid == 0: for reference in anime.reference_set.all(): if reference.url.startswith( 'http://anidb.net') or reference.url.startswith( 'https://anidb.net'): query = urlparse(reference.url).query anidb_aid = parse_qs(query).get('aid') if anidb_aid: anime.anidb_aid = anidb_aid[0] anime.save() todo = Work.objects.filter(category__slug='anime', id=anime_id, anidb_aid__gt=0) else: todo = Work.objects\ .only('pk', 'title', 'poster', 'nsfw')\ .annotate(rating_count=Count('rating'))\ .filter(category__slug=category, rating_count__gte=6)\ .exclude(anidb_aid=0)\ .order_by('-rating_count') a = AniDB('mangakihttp', 1) i = 0 for anime in todo: i += 1 if i < start: continue print(i, ':', anime.title, anime.id) creators = a.get(anime.anidb_aid).creators print(creators) staff_map = dict( Role.objects.filter( slug__in=['author', 'director', 'composer']).values_list( 'slug', 'pk')) for creator in creators.findAll('name'): artist = get_or_create_artist(creator.string) if creator['type'] == 'Direction': staff_id = 'director' elif creator['type'] == 'Music': staff_id = 'composer' elif creator['type'] == 'Original Work' or creator[ 'type'] == 'Story Composition': staff_id = 'author' else: staff_id = None if staff_id is not None: Staff.objects.get_or_create(work=anime, role_id=staff_map[staff_id], artist=artist) anime.save()
def handle(self, *args, **options): category = 'anime'; work_query = 'SELECT mangaki_{category}.work_ptr_id, mangaki_work.id, mangaki_work.title, mangaki_work.poster, mangaki_work.nsfw, COUNT(mangaki_work.id) rating_count FROM mangaki_{category}, mangaki_work, mangaki_rating WHERE mangaki_{category}.work_ptr_id = mangaki_work.id AND mangaki_rating.work_id = mangaki_work.id AND mangaki_{category}.anidb_aid = 0 GROUP BY mangaki_work.id, mangaki_{category}.work_ptr_id HAVING COUNT(mangaki_work.id) >= {min_ratings} ORDER BY {order_by}' a = AniDB('mangakihttp', 1) for anime in Anime.objects.raw(work_query.format(category=category, min_ratings=6, order_by='rating_count DESC')): print(anime.title, anime.id) for proposal in a.search(r'\%s' % anime.title): print(proposal) anidb_aid = input('Which one? ') if anidb_aid == 'q': continue anime.anidb_aid = int(anidb_aid) anime.save()
def handle(self, *args, **options): q = Work.objects\ .only('pk', 'title', 'poster', 'nsfw')\ .annotate(rating_count=Count('rating'))\ .filter(anidb_aid=0, category__slug='anime', rating_count__gte=6)\ .order_by('-rating_count') a = AniDB('mangakihttp', 1) for anime in q: print(anime.title, anime.id) for proposal in a.search(r'\%s' % anime.title): print(proposal) anidb_aid = input('Which one? ') if anidb_aid == 'q': continue anime.anidb_aid = int(anidb_aid) anime.save()
def handle(self, *args, **options): category = 'anime' work_query = 'SELECT mangaki_{category}.work_ptr_id, mangaki_work.id, mangaki_work.title, mangaki_work.poster, mangaki_work.nsfw, COUNT(mangaki_work.id) rating_count FROM mangaki_{category}, mangaki_work, mangaki_rating WHERE mangaki_{category}.work_ptr_id = mangaki_work.id AND mangaki_rating.work_id = mangaki_work.id AND mangaki_{category}.anidb_aid = 0 GROUP BY mangaki_work.id, mangaki_{category}.work_ptr_id HAVING COUNT(mangaki_work.id) >= {min_ratings} ORDER BY {order_by}' a = AniDB('mangakihttp', 1) for anime in Anime.objects.raw( work_query.format(category=category, min_ratings=6, order_by='rating_count DESC')): print(anime.title, anime.id) for proposal in a.search(r'\%s' % anime.title): print(proposal) anidb_aid = input('Which one? ') if anidb_aid == 'q': continue anime.anidb_aid = int(anidb_aid) anime.save()
def handle(self, *args, **options): category = 'anime' start = 0 if len(sys.argv) > 2: if sys.argv[2] == 'id': anime_id = sys.argv[3] anime = Anime.objects.get(id=anime_id) if anime.anidb_aid == 0: for reference in anime.reference_set.all(): if reference.url.startswith('http://anidb.net'): query = urlparse(reference.url).query anidb_aid = parse_qs(query).get('aid') if anidb_aid: anime.anidb_aid = anidb_aid[0] anime.save() todo = Anime.objects.filter(id=anime_id, anidb_aid__gt=0) else: work_query = 'SELECT mangaki_{category}.work_ptr_id, mangaki_work.id, mangaki_work.title, mangaki_work.poster, mangaki_work.nsfw, COUNT(mangaki_work.id) rating_count FROM mangaki_{category}, mangaki_work, mangaki_rating WHERE mangaki_{category}.work_ptr_id = mangaki_work.id AND mangaki_rating.work_id = mangaki_work.id AND mangaki_{category}.anidb_aid > 0 GROUP BY mangaki_work.id, mangaki_{category}.work_ptr_id HAVING COUNT(mangaki_work.id) >= {min_ratings} ORDER BY {order_by}' todo = Anime.objects.raw( work_query.format(category=category, min_ratings=6, order_by='rating_count DESC')) if sys.argv[2] == 'from': start = int(sys.argv[3]) a = AniDB('mangakihttp', 1) i = 0 for anime in todo: i += 1 if i < start: continue print(i, ':', anime.title, anime.id) creators = a.get(anime.anidb_aid).creators print(creators) for creator in creators.findAll('name'): if creator['type'] == 'Direction': try_replace(anime, 'director', creator.string) elif creator['type'] == 'Music': try_replace(anime, 'composer', creator.string) elif creator['type'] == 'Original Work' or creator[ 'type'] == 'Story Composition': try_replace(anime, 'author', creator.string) anime.save()
def update_tags_via_anidb(self, request, queryset): if request.POST.get("post"): chosen_ids = request.POST.getlist('checks') for anime_id in chosen_ids: anime = Work.objects.get(id=anime_id) a = AniDB('mangakihttp', 1) retrieve_tags = anime.retrieve_tags(a) deleted_tags = retrieve_tags["deleted_tags"] added_tags = retrieve_tags["added_tags"] updated_tags = retrieve_tags["updated_tags"] anime.update_tags(deleted_tags, added_tags, updated_tags) self.message_user(request, "Modifications sur les tags faites") return None for anime in queryset.select_related("category"): if anime.category.slug != 'anime': self.message_user(request, "%s n'est pas un anime. La recherche des tags via AniDB n'est possible que pour les animes " % anime.title) self.message_user(request, "Vous avez un filtre à votre droite pour avoir les animes avec un anidb_aid") return None elif not anime.anidb_aid: self.message_user(request, "%s n'a pas de lien actuel avec la base d'aniDB (pas d'anidb_aid)" % anime.title) self.message_user(request, "Vous avez un filtre à votre droite pour avoir les animes avec un anidb_aid") return None all_information = {} for anime in queryset: a = AniDB('mangakihttp', 1) retrieve_tags = anime.retrieve_tags(a) deleted_tags = retrieve_tags["deleted_tags"] added_tags = retrieve_tags["added_tags"] updated_tags = retrieve_tags["updated_tags"] kept_tags = retrieve_tags["kept_tags"] all_information[anime.id] = {'title': anime.title, 'deleted_tags': deleted_tags.items(), 'added_tags': added_tags.items(), 'updated_tags': updated_tags.items(), "kept_tags": kept_tags.items()} context = { 'all_information': all_information.items(), 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, } return TemplateResponse(request, "admin/update_tags_via_anidb.html", context)
def handle(self, *args, **options): category = 'anime' start = 0 if options.get('id'): anime_id = options.get('id')[0] anime = Anime.objects.get(id=anime_id) if anime.anidb_aid == 0: for reference in anime.reference_set.all(): if reference.url.startswith('http://anidb.net'): query = urlparse(reference.url).query anidb_aid = parse_qs(query).get('aid') if anidb_aid: anime.anidb_aid = anidb_aid[0] anime.save() todo = Anime.objects.filter(id=anime_id, anidb_aid__gt=0) else: work_query = 'SELECT mangaki_{category}.work_ptr_id, mangaki_work.id, mangaki_work.title, mangaki_work.poster, mangaki_work.nsfw, COUNT(mangaki_work.id) rating_count FROM mangaki_{category}, mangaki_work, mangaki_rating WHERE mangaki_{category}.work_ptr_id = mangaki_work.id AND mangaki_rating.work_id = mangaki_work.id AND mangaki_{category}.anidb_aid > 0 GROUP BY mangaki_work.id, mangaki_{category}.work_ptr_id HAVING COUNT(mangaki_work.id) >= {min_ratings} ORDER BY {order_by}' todo = Anime.objects.raw(work_query.format(category=category, min_ratings=6, order_by='rating_count DESC')) """if sys.argv[2] == 'from': start = int(sys.argv[3])""" a = AniDB('mangakihttp', 1) i = 0 for anime in todo: i += 1 if i < start: continue print(i, ':', anime.title, anime.id) creators = a.get(anime.anidb_aid).creators print(creators) for creator in creators.findAll('name'): if creator['type'] == 'Direction': try_replace(anime, 'director', creator.string) elif creator['type'] == 'Music': try_replace(anime, 'composer', creator.string) elif creator['type'] == 'Original Work' or creator['type'] == 'Story Composition': try_replace(anime, 'author', creator.string) anime.save()
def setUp(self): self.anidb = AniDB('test_client', 1) self.no_anidb = AniDB() self.search_fixture = self.read_fixture('search_sangatsu_no_lion.xml')
class AniDBTest(TestCase): @staticmethod def create_anime(**kwargs): anime = Category.objects.get(slug='anime') return Work.objects.create(category=anime, **kwargs) @staticmethod def read_fixture(filename): with open(os.path.join(settings.TEST_DATA_DIR, filename), 'r', encoding='utf-8') as f: return f.read() def setUp(self): self.anidb = AniDB('test_client', 1) self.no_anidb = AniDB() self.search_fixture = self.read_fixture('search_sangatsu_no_lion.xml') def test_to_python_datetime(self): self.assertEqual(to_python_datetime('2017-12-25'), datetime(2017, 12, 25, 0, 0)) self.assertEqual(to_python_datetime('2017-12'), datetime(2017, 12, 1, 0, 0)) self.assertEqual(to_python_datetime('2017'), datetime(2017, 1, 1, 0, 0)) self.assertRaises(ValueError, to_python_datetime, '2017-25') def test_missing_client(self): self.assertRaises(RuntimeError, self.no_anidb._request, 'dummypage') self.assertFalse(self.no_anidb.is_available) @responses.activate def test_anidb_search(self): responses.add( responses.GET, AniDB.SEARCH_URL, body=self.search_fixture, status=200, content_type='application/xml' ) anime_query = 'sangatsu no lion' results = self.anidb.search(q=anime_query) self.assertEqual(len(results), 2) self.assertEqual(len(responses.calls), 1) @responses.activate def test_anidb_get_methods(self): responses.add( responses.GET, AniDB.BASE_URL, body=self.read_fixture('anidb/sangatsu_no_lion.xml'), status=200, content_type='application/xml' ) titles, main_title = self.anidb.get_titles(anidb_aid=11606) creators, studio = self.anidb.get_creators(anidb_aid=11606) tags = self.anidb.get_tags(anidb_aid=11606) related_animes = self.anidb.get_related_animes(anidb_aid=11606) self.assertEqual(len(titles), 9) self.assertEqual(main_title, 'Sangatsu no Lion') self.assertEqual(len(creators), 4) self.assertEqual(studio.title, 'Shaft') self.assertEqual(len(tags), 30) self.assertEqual(len(related_animes), 2) @responses.activate def test_anidb_get_animes(self): # Fake an artist entry with no AniDB creator ID that will be filled by retrieving Sangatsu artist = Artist(name="Shinbou Akiyuki").save() filenames = ['anidb/sangatsu_no_lion.xml', 'anidb/sangatsu_no_lion.xml', 'anidb/hibike_euphonium.xml'] with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps: for filename in filenames: rsps.add( responses.GET, AniDB.BASE_URL, body=self.read_fixture(filename), status=200, content_type='application/xml' ) sangatsu = self.anidb.get_or_update_work(11606) tags_sangatsu_from_anidb = self.anidb.get_tags(11606) tags_diff_sangatsu = diff_between_anidb_and_local_tags(sangatsu, tags_sangatsu_from_anidb) hibike = self.anidb.get_or_update_work(10889) # Retrieve tags tags_sangatsu = set(Work.objects.get(pk=sangatsu.pk).taggedwork_set.all().values_list('tag__title', flat=True)) tags_hibike = set(Work.objects.get(pk=hibike.pk).taggedwork_set.all().values_list('tag__title', flat=True)) shared_tags = tags_sangatsu.intersection(tags_hibike) # Checks on tags self.assertEqual(len(tags_sangatsu), 30) self.assertEqual(len(tags_hibike), 38) self.assertEqual(len(shared_tags), 18) # Check for Sangatsu's informations self.assertEqual(sangatsu.title, 'Sangatsu no Lion') self.assertEqual(sangatsu.nb_episodes, 22) self.assertEqual(sangatsu.studio.title, 'Shaft') self.assertEqual(sangatsu.date, datetime(2016, 10, 8, 0, 0)) self.assertEqual(sangatsu.end_date, datetime(2017, 3, 18, 0, 0)) # Check for Sangatsu's staff staff_sangatsu = Work.objects.get(pk=sangatsu.pk).staff_set.all().values_list('artist__name', flat=True) self.assertCountEqual(staff_sangatsu, ['Umino Chika', 'Hashimoto Yukari', 'Shinbou Akiyuki', 'Okada Kenjirou']) # Check retrieved tags from AniDB self.assertEqual(len(tags_diff_sangatsu["deleted_tags"]), 0) self.assertEqual(len(tags_diff_sangatsu["added_tags"]), 0) self.assertEqual(len(tags_diff_sangatsu["updated_tags"]), 0) self.assertEqual(len(tags_diff_sangatsu["kept_tags"]), len(tags_sangatsu)) # Check for no artist duplication artist = Artist.objects.filter(name="Shinbou Akiyuki") self.assertEqual(artist.count(), 1) self.assertEqual(artist.first().anidb_creator_id, 59) @responses.activate def test_anidb_duplicate_anime_id(self): for _ in range(2): responses.add( responses.GET, AniDB.BASE_URL, body=self.read_fixture('anidb/hibike_euphonium.xml'), status=200, content_type='application/xml' ) self.create_anime(title='Hibike! Euphonium', anidb_aid=10889) self.create_anime(title='Hibike! Euphonium Duplicate', anidb_aid=10889) self.anidb.get_or_update_work(10889) self.assertIs(self.anidb.get_or_update_work(10889), None) @responses.activate def test_anidb_nsfw(self): animes = {} animes_sources = { # Not NSFW at all 'anidb/sangatsu_no_lion.xml': (11606, 'Sangatsu no Lion'), 'anidb/hibike_euphonium.xml': (10889, 'Hibike! Euphonium'), # Totally NSFW (restricted on AniDB) 'anidb/boku_no_piko.xml': (4544, 'Boku no Piko'), 'anidb/bible_black.xml': (528, 'Bible Black'), # Should be marked NSFW 'anidb/r15.xml': (8396, 'R-15'), 'anidb/astarotte_no_omocha_ex.xml': (8560, 'Astarotte no Omocha! EX'), 'anidb/aki_sora.xml': (6782, 'Aki Sora'), # Shouldn't be marked NSFW 'anidb/punchline.xml': (10948, 'Punch Line'), 'anidb/panty_stocking.xml': (7529, 'Panty & Stocking with Garterbelt'), 'anidb/shimoneta.xml': (10888, 'Shimoneta to Iu Gainen ga Sonzai Shinai Taikutsu na Sekai') } are_nsfw = ['anidb/boku_no_piko.xml', 'anidb/bible_black.xml', 'anidb/r15.xml', 'anidb/astarotte_no_omocha_ex.xml', 'anidb/aki_sora.xml'] are_sfw = ['anidb/sangatsu_no_lion.xml', 'anidb/hibike_euphonium.xml', 'anidb/punchline.xml', 'anidb/panty_stocking.xml', 'anidb/shimoneta.xml'] with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps: for filename, infos in animes_sources.items(): rsps.add( responses.GET, AniDB.BASE_URL, body=self.read_fixture(filename), status=200, content_type='application/xml' ) animes[filename] = self.anidb.get_or_update_work(infos[0]) for filename in are_nsfw: with self.subTest('Asserting NSFW', anime=animes_sources[filename][1]): self.assertEqual(animes[filename].title, animes_sources[filename][1]) self.assertTrue(animes[filename].nsfw) for filename in are_sfw: with self.subTest('Asserting SFW', anime=animes_sources[filename][1]): self.assertEqual(animes[filename].title, animes_sources[filename][1]) self.assertFalse(animes[filename].nsfw) @responses.activate def test_anidb_related_animes(self): animes = {} related_animes = {} animes_sources = { 'anidb/hibike_euphonium.xml': 10889, 'anidb/hibike_euphonium2.xml': 11746, 'anidb/hibike_euphonium_movie1.xml': 11747, 'anidb/hibike_euphonium_movie2.xml': 12962, 'anidb/hibike_euphonium_original_movies.xml': 13207, 'anidb/sangatsu_no_lion.xml': 11606 } with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps: for filename, anidb_aid in animes_sources.items(): for _ in range(2): rsps.add( responses.GET, AniDB.BASE_URL, body=self.read_fixture(filename), status=200, content_type='application/xml' ) animes[filename] = self.anidb.get_or_update_work(anidb_aid) related_animes[filename] = self.anidb.get_related_animes(anidb_aid=anidb_aid) # Ran once in get_or_update_work but ran again to check that it does not cause errors for filename in animes_sources: self.anidb._build_related_animes(animes[filename], related_animes[filename]) relations = RelatedWork.objects.filter( child_work__anidb_aid__in=animes_sources.values(), parent_work__anidb_aid__in=animes_sources.values() ) # Checks that anime are created if missing but not all data is retrieved from AniDB self.assertEqual(Work.objects.get(title='Sangatsu no Lion meets Bump of Chicken').ext_synopsis, '') self.assertNotEqual(Work.objects.get(title='Sangatsu no Lion').ext_synopsis, '') # Checks on relations self.assertTrue(relations.filter(child_work__anidb_aid=11746, parent_work__anidb_aid=10889, type='sequel').exists()) self.assertTrue(relations.filter(child_work__anidb_aid=10889, parent_work__anidb_aid=11746, type='prequel').exists()) self.assertTrue(relations.filter(child_work__anidb_aid=11747, parent_work__anidb_aid=10889, type='summary').exists()) self.assertTrue(relations.filter(child_work__anidb_aid=10889, parent_work__anidb_aid=11747, type='full_story').exists()) self.assertTrue(relations.filter(child_work__anidb_aid=13207, parent_work__anidb_aid=11746, type='sequel').exists()) self.assertTrue(relations.filter(child_work__anidb_aid=11746, parent_work__anidb_aid=13207, type='prequel').exists()) self.assertTrue(relations.filter(child_work__anidb_aid=12962, parent_work__anidb_aid=11746, type='summary').exists()) self.assertTrue(relations.filter(child_work__anidb_aid=11746, parent_work__anidb_aid=12962, type='full_story').exists())
def handle(self, *args, **options): category = 'anime' start = 0 if options.get('id'): anime_id = options.get('id')[0] anime = Work.objects.filter(category__slug='anime').get( id=anime_id) if anime.anidb_aid == 0: for reference in anime.reference_set.all(): if reference.url.startswith( 'http://anidb.net') or reference.url.startswith( 'https://anidb.net'): query = urlparse(reference.url).query anidb_aid = parse_qs(query).get('aid') if anidb_aid: anime.anidb_aid = anidb_aid[0] anime.save() todo = Work.objects.filter(category__slug='anime', id=anime_id, anidb_aid__gt=0) else: todo = Work.objects\ .only('pk', 'title', 'ext_poster', 'nsfw')\ .annotate(rating_count=Count('rating'))\ .filter(category__slug=category, rating_count__gte=6)\ .exclude(anidb_aid=0)\ .order_by('-rating_count') a = AniDB('mangakihttp', 1) i = 0 for anime in todo: i += 1 if i < start: continue print(i, ':', anime.title, anime.id) creators = a.get(anime.anidb_aid).creators worktitles = a.get(anime.anidb_aid).worktitles for worktitle in worktitles: language = Language.objects.get(iso639=worktitle[2]) WorkTitle.objects.get_or_create(work=anime, title=worktitle[0], language=language, type=worktitle[1]) retrieve_tags = anime.retrieve_tags(a) deleted_tags = retrieve_tags["deleted_tags"] added_tags = retrieve_tags["added_tags"] updated_tags = retrieve_tags["updated_tags"] kept_tags = retrieve_tags["kept_tags"] print(anime.title + ":") if deleted_tags: print("\n\tLes tags enlevés sont :") for tag, weight in deleted_tags.items(): print('\t\t{}: {} '.format(tag, weight)) if added_tags: print("\n\tLes tags totalement nouveaux sont :") for tag, weight in added_tags.items(): print('\t\t{}: {} '.format(tag, weight)) if updated_tags: print("\n\tLes tags modifiés sont :") for tag, weight in updated_tags.items(): print('\t\t{}: {} -> {}'.format(tag, weight[0], weight[1])) if kept_tags: print("\n\tLes tags non modifiés/restés identiques sont :") for tag, weight in kept_tags.items(): print('\t\t{}: {} '.format(tag, weight)) choice = input("Voulez-vous réaliser ces changements [y/n] : ") if choice == 'n': print("\nOk, aucun changement ne va être fait") elif choice == 'y': anime.update_tags(deleted_tags, added_tags, updated_tags) staff_map = dict( Role.objects.filter( slug__in=['author', 'director', 'composer']).values_list( 'slug', 'pk')) for creator in creators.findAll('name'): artist = get_or_create_artist(creator.string) if creator['type'] == 'Direction': staff_id = 'director' elif creator['type'] == 'Music': staff_id = 'composer' elif creator['type'] == 'Original Work' or creator[ 'type'] == 'Story Composition': staff_id = 'author' else: staff_id = None if staff_id is not None: Staff.objects.get_or_create(work=anime, role_id=staff_map[staff_id], artist=artist) anime.save()
def setUp(self): # FIXME: The defaults for editor and studio in Work requires those to # exist, or else foreign key constraints fail. Editor.objects.create(pk=1) Studio.objects.create(pk=1) self.anidb = AniDB('mangakihttp', 1)