def test_anidb_get_animes(self): # Fake an artist entry with no AniDB creator ID that will be filled by retrieving Sangatsu artist = Artist(name="Shinbou Akiyuki").save() filenames = ['anidb/sangatsu_no_lion.xml', 'anidb/sangatsu_no_lion.xml', 'anidb/hibike_euphonium.xml'] with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps: for filename in filenames: rsps.add( responses.GET, AniDB.BASE_URL, body=self.read_fixture(filename), status=200, content_type='application/xml' ) sangatsu = self.anidb.get_or_update_work(11606) tags_sangatsu_from_anidb = self.anidb.get_tags(11606) tags_diff_sangatsu = diff_between_anidb_and_local_tags(sangatsu, tags_sangatsu_from_anidb) hibike = self.anidb.get_or_update_work(10889) # Retrieve tags tags_sangatsu = set(Work.objects.get(pk=sangatsu.pk).taggedwork_set.all().values_list('tag__title', flat=True)) tags_hibike = set(Work.objects.get(pk=hibike.pk).taggedwork_set.all().values_list('tag__title', flat=True)) shared_tags = tags_sangatsu.intersection(tags_hibike) # Checks on tags self.assertEqual(len(tags_sangatsu), 30) self.assertEqual(len(tags_hibike), 38) self.assertEqual(len(shared_tags), 18) # Check for Sangatsu's informations self.assertEqual(sangatsu.title, 'Sangatsu no Lion') self.assertEqual(sangatsu.nb_episodes, 22) self.assertEqual(sangatsu.studio.title, 'Shaft') self.assertEqual(sangatsu.date, datetime(2016, 10, 8, 0, 0)) self.assertEqual(sangatsu.end_date, datetime(2017, 3, 18, 0, 0)) # Check for Sangatsu's staff staff_sangatsu = Work.objects.get(pk=sangatsu.pk).staff_set.all().values_list('artist__name', flat=True) self.assertCountEqual(staff_sangatsu, ['Umino Chika', 'Hashimoto Yukari', 'Shinbou Akiyuki', 'Okada Kenjirou']) # Check retrieved tags from AniDB self.assertEqual(len(tags_diff_sangatsu["deleted_tags"]), 0) self.assertEqual(len(tags_diff_sangatsu["added_tags"]), 0) self.assertEqual(len(tags_diff_sangatsu["updated_tags"]), 0) self.assertEqual(len(tags_diff_sangatsu["kept_tags"]), len(tags_sangatsu)) # Check for no artist duplication artist = Artist.objects.filter(name="Shinbou Akiyuki") self.assertEqual(artist.count(), 1) self.assertEqual(artist.first().anidb_creator_id, 59)
def update_tags_via_anidb(self, request, queryset): works = queryset.all() if request.POST.get('confirm'): # Updating tags has been confirmed to_update_work_ids = set(map(int, request.POST.getlist('to_update_work_ids'))) nb_updates = len(to_update_work_ids) work_ids = list(map(int, request.POST.getlist('work_ids'))) tag_titles = request.POST.getlist('tag_titles') tag_weights = list(map(int, request.POST.getlist('weights'))) tag_anidb_tag_ids = list(map(int, request.POST.getlist('anidb_tag_ids'))) tags = list(map(AniDBTag, tag_titles, tag_weights, tag_anidb_tag_ids)) # Checkboxes to know which tags have to be kept regardless of their pending status tag_checkboxes = request.POST.getlist('tag_checkboxes') tags_to_process = set(tuple(map(int, tag_checkbox.split(':'))) for tag_checkbox in tag_checkboxes) # Make a dict with work_id -> tags to keep tags_final = {} for index, work_id in enumerate(work_ids): if work_id not in to_update_work_ids: continue if work_id not in tags_final: tags_final[work_id] = [] if (work_id, tags[index].anidb_tag_id) in tags_to_process: tags_final[work_id].append(tags[index]) # Process selected tags for works that have been selected for work in works: if work.id in to_update_work_ids: client.update_tags(work, tags_final[work.id]) if nb_updates == 0: self.message_user(request, "Aucune oeuvre n'a été marquée comme devant être mise à jour.", level=messages.WARNING) elif nb_updates == 1: self.message_user(request, "Mise à jour des tags effectuée pour une œuvre.") else: self.message_user(request, "Mise à jour des tags effectuée pour {} œuvres.".format(nb_updates)) return None # Check for works with missing AniDB AID if not all(work.anidb_aid for work in works): self.message_user(request, """Certains de vos choix ne possèdent pas d'identifiant AniDB. Le rafraichissement de leurs tags a été omis. (Détails: {})""" .format(", ".join(map(lambda w: w.title, filter(lambda w: not w.anidb_aid, works)))), level=messages.WARNING) # Retrieve and send tags information to the appropriate form all_information = {} for index, work in enumerate(works, start=1): if work.anidb_aid: if index % 25 == 0: logger.info('(AniDB refresh): Sleeping...') time.sleep(1) # Don't spam AniDB. anidb_tags = client.get_tags(anidb_aid=work.anidb_aid) tags_diff = diff_between_anidb_and_local_tags(work, anidb_tags) tags_count = 0 for tags_info in tags_diff.values(): tags_count += len(tags_info) if tags_count > 0: all_information[work.id] = { 'title': work.title, 'deleted_tags': tags_diff["deleted_tags"], 'added_tags': tags_diff["added_tags"], 'updated_tags': tags_diff["updated_tags"], 'kept_tags': tags_diff["kept_tags"] } if all_information: context = { 'all_information': all_information.items(), 'queryset': queryset, 'opts': TaggedWork._meta, 'action': 'update_tags_via_anidb', 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME } return TemplateResponse(request, "admin/update_tags_via_anidb.html", context) else: self.message_user(request, "Aucune des œuvres sélectionnées n'a subit de mise à jour des tags chez AniDB.", level=messages.WARNING) return None
def handle(self, *args, **options): category = 'anime' start = 0 if options.get('id'): anime_id = options.get('id')[0] anime = Work.objects.filter(category__slug='anime').get(id=anime_id) if anime.anidb_aid == 0: for reference in anime.reference_set.all(): if reference.url.startswith('http://anidb.net') or reference.url.startswith('https://anidb.net'): query = urlparse(reference.url).query anidb_aid = parse_qs(query).get('aid') if anidb_aid: anime.anidb_aid = anidb_aid[0] anime.save() todo = Work.objects.filter(category__slug='anime', id=anime_id, anidb_aid__gt=0) else: todo = Work.objects\ .only('pk', 'title', 'ext_poster', 'nsfw')\ .annotate(rating_count=Count('rating'))\ .filter(category__slug=category, rating_count__gte=6)\ .exclude(anidb_aid=0)\ .order_by('-rating_count') a = client i = 0 for anime in todo: i += 1 if i < start: continue print(i, ':', anime.title, anime.id) creators = a.get(anime.anidb_aid).creators worktitles = a.get(anime.anidb_aid).worktitles for worktitle in worktitles: language = Language.objects.get(iso639=worktitle[2]) WorkTitle.objects.get_or_create(work=anime, title=worktitle[0], language=language, type=worktitle[1]) anidb_tags = client.get_tags(anidb_aid=anime.anidb_aid) tags_diff = diff_between_anidb_and_local_tags(anime, anidb_tags) deleted_tags = tags_diff["deleted_tags"] added_tags = tags_diff["added_tags"] updated_tags = tags_diff["updated_tags"] kept_tags = tags_diff["kept_tags"] print(anime.title+":") if deleted_tags: print("\n\tLes tags enlevés sont :") for tag in deleted_tags: print('\t\t[AniDB Tag ID #{}] {}: {} '.format(tag.anidb_tag_id, tag.title, tag.weight)) if added_tags: print("\n\tLes tags totalement nouveaux sont :") for tag in added_tags: print('\t\t[AniDB Tag ID #{}] {}: {} '.format(tag.anidb_tag_id, tag.title, tag.weight)) if updated_tags: print("\n\tLes tags modifiés sont :") for tag in updated_tags: print('\t\t[AniDB Tag ID #{}] {}: {} '.format(tag.anidb_tag_id, tag.title, tag.weight)) if kept_tags: print("\n\tLes tags identiques sont :") for tag in kept_tags: print('\t\t[AniDB Tag ID #{}] {}: {} '.format(tag.anidb_tag_id, tag.title, tag.weight)) choice = input("Voulez-vous réaliser ces changements [y/n] : ") if choice == 'n': print("\nOk, aucun changement ne va être fait") elif choice == 'y': all_tags = deleted_tags + added_tags + updated_tags + kept_tags client.update_tags(anime, all_tags) staff_map = dict(Role.objects.filter(slug__in=['author', 'director', 'composer']).values_list('slug', 'pk')) for creator in creators.findAll('name'): artist = get_or_create_artist(creator.string) if creator['type'] == 'Direction': staff_id = 'director' elif creator['type'] == 'Music': staff_id = 'composer' elif creator['type'] == 'Original Work' or creator['type'] == 'Story Composition': staff_id = 'author' else: staff_id = None if staff_id is not None: Staff.objects.get_or_create(work=anime, role_id=staff_map[staff_id], artist=artist) anime.save()