def handle(self, *args, **options):
category = 'anime'
start = 0
if options.get('id'):
anime_id = options.get('id')[0]
anime = Work.objects.filter(category__slug='anime').get(
id=anime_id)
if anime.anidb_aid == 0:
for reference in anime.reference_set.all():
if reference.url.startswith(
'http://anidb.net') or reference.url.startswith(
'https://anidb.net'):
query = urlparse(reference.url).query
anidb_aid = parse_qs(query).get('aid')
if anidb_aid:
anime.anidb_aid = anidb_aid[0]
anime.save()
todo = Work.objects.filter(category__slug='anime',
id=anime_id,
anidb_aid__gt=0)
else:
todo = Work.objects\
.only('pk', 'title', 'poster', 'nsfw')\
.annotate(rating_count=Count('rating'))\
.filter(category__slug=category, rating_count__gte=6)\
.exclude(anidb_aid=0)\
.order_by('-rating_count')
a = AniDB('mangakihttp', 1)
i = 0
for anime in todo:
i += 1
if i < start:
continue
print(i, ':', anime.title, anime.id)
creators = a.get(anime.anidb_aid).creators
print(creators)
staff_map = dict(
Role.objects.filter(
slug__in=['author', 'director', 'composer']).values_list(
'slug', 'pk'))
for creator in creators.findAll('name'):
artist = get_or_create_artist(creator.string)
if creator['type'] == 'Direction':
staff_id = 'director'
elif creator['type'] == 'Music':
staff_id = 'composer'
elif creator['type'] == 'Original Work' or creator[
'type'] == 'Story Composition':
staff_id = 'author'
else:
staff_id = None
if staff_id is not None:
Staff.objects.get_or_create(work=anime,
role_id=staff_map[staff_id],
artist=artist)
anime.save()
def handle(self, *args, **options):
category = 'anime';
work_query = 'SELECT mangaki_{category}.work_ptr_id, mangaki_work.id, mangaki_work.title, mangaki_work.poster, mangaki_work.nsfw, COUNT(mangaki_work.id) rating_count FROM mangaki_{category}, mangaki_work, mangaki_rating WHERE mangaki_{category}.work_ptr_id = mangaki_work.id AND mangaki_rating.work_id = mangaki_work.id AND mangaki_{category}.anidb_aid = 0 GROUP BY mangaki_work.id, mangaki_{category}.work_ptr_id HAVING COUNT(mangaki_work.id) >= {min_ratings} ORDER BY {order_by}'
a = AniDB('mangakihttp', 1)
for anime in Anime.objects.raw(work_query.format(category=category, min_ratings=6, order_by='rating_count DESC')):
print(anime.title, anime.id)
for proposal in a.search(r'\%s' % anime.title):
print(proposal)
anidb_aid = input('Which one? ')
if anidb_aid == 'q':
continue
anime.anidb_aid = int(anidb_aid)
anime.save()
def handle(self, *args, **options):
q = Work.objects\
.only('pk', 'title', 'poster', 'nsfw')\
.annotate(rating_count=Count('rating'))\
.filter(anidb_aid=0, category__slug='anime', rating_count__gte=6)\
.order_by('-rating_count')
a = AniDB('mangakihttp', 1)
for anime in q:
print(anime.title, anime.id)
for proposal in a.search(r'\%s' % anime.title):
print(proposal)
anidb_aid = input('Which one? ')
if anidb_aid == 'q':
continue
anime.anidb_aid = int(anidb_aid)
anime.save()
def handle(self, *args, **options):
category = 'anime'
work_query = 'SELECT mangaki_{category}.work_ptr_id, mangaki_work.id, mangaki_work.title, mangaki_work.poster, mangaki_work.nsfw, COUNT(mangaki_work.id) rating_count FROM mangaki_{category}, mangaki_work, mangaki_rating WHERE mangaki_{category}.work_ptr_id = mangaki_work.id AND mangaki_rating.work_id = mangaki_work.id AND mangaki_{category}.anidb_aid = 0 GROUP BY mangaki_work.id, mangaki_{category}.work_ptr_id HAVING COUNT(mangaki_work.id) >= {min_ratings} ORDER BY {order_by}'
a = AniDB('mangakihttp', 1)
for anime in Anime.objects.raw(
work_query.format(category=category,
min_ratings=6,
order_by='rating_count DESC')):
print(anime.title, anime.id)
for proposal in a.search(r'\%s' % anime.title):
print(proposal)
anidb_aid = input('Which one? ')
if anidb_aid == 'q':
continue
anime.anidb_aid = int(anidb_aid)
anime.save()
def handle(self, *args, **options):
category = 'anime'
start = 0
if len(sys.argv) > 2:
if sys.argv[2] == 'id':
anime_id = sys.argv[3]
anime = Anime.objects.get(id=anime_id)
if anime.anidb_aid == 0:
for reference in anime.reference_set.all():
if reference.url.startswith('http://anidb.net'):
query = urlparse(reference.url).query
anidb_aid = parse_qs(query).get('aid')
if anidb_aid:
anime.anidb_aid = anidb_aid[0]
anime.save()
todo = Anime.objects.filter(id=anime_id, anidb_aid__gt=0)
else:
work_query = 'SELECT mangaki_{category}.work_ptr_id, mangaki_work.id, mangaki_work.title, mangaki_work.poster, mangaki_work.nsfw, COUNT(mangaki_work.id) rating_count FROM mangaki_{category}, mangaki_work, mangaki_rating WHERE mangaki_{category}.work_ptr_id = mangaki_work.id AND mangaki_rating.work_id = mangaki_work.id AND mangaki_{category}.anidb_aid > 0 GROUP BY mangaki_work.id, mangaki_{category}.work_ptr_id HAVING COUNT(mangaki_work.id) >= {min_ratings} ORDER BY {order_by}'
todo = Anime.objects.raw(
work_query.format(category=category,
min_ratings=6,
order_by='rating_count DESC'))
if sys.argv[2] == 'from':
start = int(sys.argv[3])
a = AniDB('mangakihttp', 1)
i = 0
for anime in todo:
i += 1
if i < start:
continue
print(i, ':', anime.title, anime.id)
creators = a.get(anime.anidb_aid).creators
print(creators)
for creator in creators.findAll('name'):
if creator['type'] == 'Direction':
try_replace(anime, 'director', creator.string)
elif creator['type'] == 'Music':
try_replace(anime, 'composer', creator.string)
elif creator['type'] == 'Original Work' or creator[
'type'] == 'Story Composition':
try_replace(anime, 'author', creator.string)
anime.save()
def update_tags_via_anidb(self, request, queryset):
if request.POST.get("post"):
chosen_ids = request.POST.getlist('checks')
for anime_id in chosen_ids:
anime = Work.objects.get(id=anime_id)
a = AniDB('mangakihttp', 1)
retrieve_tags = anime.retrieve_tags(a)
deleted_tags = retrieve_tags["deleted_tags"]
added_tags = retrieve_tags["added_tags"]
updated_tags = retrieve_tags["updated_tags"]
anime.update_tags(deleted_tags, added_tags, updated_tags)
self.message_user(request, "Modifications sur les tags faites")
return None
for anime in queryset.select_related("category"):
if anime.category.slug != 'anime':
self.message_user(request, "%s n'est pas un anime. La recherche des tags via AniDB n'est possible que pour les animes " % anime.title)
self.message_user(request, "Vous avez un filtre à votre droite pour avoir les animes avec un anidb_aid")
return None
elif not anime.anidb_aid:
self.message_user(request, "%s n'a pas de lien actuel avec la base d'aniDB (pas d'anidb_aid)" % anime.title)
self.message_user(request, "Vous avez un filtre à votre droite pour avoir les animes avec un anidb_aid")
return None
all_information = {}
for anime in queryset:
a = AniDB('mangakihttp', 1)
retrieve_tags = anime.retrieve_tags(a)
deleted_tags = retrieve_tags["deleted_tags"]
added_tags = retrieve_tags["added_tags"]
updated_tags = retrieve_tags["updated_tags"]
kept_tags = retrieve_tags["kept_tags"]
all_information[anime.id] = {'title': anime.title, 'deleted_tags': deleted_tags.items(), 'added_tags': added_tags.items(), 'updated_tags': updated_tags.items(), "kept_tags": kept_tags.items()}
context = {
'all_information': all_information.items(),
'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME,
}
return TemplateResponse(request, "admin/update_tags_via_anidb.html", context)
def handle(self, *args, **options):
category = 'anime'
start = 0
if options.get('id'):
anime_id = options.get('id')[0]
anime = Anime.objects.get(id=anime_id)
if anime.anidb_aid == 0:
for reference in anime.reference_set.all():
if reference.url.startswith('http://anidb.net'):
query = urlparse(reference.url).query
anidb_aid = parse_qs(query).get('aid')
if anidb_aid:
anime.anidb_aid = anidb_aid[0]
anime.save()
todo = Anime.objects.filter(id=anime_id, anidb_aid__gt=0)
else:
work_query = 'SELECT mangaki_{category}.work_ptr_id, mangaki_work.id, mangaki_work.title, mangaki_work.poster, mangaki_work.nsfw, COUNT(mangaki_work.id) rating_count FROM mangaki_{category}, mangaki_work, mangaki_rating WHERE mangaki_{category}.work_ptr_id = mangaki_work.id AND mangaki_rating.work_id = mangaki_work.id AND mangaki_{category}.anidb_aid > 0 GROUP BY mangaki_work.id, mangaki_{category}.work_ptr_id HAVING COUNT(mangaki_work.id) >= {min_ratings} ORDER BY {order_by}'
todo = Anime.objects.raw(work_query.format(category=category, min_ratings=6, order_by='rating_count DESC'))
"""if sys.argv[2] == 'from':
start = int(sys.argv[3])"""
a = AniDB('mangakihttp', 1)
i = 0
for anime in todo:
i += 1
if i < start:
continue
print(i, ':', anime.title, anime.id)
creators = a.get(anime.anidb_aid).creators
print(creators)
for creator in creators.findAll('name'):
if creator['type'] == 'Direction':
try_replace(anime, 'director', creator.string)
elif creator['type'] == 'Music':
try_replace(anime, 'composer', creator.string)
elif creator['type'] == 'Original Work' or creator['type'] == 'Story Composition':
try_replace(anime, 'author', creator.string)
anime.save()
def setUp(self):
self.anidb = AniDB('test_client', 1)
self.no_anidb = AniDB()
self.search_fixture = self.read_fixture('search_sangatsu_no_lion.xml')
class AniDBTest(TestCase):
@staticmethod
def create_anime(**kwargs):
anime = Category.objects.get(slug='anime')
return Work.objects.create(category=anime, **kwargs)
@staticmethod
def read_fixture(filename):
with open(os.path.join(settings.TEST_DATA_DIR, filename), 'r', encoding='utf-8') as f:
return f.read()
def setUp(self):
self.anidb = AniDB('test_client', 1)
self.no_anidb = AniDB()
self.search_fixture = self.read_fixture('search_sangatsu_no_lion.xml')
def test_to_python_datetime(self):
self.assertEqual(to_python_datetime('2017-12-25'), datetime(2017, 12, 25, 0, 0))
self.assertEqual(to_python_datetime('2017-12'), datetime(2017, 12, 1, 0, 0))
self.assertEqual(to_python_datetime('2017'), datetime(2017, 1, 1, 0, 0))
self.assertRaises(ValueError, to_python_datetime, '2017-25')
def test_missing_client(self):
self.assertRaises(RuntimeError, self.no_anidb._request, 'dummypage')
self.assertFalse(self.no_anidb.is_available)
@responses.activate
def test_anidb_search(self):
responses.add(
responses.GET,
AniDB.SEARCH_URL,
body=self.search_fixture,
status=200,
content_type='application/xml'
)
anime_query = 'sangatsu no lion'
results = self.anidb.search(q=anime_query)
self.assertEqual(len(results), 2)
self.assertEqual(len(responses.calls), 1)
@responses.activate
def test_anidb_get_methods(self):
responses.add(
responses.GET,
AniDB.BASE_URL,
body=self.read_fixture('anidb/sangatsu_no_lion.xml'),
status=200,
content_type='application/xml'
)
titles, main_title = self.anidb.get_titles(anidb_aid=11606)
creators, studio = self.anidb.get_creators(anidb_aid=11606)
tags = self.anidb.get_tags(anidb_aid=11606)
related_animes = self.anidb.get_related_animes(anidb_aid=11606)
self.assertEqual(len(titles), 9)
self.assertEqual(main_title, 'Sangatsu no Lion')
self.assertEqual(len(creators), 4)
self.assertEqual(studio.title, 'Shaft')
self.assertEqual(len(tags), 30)
self.assertEqual(len(related_animes), 2)
@responses.activate
def test_anidb_get_animes(self):
# Fake an artist entry with no AniDB creator ID that will be filled by retrieving Sangatsu
artist = Artist(name="Shinbou Akiyuki").save()
filenames = ['anidb/sangatsu_no_lion.xml', 'anidb/sangatsu_no_lion.xml', 'anidb/hibike_euphonium.xml']
with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
for filename in filenames:
rsps.add(
responses.GET,
AniDB.BASE_URL,
body=self.read_fixture(filename),
status=200,
content_type='application/xml'
)
sangatsu = self.anidb.get_or_update_work(11606)
tags_sangatsu_from_anidb = self.anidb.get_tags(11606)
tags_diff_sangatsu = diff_between_anidb_and_local_tags(sangatsu, tags_sangatsu_from_anidb)
hibike = self.anidb.get_or_update_work(10889)
# Retrieve tags
tags_sangatsu = set(Work.objects.get(pk=sangatsu.pk).taggedwork_set.all().values_list('tag__title', flat=True))
tags_hibike = set(Work.objects.get(pk=hibike.pk).taggedwork_set.all().values_list('tag__title', flat=True))
shared_tags = tags_sangatsu.intersection(tags_hibike)
# Checks on tags
self.assertEqual(len(tags_sangatsu), 30)
self.assertEqual(len(tags_hibike), 38)
self.assertEqual(len(shared_tags), 18)
# Check for Sangatsu's informations
self.assertEqual(sangatsu.title, 'Sangatsu no Lion')
self.assertEqual(sangatsu.nb_episodes, 22)
self.assertEqual(sangatsu.studio.title, 'Shaft')
self.assertEqual(sangatsu.date, datetime(2016, 10, 8, 0, 0))
self.assertEqual(sangatsu.end_date, datetime(2017, 3, 18, 0, 0))
# Check for Sangatsu's staff
staff_sangatsu = Work.objects.get(pk=sangatsu.pk).staff_set.all().values_list('artist__name', flat=True)
self.assertCountEqual(staff_sangatsu, ['Umino Chika', 'Hashimoto Yukari', 'Shinbou Akiyuki', 'Okada Kenjirou'])
# Check retrieved tags from AniDB
self.assertEqual(len(tags_diff_sangatsu["deleted_tags"]), 0)
self.assertEqual(len(tags_diff_sangatsu["added_tags"]), 0)
self.assertEqual(len(tags_diff_sangatsu["updated_tags"]), 0)
self.assertEqual(len(tags_diff_sangatsu["kept_tags"]), len(tags_sangatsu))
# Check for no artist duplication
artist = Artist.objects.filter(name="Shinbou Akiyuki")
self.assertEqual(artist.count(), 1)
self.assertEqual(artist.first().anidb_creator_id, 59)
@responses.activate
def test_anidb_duplicate_anime_id(self):
for _ in range(2):
responses.add(
responses.GET,
AniDB.BASE_URL,
body=self.read_fixture('anidb/hibike_euphonium.xml'),
status=200,
content_type='application/xml'
)
self.create_anime(title='Hibike! Euphonium', anidb_aid=10889)
self.create_anime(title='Hibike! Euphonium Duplicate', anidb_aid=10889)
self.anidb.get_or_update_work(10889)
self.assertIs(self.anidb.get_or_update_work(10889), None)
@responses.activate
def test_anidb_nsfw(self):
animes = {}
animes_sources = {
# Not NSFW at all
'anidb/sangatsu_no_lion.xml': (11606, 'Sangatsu no Lion'),
'anidb/hibike_euphonium.xml': (10889, 'Hibike! Euphonium'),
# Totally NSFW (restricted on AniDB)
'anidb/boku_no_piko.xml': (4544, 'Boku no Piko'),
'anidb/bible_black.xml': (528, 'Bible Black'),
# Should be marked NSFW
'anidb/r15.xml': (8396, 'R-15'),
'anidb/astarotte_no_omocha_ex.xml': (8560, 'Astarotte no Omocha! EX'),
'anidb/aki_sora.xml': (6782, 'Aki Sora'),
# Shouldn't be marked NSFW
'anidb/punchline.xml': (10948, 'Punch Line'),
'anidb/panty_stocking.xml': (7529, 'Panty & Stocking with Garterbelt'),
'anidb/shimoneta.xml': (10888, 'Shimoneta to Iu Gainen ga Sonzai Shinai Taikutsu na Sekai')
}
are_nsfw = ['anidb/boku_no_piko.xml', 'anidb/bible_black.xml',
'anidb/r15.xml', 'anidb/astarotte_no_omocha_ex.xml',
'anidb/aki_sora.xml']
are_sfw = ['anidb/sangatsu_no_lion.xml', 'anidb/hibike_euphonium.xml',
'anidb/punchline.xml', 'anidb/panty_stocking.xml',
'anidb/shimoneta.xml']
with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
for filename, infos in animes_sources.items():
rsps.add(
responses.GET,
AniDB.BASE_URL,
body=self.read_fixture(filename),
status=200,
content_type='application/xml'
)
animes[filename] = self.anidb.get_or_update_work(infos[0])
for filename in are_nsfw:
with self.subTest('Asserting NSFW', anime=animes_sources[filename][1]):
self.assertEqual(animes[filename].title, animes_sources[filename][1])
self.assertTrue(animes[filename].nsfw)
for filename in are_sfw:
with self.subTest('Asserting SFW', anime=animes_sources[filename][1]):
self.assertEqual(animes[filename].title, animes_sources[filename][1])
self.assertFalse(animes[filename].nsfw)
@responses.activate
def test_anidb_related_animes(self):
animes = {}
related_animes = {}
animes_sources = {
'anidb/hibike_euphonium.xml': 10889,
'anidb/hibike_euphonium2.xml': 11746,
'anidb/hibike_euphonium_movie1.xml': 11747,
'anidb/hibike_euphonium_movie2.xml': 12962,
'anidb/hibike_euphonium_original_movies.xml': 13207,
'anidb/sangatsu_no_lion.xml': 11606
}
with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
for filename, anidb_aid in animes_sources.items():
for _ in range(2):
rsps.add(
responses.GET,
AniDB.BASE_URL,
body=self.read_fixture(filename),
status=200,
content_type='application/xml'
)
animes[filename] = self.anidb.get_or_update_work(anidb_aid)
related_animes[filename] = self.anidb.get_related_animes(anidb_aid=anidb_aid)
# Ran once in get_or_update_work but ran again to check that it does not cause errors
for filename in animes_sources:
self.anidb._build_related_animes(animes[filename], related_animes[filename])
relations = RelatedWork.objects.filter(
child_work__anidb_aid__in=animes_sources.values(),
parent_work__anidb_aid__in=animes_sources.values()
)
# Checks that anime are created if missing but not all data is retrieved from AniDB
self.assertEqual(Work.objects.get(title='Sangatsu no Lion meets Bump of Chicken').ext_synopsis, '')
self.assertNotEqual(Work.objects.get(title='Sangatsu no Lion').ext_synopsis, '')
# Checks on relations
self.assertTrue(relations.filter(child_work__anidb_aid=11746, parent_work__anidb_aid=10889, type='sequel').exists())
self.assertTrue(relations.filter(child_work__anidb_aid=10889, parent_work__anidb_aid=11746, type='prequel').exists())
self.assertTrue(relations.filter(child_work__anidb_aid=11747, parent_work__anidb_aid=10889, type='summary').exists())
self.assertTrue(relations.filter(child_work__anidb_aid=10889, parent_work__anidb_aid=11747, type='full_story').exists())
self.assertTrue(relations.filter(child_work__anidb_aid=13207, parent_work__anidb_aid=11746, type='sequel').exists())
self.assertTrue(relations.filter(child_work__anidb_aid=11746, parent_work__anidb_aid=13207, type='prequel').exists())
self.assertTrue(relations.filter(child_work__anidb_aid=12962, parent_work__anidb_aid=11746, type='summary').exists())
self.assertTrue(relations.filter(child_work__anidb_aid=11746, parent_work__anidb_aid=12962, type='full_story').exists())
def handle(self, *args, **options):
category = 'anime'
start = 0
if options.get('id'):
anime_id = options.get('id')[0]
anime = Work.objects.filter(category__slug='anime').get(
id=anime_id)
if anime.anidb_aid == 0:
for reference in anime.reference_set.all():
if reference.url.startswith(
'http://anidb.net') or reference.url.startswith(
'https://anidb.net'):
query = urlparse(reference.url).query
anidb_aid = parse_qs(query).get('aid')
if anidb_aid:
anime.anidb_aid = anidb_aid[0]
anime.save()
todo = Work.objects.filter(category__slug='anime',
id=anime_id,
anidb_aid__gt=0)
else:
todo = Work.objects\
.only('pk', 'title', 'ext_poster', 'nsfw')\
.annotate(rating_count=Count('rating'))\
.filter(category__slug=category, rating_count__gte=6)\
.exclude(anidb_aid=0)\
.order_by('-rating_count')
a = AniDB('mangakihttp', 1)
i = 0
for anime in todo:
i += 1
if i < start:
continue
print(i, ':', anime.title, anime.id)
creators = a.get(anime.anidb_aid).creators
worktitles = a.get(anime.anidb_aid).worktitles
for worktitle in worktitles:
language = Language.objects.get(iso639=worktitle[2])
WorkTitle.objects.get_or_create(work=anime,
title=worktitle[0],
language=language,
type=worktitle[1])
retrieve_tags = anime.retrieve_tags(a)
deleted_tags = retrieve_tags["deleted_tags"]
added_tags = retrieve_tags["added_tags"]
updated_tags = retrieve_tags["updated_tags"]
kept_tags = retrieve_tags["kept_tags"]
print(anime.title + ":")
if deleted_tags:
print("\n\tLes tags enlevés sont :")
for tag, weight in deleted_tags.items():
print('\t\t{}: {} '.format(tag, weight))
if added_tags:
print("\n\tLes tags totalement nouveaux sont :")
for tag, weight in added_tags.items():
print('\t\t{}: {} '.format(tag, weight))
if updated_tags:
print("\n\tLes tags modifiés sont :")
for tag, weight in updated_tags.items():
print('\t\t{}: {} -> {}'.format(tag, weight[0], weight[1]))
if kept_tags:
print("\n\tLes tags non modifiés/restés identiques sont :")
for tag, weight in kept_tags.items():
print('\t\t{}: {} '.format(tag, weight))
choice = input("Voulez-vous réaliser ces changements [y/n] : ")
if choice == 'n':
print("\nOk, aucun changement ne va être fait")
elif choice == 'y':
anime.update_tags(deleted_tags, added_tags, updated_tags)
staff_map = dict(
Role.objects.filter(
slug__in=['author', 'director', 'composer']).values_list(
'slug', 'pk'))
for creator in creators.findAll('name'):
artist = get_or_create_artist(creator.string)
if creator['type'] == 'Direction':
staff_id = 'director'
elif creator['type'] == 'Music':
staff_id = 'composer'
elif creator['type'] == 'Original Work' or creator[
'type'] == 'Story Composition':
staff_id = 'author'
else:
staff_id = None
if staff_id is not None:
Staff.objects.get_or_create(work=anime,
role_id=staff_map[staff_id],
artist=artist)
anime.save()
def setUp(self):
# FIXME: The defaults for editor and studio in Work requires those to
# exist, or else foreign key constraints fail.
Editor.objects.create(pk=1)
Studio.objects.create(pk=1)
self.anidb = AniDB('mangakihttp', 1)
