def setUp(self):
user = get_user_model().objects.create_user(username='******',
password='******')
otaku = get_user_model().objects.create_user(username='******',
password='******')
otaku2 = get_user_model().objects.create_user(username='******',
password='******')
self.anime_category = Category.objects.get(slug='anime')
manga = Category.objects.get(slug='manga')
works = [
Work(title='Anime B', nb_episodes=0, category=self.anime_category),
Work(title='Anime A', nb_episodes=1, category=self.anime_category),
Work(title='Manga B', category=manga),
Work(title='Manga A', category=manga),
]
works = Work.objects.bulk_create(works)
# This will work as long as mangaki.algo.dataset.RATED_BY_AT_LEAST <= 2
ratings = (
[Rating(user=otaku, work=work, choice='like')
for work in works] + [
Rating(user=otaku2, work=work, choice='dislike')
for work in works
] + [Rating(user=user, work=works[0], choice='dislike')])
Rating.objects.bulk_create(ratings)
if not os.path.exists(SNAPSHOT_DIR_TEST):
os.makedirs(SNAPSHOT_DIR_TEST)
def setUp(self):
self.user = get_user_model().objects.create_user(username='******',
password='******')
otaku = get_user_model().objects.create_user(username='******',
password='******')
otaku2 = get_user_model().objects.create_user(username='******',
password='******')
self.anime_category = Category.objects.get(slug='anime')
manga = Category.objects.get(slug='manga')
works = [
Work(title='Anime B', nb_episodes=0, category=self.anime_category),
Work(title='Anime A', nb_episodes=1, category=self.anime_category),
Work(title='Manga B', category=manga),
Work(title='Manga A', category=manga),
]
works = Work.objects.bulk_create(works)
self.work = works[0]
# This will work as long as zero.dataset.RATED_BY_AT_LEAST <= 2
ratings = (
[Rating(user=otaku, work=work, choice='like')
for work in works] + [
Rating(user=otaku2, work=work, choice='dislike')
for work in works
] + [Rating(user=self.user, work=works[0], choice='dislike')])
Rating.objects.bulk_create(ratings)
if not os.path.exists(ML_SNAPSHOT_ROOT_TEST):
os.makedirs(ML_SNAPSHOT_ROOT_TEST)
for key in {'svd', 'als', 'knn', 'knn-anonymous'}:
path = get_path(key)
if not os.path.exists(path):
os.makedirs(path)
def signup(self, request, user):
if self.cleaned_data['import_ratings']:
ratings = get_anonymous_ratings(request.session)
clear_anonymous_ratings(request.session)
Rating.objects.bulk_create([
Rating(user=user, work_id=work_id, choice=choice)
for work_id, choice in ratings.items()
])
def get_profile_ratings(request,
category: str,
already_seen: bool,
can_see: bool,
is_anonymous: bool,
user: User) -> Tuple[List[Rating], Counter]:
counts = Counter()
if is_anonymous:
ratings = []
anon_ratings = get_anonymous_ratings(request.session)
works_per_pk = Work.objects.select_related('category').in_bulk(anon_ratings.keys())
for pk, choice in anon_ratings.items():
rating = Rating()
rating.work = works_per_pk[pk]
rating.choice = choice
seen_work = rating.choice not in SEE_CHOICES['unseen']
count_key = 'seen_{}' if seen_work else 'unseen_{}'
counts[count_key.format(rating.work.category.slug)] += 1
if already_seen == seen_work and rating.work.category.slug == category:
ratings.append(rating)
elif can_see:
ratings = list(
Rating.objects
.filter(user=user,
work__category__slug=category,
choice__in=SEE_CHOICES['seen'] if already_seen else SEE_CHOICES['unseen'])
.select_related('work', 'work__category')
)
categories = Category.objects.all()
for category in categories:
qs = Rating.objects.filter(user=user,
work__category=category)
seen = qs.filter(choice__in=SEE_CHOICES['seen']).count()
unseen = qs.count() - seen
counts['seen_{}'.format(category.slug)] = seen
counts['unseen_{}'.format(category.slug)] = unseen
else:
ratings = []
return ratings, counts
def import_mal(mal_username, mangaki_username):
"""
Import myAnimeList by username
"""
MAL_URL = 'http://myanimelist.net/malappinfo.php?u=%s&status=all&type=anime' % mal_username
HEADERS = {'X-Real-IP': random_ip(), 'User-Agent': MAL_USER_AGENT}
r = requests.get(MAL_URL, headers=HEADERS)
xml = ET.fromstring(r.text)
user = User.objects.get(username=mangaki_username)
nb_added = 0
fails = []
for entry in xml:
if entry.tag == 'anime':
title = entry.find('series_title').text
poster = entry.find('series_image').text
score = int(entry.find('my_score').text)
mal_id = entry.find('series_animedb_id').text
try:
animes = Work.objects.filter(category__slug='anime')
try:
anime = animes.get(title=title)
except Work.DoesNotExist:
if animes.filter(poster=poster).count() == 1:
anime = Work.objects.get(poster=poster)
elif animes.filter(poster=poster).count() >= 2:
raise Exception(
'Integrity violation: found two or more works with the same poster, do you come from the past?'
)
else:
entries = lookup_mal_api(title)
retrieve_anime(entries)
anime = animes.get(poster=poster)
if anime:
if not Rating.objects.filter(user=user,
work=anime).count():
if 7 <= score <= 10:
choice = 'like'
elif 5 <= score <= 6:
choice = 'neutral'
elif score > 0:
choice = 'dislike'
else:
continue
Rating(user=user, work=anime, choice=choice).save()
nb_added += 1
except Exception as e:
print(e)
SearchIssue(user=user,
title=title,
poster=poster,
mal_id=mal_id,
score=score).save()
fails.append(title)
return nb_added, fails
def signup(self, request, user):
if self.cleaned_data['import_ratings']:
ratings = get_anonymous_ratings(request.session)
clear_anonymous_ratings(request.session)
Rating.objects.bulk_create([
Rating(user=user, work_id=work_id, choice=choice)
for work_id, choice in ratings.items()
])
Profile.objects.filter(id=user.profile.pk).update(
newsletter_ok=self.cleaned_data['newsletter_ok'],
research_ok=self.cleaned_data['research_ok'])
def import_mal(mal_username: str, mangaki_username: str):
"""
Import myAnimeList by username
"""
user = User.objects.get(username=mangaki_username)
fails = []
mangaki_lists = {
MALWorks.animes: Work.objects.filter(category__slug='anime'),
MALWorks.mangas: Work.objects.filter(category__slug='manga')
}
scores = {}
willsee = set()
wontsee = set()
for work_type in SUPPORTED_MANGAKI_WORKS:
user_works = set(client.list_works_from_a_user(work_type,
mal_username))
logger.info('Fetching {} works from {}\'s MAL.'.format(
len(user_works), mal_username))
for user_work in user_works:
try:
work = get_or_create_from_mal(mangaki_lists[work_type],
work_type, user_work.title,
user_work.synonyms,
user_work.poster)
if (work and
not any(work.id in container
for container in (scores, wontsee, willsee))):
if user_work.status == MALStatus.completed:
scores[work.id] = user_work.score
elif user_work.status == MALStatus.dropped:
wontsee.add(work.id)
elif user_work.status == MALStatus.plan_to_watch:
willsee.add(work.id)
except Exception:
logger.exception(
'Failure to fetch the work from MAL and import it into the Mangaki database.'
)
SearchIssue(user=user,
title=user_work.title,
poster=user_work.poster,
mal_id=user_work.mal_id,
score=user_work.score).save()
fails.append(user_work.title)
existing_ratings = (Rating.objects.filter(
user=user, work__in=scores.keys()).values_list('work',
flat=True).all())
for related_work_id in existing_ratings:
del scores[related_work_id]
ratings = []
for work_id, score in scores.items():
choice = compute_rating_choice_from_mal_score(score)
if not choice:
raise RuntimeError('No choice was deduced from MAL score!')
rating = Rating(user=user, choice=choice, work_id=work_id)
ratings.append(rating)
for work_id in willsee:
rating = Rating(user=user, choice='willsee', work_id=work_id)
ratings.append(rating)
for work_id in wontsee:
rating = Rating(user=user, choice='wontsee', work_id=work_id)
ratings.append(rating)
Rating.objects.bulk_create(ratings)
return len(ratings), fails
def setUp(self):
self.user = get_user_model().objects.create_superuser(
username='******', password='******', email='*****@*****.**')
self.users = []
for username in 'ABCD':
self.users.append(get_user_model().objects.create_user(
username=username, password='******'))
today = datetime.now()
yesterday = datetime.now() - timedelta(1)
tomorrow = datetime.now() + timedelta(1)
anime = Category.objects.get(slug='anime')
Work.objects.bulk_create([
Work(title='Sangatsu no Lion', category=anime) for _ in range(10)
])
Work.objects.create(title='Sangatsu no Lion',
category=anime,
nb_episodes=22)
self.work_ids = Work.objects.values_list('id', flat=True)
# Admin rated every movie
Rating.objects.bulk_create([
Rating(work_id=work_id, user=self.user, choice='like')
for work_id in self.work_ids
])
the_artist = Artist.objects.create(name='Yoko Kanno')
references = []
for work_id in self.work_ids:
references.extend(
Reference.objects.bulk_create([
Reference(work_id=work_id,
source='MAL',
identifier=31646,
url='https://myanimelist.net/anime/31646'),
Reference(
work_id=work_id,
source='AniDB',
identifier=11606,
url=
'https://anidb.net/perl-bin/animedb.pl?show=anime&aid=11606'
)
]))
roles = Role.objects.bulk_create([
Role(name='Director', slug='xxx'),
Role(name='Composer', slug='yyy')
])
Staff.objects.bulk_create([
Staff(work_id=self.work_ids[0], artist=the_artist, role=roles[0]),
Staff(work_id=self.work_ids[1], artist=the_artist, role=roles[0]),
Staff(work_id=self.work_ids[1], artist=the_artist, role=roles[1])
])
genres = Genre.objects.bulk_create(
[Genre(title='SF'),
Genre(title='Slice of life')])
Work.objects.get(id=self.work_ids[0]).genre.add(genres[0])
Work.objects.get(id=self.work_ids[1]).genre.add(genres[0])
Work.objects.get(id=self.work_ids[1]).genre.add(genres[1])
# Rating are built so that after merge, only the favorites should be kept
Rating.objects.bulk_create([
Rating(work_id=self.work_ids[0],
user=self.users[0],
choice='like',
date=today),
Rating(work_id=self.work_ids[1],
user=self.users[0],
choice='favorite',
date=tomorrow),
Rating(work_id=self.work_ids[2],
user=self.users[0],
choice='dislike',
date=yesterday),
Rating(work_id=self.work_ids[1],
user=self.users[1],
choice='favorite',
date=today),
Rating(work_id=self.work_ids[0],
user=self.users[2],
choice='favorite',
date=today),
Rating(work_id=self.work_ids[2],
user=self.users[2],
choice='like',
date=yesterday),
Rating(work_id=self.work_ids[0],
user=self.users[3],
choice='favorite',
date=yesterday)
])
Rating.objects.filter(work_id=self.work_ids[1],
user=self.users[0]).update(date=tomorrow)
Rating.objects.filter(work_id=self.work_ids[2],
user=self.users[0]).update(date=yesterday)
Rating.objects.filter(work_id=self.work_ids[2],
user=self.users[2]).update(date=yesterday),
Rating.objects.filter(work_id=self.work_ids[0],
user=self.users[3]).update(date=yesterday)
def import_mal(mal_username: str, mangaki_username: str, update_callback=None):
"""
Import myAnimeList by username
"""
user = User.objects.get(username=mangaki_username)
fails = []
mangaki_lists = {
MALWorks.animes: Work.objects.filter(category__slug='anime'),
MALWorks.mangas: Work.objects.filter(category__slug='manga')
}
scores = {}
willsee = set()
wontsee = set()
for work_type in SUPPORTED_MANGAKI_WORKS:
user_works = set(client.list_works_from_a_user(work_type,
mal_username))
logger.info('Fetching {} works from {}\'s MAL.'.format(
len(user_works), mal_username))
for current_index, user_work in enumerate(user_works):
try:
work = get_or_create_from_mal(mangaki_lists[work_type],
work_type, user_work.title,
user_work.synonyms,
user_work.poster,
user_work.mal_id)
if (work and
not any(work.id in container
for container in (scores, wontsee, willsee))):
if user_work.status == MALStatus.completed:
scores[work.id] = user_work.score
elif user_work.status == MALStatus.dropped:
wontsee.add(work.id)
elif user_work.status == MALStatus.plan_to_watch:
willsee.add(work.id)
if update_callback:
update_callback(len(user_works), current_index + 1,
user_work.title)
except Exception:
logger.exception(
'Failure to fetch the work from MAL and import it into the Mangaki database.'
)
fails.append(user_work.title)
# MAL is the source of truth for further imports, rather than our own database of ratings.
Rating.objects.filter(user=user,
work__in=list(scores.keys()) +
list(willsee | wontsee)).delete()
ratings = []
for work_id, score in scores.items():
choice = compute_rating_choice_from_mal_score(score)
if not choice:
raise RuntimeError('No choice was deduced from MAL score!')
rating = Rating(user=user, choice=choice, work_id=work_id)
ratings.append(rating)
for work_id in willsee:
rating = Rating(user=user, choice='willsee', work_id=work_id)
ratings.append(rating)
for work_id in wontsee:
rating = Rating(user=user, choice='wontsee', work_id=work_id)
ratings.append(rating)
Rating.objects.bulk_create(ratings)
return len(ratings), fails
