def select_reply_review(args):
"""Select reply collection"""
is_parameter_exists([constants.ID], args)
# Review Id
review_id = args[constants.ID]
# Request User
request_user = args[constants.USER]
# Page Number
page_number = 1 if constants.PAGE_NUMBER not in args else int(
args[constants.PAGE_NUMBER])
# Replies Queryset
queryset = ReplyReview.objects.filter(review_id=review_id)
# Replies
review_replies = get_results_from_queryset(queryset, 10, page_number)
# is_finished
is_finished = not review_replies.has_next()
reply_ids = [review_reply.reply_id for review_reply in review_replies]
replies, _ = __get_replies(Q(id__in=reply_ids), request_user, 10)
return replies, page_number, is_finished
def select_paper_like(args):
"""Select Paper Like"""
# Request User
request_user = args[constants.USER]
# Page Number
page_number = 1 if constants.PAGE_NUMBER not in args else int(args[constants.PAGE_NUMBER])
# Papers Queryset
queryset = PaperLike.objects.filter(Q(user_id=request_user.id)).order_by(
'-creation_date').values_list('paper_id', flat=True)
# Paper Ids
paper_ids = get_results_from_queryset(queryset, 10, page_number)
# is_finished
is_finished = not paper_ids.has_next()
# need to maintain the order
preserved = Case(*[When(pk=pk, then=pos) for pos, pk in enumerate(paper_ids)])
# Papers
papers, _ = __get_papers(Q(id__in=paper_ids), request_user, 10, preserved)
return papers, page_number, is_finished
def select_user_search_collection(args):
"""Search Users Not in Collection"""
is_parameter_exists([constants.TEXT, constants.COLLECTION_ID], args)
# Collection ID
collection_id = args[constants.COLLECTION_ID]
# Request User
request_user = args[constants.USER]
# Search Keyword
keyword = args[constants.TEXT]
# Page Number
page_number = 1 if constants.PAGE_NUMBER not in args else int(
args[constants.PAGE_NUMBER])
# User Queryset
queryset = User.objects.annotate(
is_in_collection=__is_in_collection('id', collection_id)).filter(
username__icontains=keyword,
is_in_collection=False).values_list('id', flat=True)
# User Ids
user_ids = get_results_from_queryset(queryset, 10, page_number)
# is_finished
is_finished = not user_ids.has_next()
# Users
users, _, _ = __get_users(Q(id__in=user_ids), request_user, 10)
return users, page_number, is_finished
def select_paper_collection(args):
"""Select Collections' Paper"""
is_parameter_exists([
constants.ID
], args)
# Request User
request_user = args[constants.USER]
# Collection Id
collection_id = args[constants.ID]
# Page Number
page_number = 1 if constants.PAGE_NUMBER not in args else int(args[constants.PAGE_NUMBER])
# Papers Queryset
queryset = CollectionPaper.objects.filter(
collection_id=collection_id
)
# Papers
collection_papers = get_results_from_queryset(queryset, 10, page_number)
# is_finished
is_finished = not collection_papers.has_next()
paper_ids = [collection_paper.paper_id for collection_paper in collection_papers]
papers, _ = __get_papers(Q(id__in=paper_ids), request_user, 10)
return papers, page_number, is_finished
def select_user_search(args):
"""Select User Search"""
is_parameter_exists([constants.TEXT], args)
# Request User
request_user = args[constants.USER]
# Search Keyword
keyword = args[constants.TEXT]
# Page Number
page_number = 1 if constants.PAGE_NUMBER not in args else int(
args[constants.PAGE_NUMBER])
# User Queryset
queryset = User.objects.filter(Q(username__icontains=keyword)).values_list(
'id', flat=True)
total_count = queryset.count() # count whole users
# User Ids
user_ids = get_results_from_queryset(queryset, 10, page_number)
# is_finished
is_finished = not user_ids.has_next()
# Users
users, _, _ = __get_users(Q(id__in=user_ids), request_user, 10)
return users, page_number, is_finished, total_count
def select_user_following_collection(args):
"""Get Following Users Not in Collection"""
is_parameter_exists([constants.COLLECTION_ID], args)
# Collection ID
collection_id = args[constants.COLLECTION_ID]
# Request User
request_user = args[constants.USER]
# Page Number
page_number = 1 if constants.PAGE_NUMBER not in args else int(
args[constants.PAGE_NUMBER])
# QuerySet
queryset = UserFollow.objects.annotate(is_in_collection=__is_in_collection(
'followed_user', collection_id)).filter(
following_user=request_user.id,
is_in_collection=False).values_list('followed_user', flat=True)
# User Ids
user_ids = get_results_from_queryset(queryset, 10, page_number)
# is_finished
is_finished = not user_ids.has_next()
# Filter Query
filter_query = Q(id__in=user_ids)
# Users
users, _, _ = __get_users(filter_query, request_user, 10)
return users, page_number, is_finished
def __get_reviews(filter_query,
request_user,
count,
params=None,
page_number=1):
"""Get Reviews By Query"""
params = {} if params is None else params
order_by = '-pk' if constants.ORDER_BY not in params else params[
constants.ORDER_BY]
queryset = Review.objects.filter(filter_query).annotate(
is_liked=__is_review_liked('id', request_user)).order_by(order_by)
total_count = queryset.count() if constants.TOTAL_COUNT in params else None
reviews = get_results_from_queryset(queryset, count, page_number)
is_finished = True
if count is not None:
is_finished = not reviews.has_next()
pagination_value = reviews[len(reviews) - 1].id if reviews else 0
reviews = __pack_reviews(reviews, request_user)
return reviews, pagination_value, is_finished, total_count
def __get_keywords(filter_query):
"""Get Keywords By Query"""
queryset = Keyword.objects.filter(
filter_query
)
keywords = get_results_from_queryset(queryset, None)
keywords = __pack_keywords(keywords)
return keywords
def __get_publications(filter_query):
"""Get Publications"""
queryset = Publication.objects.filter(
filter_query
)
publications = get_results_from_queryset(queryset, None)
publications = __pack_publications(publications)
return publications
def __get_publishers(filter_query):
"""Get Publishers By Query"""
queryset = Publisher.objects.filter(
filter_query
)
publishers = get_results_from_queryset(queryset, None)
publishers = __pack_publisher(publishers)
return publishers
def __get_authors(filter_query):
"""Get Authors By Query"""
queryset = Author.objects.filter(
filter_query
)
authors = get_results_from_queryset(queryset, None)
authors = __pack_authors(authors)
return authors
def __get_replies(filter_query, request_user, count):
"""Get Replies By Query"""
queryset = Reply.objects.filter(filter_query).annotate(
is_liked=__is_reply_liked('id', request_user))
replies = get_results_from_queryset(queryset, count)
pagination_value = replies[len(replies) - 1].id if replies else 0
replies = __pack_replies(replies, request_user)
return replies, pagination_value
def select_user_collection(args):
"""Get Users of the given Collection"""
is_parameter_exists([constants.ID], args)
collection_id = args[constants.ID]
request_user = args[constants.USER]
# Decide if results include request_user if she or he is one of members
includes_me = True if constants.INCLUDES_ME not in args else json.loads(
args[constants.INCLUDES_ME])
# Page Number
page_number = 1 if constants.PAGE_NUMBER not in args else int(
args[constants.PAGE_NUMBER])
# Check Collection Id
if not Collection.objects.filter(id=collection_id).exists():
raise ApiError(constants.NOT_EXIST_OBJECT)
# Members QuerySet (except 'pending')
if includes_me:
query = Q(
collection_id=collection_id) & ~Q(type=COLLECTION_USER_TYPE[2])
else:
query = Q(collection_id=collection_id) & ~Q(
user_id=request_user.id) & ~Q(type=COLLECTION_USER_TYPE[2])
queryset = CollectionUser.objects.filter(query)
# Members(including owner) Of Collections
collection_members = get_results_from_queryset(queryset, 10, page_number)
# is_finished
is_finished = not collection_members.has_next()
# Member Ids
member_ids = [
collection_member.user_id for collection_member in collection_members
]
member_ids = list(set(member_ids))
# Get Members
params = {constants.COLLECTION_ID: collection_id}
members, _, _ = __get_users(Q(id__in=member_ids),
request_user,
10,
params=params)
return members, page_number, is_finished
def __get_collections(filter_query,
request_user,
count,
params=None,
page_number=1):
"""Get Collections By Query"""
params = {} if params is None else params
paper_id = None if constants.PAPER_ID not in params else params[
constants.PAPER_ID]
order_by = '-pk' if constants.ORDER_BY not in params else params[
constants.ORDER_BY]
target_user_id = request_user.id if constants.USER_ID not in params else params[
constants.USER_ID]
queryset = Collection.objects.annotate(
is_user_collection=__is_member('id', target_user_id),
is_member=__is_member('id', request_user.id),
is_shared=__is_shared(
'id', target_user_id)).filter(filter_query).annotate(
is_liked=__is_collection_liked('id', request_user),
is_owned=__is_collection_owned('id', request_user),
contains_paper=__contains_paper('id', paper_id),
owner_id=Subquery(CollectionUser.objects.filter(
collection_id=OuterRef('id'),
type=COLLECTION_USER_TYPE[0]).values('user_id')[:1],
output_field=models.IntegerField()),
collection_user_type=Subquery(
CollectionUser.objects.filter(
collection_id=OuterRef('id'),
user_id=request_user.id,
).values('type')[:1],
output_field=models.CharField())).order_by(order_by)
total_count = queryset.count() if constants.TOTAL_COUNT in params else None
collections = get_results_from_queryset(queryset, count, page_number)
is_finished = True
if count is not None:
is_finished = not collections.has_next()
pagination_value = collections[len(collections) -
1].id if collections else 0
collections = __pack_collections(collections,
request_user,
paper_id=paper_id)
return collections, pagination_value, is_finished, total_count
def __get_papers(filter_query, request_user, count, order_by='-pk'):
"""Get Papers By Query"""
queryset = Paper.objects.filter(
filter_query
).annotate(
is_liked=__is_paper_liked('id', request_user)
).order_by(order_by)
papers = get_results_from_queryset(queryset, count)
pagination_value = papers[len(papers) - 1].id if papers else 0
papers = __pack_papers(papers, request_user)
return papers, pagination_value
def select_recommendation(args):
"""Get Recommendations"""
request_user = args[constants.USER]
page_number = 1 if constants.PAGE_NUMBER not in args else int(args[constants.PAGE_NUMBER])
recommendation_queryset = UserRecommendation.objects.filter(user_id=request_user.id).order_by('rank')
recommendations = get_results_from_queryset(recommendation_queryset, 30, page_number)
is_finished = not recommendations.has_next()
recommendations = __pack_recommendations(recommendations, request_user)
return recommendations, page_number, is_finished
def select_keyword_init(args):
"""Get keywords init"""
page_number = 1 if constants.PAGE_NUMBER not in args else int(args[constants.PAGE_NUMBER])
keyword_queryset = (Keyword.objects.annotate(num_papers=Count('papers'))
.filter(num_papers__gt=9, num_papers__lt=100))
keywords = get_results_from_queryset(keyword_queryset, 20, page_number)
is_finished = not keywords.has_next()
keywords = paper_utils.pack_keywords(keywords)
return keywords, page_number, is_finished
def __get_users(filter_query, request_user, count, params=None):
"""Get Users By Query"""
params = {} if params is None else params
collection_id = None if constants.COLLECTION_ID not in params else params[
constants.COLLECTION_ID]
queryset = User.objects.filter(filter_query).annotate(
is_following=__is_following('id', request_user),
is_followed=__is_followed('id', request_user))
# FIXME: This function needs refactoring so that it works like that of collections/utils or reviews/utils
total_count = queryset.count() if constants.TOTAL_COUNT in params else None
users = get_results_from_queryset(queryset, count)
pagination_value = users[len(users) - 1].id if users else 0
users = __pack_users(users, request_user, collection_id=collection_id)
return users, pagination_value, total_count
def select_subscriptions(args):
"""Get Subscriptions of the current User"""
request_user = args[constants.USER]
page_number = 1 if constants.PAGE_NUMBER not in args else int(
args[constants.PAGE_NUMBER])
# get the list of users that this user is following
followings_queryset = UserFollow.objects.filter(
following_user=request_user.id).values_list('followed_user', flat=True)
# anonymous reviews should not be seen
anonymous_reviews = Review.objects.filter(anonymous=True).values_list(
'id', flat=True)
# private collections where the user is not a member should not be seen
private_collections = Collection.objects.annotate(
is_member=__is_member('id', request_user.id)).filter(
type="private", is_member=False).values_list('id', flat=True)
# Subscription QuerySet
subscription_queryset = Subscription.objects.filter(
Q(actor__in=followings_queryset)).exclude(
action_object_content_type__model="Review",
action_object_object_id__in=anonymous_reviews,
).exclude(
action_object_content_type__model="Collection",
action_object_object_id__in=private_collections,
)
subscriptions = get_results_from_queryset(subscription_queryset, 20,
page_number)
# is_finished
is_finished = not subscriptions.has_next()
subscriptions = __pack_subscriptions(subscriptions, request_user)
return subscriptions, page_number, is_finished
def select_notifications(args):
"""Get Notifications of the current User"""
# Request User
request_user = args[constants.USER] if constants.USER in args else None
# Page Number
page_number = 1 if constants.PAGE_NUMBER not in args else int(args[constants.PAGE_NUMBER])
# User
user = User.objects.get(pk=request_user.id)
# Notification QuerySet
queryset = user.notifications.unread().filter(~Q(actor_object_id=request_user.id))
total_count = queryset.count()
notifications = get_results_from_queryset(queryset, 10, page_number)
is_finished = not notifications.has_next()
notifications = __pack_notifications(notifications)
return notifications, page_number, is_finished, total_count
def __get_papers_search(result_ids, request_user, source_to_id, external, is_finished):
"""Get Papers By Query"""
# need to maintain the order
preserved = Case(*[When(pk=pk, then=pos) for pos, pk in enumerate(result_ids)])
queryset = Paper.objects.filter(
Q(id__in=result_ids)
).annotate(
is_liked=__is_paper_liked('id', request_user)
).order_by(preserved)
if external:
# if external, give all ready search results
papers = queryset
else:
papers = get_results_from_queryset(queryset, SEARCH_COUNT)
is_finished = len(papers) < SEARCH_COUNT if SEARCH_COUNT and papers else True
pagination_value = papers[len(papers) - 1].id if papers else 0
papers = __pack_papers(papers, request_user, source_to_id=source_to_id)
return papers, pagination_value, is_finished
def select_user_followed(args):
"""Select User’s Followers"""
is_parameter_exists([constants.ID], args)
# Request User
request_user = args[constants.USER]
# Requested User ID
requested_user_id = args[constants.ID]
# Page Number
page_number = 1 if constants.PAGE_NUMBER not in args else int(
args[constants.PAGE_NUMBER])
# Check User Id
if not User.objects.filter(id=requested_user_id).exists():
raise ApiError(constants.NOT_EXIST_OBJECT)
# Follower QuerySet
queryset = UserFollow.objects.filter(
followed_user=requested_user_id).values_list('following_user',
flat=True)
# User Ids
user_ids = get_results_from_queryset(queryset, 10, page_number)
# is_finished
is_finished = not user_ids.has_next()
# Filter Query
filter_query = Q(id__in=user_ids)
# Users
users, _, _ = __get_users(filter_query, request_user, 10)
return users, page_number, is_finished
def select_paper_search(args):
"""Select Paper Search"""
is_parameter_exists([
constants.TEXT
], args)
# Request User
request_user = args[constants.USER]
# Search Keyword
keyword = args[constants.TEXT]
# Page Number
page_number = 1 if constants.PAGE_NUMBER not in args else int(args[constants.PAGE_NUMBER])
# Check cache
cache_key = keyword + '_' + str(page_number)
cache_result = cache.get(cache_key)
if cache_result:
logging.info("CACHE HIT: %s", cache_key)
result_ids, paper_ids, external, is_finished = cache_result
else:
logging.info("CACHE MISS: %s", cache_key)
paper_ids = {
'papersfeed': [], # only used when all external search fails
'arxiv': [],
'crossref': [],
}
is_finished_dict = {
'arxiv': False,
'crossref': False,
}
future_to_source = {}
with concurrent.futures.ThreadPoolExecutor() as executor:
future_to_source[executor.submit(exploit_arxiv, keyword, page_number)] = 'arxiv'
future_to_source[executor.submit(exploit_crossref, keyword, page_number)] = 'crossref'
for future in concurrent.futures.as_completed(future_to_source):
source = future_to_source[future]
try:
paper_ids[source], is_finished_dict[source] = future.result()
except Exception as exc: # pylint: disable=broad-except
logging.warning("[%s] generated an exception: %s", source, exc)
external = True
# if there are both results (len <= SEARCH_RESULT * 2)
if paper_ids['arxiv'] and paper_ids['crossref']:
result_ids = paper_ids['arxiv'] + paper_ids['crossref']
is_finished = is_finished_dict['arxiv'] and is_finished_dict['crossref']
# if only there are results of Crossref (len <= SEARCH_COUNT)
elif not paper_ids['arxiv'] and paper_ids['crossref']:
result_ids = paper_ids['crossref']
is_finished = is_finished_dict['crossref']
# if only there are results of arXiv (len <= SEARCH_COUNT)
elif not paper_ids['crossref'] and paper_ids['arxiv']:
result_ids = paper_ids['arxiv']
is_finished = is_finished_dict['arxiv']
# if cannot get any results from external sources
else:
external = False
logging.info("[naive-search] Searching in PapersFeed DB")
# Papers Queryset
queryset = Paper.objects.filter(Q(title__icontains=keyword) | Q(abstract__icontains=keyword)) \
.values_list('id', 'abstract')
# Check if the papers have keywords. If not, try extracting keywords this time
abstracts = {}
for paper_id, abstract in queryset:
paper_ids['papersfeed'].append(paper_id)
if not PaperKeyword.objects.filter(paper_id=paper_id).exists():
abstracts[paper_id] = abstract
if abstracts:
__extract_keywords_from_abstract(abstracts)
# Paper Ids
result_ids = get_results_from_queryset(paper_ids['papersfeed'], SEARCH_COUNT, page_number)
# is_finished (tentative)
is_finished = False
# Cache set
cache.set(cache_key, (result_ids, paper_ids, external, is_finished), timeout=None)
logging.info("CACHE SET: %s", cache_key)
# Papers - Sometimes, there can be duplicated ids in result_ids. Thus, len(papers) < len(result_ids) is possible.
papers, _, is_finished = __get_papers_search(result_ids, request_user, paper_ids, external, is_finished)
return papers, page_number, is_finished
