def test_pipeline_graph_vertex_chain_dict_to_list_internal(self):
pipeline_test = pipeline.Pipeline(
self.graph_test_2.get_vertices_iterator(), [
pipes.GraphVertexPipe(),
pipes.ElementPropertiesPipe(internal=True)
])
expected_output = [{
'_id': 1,
'_label': 'test_label',
'test_field': 8
}, {
'_id': 2,
'_label': 'test_label',
'test_field': 10
}, {
'_id': 3,
'_label': 'test_label',
'test_field': 12,
'time': 'now'
}]
self.assertEqual(pipeline_test.to_list(), expected_output)
self.assertEqual(
str(pipeline_test),
'[DictKeyValueIterator(3), GraphVertexPipe(), ElementPropertiesPipe(internal:True)]'
)
def test_pipeline_refresh_as_pipes(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
result = pipeline_test.refresh_as_pipes()
self.assertEqual(result, None)
def test_pipeline_exclude_limit(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
self.assertEqual(pipeline_test.to_list(), [2, 3, 4, 5, 6])
def get_storefront_most_popular(request_profile,
pre_fetch=True,
ordering=None):
"""
Get Most Popular Listings for storefront
"""
username = request_profile.user.username
# Get most popular listings via a weighted average
most_popular_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings(
username).filter(approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False).order_by('-avg_rate',
'-total_reviews')
most_popular_listings = pipeline.Pipeline(
recommend_utils.ListIterator(
[listing for listing in most_popular_listings_raw]), [
pipes.ListingPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(36)
]).to_list()
sorted_most_popular_listings = custom_sort_listings(
most_popular_listings, ordering)
return sorted_most_popular_listings
def test_pipeline_limit(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.LimitPipe(5)])
self.assertEqual(pipeline_test.to_list(), [1, 2, 3, 4, 5])
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.LimitPipe(2)])
self.assertEqual(pipeline_test.to_list(), [1, 2])
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3]), [pipes.LimitPipe(5)])
self.assertEqual(pipeline_test.to_list(), [1, 2, 3])
def test_pipeline_remove(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
self.assertRaises(recommend_utils.UnsupportedOperationException,
pipeline_test.remove)
def test_pipeline_get_starts(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
result = pipeline_test.get_starts()
self.assertEqual(str(result), 'ListIterator(7)')
def test_pipeline_graph_vertex_while(self):
pipeline_test = pipeline.Pipeline(
self.graph_test_1.get_vertices_iterator(),
[pipes.GraphVertexPipe()])
list_out = self._iterate_pipeline(pipeline_test)
self.assertEquals(str(list_out),
'[Vertex(test_label), Vertex(test_label)]')
def _create_folder_listing_queries_and_serialized(request_profile, folder_bookmark_entry, is_parent=None, ordering_fields=None, serializer_class=None, request=None):
"""
Create queries to get folder and listing data
Args:
request_profile:
used to filter profile bookmarks
request:
used for serializer to create urls
folder_bookmark_entry:
is_parent:
ordering_fields:
"""
is_parent = is_parent if is_parent else False
ordering_fields = ordering_fields if ordering_fields else ['created_date']
folder_query = models.BookmarkEntry.objects.filter(
type=FOLDER_TYPE,
bookmark_parent__bookmark_permission__profile=request_profile # Validate to make sure user can see folder
)
# for_profile should do `private listing` check for listing
listing_query = models.BookmarkEntry.objects.for_profile_minus_security_marking(request_profile).filter(
type=LISTING_TYPE,
bookmark_parent__bookmark_permission__profile=request_profile # Validate to make sure user can see folder,
)
folder_order_fields = []
listing_order_fields = []
for ordering_field in ordering_fields:
if ordering_field in FOLDER_QUERY_ORDER_MAPPING:
folder_order_fields.append(FOLDER_QUERY_ORDER_MAPPING[ordering_field])
if ordering_field in LISTING_QUERY_ORDER_MAPPING:
listing_order_fields.append(LISTING_QUERY_ORDER_MAPPING[ordering_field])
folder_query = folder_query.order_by(*folder_order_fields)
listing_query = listing_query.order_by(*listing_order_fields)
if is_parent:
folder_query = folder_query.filter(id=folder_bookmark_entry.id)
listing_query = listing_query.filter(id=folder_bookmark_entry.id)
else:
folder_query = folder_query.filter(bookmark_parent=folder_bookmark_entry)
listing_query = listing_query.filter(bookmark_parent=folder_bookmark_entry)
folder_data = serializer_class(folder_query, many=True, context={'request': request}).data
listing_data = serializer_class(listing_query, many=True, context={'request': request}).data
# Filter out all listings that request_profile does not have access to
listing_data_after_filter = pipeline.Pipeline(recommend_utils.ListIterator(listing_data),
[pipes.BookmarkListingDictPostSecurityMarkingCheckPipe(request_profile.user.username)]).to_list()
return {"folder_data": folder_data,
"listing_data_after_filter": listing_data_after_filter}
def test_pipeline_graph_vertex_chain_to_list(self):
pipeline_test = pipeline.Pipeline(
self.graph_test_1.get_vertices_iterator(),
[pipes.GraphVertexPipe(),
pipes.ElementIdPipe()])
self.assertEqual(pipeline_test.to_list(), [1, 2])
self.assertEqual(
str(pipeline_test),
'[DictKeyValueIterator(2), GraphVertexPipe(), ElementIdPipe()]')
def test_pipeline_count(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
result = pipeline_test.count()
self.assertEqual(result, 5)
result = pipeline_test.count()
self.assertEqual(result, 0)
def test_pipeline_get_pipes(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
result = ', '.join([str(pipe) for pipe in pipeline_test.get_pipes()])
self.assertEqual(
str(result),
'ListIterator(7), ExcludePipe(), LimitPipe(limit_number:5)')
def test_pipeline_capitalize(self):
caps_pipe = pipes.CapitalizePipe()
pipeline_test = pipeline.Pipeline()
pipeline_test.add_pipe(caps_pipe)
pipeline_test.set_starts(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']))
list_out = self._iterate_pipeline(pipeline_test)
self.assertEqual(list_out, ['THIS', 'IS', 'THE', 'TEST'])
def test_pipeline_graph_vertex_chain_to_list(self):
graph = Graph()
graph.add_vertex('test_label', {'test_field': 1})
graph.add_vertex('test_label', {'test_field': 2})
pipeline_test = pipeline.Pipeline(
graph.get_vertices_iterator(),
[pipes.GraphVertexPipe(),
pipes.ElementIdPipe()])
self.assertEqual(pipeline_test.to_list(), [1, 2])
def list(self, request, listing_pk=None):
queryset = self.filter_queryset(self.get_queryset(listing_pk))
serializer = serializers.ListingSerializer(
queryset, context={'request': request}, many=True)
similar_listings = pipeline.Pipeline(
recommend_utils.ListIterator(serializer.data), [
pipes.ListingDictPostSecurityMarkingCheckPipe(
self.request.user.username),
pipes.LimitPipe(10)
]).to_list()
return Response(similar_listings)
def test_pipeline_chain_2(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']),
[pipes.CapitalizePipe(), pipes.LenPipe()])
list_out = []
try:
while pipeline_test.has_next():
current_object = pipeline_test.next()
list_out.append(current_object)
except recommend_utils.FastNoSuchElementException:
# Ignore FastNoSuchElementException
pass
self.assertEqual(list_out, [4, 2, 3, 4])
def list(self, request):
queryset = self.get_queryset()
serializer = serializers.ImageSerializer(queryset,
many=True,
context={'request': request})
serializer_iterator = recommend_utils.ListIterator(serializer.data)
pipeline_list = [
pipes.ListingDictPostSecurityMarkingCheckPipe(
request.user.username)
]
recommended_listings = pipeline.Pipeline(serializer_iterator,
pipeline_list).to_list()
return Response(recommended_listings)
def get_storefront_recent(request_profile, pre_fetch=True):
"""
Get Recent Listings for storefront
"""
username = request_profile.user.username
# Get Recent Listings
recent_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings(
username).order_by('-approved_date').filter(
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False)
recent_listings = pipeline.Pipeline(recommend_utils.ListIterator([listing for listing in recent_listings_raw]),
[pipes.ListingPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(24)]).to_list()
return recent_listings
def get_storefront_featured(request_profile, pre_fetch=True):
"""
Get Featured Listings for storefront
"""
username = request_profile.user.username
# Get Featured Listings
featured_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings(
username).filter(
is_featured=True,
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False).order_by(F('featured_date').desc(nulls_last=True))
featured_listings = pipeline.Pipeline(recommend_utils.ListIterator([listing for listing in featured_listings_raw]),
[pipes.ListingPostSecurityMarkingCheckPipe(username)]).to_list()
return featured_listings
def test_pipeline_graph_vertex_while(self):
graph = Graph()
graph.add_vertex('test_label', {'test_field': 1})
graph.add_vertex('test_label', {'test_field': 2})
pipeline_test = pipeline.Pipeline(graph.get_vertices_iterator(),
[pipes.GraphVertexPipe()])
try:
list_out = []
while pipeline_test.has_next():
current_object = pipeline_test.next()
list_out.append(current_object.id)
except recommend_utils.FastNoSuchElementException:
# Ignore FastNoSuchElementException
pass
self.assertEqual(list_out, [1, 2])
def test_pipeline_1(self):
caps_pipe = pipes.CapitalizePipe()
pipeline_test = pipeline.Pipeline()
pipeline_test.add_pipe(caps_pipe)
pipeline_test.set_starts(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']))
list_out = []
try:
while pipeline_test.has_next():
current_object = pipeline_test.next()
list_out.append(current_object)
except recommend_utils.FastNoSuchElementException:
# Ignore FastNoSuchElementException
pass
self.assertEqual(list_out, ['THIS', 'IS', 'THE', 'TEST'])
def test_pipeline_graph_vertex_chain_dict_to_list(self):
graph = Graph()
graph.add_vertex('test_label', {'test_field': 8})
graph.add_vertex('test_label', {'test_field': 10})
graph.add_vertex('test_label', {'test_field': 12, 'time': 'now'})
pipeline_test = pipeline.Pipeline(
graph.get_vertices_iterator(),
[pipes.GraphVertexPipe(),
pipes.ElementPropertiesPipe()])
output = [{
'test_field': 8
}, {
'test_field': 10
}, {
'test_field': 12,
'time': 'now'
}]
self.assertEqual(pipeline_test.to_list(), output)
def _get_recommended_listings(self):
profile_username = self.profile_instance.user.username
# Post security_marking check - lazy loading
pipeline_list = [
pipes.ListingPostSecurityMarkingCheckPipe(profile_username),
pipes.LimitPipe(10)
]
if self.randomize_recommended:
pipeline_list.insert(0, pipes.JitterPipe())
recommended_listings_iterator = recommend_utils.ListIterator(
self.recommended_listings_raw)
self.recommended_listings = pipeline.Pipeline(
recommended_listings_iterator, pipeline_list).to_list()
self.recommended_listings_ids = [
listing.id for listing in self.recommended_listings
]
def get_storefront_featured(username, pre_fetch=True):
"""
Get Featured Listings for storefront
"""
# Get Featured Listings
featured_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings(
username).filter(is_featured=True,
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False)
if pre_fetch:
featured_listings_raw = listing_serializers.ListingSerializer.setup_eager_loading(
featured_listings_raw)
featured_listings = pipeline.Pipeline(
recommend_utils.ListIterator(
[listing for listing in featured_listings_raw]),
[pipes.ListingPostSecurityMarkingCheckPipe(username)]).to_list()
return featured_listings
def get_storefront_most_popular(username, pre_fetch=True):
"""
Get Most Popular Listings for storefront
"""
# Get most popular listings via a weighted average
most_popular_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings(
username).filter(approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False).order_by('-avg_rate',
'-total_reviews')
if pre_fetch:
most_popular_listings_raw = listing_serializers.ListingSerializer.setup_eager_loading(
most_popular_listings_raw)
most_popular_listings = pipeline.Pipeline(
recommend_utils.ListIterator(
[listing for listing in most_popular_listings_raw]), [
pipes.ListingPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(36)
]).to_list()
return most_popular_listings
def get_storefront_recommended(username, pre_fetch=True):
"""
Get Recommended Listings for storefront
"""
extra_data = {}
profile = models.Profile.objects.get(user__username=username)
if not profile.is_beta_user():
return [], extra_data
# Retrieve List of Recommended Apps for profile:
listing_ids_list, recommended_entry_data = get_recommendation_listing_ids(
profile)
extra_data['recommended_entry_data'] = recommended_entry_data
# Retrieve Profile Bookmarks and remove bookmarked from recommendation list
bookmarked_apps_list = set([
application_library_entry.listing.id for application_library_entry in
models.ApplicationLibraryEntry.objects.for_user(username)
])
listing_ids_list_temp = []
for current_listing_id in listing_ids_list:
if current_listing_id not in bookmarked_apps_list:
listing_ids_list_temp.append(current_listing_id)
listing_ids_list = listing_ids_list_temp
# Send new recommendation list minus bookmarked apps to User Interface
recommended_listings_queryset = models.Listing.objects.for_user_organization_minus_security_markings(
username).filter(pk__in=listing_ids_list,
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False).all()
if pre_fetch:
recommended_listings_queryset = listing_serializers.ListingSerializer.setup_eager_loading(
recommended_listings_queryset)
# Fix Order of Recommendations
id_recommended_object_mapper = {}
for recommendation_entry in recommended_listings_queryset:
id_recommended_object_mapper[
recommendation_entry.id] = recommendation_entry
# recommended_listings_raw = [id_recommended_object_mapper[listing_id] for listing_id in listing_ids_list]
recommended_listings_raw = []
for listing_id in listing_ids_list:
if listing_id in id_recommended_object_mapper:
recommended_listings_raw.append(
id_recommended_object_mapper[listing_id])
# Post security_marking check - lazy loading
recommended_listings = pipeline.Pipeline(
recommend_utils.ListIterator([
recommendations_listing
for recommendations_listing in recommended_listings_raw
]), [
pipes.JitterPipe(),
pipes.ListingPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(10)
]).to_list()
return recommended_listings, extra_data
def get_storefront_new(username, request):
"""
Returns data for /storefront api invocation including:
* recommended listings (max=10)
* featured listings (max=12)
* recent (new) listings (max=24)
* most popular listings (max=36)
Args:
username
Returns:
{
'recommended': [Listing],
'featured': [Listing],
'recent': [Listing],
'most_popular': [Listing]
}
"""
extra_data = {}
profile = models.Profile.objects.get(user__username=username)
if profile.highest_role() == 'APPS_MALL_STEWARD':
exclude_orgs = []
elif profile.highest_role() == 'ORG_STEWARD':
user_orgs = profile.stewarded_organizations.all()
user_orgs = [i.title for i in user_orgs]
exclude_orgs = [
agency.title
for agency in models.Agency.objects.exclude(title__in=user_orgs)
]
else:
user_orgs = profile.organizations.all()
user_orgs = [i.title for i in user_orgs]
exclude_orgs = [
agency.title
for agency in models.Agency.objects.exclude(title__in=user_orgs)
]
current_listings = get_user_listings(username, request, exclude_orgs)
# Get Recommended Listings for owner
if profile.is_beta_user():
recommendation_listing_ids, recommended_entry_data = get_recommendation_listing_ids(
profile)
listing_ids_list = set(recommendation_listing_ids)
recommended_listings_raw = []
for current_listing in current_listings:
if current_listing['id'] in listing_ids_list:
recommended_listings_raw.append(current_listing)
recommended_listings = pipeline.Pipeline(
recommend_utils.ListIterator(recommended_listings_raw), [
pipes.JitterPipe(),
pipes.ListingDictPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(10)
]).to_list()
else:
recommended_listings = []
# Get Featured Listings
featured_listings = pipeline.Pipeline(
recommend_utils.ListIterator(current_listings), [
pipes.ListingDictPostSecurityMarkingCheckPipe(username,
featured=True),
pipes.LimitPipe(12)
]).to_list()
# Get Recent Listings
recent_listings = pipeline.Pipeline(
recommend_utils.ListIterator(current_listings), [
pipes.ListingDictPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(24)
]).to_list()
most_popular_listings = pipeline.Pipeline(
recommend_utils.ListIterator(
sorted(current_listings,
key=lambda k: (k['avg_rate'], ['total_reviews']),
reverse=True)), [
pipes.ListingDictPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(36)
]).to_list()
# TODO 2PI filtering
data = {
'recommended': recommended_listings,
'featured': featured_listings,
'recent': recent_listings,
'most_popular': most_popular_listings
}
return data, extra_data
def test_pipeline_capitalize_len_list(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']),
[pipes.CapitalizePipe(), pipes.LenPipe()])
self.assertEqual(pipeline_test.to_list(), [4, 2, 3, 4])
def test_pipeline_capitalize_len(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']),
[pipes.CapitalizePipe(), pipes.LenPipe()])
list_out = self._iterate_pipeline(pipeline_test)
self.assertEqual(list_out, [4, 2, 3, 4])
def test_pipeline_iterate_exception(self):
pipeline_test = pipeline.Pipeline()
with self.assertRaisesRegex(Exception, 'No Start Iterator set') as err:
pipeline_test.iterate()
