def test_pipeline_graph_vertex_chain_dict_to_list_internal(self): pipeline_test = pipeline.Pipeline( self.graph_test_2.get_vertices_iterator(), [ pipes.GraphVertexPipe(), pipes.ElementPropertiesPipe(internal=True) ]) expected_output = [{ '_id': 1, '_label': 'test_label', 'test_field': 8 }, { '_id': 2, '_label': 'test_label', 'test_field': 10 }, { '_id': 3, '_label': 'test_label', 'test_field': 12, 'time': 'now' }] self.assertEqual(pipeline_test.to_list(), expected_output) self.assertEqual( str(pipeline_test), '[DictKeyValueIterator(3), GraphVertexPipe(), ElementPropertiesPipe(internal:True)]' )
def test_pipeline_refresh_as_pipes(self): pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]), [pipes.ExcludePipe([1]), pipes.LimitPipe(5)]) result = pipeline_test.refresh_as_pipes() self.assertEqual(result, None)
def test_pipeline_exclude_limit(self): pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]), [pipes.ExcludePipe([1]), pipes.LimitPipe(5)]) self.assertEqual(pipeline_test.to_list(), [2, 3, 4, 5, 6])
def get_storefront_most_popular(request_profile, pre_fetch=True, ordering=None): """ Get Most Popular Listings for storefront """ username = request_profile.user.username # Get most popular listings via a weighted average most_popular_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings( username).filter(approval_status=models.Listing.APPROVED, is_enabled=True, is_deleted=False).order_by('-avg_rate', '-total_reviews') most_popular_listings = pipeline.Pipeline( recommend_utils.ListIterator( [listing for listing in most_popular_listings_raw]), [ pipes.ListingPostSecurityMarkingCheckPipe(username), pipes.LimitPipe(36) ]).to_list() sorted_most_popular_listings = custom_sort_listings( most_popular_listings, ordering) return sorted_most_popular_listings
def test_pipeline_limit(self): pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]), [pipes.LimitPipe(5)]) self.assertEqual(pipeline_test.to_list(), [1, 2, 3, 4, 5]) pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]), [pipes.LimitPipe(2)]) self.assertEqual(pipeline_test.to_list(), [1, 2]) pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator([1, 2, 3]), [pipes.LimitPipe(5)]) self.assertEqual(pipeline_test.to_list(), [1, 2, 3])
def test_pipeline_remove(self): pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]), [pipes.ExcludePipe([1]), pipes.LimitPipe(5)]) self.assertRaises(recommend_utils.UnsupportedOperationException, pipeline_test.remove)
def test_pipeline_get_starts(self): pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]), [pipes.ExcludePipe([1]), pipes.LimitPipe(5)]) result = pipeline_test.get_starts() self.assertEqual(str(result), 'ListIterator(7)')
def test_pipeline_graph_vertex_while(self): pipeline_test = pipeline.Pipeline( self.graph_test_1.get_vertices_iterator(), [pipes.GraphVertexPipe()]) list_out = self._iterate_pipeline(pipeline_test) self.assertEquals(str(list_out), '[Vertex(test_label), Vertex(test_label)]')
def _create_folder_listing_queries_and_serialized(request_profile, folder_bookmark_entry, is_parent=None, ordering_fields=None, serializer_class=None, request=None): """ Create queries to get folder and listing data Args: request_profile: used to filter profile bookmarks request: used for serializer to create urls folder_bookmark_entry: is_parent: ordering_fields: """ is_parent = is_parent if is_parent else False ordering_fields = ordering_fields if ordering_fields else ['created_date'] folder_query = models.BookmarkEntry.objects.filter( type=FOLDER_TYPE, bookmark_parent__bookmark_permission__profile=request_profile # Validate to make sure user can see folder ) # for_profile should do `private listing` check for listing listing_query = models.BookmarkEntry.objects.for_profile_minus_security_marking(request_profile).filter( type=LISTING_TYPE, bookmark_parent__bookmark_permission__profile=request_profile # Validate to make sure user can see folder, ) folder_order_fields = [] listing_order_fields = [] for ordering_field in ordering_fields: if ordering_field in FOLDER_QUERY_ORDER_MAPPING: folder_order_fields.append(FOLDER_QUERY_ORDER_MAPPING[ordering_field]) if ordering_field in LISTING_QUERY_ORDER_MAPPING: listing_order_fields.append(LISTING_QUERY_ORDER_MAPPING[ordering_field]) folder_query = folder_query.order_by(*folder_order_fields) listing_query = listing_query.order_by(*listing_order_fields) if is_parent: folder_query = folder_query.filter(id=folder_bookmark_entry.id) listing_query = listing_query.filter(id=folder_bookmark_entry.id) else: folder_query = folder_query.filter(bookmark_parent=folder_bookmark_entry) listing_query = listing_query.filter(bookmark_parent=folder_bookmark_entry) folder_data = serializer_class(folder_query, many=True, context={'request': request}).data listing_data = serializer_class(listing_query, many=True, context={'request': request}).data # Filter out all listings that request_profile does not have access to listing_data_after_filter = pipeline.Pipeline(recommend_utils.ListIterator(listing_data), [pipes.BookmarkListingDictPostSecurityMarkingCheckPipe(request_profile.user.username)]).to_list() return {"folder_data": folder_data, "listing_data_after_filter": listing_data_after_filter}
def test_pipeline_graph_vertex_chain_to_list(self): pipeline_test = pipeline.Pipeline( self.graph_test_1.get_vertices_iterator(), [pipes.GraphVertexPipe(), pipes.ElementIdPipe()]) self.assertEqual(pipeline_test.to_list(), [1, 2]) self.assertEqual( str(pipeline_test), '[DictKeyValueIterator(2), GraphVertexPipe(), ElementIdPipe()]')
def test_pipeline_count(self): pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]), [pipes.ExcludePipe([1]), pipes.LimitPipe(5)]) result = pipeline_test.count() self.assertEqual(result, 5) result = pipeline_test.count() self.assertEqual(result, 0)
def test_pipeline_get_pipes(self): pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]), [pipes.ExcludePipe([1]), pipes.LimitPipe(5)]) result = ', '.join([str(pipe) for pipe in pipeline_test.get_pipes()]) self.assertEqual( str(result), 'ListIterator(7), ExcludePipe(), LimitPipe(limit_number:5)')
def test_pipeline_capitalize(self): caps_pipe = pipes.CapitalizePipe() pipeline_test = pipeline.Pipeline() pipeline_test.add_pipe(caps_pipe) pipeline_test.set_starts( recommend_utils.ListIterator(['this', 'is', 'the', 'test'])) list_out = self._iterate_pipeline(pipeline_test) self.assertEqual(list_out, ['THIS', 'IS', 'THE', 'TEST'])
def test_pipeline_graph_vertex_chain_to_list(self): graph = Graph() graph.add_vertex('test_label', {'test_field': 1}) graph.add_vertex('test_label', {'test_field': 2}) pipeline_test = pipeline.Pipeline( graph.get_vertices_iterator(), [pipes.GraphVertexPipe(), pipes.ElementIdPipe()]) self.assertEqual(pipeline_test.to_list(), [1, 2])
def list(self, request, listing_pk=None): queryset = self.filter_queryset(self.get_queryset(listing_pk)) serializer = serializers.ListingSerializer( queryset, context={'request': request}, many=True) similar_listings = pipeline.Pipeline( recommend_utils.ListIterator(serializer.data), [ pipes.ListingDictPostSecurityMarkingCheckPipe( self.request.user.username), pipes.LimitPipe(10) ]).to_list() return Response(similar_listings)
def test_pipeline_chain_2(self): pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator(['this', 'is', 'the', 'test']), [pipes.CapitalizePipe(), pipes.LenPipe()]) list_out = [] try: while pipeline_test.has_next(): current_object = pipeline_test.next() list_out.append(current_object) except recommend_utils.FastNoSuchElementException: # Ignore FastNoSuchElementException pass self.assertEqual(list_out, [4, 2, 3, 4])
def list(self, request): queryset = self.get_queryset() serializer = serializers.ImageSerializer(queryset, many=True, context={'request': request}) serializer_iterator = recommend_utils.ListIterator(serializer.data) pipeline_list = [ pipes.ListingDictPostSecurityMarkingCheckPipe( request.user.username) ] recommended_listings = pipeline.Pipeline(serializer_iterator, pipeline_list).to_list() return Response(recommended_listings)
def get_storefront_recent(request_profile, pre_fetch=True): """ Get Recent Listings for storefront """ username = request_profile.user.username # Get Recent Listings recent_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings( username).order_by('-approved_date').filter( approval_status=models.Listing.APPROVED, is_enabled=True, is_deleted=False) recent_listings = pipeline.Pipeline(recommend_utils.ListIterator([listing for listing in recent_listings_raw]), [pipes.ListingPostSecurityMarkingCheckPipe(username), pipes.LimitPipe(24)]).to_list() return recent_listings
def get_storefront_featured(request_profile, pre_fetch=True): """ Get Featured Listings for storefront """ username = request_profile.user.username # Get Featured Listings featured_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings( username).filter( is_featured=True, approval_status=models.Listing.APPROVED, is_enabled=True, is_deleted=False).order_by(F('featured_date').desc(nulls_last=True)) featured_listings = pipeline.Pipeline(recommend_utils.ListIterator([listing for listing in featured_listings_raw]), [pipes.ListingPostSecurityMarkingCheckPipe(username)]).to_list() return featured_listings
def test_pipeline_graph_vertex_while(self): graph = Graph() graph.add_vertex('test_label', {'test_field': 1}) graph.add_vertex('test_label', {'test_field': 2}) pipeline_test = pipeline.Pipeline(graph.get_vertices_iterator(), [pipes.GraphVertexPipe()]) try: list_out = [] while pipeline_test.has_next(): current_object = pipeline_test.next() list_out.append(current_object.id) except recommend_utils.FastNoSuchElementException: # Ignore FastNoSuchElementException pass self.assertEqual(list_out, [1, 2])
def test_pipeline_1(self): caps_pipe = pipes.CapitalizePipe() pipeline_test = pipeline.Pipeline() pipeline_test.add_pipe(caps_pipe) pipeline_test.set_starts( recommend_utils.ListIterator(['this', 'is', 'the', 'test'])) list_out = [] try: while pipeline_test.has_next(): current_object = pipeline_test.next() list_out.append(current_object) except recommend_utils.FastNoSuchElementException: # Ignore FastNoSuchElementException pass self.assertEqual(list_out, ['THIS', 'IS', 'THE', 'TEST'])
def test_pipeline_graph_vertex_chain_dict_to_list(self): graph = Graph() graph.add_vertex('test_label', {'test_field': 8}) graph.add_vertex('test_label', {'test_field': 10}) graph.add_vertex('test_label', {'test_field': 12, 'time': 'now'}) pipeline_test = pipeline.Pipeline( graph.get_vertices_iterator(), [pipes.GraphVertexPipe(), pipes.ElementPropertiesPipe()]) output = [{ 'test_field': 8 }, { 'test_field': 10 }, { 'test_field': 12, 'time': 'now' }] self.assertEqual(pipeline_test.to_list(), output)
def _get_recommended_listings(self): profile_username = self.profile_instance.user.username # Post security_marking check - lazy loading pipeline_list = [ pipes.ListingPostSecurityMarkingCheckPipe(profile_username), pipes.LimitPipe(10) ] if self.randomize_recommended: pipeline_list.insert(0, pipes.JitterPipe()) recommended_listings_iterator = recommend_utils.ListIterator( self.recommended_listings_raw) self.recommended_listings = pipeline.Pipeline( recommended_listings_iterator, pipeline_list).to_list() self.recommended_listings_ids = [ listing.id for listing in self.recommended_listings ]
def get_storefront_featured(username, pre_fetch=True): """ Get Featured Listings for storefront """ # Get Featured Listings featured_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings( username).filter(is_featured=True, approval_status=models.Listing.APPROVED, is_enabled=True, is_deleted=False) if pre_fetch: featured_listings_raw = listing_serializers.ListingSerializer.setup_eager_loading( featured_listings_raw) featured_listings = pipeline.Pipeline( recommend_utils.ListIterator( [listing for listing in featured_listings_raw]), [pipes.ListingPostSecurityMarkingCheckPipe(username)]).to_list() return featured_listings
def get_storefront_most_popular(username, pre_fetch=True): """ Get Most Popular Listings for storefront """ # Get most popular listings via a weighted average most_popular_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings( username).filter(approval_status=models.Listing.APPROVED, is_enabled=True, is_deleted=False).order_by('-avg_rate', '-total_reviews') if pre_fetch: most_popular_listings_raw = listing_serializers.ListingSerializer.setup_eager_loading( most_popular_listings_raw) most_popular_listings = pipeline.Pipeline( recommend_utils.ListIterator( [listing for listing in most_popular_listings_raw]), [ pipes.ListingPostSecurityMarkingCheckPipe(username), pipes.LimitPipe(36) ]).to_list() return most_popular_listings
def get_storefront_recommended(username, pre_fetch=True): """ Get Recommended Listings for storefront """ extra_data = {} profile = models.Profile.objects.get(user__username=username) if not profile.is_beta_user(): return [], extra_data # Retrieve List of Recommended Apps for profile: listing_ids_list, recommended_entry_data = get_recommendation_listing_ids( profile) extra_data['recommended_entry_data'] = recommended_entry_data # Retrieve Profile Bookmarks and remove bookmarked from recommendation list bookmarked_apps_list = set([ application_library_entry.listing.id for application_library_entry in models.ApplicationLibraryEntry.objects.for_user(username) ]) listing_ids_list_temp = [] for current_listing_id in listing_ids_list: if current_listing_id not in bookmarked_apps_list: listing_ids_list_temp.append(current_listing_id) listing_ids_list = listing_ids_list_temp # Send new recommendation list minus bookmarked apps to User Interface recommended_listings_queryset = models.Listing.objects.for_user_organization_minus_security_markings( username).filter(pk__in=listing_ids_list, approval_status=models.Listing.APPROVED, is_enabled=True, is_deleted=False).all() if pre_fetch: recommended_listings_queryset = listing_serializers.ListingSerializer.setup_eager_loading( recommended_listings_queryset) # Fix Order of Recommendations id_recommended_object_mapper = {} for recommendation_entry in recommended_listings_queryset: id_recommended_object_mapper[ recommendation_entry.id] = recommendation_entry # recommended_listings_raw = [id_recommended_object_mapper[listing_id] for listing_id in listing_ids_list] recommended_listings_raw = [] for listing_id in listing_ids_list: if listing_id in id_recommended_object_mapper: recommended_listings_raw.append( id_recommended_object_mapper[listing_id]) # Post security_marking check - lazy loading recommended_listings = pipeline.Pipeline( recommend_utils.ListIterator([ recommendations_listing for recommendations_listing in recommended_listings_raw ]), [ pipes.JitterPipe(), pipes.ListingPostSecurityMarkingCheckPipe(username), pipes.LimitPipe(10) ]).to_list() return recommended_listings, extra_data
def get_storefront_new(username, request): """ Returns data for /storefront api invocation including: * recommended listings (max=10) * featured listings (max=12) * recent (new) listings (max=24) * most popular listings (max=36) Args: username Returns: { 'recommended': [Listing], 'featured': [Listing], 'recent': [Listing], 'most_popular': [Listing] } """ extra_data = {} profile = models.Profile.objects.get(user__username=username) if profile.highest_role() == 'APPS_MALL_STEWARD': exclude_orgs = [] elif profile.highest_role() == 'ORG_STEWARD': user_orgs = profile.stewarded_organizations.all() user_orgs = [i.title for i in user_orgs] exclude_orgs = [ agency.title for agency in models.Agency.objects.exclude(title__in=user_orgs) ] else: user_orgs = profile.organizations.all() user_orgs = [i.title for i in user_orgs] exclude_orgs = [ agency.title for agency in models.Agency.objects.exclude(title__in=user_orgs) ] current_listings = get_user_listings(username, request, exclude_orgs) # Get Recommended Listings for owner if profile.is_beta_user(): recommendation_listing_ids, recommended_entry_data = get_recommendation_listing_ids( profile) listing_ids_list = set(recommendation_listing_ids) recommended_listings_raw = [] for current_listing in current_listings: if current_listing['id'] in listing_ids_list: recommended_listings_raw.append(current_listing) recommended_listings = pipeline.Pipeline( recommend_utils.ListIterator(recommended_listings_raw), [ pipes.JitterPipe(), pipes.ListingDictPostSecurityMarkingCheckPipe(username), pipes.LimitPipe(10) ]).to_list() else: recommended_listings = [] # Get Featured Listings featured_listings = pipeline.Pipeline( recommend_utils.ListIterator(current_listings), [ pipes.ListingDictPostSecurityMarkingCheckPipe(username, featured=True), pipes.LimitPipe(12) ]).to_list() # Get Recent Listings recent_listings = pipeline.Pipeline( recommend_utils.ListIterator(current_listings), [ pipes.ListingDictPostSecurityMarkingCheckPipe(username), pipes.LimitPipe(24) ]).to_list() most_popular_listings = pipeline.Pipeline( recommend_utils.ListIterator( sorted(current_listings, key=lambda k: (k['avg_rate'], ['total_reviews']), reverse=True)), [ pipes.ListingDictPostSecurityMarkingCheckPipe(username), pipes.LimitPipe(36) ]).to_list() # TODO 2PI filtering data = { 'recommended': recommended_listings, 'featured': featured_listings, 'recent': recent_listings, 'most_popular': most_popular_listings } return data, extra_data
def test_pipeline_capitalize_len_list(self): pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator(['this', 'is', 'the', 'test']), [pipes.CapitalizePipe(), pipes.LenPipe()]) self.assertEqual(pipeline_test.to_list(), [4, 2, 3, 4])
def test_pipeline_capitalize_len(self): pipeline_test = pipeline.Pipeline( recommend_utils.ListIterator(['this', 'is', 'the', 'test']), [pipes.CapitalizePipe(), pipes.LenPipe()]) list_out = self._iterate_pipeline(pipeline_test) self.assertEqual(list_out, [4, 2, 3, 4])
def test_pipeline_iterate_exception(self): pipeline_test = pipeline.Pipeline() with self.assertRaisesRegex(Exception, 'No Start Iterator set') as err: pipeline_test.iterate()