def test_pipeline_exclude_limit(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
self.assertEqual(pipeline_test.to_list(), [2, 3, 4, 5, 6])
def test_pipeline_refresh_as_pipes(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
result = pipeline_test.refresh_as_pipes()
self.assertEqual(result, None)
def get_storefront_most_popular(request_profile,
pre_fetch=True,
ordering=None):
"""
Get Most Popular Listings for storefront
"""
username = request_profile.user.username
# Get most popular listings via a weighted average
most_popular_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings(
username).filter(approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False).order_by('-avg_rate',
'-total_reviews')
most_popular_listings = pipeline.Pipeline(
recommend_utils.ListIterator(
[listing for listing in most_popular_listings_raw]), [
pipes.ListingPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(36)
]).to_list()
sorted_most_popular_listings = custom_sort_listings(
most_popular_listings, ordering)
return sorted_most_popular_listings
def test_pipeline_get_starts(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
result = pipeline_test.get_starts()
self.assertEqual(str(result), 'ListIterator(7)')
def test_pipe_each_key_mixed_none(self):
each_key_pipe = pipes.EachKeyPipe('testKey')
data = [{"key2": [5, 7]}, {"testKey": [4, 8, 5, 2]}, {"testKey": None}]
each_key_pipe.set_starts(recommend_utils.ListIterator(data))
list_out = self._iterate_pipe(each_key_pipe)
self.assertEqual(list_out, [4, 8, 5, 2])
def test_pipeline_limit(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.LimitPipe(5)])
self.assertEqual(pipeline_test.to_list(), [1, 2, 3, 4, 5])
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.LimitPipe(2)])
self.assertEqual(pipeline_test.to_list(), [1, 2])
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3]), [pipes.LimitPipe(5)])
self.assertEqual(pipeline_test.to_list(), [1, 2, 3])
def test_pipeline_remove(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
self.assertRaises(recommend_utils.UnsupportedOperationException,
pipeline_test.remove)
def test_capitalize_pipe(self):
current_pipe = pipes.CapitalizePipe()
current_pipe.set_starts(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']))
list_out = self._iterate_pipe(current_pipe)
self.assertEqual(list_out, ['THIS', 'IS', 'THE', 'TEST'])
self.assertEqual(str(current_pipe), 'CapitalizePipe()')
def _create_folder_listing_queries_and_serialized(request_profile, folder_bookmark_entry, is_parent=None, ordering_fields=None, serializer_class=None, request=None):
"""
Create queries to get folder and listing data
Args:
request_profile:
used to filter profile bookmarks
request:
used for serializer to create urls
folder_bookmark_entry:
is_parent:
ordering_fields:
"""
is_parent = is_parent if is_parent else False
ordering_fields = ordering_fields if ordering_fields else ['created_date']
folder_query = models.BookmarkEntry.objects.filter(
type=FOLDER_TYPE,
bookmark_parent__bookmark_permission__profile=request_profile # Validate to make sure user can see folder
)
# for_profile should do `private listing` check for listing
listing_query = models.BookmarkEntry.objects.for_profile_minus_security_marking(request_profile).filter(
type=LISTING_TYPE,
bookmark_parent__bookmark_permission__profile=request_profile # Validate to make sure user can see folder,
)
folder_order_fields = []
listing_order_fields = []
for ordering_field in ordering_fields:
if ordering_field in FOLDER_QUERY_ORDER_MAPPING:
folder_order_fields.append(FOLDER_QUERY_ORDER_MAPPING[ordering_field])
if ordering_field in LISTING_QUERY_ORDER_MAPPING:
listing_order_fields.append(LISTING_QUERY_ORDER_MAPPING[ordering_field])
folder_query = folder_query.order_by(*folder_order_fields)
listing_query = listing_query.order_by(*listing_order_fields)
if is_parent:
folder_query = folder_query.filter(id=folder_bookmark_entry.id)
listing_query = listing_query.filter(id=folder_bookmark_entry.id)
else:
folder_query = folder_query.filter(bookmark_parent=folder_bookmark_entry)
listing_query = listing_query.filter(bookmark_parent=folder_bookmark_entry)
folder_data = serializer_class(folder_query, many=True, context={'request': request}).data
listing_data = serializer_class(listing_query, many=True, context={'request': request}).data
# Filter out all listings that request_profile does not have access to
listing_data_after_filter = pipeline.Pipeline(recommend_utils.ListIterator(listing_data),
[pipes.BookmarkListingDictPostSecurityMarkingCheckPipe(request_profile.user.username)]).to_list()
return {"folder_data": folder_data,
"listing_data_after_filter": listing_data_after_filter}
def test_pipeline_count(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
result = pipeline_test.count()
self.assertEqual(result, 5)
result = pipeline_test.count()
self.assertEqual(result, 0)
def test_pipeline_get_pipes(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator([1, 2, 3, 4, 5, 6, 7]),
[pipes.ExcludePipe([1]),
pipes.LimitPipe(5)])
result = ', '.join([str(pipe) for pipe in pipeline_test.get_pipes()])
self.assertEqual(
str(result),
'ListIterator(7), ExcludePipe(), LimitPipe(limit_number:5)')
def test_distinct_pipe(self):
current_pipe = pipes.DistinctPipe()
current_pipe.set_starts(
recommend_utils.ListIterator(
['is', 'this', 'is', 'the', 'test', 'this']))
list_out = self._iterate_pipe(current_pipe)
self.assertEqual(list_out, ['is', 'this', 'the', 'test'])
self.assertEqual(str(current_pipe), 'DistinctPipe()')
def test_pipeline_capitalize(self):
caps_pipe = pipes.CapitalizePipe()
pipeline_test = pipeline.Pipeline()
pipeline_test.add_pipe(caps_pipe)
pipeline_test.set_starts(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']))
list_out = self._iterate_pipeline(pipeline_test)
self.assertEqual(list_out, ['THIS', 'IS', 'THE', 'TEST'])
def test_pipe_cap_len_chain(self):
caps_pipe = pipes.CapitalizePipe()
len_pipe = pipes.LenPipe()
caps_pipe.set_starts(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']))
len_pipe.set_starts(caps_pipe)
list_out = self._iterate_pipe(len_pipe)
self.assertEqual(list_out, [4, 2, 3, 4])
def list(self, request, listing_pk=None):
queryset = self.filter_queryset(self.get_queryset(listing_pk))
serializer = serializers.ListingSerializer(
queryset, context={'request': request}, many=True)
similar_listings = pipeline.Pipeline(
recommend_utils.ListIterator(serializer.data), [
pipes.ListingDictPostSecurityMarkingCheckPipe(
self.request.user.username),
pipes.LimitPipe(10)
]).to_list()
return Response(similar_listings)
def test_list_iterator(self):
list_iterator = recommend_utils.ListIterator([1, 2, 3])
list_out = []
try:
while list_iterator.has_next():
list_out.append(list_iterator.next())
except recommend_utils.FastNoSuchElementException:
# Ignore FastNoSuchElementException
pass
self.assertEqual(list_out, [1, 2, 3])
def load_yaml_file(self, listing_file_name=None):
listing_file_path = os.path.join(TEST_DATA_PATH, listing_file_name or 'listings.yaml')
listings_data = []
with open(listing_file_path, 'r') as stream:
try:
listings_data = yaml.load(stream)
except yaml.YAMLError as exc:
print(exc)
raise
self.pipeline.add_pipe(recommend_utils.ListIterator(listings_data))
return self
def test_pipeline_chain_2(self):
pipeline_test = pipeline.Pipeline(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']),
[pipes.CapitalizePipe(), pipes.LenPipe()])
list_out = []
try:
while pipeline_test.has_next():
current_object = pipeline_test.next()
list_out.append(current_object)
except recommend_utils.FastNoSuchElementException:
# Ignore FastNoSuchElementException
pass
self.assertEqual(list_out, [4, 2, 3, 4])
def list(self, request):
queryset = self.get_queryset()
serializer = serializers.ImageSerializer(queryset,
many=True,
context={'request': request})
serializer_iterator = recommend_utils.ListIterator(serializer.data)
pipeline_list = [
pipes.ListingDictPostSecurityMarkingCheckPipe(
request.user.username)
]
recommended_listings = pipeline.Pipeline(serializer_iterator,
pipeline_list).to_list()
return Response(recommended_listings)
def get_storefront_recent(request_profile, pre_fetch=True):
"""
Get Recent Listings for storefront
"""
username = request_profile.user.username
# Get Recent Listings
recent_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings(
username).order_by('-approved_date').filter(
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False)
recent_listings = pipeline.Pipeline(recommend_utils.ListIterator([listing for listing in recent_listings_raw]),
[pipes.ListingPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(24)]).to_list()
return recent_listings
def test_pipe_cap(self):
caps_pipe = pipes.CapitalizePipe()
caps_pipe.set_starts(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']))
list_out = []
try:
while caps_pipe.has_next():
current_object = caps_pipe.next()
list_out.append(current_object)
except recommend_utils.FastNoSuchElementException:
# Ignore FastNoSuchElementException
pass
self.assertEqual(list_out, ['THIS', 'IS', 'THE', 'TEST'])
def get_storefront_featured(request_profile, pre_fetch=True):
"""
Get Featured Listings for storefront
"""
username = request_profile.user.username
# Get Featured Listings
featured_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings(
username).filter(
is_featured=True,
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False).order_by(F('featured_date').desc(nulls_last=True))
featured_listings = pipeline.Pipeline(recommend_utils.ListIterator([listing for listing in featured_listings_raw]),
[pipes.ListingPostSecurityMarkingCheckPipe(username)]).to_list()
return featured_listings
def test_side_effect_pipe(self):
side_effect_objects = []
def func1(current_object):
side_effect_objects.append(len(current_object))
current_pipe = pipes.SideEffectPipe(func1)
current_pipe.set_starts(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']))
list_out = self._iterate_pipe(current_pipe)
self.assertEqual(list_out, ['this', 'is', 'the', 'test'])
self.assertEqual(side_effect_objects, [4, 2, 3, 4])
self.assertEqual(str(current_pipe),
'SideEffectPipe(function:<class \'function\'>)')
def v(self, vertex_id, *vertex_ids):
"""
Get Vertex by internal id
"""
self.pipeline.add_pipe(pipes.GraphVertexPipe())
if not self.pipeline.starts:
out_list = []
if self.graph.vertices.get(vertex_id):
out_list.append(self.graph.vertices.get(vertex_id))
for current_vertex_id in vertex_ids:
if self.graph.vertices.get(current_vertex_id):
out_list.append(self.graph.vertices.get(current_vertex_id))
self.pipeline.set_starts(recommend_utils.ListIterator(out_list))
return self
def test_pipe_cap_len_chain(self):
caps_pipe = pipes.CapitalizePipe()
len_pipe = pipes.LenPipe()
caps_pipe.set_starts(
recommend_utils.ListIterator(['this', 'is', 'the', 'test']))
len_pipe.set_starts(caps_pipe)
list_out = []
try:
while len_pipe.has_next(): # Make this refresh has loop
current_object = len_pipe.next()
list_out.append(current_object)
except recommend_utils.FastNoSuchElementException:
# Ignore FastNoSuchElementException
pass
self.assertEqual(list_out, [4, 2, 3, 4])
def _get_recommended_listings(self):
profile_username = self.profile_instance.user.username
# Post security_marking check - lazy loading
pipeline_list = [
pipes.ListingPostSecurityMarkingCheckPipe(profile_username),
pipes.LimitPipe(10)
]
if self.randomize_recommended:
pipeline_list.insert(0, pipes.JitterPipe())
recommended_listings_iterator = recommend_utils.ListIterator(
self.recommended_listings_raw)
self.recommended_listings = pipeline.Pipeline(
recommended_listings_iterator, pipeline_list).to_list()
self.recommended_listings_ids = [
listing.id for listing in self.recommended_listings
]
def get_storefront_featured(username, pre_fetch=True):
"""
Get Featured Listings for storefront
"""
# Get Featured Listings
featured_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings(
username).filter(is_featured=True,
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False)
if pre_fetch:
featured_listings_raw = listing_serializers.ListingSerializer.setup_eager_loading(
featured_listings_raw)
featured_listings = pipeline.Pipeline(
recommend_utils.ListIterator(
[listing for listing in featured_listings_raw]),
[pipes.ListingPostSecurityMarkingCheckPipe(username)]).to_list()
return featured_listings
def process_next_start(self):
"""
EachKeyPipe
"""
while True:
if self.next_end.has_next():
try:
while True:
return self.next_end.next()
except recommend_utils.FastNoSuchElementException:
# Ignore FastNoSuchElementException
pass
else:
current_dict = self.starts.next()
if self.key in current_dict:
current_list = current_dict[self.key]
if current_list:
self.next_end = recommend_utils.ListIterator(
current_list)
def get_storefront_most_popular(username, pre_fetch=True):
"""
Get Most Popular Listings for storefront
"""
# Get most popular listings via a weighted average
most_popular_listings_raw = models.Listing.objects.for_user_organization_minus_security_markings(
username).filter(approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False).order_by('-avg_rate',
'-total_reviews')
if pre_fetch:
most_popular_listings_raw = listing_serializers.ListingSerializer.setup_eager_loading(
most_popular_listings_raw)
most_popular_listings = pipeline.Pipeline(
recommend_utils.ListIterator(
[listing for listing in most_popular_listings_raw]), [
pipes.ListingPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(36)
]).to_list()
return most_popular_listings
def get_storefront_recommended(username, pre_fetch=True):
"""
Get Recommended Listings for storefront
"""
extra_data = {}
profile = models.Profile.objects.get(user__username=username)
if not profile.is_beta_user():
return [], extra_data
# Retrieve List of Recommended Apps for profile:
listing_ids_list, recommended_entry_data = get_recommendation_listing_ids(
profile)
extra_data['recommended_entry_data'] = recommended_entry_data
# Retrieve Profile Bookmarks and remove bookmarked from recommendation list
bookmarked_apps_list = set([
application_library_entry.listing.id for application_library_entry in
models.ApplicationLibraryEntry.objects.for_user(username)
])
listing_ids_list_temp = []
for current_listing_id in listing_ids_list:
if current_listing_id not in bookmarked_apps_list:
listing_ids_list_temp.append(current_listing_id)
listing_ids_list = listing_ids_list_temp
# Send new recommendation list minus bookmarked apps to User Interface
recommended_listings_queryset = models.Listing.objects.for_user_organization_minus_security_markings(
username).filter(pk__in=listing_ids_list,
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False).all()
if pre_fetch:
recommended_listings_queryset = listing_serializers.ListingSerializer.setup_eager_loading(
recommended_listings_queryset)
# Fix Order of Recommendations
id_recommended_object_mapper = {}
for recommendation_entry in recommended_listings_queryset:
id_recommended_object_mapper[
recommendation_entry.id] = recommendation_entry
# recommended_listings_raw = [id_recommended_object_mapper[listing_id] for listing_id in listing_ids_list]
recommended_listings_raw = []
for listing_id in listing_ids_list:
if listing_id in id_recommended_object_mapper:
recommended_listings_raw.append(
id_recommended_object_mapper[listing_id])
# Post security_marking check - lazy loading
recommended_listings = pipeline.Pipeline(
recommend_utils.ListIterator([
recommendations_listing
for recommendations_listing in recommended_listings_raw
]), [
pipes.JitterPipe(),
pipes.ListingPostSecurityMarkingCheckPipe(username),
pipes.LimitPipe(10)
]).to_list()
return recommended_listings, extra_data