def test_query_followed_by_query_signature(self):
""" This is the simplest case where one is followed by other, and hence there is no overlapping """
with QueryProfiler(QueryProfilerLevel.QUERY) as qp_query:
str(Pizza.objects.all())
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level, QueryProfilerLevel.QUERY)
self.assertIsNone(
data_collector_thread_local_storage._current_query_profiler_level)
with QueryProfiler(
QueryProfilerLevel.QUERY_SIGNATURE) as qp_query_signature:
str(Pizza.objects.all())
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level,
QueryProfilerLevel.QUERY_SIGNATURE)
self.assertIsNone(
data_collector_thread_local_storage._current_query_profiler_level)
self.assertIsNotNone(qp_query_signature.query_profiled_data.summary.
potential_n_plus1_query_count)
self.assertIsNone(
qp_query.query_profiled_data.summary.potential_n_plus1_query_count)
self.assertEqual(
qp_query_signature.query_profiled_data.summary.total_query_count,
4)
self.assertTrue(qp_query.query_profiled_data.summary.total_query_count,
4)
def test_query_signature_nesting_query_one_level(self):
""" We call query inside the profiler with query_signature type """
with QueryProfiler(
QueryProfilerLevel.QUERY_SIGNATURE) as qp_query_signature:
str(Pizza.objects.all())
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level,
QueryProfilerLevel.QUERY_SIGNATURE)
with QueryProfiler(QueryProfilerLevel.QUERY) as qp_query:
str(Pizza.objects.all())
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level,
QueryProfilerLevel.QUERY_SIGNATURE)
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level,
QueryProfilerLevel.QUERY_SIGNATURE)
self.assertIsNone(
data_collector_thread_local_storage._current_query_profiler_level)
self.assertIsNotNone(qp_query_signature.query_profiled_data.summary.
potential_n_plus1_query_count)
self.assertIsNotNone(
qp_query.query_profiled_data.summary.potential_n_plus1_query_count)
self.assertTrue(qp_query.query_profiled_data.summary.total_query_count,
4)
self.assertEqual(
qp_query_signature.query_profiled_data.summary.total_query_count,
4 + 4)
def test_spicy_toppings_db_filtering(self):
""" Test to verify that no amount of prefetching would help if we do db filtering"""
# STEP 1: Without any prefetching
pizzas = Pizza.objects.all()
with QueryProfiler(QueryProfilerLevel.QUERY_SIGNATURE
) as qp_pizzas_without_prefetch:
for pizza in pizzas:
pizza.spicy_toppings_db_filtering()
query_signature_to_query_signature_statistics = \
qp_pizzas_without_prefetch.query_profiled_data.query_signature_to_query_signature_statistics
self.assertEqual(len(query_signature_to_query_signature_statistics), 2)
checked_for_table_pizza = checked_for_table_toppings = False
for query_signature, query_signature_statistics in query_signature_to_query_signature_statistics.items(
):
if query_signature_statistics.frequency == 1:
self.assertEqual(query_signature.analysis,
QuerySignatureAnalyzeResult.UNKNOWN)
checked_for_table_pizza = True
elif query_signature_statistics.frequency == 3:
self.assertEqual(query_signature.analysis,
QuerySignatureAnalyzeResult.FILTER)
checked_for_table_toppings = True
self.assertTrue(checked_for_table_pizza)
self.assertTrue(checked_for_table_toppings)
# STEP 2: Lets try prefetching
pizzas = Pizza.objects.prefetch_related('toppings').all()
with QueryProfiler(
QueryProfilerLevel.QUERY_SIGNATURE) as qp_pizzas_wit_prefetch:
for pizza in pizzas:
pizza.spicy_toppings_db_filtering()
query_signature_to_query_signature_statistics = \
qp_pizzas_wit_prefetch.query_profiled_data.query_signature_to_query_signature_statistics
# Number of queries have increased by 1, since we did prefetching which could not be used
self.assertEqual(len(query_signature_to_query_signature_statistics), 3)
frequencies = [
query_signature_statistics.frequency for query_signature_statistics
in query_signature_to_query_signature_statistics.values()
]
query_signature_analysis = {
query_signature.analysis
for query_signature in
query_signature_to_query_signature_statistics.keys()
}
self.assertListEqual(frequencies, [1, 1, 3])
self.assertSetEqual(
query_signature_analysis, {
QuerySignatureAnalyzeResult.FILTER,
QuerySignatureAnalyzeResult.PREFETCHED_RELATED,
QuerySignatureAnalyzeResult.UNKNOWN
})
def test_complex_nesting(self):
with QueryProfiler(
QueryProfilerLevel.QUERY_SIGNATURE) as qp_query_signature_1:
str(Pizza.objects.all())
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level,
QueryProfilerLevel.QUERY_SIGNATURE)
with QueryProfiler(QueryProfilerLevel.QUERY) as qp_query_1:
str(Pizza.objects.all())
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level,
QueryProfilerLevel.QUERY_SIGNATURE)
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level,
QueryProfilerLevel.QUERY_SIGNATURE)
with QueryProfiler(QueryProfilerLevel.QUERY) as qp_query_2:
str(Pizza.objects.all())
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level,
QueryProfilerLevel.QUERY_SIGNATURE)
with QueryProfiler(QueryProfilerLevel.QUERY_SIGNATURE) as _:
str(Pizza.objects.all())
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level,
QueryProfilerLevel.QUERY_SIGNATURE)
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level,
QueryProfilerLevel.QUERY_SIGNATURE)
self.assertIsNone(
data_collector_thread_local_storage._current_query_profiler_level)
self.assertIsNotNone(qp_query_signature_1.query_profiled_data.summary.
potential_n_plus1_query_count)
self.assertEqual(
qp_query_signature_1.query_profiled_data.summary.total_query_count,
4 + 4 + 4 + 4)
self.assertIsNotNone(qp_query_1.query_profiled_data.summary.
potential_n_plus1_query_count)
self.assertTrue(
qp_query_1.query_profiled_data.summary.total_query_count, 4)
self.assertIsNotNone(qp_query_2.query_profiled_data.summary.
potential_n_plus1_query_count)
self.assertTrue(
qp_query_2.query_profiled_data.summary.total_query_count, 4)
def test_after_applying_prefetch(self):
pizzas = Pizza.objects.prefetch_related('toppings').all()
# Pizzas __str__ references toppings, which we have now prefetched
with QueryProfiler(
QueryProfilerLevel.QUERY_SIGNATURE) as qp_pizzas_with_prefetch:
for pizza in pizzas:
str(pizza) # We can also print it
query_signature_to_query_signature_statistics = \
qp_pizzas_with_prefetch.query_profiled_data.query_signature_to_query_signature_statistics
self.assertEqual(len(query_signature_to_query_signature_statistics), 2)
frequencies = {
query_signature_statistics.frequency
for query_signature_statistics in
query_signature_to_query_signature_statistics.values()
}
query_signature_analysis = {
query_signature.analysis
for query_signature in
query_signature_to_query_signature_statistics.keys()
}
self.assertSetEqual(frequencies, {1})
self.assertSetEqual(
query_signature_analysis, {
QuerySignatureAnalyzeResult.UNKNOWN,
QuerySignatureAnalyzeResult.PREFETCHED_RELATED
})
def test_missing_prefetch(self):
pizzas = Pizza.objects.all()
# Pizzas __str__ references toppings, which we have not prefetched
with QueryProfiler(QueryProfilerLevel.QUERY_SIGNATURE
) as qp_pizzas_without_prefetch:
for pizza in pizzas:
str(pizza) # We can also print it
query_signature_to_query_signature_statistics = \
qp_pizzas_without_prefetch.query_profiled_data.query_signature_to_query_signature_statistics
# The first query would be for pizzas, and second one would be the call to toppings
self.assertEqual(len(query_signature_to_query_signature_statistics), 2)
checked_for_table_pizza = checked_for_table_toppings = False
for query_signature, query_signature_statistics in query_signature_to_query_signature_statistics.items(
):
if query_signature_statistics.frequency == 1:
self.assertEqual(query_signature.analysis,
QuerySignatureAnalyzeResult.UNKNOWN)
checked_for_table_pizza = True
elif query_signature_statistics.frequency == 3:
self.assertEqual(
query_signature.analysis,
QuerySignatureAnalyzeResult.MISSING_PREFETCH_RELATED)
checked_for_table_toppings = True
self.assertTrue(checked_for_table_pizza)
self.assertTrue(checked_for_table_toppings)
def start_profiler():
scraper = WildberriesItemScraper()
category = ItemCategory.objects.get(id=2739)
# category = scraper._get_category()
with QueryProfiler(QueryProfilerLevel.QUERY_SIGNATURE) as qp:
scraper._process_all_pages(category, counter=1, debug=True)
print(qp.query_profiled_data.summary)
print('\n')
def __call__(self, request: HttpRequest) -> HttpResponseBase:
# Check if we have to enable query profiler or not.
query_profiler_level: Optional[
QueryProfilerLevel] = settings.DJANGO_QUERY_PROFILER_LEVEL_FUNC(
request)
if not query_profiler_level:
return self.get_response(request)
start_time: float = time()
'''
Lets clear all the storage related to this thread. This is not strictly needed, but just as a safety measure
As a side effect, this implies that we *CANNOT* use this middleware twice for a request
'''
with QueryProfiler(query_profiler_level,
clear_thread_local=True) as query_profiler:
response = self.get_response(request)
query_profiled_data: QueryProfiledData = query_profiler.query_profiled_data
try:
# Pickling the object, and saving to redis
redis_key: str = redis_utils.store_data(query_profiled_data)
query_profiled_detail_relative_url: str = reverse(
query_profiler_url.GET_QUERY_PROFILED_DATA_NAME,
args=[redis_key, query_profiler_level.name])
query_profiled_detail_absolute_url: str = request.build_absolute_uri(
query_profiled_detail_relative_url)
detailed_view_link_text: str = query_profiler_level.name.lower()
except Exception as ex:
# The exception can happen because of two reasons:
# 1. redis throws exception
# 2. detailed_view_url not setup in urls.py
if settings.DJANGO_QUERY_PROFILER_IGNORE_DETAILED_VIEW_EXCEPTION:
query_profiled_detail_absolute_url: str = DETAILED_VIEW_EXCEPTION_URL
detailed_view_link_text: str = DETAILED_VIEW_EXCEPTION_LINK_TEXT
else:
raise ex
# Setting all headers that the chrome plugin require
response[ChromePluginData.QUERY_PROFILED_SUMMARY_DATA] = json.dumps(
query_profiled_data.summary.as_dict())
response[
ChromePluginData.
QUERY_PROFILED_DETAILED_URL] = query_profiled_detail_absolute_url
response[
ChromePluginData.
TIME_SPENT_PROFILING_IN_MICROS] = query_profiled_data.time_spent_profiling_in_micros
response[ChromePluginData.TOTAL_SERVER_TIME_IN_MILLIS] = int(
(time() - start_time) * 1000)
response[
ChromePluginData.
QUERY_PROFILER_DETAILED_VIEW_LINK_TEXT] = detailed_view_link_text
settings.DJANGO_QUERY_PROFILER_POST_PROCESSOR(query_profiled_data,
request, response)
return response
def test_no_data_profiler_summary(self):
""" Tests for the case when we don't have any data """
with QueryProfiler(QueryProfilerLevel.QUERY) as empty_data:
list(Topping.objects.all())
summary_dict: Dict = empty_data.query_profiled_data.summary.as_dict()
self.assertEqual(summary_dict[SqlStatement.SELECT.name], 1)
self.assertEqual(summary_dict[SqlStatement.INSERT.name], 0)
self.assertEqual(summary_dict[SqlStatement.UPDATE.name], 0)
self.assertEqual(summary_dict[SqlStatement.DELETE.name], 0)
self.assertEqual(summary_dict[SqlStatement.TRANSACTIONALS.name], 0)
def test_query_nested_by_another_query(self):
with QueryProfiler(QueryProfilerLevel.QUERY):
str(Pizza.objects.all())
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level, QueryProfilerLevel.QUERY)
with QueryProfiler(QueryProfilerLevel.QUERY) as qp_query_nested:
str(Pizza.objects.all())
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level, QueryProfilerLevel.QUERY)
self.assertEqual(
data_collector_thread_local_storage.
_current_query_profiler_level, QueryProfilerLevel.QUERY)
self.assertIsNone(qp_query_nested.query_profiled_data.summary.
potential_n_plus1_query_count)
self.assertIsNotNone(str(data_collector_thread_local_storage)
) # Not throwing exception is the test
def test_bulk_create_toppings(self):
""" Sql statements when bulk inserting """
with QueryProfiler(QueryProfilerLevel.QUERY) as qp_bulk_create:
bulk_create_toppings()
summary_dict: Dict = qp_bulk_create.query_profiled_data.summary.as_dict(
)
self.assertEqual(summary_dict[SqlStatement.SELECT.name], 0)
self.assertEqual(summary_dict[SqlStatement.INSERT.name], 1)
self.assertEqual(summary_dict[SqlStatement.UPDATE.name], 0)
self.assertEqual(summary_dict[SqlStatement.DELETE.name], 0)
self.assertEqual(summary_dict[SqlStatement.TRANSACTIONALS.name], 0)
def test_delete_toppings(self):
""" We would delete toppings in two ways and see the count of sql statements """
bulk_create_toppings()
with QueryProfiler(QueryProfilerLevel.QUERY) as qp_bulk_delete:
Topping.objects.all().delete()
summary_dict: Dict = qp_bulk_delete.query_profiled_data.summary.as_dict(
)
# Since topping is also has a many-to-many relationship, it has to delete the through table
self.assertEqual(summary_dict[SqlStatement.SELECT.name], 1)
self.assertEqual(summary_dict[SqlStatement.INSERT.name], 0)
self.assertEqual(summary_dict[SqlStatement.UPDATE.name], 0)
self.assertEqual(summary_dict[SqlStatement.DELETE.name], 2)
self.assertEqual(summary_dict[SqlStatement.TRANSACTIONALS.name], 0)
def test_non_bulk_create_toppings(self):
""" Sql statements when creating without bulk_create"""
with QueryProfiler(QueryProfilerLevel.QUERY) as qp_create:
Topping.objects.create(name='olives', is_spicy=False)
Topping.objects.create(name='pineapple', is_spicy=False)
Topping.objects.create(name='pepperoni', is_spicy=True)
Topping.objects.create(name='canadian_bacon', is_spicy=True)
Topping.objects.create(name='mozzarella', is_spicy=False)
summary_dict: Dict = qp_create.query_profiled_data.summary.as_dict()
self.assertEqual(summary_dict[SqlStatement.SELECT.name], 0)
self.assertEqual(summary_dict[SqlStatement.INSERT.name], 5)
self.assertEqual(summary_dict[SqlStatement.UPDATE.name], 0)
self.assertEqual(summary_dict[SqlStatement.DELETE.name], 0)
self.assertEqual(summary_dict[SqlStatement.TRANSACTIONALS.name], 0)
def test_missing_select_related(self):
"""
This function tests various select_related and prefetch_related on the function
food.models.py#toppings_of_best_pizza_serving_restaurants
"""
# STEP 1: No select_related/prefetch_related
pizza = Pizza.objects.first()
with QueryProfiler(QueryProfilerLevel.QUERY_SIGNATURE
) as qp_no_prefetch_select_related:
str(pizza.toppings_of_best_pizza_serving_restaurants())
query_signature_to_query_signature_statistics = \
qp_no_prefetch_select_related.query_profiled_data.query_signature_to_query_signature_statistics
query_signature_analysis = [
query_signature.analysis for query_signature in
query_signature_to_query_signature_statistics.keys()
]
# We are missing prefetch to restaurants, which in turn is missing select_related to best_pizza, and then
# prefetch to toppings
expected_query_signature_analysis = [
QuerySignatureAnalyzeResult.MISSING_PREFETCH_RELATED,
QuerySignatureAnalyzeResult.MISSING_SELECT_RELATED,
QuerySignatureAnalyzeResult.MISSING_PREFETCH_RELATED
]
self.assertListEqual(query_signature_analysis,
expected_query_signature_analysis)
# STEP 2: Missing just the select_related to best_pizza and prefetch to toppings
pizza = Pizza.objects.prefetch_related('restaurants').first()
with QueryProfiler(
QueryProfilerLevel.QUERY_SIGNATURE) as qp_no_select_related:
str(pizza.toppings_of_best_pizza_serving_restaurants())
query_signature_to_query_signature_statistics = \
qp_no_select_related.query_profiled_data.query_signature_to_query_signature_statistics
query_signature_analysis = [
query_signature.analysis for query_signature in
query_signature_to_query_signature_statistics.keys()
]
# We have prefetch to restaurants, but restaurant is missing select_related to best_pizza, and then
# prefetch to toppings
expected_query_signature_analysis = [
QuerySignatureAnalyzeResult.MISSING_SELECT_RELATED,
QuerySignatureAnalyzeResult.MISSING_PREFETCH_RELATED
]
self.assertListEqual(query_signature_analysis,
expected_query_signature_analysis)
# STEP 3: Adding the select_related to best_pizza but still missing prefetch to toppings
pizza = Pizza.objects.prefetch_related(
Prefetch('restaurants',
queryset=Restaurant.objects.select_related(
'best_pizza'))).first()
with QueryProfiler(
QueryProfilerLevel.QUERY_SIGNATURE) as qp_no_select_related:
str(pizza.toppings_of_best_pizza_serving_restaurants())
query_signature_to_query_signature_statistics = \
qp_no_select_related.query_profiled_data.query_signature_to_query_signature_statistics
query_signature_analysis = [
query_signature.analysis for query_signature in
query_signature_to_query_signature_statistics.keys()
]
# We have prefetch to restaurants, but restaurant is missing select_related to best_pizza, and then
# prefetch to toppings
expected_query_signature_analysis = [
QuerySignatureAnalyzeResult.MISSING_PREFETCH_RELATED
]
self.assertListEqual(query_signature_analysis,
expected_query_signature_analysis)