def _load_backend(self):
"""Load the caching backend.
This will replace the current caching backend with a newly loaded
one, based on the stored cache name.
Only one thread at a time can load the cache backend. A counter
is kept that keeps the load generation number. If several threads
try to reload the backend at once, only one will succeed in doing
so for that generation.
"""
cur_load_gen = self._load_gen
with self._load_lock:
if self._load_gen == cur_load_gen:
try:
# Django >= 1.7
from django.core.cache import caches
self._backend = caches[self._cache_name]
except ImportError:
# Django < 1.7
from django.core.cache import get_cache
self._backend = get_cache(self._cache_name)
# get_cache will attempt to connect to 'close', which we don't
# want. Instead, go and disconnect this.
request_finished.disconnect(self._backend.close)
self._load_gen = cur_load_gen + 1
def closing_iterator_wrapper(iterable, close):
try:
yield from iterable
finally:
request_finished.disconnect(close_old_connections)
close() # will fire request_finished
request_finished.connect(close_old_connections)
def closing_iterator_wrapper(iterable, close):
try:
yield from iterable
finally:
request_finished.disconnect(close_old_connections)
close() # will fire request_finished
request_finished.connect(close_old_connections)
def _load_backend(self):
"""Load the caching backend.
This will replace the current caching backend with a newly loaded
one, based on the stored cache name.
Only one thread at a time can load the cache backend. A counter
is kept that keeps the load generation number. If several threads
try to reload the backend at once, only one will succeed in doing
so for that generation.
"""
cur_load_gen = self._load_gen
with self._load_lock:
if self._load_gen == cur_load_gen:
try:
# Django >= 1.7
from django.core.cache import caches
self._backend = caches[self._cache_name]
except ImportError:
# Django < 1.7
from django.core.cache import get_cache
self._backend = get_cache(self._cache_name)
# get_cache will attempt to connect to 'close', which we don't
# want. Instead, go and disconnect this.
request_finished.disconnect(self._backend.close)
self._load_gen = cur_load_gen + 1
def register_handlers(client):
from django.core.signals import got_request_exception, request_started, request_finished
from elasticapm.contrib.django.handlers import exception_handler
# Connect to Django's internal signal handlers
got_request_exception.disconnect(dispatch_uid=ERROR_DISPATCH_UID)
got_request_exception.connect(partial(exception_handler, client),
dispatch_uid=ERROR_DISPATCH_UID,
weak=False)
request_started.disconnect(dispatch_uid=REQUEST_START_DISPATCH_UID)
request_started.connect(partial(_request_started_handler, client),
dispatch_uid=REQUEST_START_DISPATCH_UID,
weak=False)
request_finished.disconnect(dispatch_uid=REQUEST_FINISH_DISPATCH_UID)
request_finished.connect(
lambda sender, **kwargs: client.end_transaction()
if _should_start_transaction(client) else None,
dispatch_uid=REQUEST_FINISH_DISPATCH_UID,
weak=False,
)
# If we can import celery, register ourselves as exception handler
try:
import celery # noqa F401
from elasticapm.contrib.celery import register_exception_tracking
try:
register_exception_tracking(client)
except Exception as e:
client.logger.exception(
"Failed installing django-celery hook: %s" % e)
except ImportError:
client.logger.debug("Not instrumenting Celery, couldn't import")
def __call__(self, environ):
# Set up middleware if needed. We couldn't do this earlier, because
# settings weren't available.
if self._request_middleware is None:
self.load_middleware()
request_started.disconnect(close_old_connections)
request_started.send(sender=self.__class__)
request_started.connect(close_old_connections)
request = WSGIRequest(environ)
# sneaky little hack so that we can easily get round
# CsrfViewMiddleware. This makes life easier, and is probably
# required for backwards compatibility with external tests against
# admin views.
request._dont_enforce_csrf_checks = not self.enforce_csrf_checks
# Request goes through middleware.
response = self.get_response(request)
# Attach the originating request to the response so that it could be
# later retrieved.
response.request_instance = request
# We're emulating a WSGI server; we must call the close method
# on completion.
if response.streaming:
response.streaming_content = closing_iterator_wrapper(
response.streaming_content, response.close)
else:
request_finished.disconnect(close_old_connections)
response.close() # will fire request_finished
request_finished.connect(close_old_connections)
return response
def encode_video(self, **kwargs):
from osso.videmus.tasks import encode_video
encode_video.delay(self.__class__, self.pk)
if hasattr(self, 'signal_uuid'):
request_finished.disconnect(dispatch_uid='encode_video_%s' %
self.signal_uuid,
weak=False)
async def test_request_lifecycle_signals_dispatched_with_thread_sensitive(
self):
class SignalHandler:
"""Track threads handler is dispatched on."""
threads = []
def __call__(self, **kwargs):
self.threads.append(threading.current_thread())
signal_handler = SignalHandler()
request_started.connect(signal_handler)
request_finished.connect(signal_handler)
# Perform a basic request.
application = get_asgi_application()
scope = self.async_request_factory._base_scope(path='/')
communicator = ApplicationCommunicator(application, scope)
await communicator.send_input({'type': 'http.request'})
response_start = await communicator.receive_output()
self.assertEqual(response_start['type'], 'http.response.start')
self.assertEqual(response_start['status'], 200)
response_body = await communicator.receive_output()
self.assertEqual(response_body['type'], 'http.response.body')
self.assertEqual(response_body['body'], b'Hello World!')
# Give response.close() time to finish.
await communicator.wait()
# At this point, AsyncToSync does not have a current executor. Thus
# SyncToAsync falls-back to .single_thread_executor.
target_thread = next(iter(SyncToAsync.single_thread_executor._threads))
request_started_thread, request_finished_thread = signal_handler.threads
self.assertEqual(request_started_thread, target_thread)
self.assertEqual(request_finished_thread, target_thread)
request_started.disconnect(signal_handler)
request_finished.disconnect(signal_handler)
async def test_request_lifecycle_signals_dispatched_with_thread_sensitive(
self):
class SignalHandler:
"""Track threads handler is dispatched on."""
threads = []
def __call__(self, **kwargs):
self.threads.append(threading.current_thread())
signal_handler = SignalHandler()
request_started.connect(signal_handler)
request_finished.connect(signal_handler)
# Perform a basic request.
application = get_asgi_application()
scope = self.async_request_factory._base_scope(path="/")
communicator = ApplicationCommunicator(application, scope)
await communicator.send_input({"type": "http.request"})
response_start = await communicator.receive_output()
self.assertEqual(response_start["type"], "http.response.start")
self.assertEqual(response_start["status"], 200)
response_body = await communicator.receive_output()
self.assertEqual(response_body["type"], "http.response.body")
self.assertEqual(response_body["body"], b"Hello World!")
# Give response.close() time to finish.
await communicator.wait()
# AsyncToSync should have executed the signals in the same thread.
request_started_thread, request_finished_thread = signal_handler.threads
self.assertEqual(request_started_thread, request_finished_thread)
request_started.disconnect(signal_handler)
request_finished.disconnect(signal_handler)
def __call__(self, environ):
from django.conf import settings
# Set up middleware if needed. We couldn't do this earlier, because
# settings weren't available.
if self._request_middleware is None:
self.load_middleware()
request_started.disconnect(close_old_connections)
request_started.send(sender=self.__class__)
request_started.connect(close_old_connections)
request = WSGIRequest(environ)
# sneaky little hack so that we can easily get round
# CsrfViewMiddleware. This makes life easier, and is probably
# required for backwards compatibility with external tests against
# admin views.
request._dont_enforce_csrf_checks = not self.enforce_csrf_checks
response = self.get_response(request)
# We're emulating a WSGI server; we must call the close method
# on completion.
if response.streaming:
response.streaming_content = closing_iterator_wrapper(response.streaming_content, response.close)
else:
request_finished.disconnect(close_old_connections)
response.close() # will fire request_finished
request_finished.connect(close_old_connections)
return response
def request_finished(self, **kwargs):
from raven.contrib.django.models import client
if getattr(self, '_txid', None):
client.transaction.pop(self._txid)
self._txid = None
request_finished.disconnect(self.request_finished)
def closing_iterator_wrapper(iterable, close):
try:
for item in iterable:
yield item
finally:
request_finished.disconnect(close_connection)
close() # will fire request_finished
request_finished.connect(close_connection)
def request_finished(self, **kwargs):
from raven.contrib.django.models import client
if getattr(self, '_txid', None):
client.transaction.pop(self._txid)
self._txid = None
request_finished.disconnect(self.request_finished)
def closing_iterator_wrapper(iterable, close):
try:
for item in iterable:
yield item
finally:
request_finished.disconnect(close_connection)
close() # will fire request_finished
request_finished.connect(close_connection)
def setUp(self):
request_finished.disconnect(es.hold.process,
dispatch_uid='process_es_tasks_on_finish')
self.gia_mock = mock.patch(
'mkt.developers.tasks.generate_image_assets').__enter__()
self.fi_mock = mock.patch(
'mkt.developers.tasks.fetch_icon').__enter__()
self.user = self.get_user()
assert self.client.login(username=self.user.email, password='******')
def setUp(self):
request_finished.disconnect(es.hold.process,
dispatch_uid='process_es_tasks_on_finish')
self.gia_mock = mock.patch(
'mkt.developers.tasks.generate_image_assets').__enter__()
self.fi_mock = mock.patch(
'mkt.developers.tasks.fetch_icon').__enter__()
self.user = self.get_user()
assert self.client.login(username=self.user.email, password='******')
def deregister_signal_handlers():
# Disconnects the signal handlers for easy access in importers
m2m_changed.disconnect(product_category_m2m_changed_signal_handler,
sender=Product.categories.through)
post_save.disconnect(product_post_save_signal_handler, sender=Product)
post_delete.disconnect(product_post_delete_signal_handler, sender=Product)
post_save.disconnect(category_change_handler, sender=Category)
post_delete.disconnect(category_change_handler, sender=Category)
if settings.HANDLE_STOCKRECORD_CHANGES:
post_save.disconnect(stockrecord_change_handler, sender=StockRecord)
post_delete.disconnect(stockrecord_post_delete_handler,
sender=StockRecord)
request_finished.disconnect(update_index.synchronize_searchindex)
def setUp(self):
request_finished.disconnect(es.hold.process, dispatch_uid="process_es_tasks_on_finish")
super(BasePackagedAppTest, self).setUp()
self.app = Webapp.objects.get(pk=337141)
self.app.update(is_packaged=True)
self.version = self.app.current_version
self.file = self.version.all_files[0]
self.file.update(filename="mozball.zip")
self.package = self.packaged_app_path("mozball.zip")
self.upload = self.get_upload(abspath=self.package)
self.upload.update(name="mozball.zip", is_webapp=True)
self.url = reverse("submit.app")
assert self.client.login(username="******", password="******")
def setUp(self):
request_finished.disconnect(es.hold.process,
dispatch_uid='process_es_tasks_on_finish')
super(BasePackagedAppTest, self).setUp()
self.app = Webapp.objects.get(pk=337141)
self.app.update(is_packaged=True)
self.version = self.app.current_version
self.file = self.version.all_files[0]
self.file.update(filename='mozball.zip')
self.package = self.packaged_app_path('mozball.zip')
self.upload = self.get_upload(abspath=self.package)
self.upload.update(name='mozball.zip', is_webapp=True)
self.url = reverse('submit.app')
assert self.client.login(username='******',
password='******')
def do_benchmark():
start_times = []
times = []
def handle_started(*args, **kwargs):
start_times.append(time())
def handle_finished(*args, **kwargs):
times.append(time() - start_times[-1])
request_started.connect(handle_started)
request_finished.connect(handle_finished)
try:
yield times
finally:
request_started.disconnect(handle_started)
request_finished.disconnect(handle_finished)
def download_file(request, file_id):
file_to_dl = [f for f in request.session['file_list']\
if f.file_id == int(file_id)][0]
f = NamedTemporaryFile(delete=False)
def test_signal(sender, **kwargs):
os.remove(f.name)
request_finished.connect(test_signal, weak=False)
request_finished.disconnect(test_signal)
if file_to_dl.metadata_labels:
f.write('<' + ' '.join(file_to_dl.metadata_labels) + '>\n')
for text in file_to_dl.texts:
if text.metadata:
f.write('<' + ' '.join(text.metadata) + '>')
f.write('\n')
for sentence in text.sentences:
for token in sentence.tokens:
f.write(token.text_id + '\t' + token.token_id + '\t' +
token.form + '\t' + token.norm + '\t' + token.lemma +
'\t' + token.upos + '\t' + token.xpos + '\t' +
token.feats + '\t' + token.ufeats + '\t' + token.head +
'\t' + token.deprel + '\t' + token.deps + '\t' +
token.misc.strip() + '\n')
f.write('\n')
f.close()
try:
UploadedFile.objects.create(md5_checksum=get_md5(file_to_dl),
normalized=file_to_dl.normalized)
except:
pass
response = HttpResponse(FileWrapper(open(f.name)),
content_type='application/force-download')
response['Content-Disposition'] = 'attachment; filename=%s' % smart_str(
file_to_dl.filename)
return response
def setUp(self):
request_finished.disconnect(es.hold.process,
dispatch_uid='process_es_tasks_on_finish')
super(BasePackagedAppTest, self).setUp()
self.app = Webapp.objects.get(pk=337141)
self.app.update(is_packaged=True)
self.version = self.app.current_version
self.file = self.version.all_files[0]
self.file.update(filename='mozball.zip')
self.package = self.packaged_app_path('mozball.zip')
self.upload = self.get_upload(abspath=self.package)
self.upload.update(name='mozball.zip', is_webapp=True)
self.url = reverse('submit.app.manifest')
assert self.client.login(username='******',
password='******')
# Complete first step.
self.client.post(reverse('submit.app.terms'),
{'read_dev_agreement': True})
def remove_tempfiles_callback(sender, **kwargs):
'''
remove older files from wms tempdir
'''
request_finished.disconnect(remove_tempfiles_callback)
delay = time.time() - 60*60*24 # 1 day delay
wms_temp_dir = os.path.join(settings.MEDIA_ROOT,'warp','wms')
for tempfile in os.listdir(wms_temp_dir):
filePath = os.path.join(wms_temp_dir,tempfile)
if os.path.getmtime(filePath) < delay:
os.remove(filePath)
warp_dir = os.path.join(settings.MEDIA_ROOT,'warp')
for tempfile in os.listdir(warp_dir):
if tempfile.endswith(('.json', '.geojson', '.xml', '.log')):
filePath = os.path.join(warp_dir,tempfile)
if os.path.getmtime(filePath) < delay:
os.remove(filePath)
async def __call__(self, scope):
# Set up middleware if needed. We couldn't do this earlier, because
# settings weren't available.
if self._middleware_chain is None:
self.load_middleware(is_async=True)
# Extract body file from the scope, if provided.
if "_body_file" in scope:
body_file = scope.pop("_body_file")
else:
body_file = FakePayload("")
request_started.disconnect(close_old_connections)
await sync_to_async(request_started.send,
thread_sensitive=False)(sender=self.__class__,
scope=scope)
request_started.connect(close_old_connections)
request = ASGIRequest(scope, body_file)
# Sneaky little hack so that we can easily get round
# CsrfViewMiddleware. This makes life easier, and is probably required
# for backwards compatibility with external tests against admin views.
request._dont_enforce_csrf_checks = not self.enforce_csrf_checks
# Request goes through middleware.
response = await self.get_response_async(request)
# Simulate behaviors of most Web servers.
conditional_content_removal(request, response)
# Attach the originating ASGI request to the response so that it could
# be later retrieved.
response.asgi_request = request
# Emulate a server by calling the close method on completion.
if response.streaming:
response.streaming_content = await sync_to_async(
closing_iterator_wrapper, thread_sensitive=False)(
response.streaming_content,
response.close,
)
else:
request_finished.disconnect(close_old_connections)
# Will fire request_finished.
await sync_to_async(response.close, thread_sensitive=False)()
request_finished.connect(close_old_connections)
return response
def __call__(self, environ):
# Set up middleware if needed. We couldn't do this earlier, because
# settings weren't available.
if self._middleware_chain is None:
self.load_middleware()
request_started.disconnect(close_old_connections)
request_started.send(sender=self.__class__, environ=environ)
request_started.connect(close_old_connections)
request = WSGIRequest(environ)
# sneaky little hack so that we can easily get round
# CsrfViewMiddleware. This makes life easier, and is probably
# required for backwards compatibility with external tests against
# admin views.
request._dont_enforce_csrf_checks = not self.enforce_csrf_checks
# Request goes through middleware.
response = self.get_response(request)
# Simulate behaviors of most Web servers.
conditional_content_removal(request, response)
# Attach the originating request to the response so that it could be
# later retrieved.
response.wsgi_request = request
# We're emulating a WSGI server; we must call the close method
# on completion.
if response.streaming:
response.streaming_content = closing_iterator_wrapper(
response.streaming_content, response.close)
else:
request_finished.disconnect(close_old_connections)
response.close() # will fire request_finished
request_finished.connect(close_old_connections)
return response
def trigger_restart(**kwargs):
from cms.signals import urls_need_reloading
request_finished.disconnect(trigger_restart, dispatch_uid=DISPATCH_UID)
urls_need_reloading.send(sender=None)
def setUp(self):
# Disable the request_finished signal during this test
# to avoid interfering with the database connection.
request_finished.disconnect(close_old_connections)
os.environ['DJANGO_SETTINGS_MODULE'] = 'stoutsd.settings'
# Google App Engine imports.
from google.appengine.ext.webapp import util
# Force Django to reload its settings.
from django.conf import settings
settings._target = None
import logging
import django.core.handlers.wsgi
import django.core.signals
import django.db
from django.core.signals import request_started, request_finished
def log_exception (*args, **kwargs):
logging.exception('Exception in request:')
# Log errors.
request_started.connect(log_exception)
# Unregister the rollback event handler.
request_finished.disconnect(django.db._rollback_on_exception)
def main():
util.run_wsgi_app(django.core.handlers.wsgi.WSGIHandler())
if __name__ == '__main__':
main()
def tearDown(self):
request_started.disconnect(self.register_started)
request_finished.disconnect(self.register_finished)
def tearDown(self):
request_started.disconnect(self.register_started)
request_finished.disconnect(self.register_finished)
def setUp(self):
# Disable the request_finished signal during this test
# to avoid interfering with the database connection.
request_finished.disconnect(close_old_connections)
from __future__ import unicode_literals
def trigger_restart(**kwargs):
from cms.signals import urls_need_reloading
request_finished.disconnect(trigger_restart, dispatch_uid=DISPATCH_UID)
urls_need_reloading.send(sender=None)
