def init_workers(self, use_workers=True):
def shutdown_handler(active_workers):
def _handler(signum, frame):
try:
for active_worker in active_workers:
active_worker.terminate()
signal.signal(signum, signal.SIG_DFL)
os.kill(os.getpid(), signum
) # Rethrow signal, this time without catching it
except Exception:
logger.exception('Error in shutdown_handler')
return _handler
if use_workers:
django_connection.close()
django_cache.close()
for idx in range(settings.JOB_EVENT_WORKERS):
queue_actual = MPQueue(settings.JOB_EVENT_MAX_QUEUE_SIZE)
w = Process(target=self.callback_worker,
args=(
queue_actual,
idx,
))
w.start()
if settings.DEBUG:
logger.info('Started worker %s' % str(idx))
self.worker_queues.append([0, queue_actual, w])
elif settings.DEBUG:
logger.warn('Started callback receiver (no workers)')
signal.signal(signal.SIGINT,
shutdown_handler([p[2] for p in self.worker_queues]))
signal.signal(signal.SIGTERM,
shutdown_handler([p[2] for p in self.worker_queues]))
def test_bulk_sms_send_multiproc(self):
# close database connections manually to make sure they're not passed
# to subprocesses
for conn in django_db_connections.all():
conn.close()
# send the messages in parallel using 10 processes. use
# multiprocessing rather than threads so we don't have to clean
# up database connections that the threads might leave open.
cache.close()
pool = multiprocessing.Pool(10)
try:
for i in range(30):
BulkMessageFactory(batch=self.batch, sms=None)
for j in range(10):
BulkMessageFactory(batch=self.batch, sms=None, deleted=True)
# 40 is the first multiple of 10 greater than or equal to 31
num_sent = send_messages(self.batch, num_to_send=40, map_=pool.map)
batch = Batch.objects.get(pk=self.batch.pk)
self.assertEqual(batch.errors, 0)
self.assertEqual(batch.status, Batch.COMPLETED)
# assert that we report the right number sent too
self.assertEqual(num_sent, 30)
self.assertEqual(len(self.outbound), 30)
finally:
pool.terminate()
pool.join()
def test_bulk_sms_send_multiproc(self):
# close database connections manually to make sure they're not passed
# to subprocesses
for conn in django_db_connections.all():
conn.close()
# send the messages in parallel using 10 processes. use
# multiprocessing rather than threads so we don't have to clean
# up database connections that the threads might leave open.
cache.close()
pool = multiprocessing.Pool(10)
try:
for i in range(30):
BulkMessageFactory(batch=self.batch, sms=None)
for j in range(10):
BulkMessageFactory(batch=self.batch, sms=None, deleted=True)
# 40 is the first multiple of 10 greater than or equal to 31
num_sent = send_messages(self.batch, num_to_send=40, map_=pool.map)
batch = Batch.objects.get(pk=self.batch.pk)
self.assertEqual(batch.errors, 0)
self.assertEqual(batch.status, Batch.COMPLETED)
# assert that we report the right number sent too
self.assertEqual(num_sent, 30)
self.assertEqual(len(self.outbound), 30)
finally:
pool.terminate()
pool.join()
def connect(self):
self.accept()
self.service_ids = self.scope['url_route']['kwargs'][
'service_ids'].split(',')
locations = get_vehicle_locations(
journey__service__in=self.service_ids)
self.send_locations(locations)
if not locations:
self.send_json([])
icarus = not any(location.journey.source_id != 75
for location in locations)
for service_id in self.service_ids:
async_to_sync(self.channel_layer.group_add)(f'service{service_id}',
self.channel_name)
if icarus:
codes = ServiceCode.objects.filter(scheme__in=schemes,
service=service_id)
codes = codes.annotate(
source_name=Replace('scheme', Value(' SIRI')))
siri_sources = SIRISource.objects.filter(
name=OuterRef('source_name'))
codes = codes.filter(Exists(siri_sources))
for code in codes:
try:
siri_one_shot(code, timezone.now(), bool(locations))
break
except Poorly:
pass
cache.close()
def handle(self, *arg, **options):
if options.get('status'):
print(Control('dispatcher').status())
return
if options.get('running'):
print(Control('dispatcher').running())
return
if options.get('reload'):
return Control('dispatcher').control({'control': 'reload'})
# It's important to close these because we're _about_ to fork, and we
# don't want the forked processes to inherit the open sockets
# for the DB and cache connections (that way lies race conditions)
django_connection.close()
django_cache.close()
# spawn a daemon thread to periodically enqueues scheduled tasks
# (like the node heartbeat)
periodic.run_continuously()
consumer = None
try:
queues = ['tower_broadcast_all', get_local_queuename()]
consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues,
AutoscalePool(min_workers=4))
consumer.run()
except KeyboardInterrupt:
logger.debug('Terminating Task Dispatcher')
if consumer:
consumer.stop()
def size(self):
val = cache.get(self.package, -1)
cache.close()
if val == -1:
val = os.path.getsize(self.apk.file.path)
cache.set(self.package, val, 30)
return val
def wrapper_cleanup_new_process(*args, **kwargs):
from awx.conf.settings import SettingsWrapper # noqa
django_connection.close()
django_cache.close()
SettingsWrapper.initialize()
return func(*args, **kwargs)
def show(request, id):
x = cache.get('poll_id__' + id)
if x:
poll = x
else:
poll = Polls.objects.filter(id=id)
cache.set('poll_id__' + id, poll)
cache.close()
if poll.count() < 1:
return HttpResponse('Not found')
else:
x = cache.get('polls_objects_get_id_%s' % id)
if x:
poll = x
else:
poll = Polls.objects.get(id=id)
cache.set('polls_objects_get_id_%s' % id, poll)
y = cache.get('options_objects_filter_poll_id_%s_order_by_-count' % id)
if y:
options = y
else:
options = Options.objects.filter(poll_id=id).order_by('-count')
cache.set('options_objects_filter_poll_id_%s_order_by_-count' % id,
options)
c_cache = cache.get('comments_poll_id__' + id)
if c_cache:
comments = c_cache
else:
comments = Comments.objects.filter(poll_id=id).order_by('-id')
cache.set('comments_poll_id__' + id, comments)
paginator = Paginator(comments, 10)
page = request.GET.get('p')
try:
comments = paginator.page(page)
except PageNotAnInteger:
comments = paginator.page(1)
except EmptyPage:
comments = paginator.page(paginator.num_pages)
u = cache.get('optionsvotes_objects_filter_poll_poll.id_%s' % poll.id)
if u:
options_c = u
else:
options_c = OptionsVotes.objects.filter(poll=poll.id).count()
cache.set('optionsvotes_objects_filter_poll_poll.id_%s' % poll.id,
options_c)
c = Context({
'poll': poll,
'options': options,
'options_c': options_c,
'comments': comments,
'user': request.user
})
c.update(csrf(request))
return render_to_response('show.html', c)
def clear_cache(request):
try:
del request.session['search-questions-post']
cache.clear()
cache.close()
except KeyError:
pass
return redirect('/')
def test_set_many(self):
if not self._run_test():
return
cache.set_many({'hello1':'world1','hello2':'world2'},30)
self.assertEquals(len(cache._logger.log), 0)
self.assertEquals(cache.get("hello1"), "world1")
self.assertEquals(len(cache._logger.log), 0)
cache.close()
self.assertEquals(len(cache._logger.log), 1)
def test_set(self):
if not self._run_test():
return
cache.set("hello","world",30)
self.assertEquals(len(cache._logger.log), 0)
self.assertEquals(cache.get("hello"), "world")
self.assertEquals(len(cache._logger.log), 0)
cache.close()
self.assertEquals(len(cache._logger.log), 1)
def finish(self, chunk = None):
super(BaseHandler, self).finish(chunk = chunk)
# Clean up django ORM connections
django.db.connection.close()
# Clean up after python-memcached
from django.core.cache import cache
if hasattr(cache, 'close'):
cache.close()
def delete_session(request):
try:
del request.session['search-questions-post']
del request.session['start-time']
del request.session['count']
cache.clear()
cache.close()
except KeyError:
pass
return redirect('/')
def handle(self, *arg, **options):
if options.get('status'):
print(Control('dispatcher').status())
return
if options.get('running'):
print(Control('dispatcher').running())
return
if options.get('reload'):
return Control('dispatcher').control({'control': 'reload'})
# It's important to close these because we're _about_ to fork, and we
# don't want the forked processes to inherit the open sockets
# for the DB and memcached connections (that way lies race conditions)
django_connection.close()
django_cache.close()
# spawn a daemon thread to periodically enqueues scheduled tasks
# (like the node heartbeat)
periodic.run_continuously()
reaper.reap()
consumer = None
# don't ship external logs inside the dispatcher's parent process
# this exists to work around a race condition + deadlock bug on fork
# in cpython itself:
# https://bugs.python.org/issue37429
AWXProxyHandler.disable()
with Connection(settings.BROKER_URL) as conn:
try:
bcast = 'tower_broadcast_all'
queues = [
Queue(q, Exchange(q), routing_key=q)
for q in (settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()])
]
queues.append(
Queue(
construct_bcast_queue_name(bcast),
exchange=Exchange(bcast, type='fanout'),
routing_key=bcast,
reply=True
)
)
consumer = AWXConsumer(
'dispatcher',
conn,
TaskWorker(),
queues,
AutoscalePool(min_workers=4)
)
consumer.run()
except KeyboardInterrupt:
logger.debug('Terminating Task Dispatcher')
if consumer:
consumer.stop()
def run():
try:
if django_settings.ASKBOT_SELF_TEST:
run_startup_tests()
except AskbotConfigError as error:
print(error)
sys.exit(1)
# close DB and cache connections to prevent issues in prefork mode
connection.close()
if hasattr(cache, 'close'):
cache.close()
def run():
try:
if django_settings.ASKBOT_SELF_TEST:
run_startup_tests()
except AskbotConfigError as error:
print(error)
sys.exit(1)
# close DB and cache connections to prevent issues in prefork mode
connection.close()
if hasattr(cache, 'close'):
cache.close()
def index(request):
x = cache.get('all_polls')
if x:
polls = x
else:
polls = Polls.objects.all().order_by('-id')
cache.set('all_polls', polls, 60)
cache.close()
c = Context({'polls': polls, 'user': request.user})
return render_to_response('index.html', c)
def test_add(self):
if not self._run_test():
return
self.assertEquals(local.storage.get(version_key("hello")), None)
cache.add("hello","world")
self.assertEquals(len(cache._logger.log), 1)
self.assertEquals(local.storage.get(version_key("hello")), "world")
cache.add("hello","world1")
self.assertEquals(local.storage.get(version_key("hello")), "world")
self.assertEquals(len(cache._logger.log), 1)
cache.close()
self.assertEquals(len(cache._logger.log), 1)
def to_cache(article_id, schema_id, hash_value, result):
article_cache = cache.get("highlighting-{0}".format(article_id))
if article_cache is None:
article_cache = {}
article_cache[schema_id] = {}
article_cache[schema_id]['hash'] = hash_value
article_cache[schema_id]['value'] = result
cache.set("highlighting-{0}".format(article_id), article_cache)
# print(article_cache)
article_cache = cache.get("highlighting-{0}".format(article_id))
# print(article_cache)
cache.close()
def finish(self, chunk = None):
super(BaseHandler, self).finish(chunk = chunk)
# Clean up django ORM connections
dj_db.connection.close()
logging.info('%d sql queries' % len(dj_db.connection.queries))
for query in dj_db.connection.queries:
logging.debug('%s [%s seconds]' % (query['sql'], query['time']))
# Clean up after python-memcached
from django.core.cache import cache
if hasattr(cache, 'close'):
cache.close()
def test_delete(self):
if not self._run_test():
return
cache.set("hello","world",30)
self._commit_and_clear_log()
self.assertEquals(cache.get("hello"),"world")
self.assertEquals(len(cache._logger.log), 1)
cache.delete("hello")
self.assertEquals(len(cache._logger.log), 1)
self.assertEquals(cache.get("hello"),None)
self.assertEquals(len(cache._logger.log), 1)
cache.close()
self.assertEquals(len(cache._logger.log), 2)
def finish(self, chunk=None):
super(BaseRedisWebSocketHandler, self).finish(chunk=chunk)
connection.close()
if False:
info('%d sql queries' % len(connection.queries))
for query in connection.queries:
debug('%s [%s seconds]' % (query['sql'], query['time']))
# Clean up cache
from django.core.cache import cache
if hasattr(cache, 'close'):
cache.close()
def finish(self, chunk = None):
super(BaseWebSocketHandler, self).finish(chunk = chunk)
# Clean up django ORM connections
django.db.connection.close()
if False:
logging.info('%d sql queries' % len(django.db.connection.queries))
for query in django.db.connection.queries:
logging.debug('%s [%s seconds]' % (query['sql'], query['time']))
# Clean up after python-memcached
from django.core.cache import cache
if hasattr(cache, 'close'):
cache.close()
def test_delete_many(self):
if not self._run_test():
return
cache.set_many({'hello1':'world1','hello2':'world2'},30)
self._commit_and_clear_log()
cache.delete_many(['hello1','hello2'])
self.assertEquals(len(cache._logger.log), 0)
cache.delete_many(['hello1','hello2'])
self.assertEquals(len(cache._logger.log), 0)
self.assertEquals(cache.get_many(['hello1','hello2']),{version_key('hello1'):None,version_key('hello2'):None})
self.assertEquals(len(cache._logger.log), 0)
cache.close()
self.assertEquals(len(cache._logger.log), 1)
def __clean_on_fork__(self):
pid = os.getpid()
# if the current pid does *not* match the value on self, it means
# that value was copied on fork, and we're now in a *forked* process;
# the *first* time we enter this code path (on setting access),
# forcibly close DB/cache sockets and set a marker so we don't run
# this code again _in this process_
#
if pid != self.__dict__['pid'] and pid not in self.__dict__['__forks__']:
self.__dict__['__forks__'][pid] = True
# It's important to close these post-fork, because we
# don't want the forked processes to inherit the open sockets
# for the DB and cache connections (that way lies race conditions)
connection.close()
django_cache.close()
def finish(self, chunk=None):
super(BaseRedisWebSocketHandler, self).finish(chunk=chunk)
connection.close()
if False:
info('%d sql queries'
% len(connection.queries))
for query in connection.queries:
debug('%s [%s seconds]' % (query['sql'],
query['time']))
# Clean up cache
from django.core.cache import cache
if hasattr(cache, 'close'):
cache.close()
def handle(self, *arg, **options):
if options.get('status'):
print Control('dispatcher').status()
return
if options.get('running'):
print Control('dispatcher').running()
return
if options.get('reload'):
return Control('dispatcher').control({'control': 'reload'})
# It's important to close these because we're _about_ to fork, and we
# don't want the forked processes to inherit the open sockets
# for the DB and memcached connections (that way lies race conditions)
django_connection.close()
django_cache.close()
beat = Process(target=self.beat)
beat.daemon = True
beat.start()
reaper.reap()
consumer = None
with Connection(settings.BROKER_URL) as conn:
try:
bcast = 'tower_broadcast_all'
queues = [
Queue(q, Exchange(q), routing_key=q)
for q in (settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()])
]
queues.append(
Queue(
construct_bcast_queue_name(bcast),
exchange=Exchange(bcast, type='fanout'),
routing_key=bcast,
reply=True
)
)
consumer = AWXConsumer(
'dispatcher',
conn,
TaskWorker(),
queues,
AutoscalePool(min_workers=4)
)
consumer.run()
except KeyboardInterrupt:
logger.debug('Terminating Task Dispatcher')
if consumer:
consumer.stop()
def up(self):
idx = len(self.workers)
# It's important to close these because we're _about_ to fork, and we
# don't want the forked processes to inherit the open sockets
# for the DB and cache connections (that way lies race conditions)
django_connection.close()
django_cache.close()
worker = self.pool_cls(self.queue_size, self.target, (idx,) + self.target_args)
self.workers.append(worker)
try:
worker.start()
except Exception:
logger.exception('could not fork')
else:
logger.debug('scaling up worker pid:{}'.format(worker.pid))
return idx, worker
def _create_pool(self):
"""
Create a multiprocessing worker pool for sending bulk messages.
"""
# close database connections manually to make sure they're not passed
# to subprocesses
logger.debug('Closing database connections before starting worker pool.')
for conn in django_db_connections.all():
conn.close()
cache.close()
# send the messages in parallel using ``concurrent_workers`` processes. use
# multiprocessing rather than threads so we don't have to clean
# up database connections that the threads might leave open.
logger.debug('Starting worker pool with {0} processes.'.format(self.concurrent_workers))
return multiprocessing.Pool(processes=self.concurrent_workers,
maxtasksperchild=MAX_MESSAGES_PER_CHILD)
def finish(self, chunk=None):
super(BaseWebSocketHandler, self).finish(chunk=chunk)
# Clean up django ORM connections
connection.close()
if False:
info("%d sql queries" % len(connection.queries))
for query in connection.queries:
debug("%s [%s seconds]" % (query["sql"], query["time"]))
# Clean up after python-memcached
from django.core.cache import cache
if hasattr(cache, "close"):
cache.close()
def clean(self):
"""
:return:
"""
# Clean up django ORM connections
django.db.connection.close()
if settings.DEBUG:
logger.info('%d sql queries' % len(django.db.connection.queries))
for query in django.db.connection.queries:
logger.debug('%s [%s seconds]' % (query['sql'], query['time']))
# Clean up after python-memcached
from django.core.cache import cache
if hasattr(cache, 'close'):
cache.close()
def process_message(self, msg, retry_count=0):
"""Process this message and return tuple (do_ack, reply_msg).
do_ack must be boolean.
reply_msg, if not None, must have the property ``next`` in the format:
exchange/routing_key
"""
fn = None
hdrs = None
try:
amqp_call_count = cache.get('amqp_call_count', 0)
if amqp_call_count % 20 == 0:
connection.close() # reset database connection after every 20 calls
amqp_call_count = 0
cache.close()
cache.set('amqp_call_count', amqp_call_count+1, 300000)
except Exception, e:
_x().exception(e)
def jail(task_id, func, args, kwargs):
"""Wraps the task in a jail, which catches all exceptions, and
saves the status and result of the task execution to the task
meta backend.
If the call was successful, it saves the result to the task result
backend, and sets the task status to ``"DONE"``.
If the call results in an exception, it saves the exception as the task
result, and sets the task status to ``"FAILURE"``.
:param task_id: The id of the task.
:param func: Callable object to execute.
:param args: List of positional args to pass on to the function.
:param kwargs: Keyword arguments mapping to pass on to the function.
:returns: the function return value on success, or
the exception instance on failure.
"""
# See: http://groups.google.com/group/django-users/browse_thread/
# thread/78200863d0c07c6d/38402e76cf3233e8?hl=en&lnk=gst&
# q=multiprocessing#38402e76cf3233e8
from django.db import connection
connection.close()
# Reset cache connection
from django.core.cache import cache
cache.close()
# Backend process cleanup
default_backend.process_cleanup()
# Convert any unicode keys in the keyword arguments to ascii.
kwargs = dict([(k.encode("utf-8"), v)
for k, v in kwargs.items()])
try:
result = func(*args, **kwargs)
except Exception, exc:
default_backend.mark_as_failure(task_id, exc)
return ExceptionInfo(sys.exc_info())
def handle(self, *arg, **options):
if options.get('status'):
print(Control('dispatcher').status())
return
if options.get('running'):
print(Control('dispatcher').running())
return
if options.get('reload'):
return Control('dispatcher').control({'control': 'reload'})
# It's important to close these because we're _about_ to fork, and we
# don't want the forked processes to inherit the open sockets
# for the DB and memcached connections (that way lies race conditions)
django_connection.close()
django_cache.close()
# spawn a daemon thread to periodically enqueues scheduled tasks
# (like the node heartbeat)
periodic.run_continuously()
reaper.reap()
consumer = None
# don't ship external logs inside the dispatcher's parent process
# this exists to work around a race condition + deadlock bug on fork
# in cpython itself:
# https://bugs.python.org/issue37429
AWXProxyHandler.disable()
try:
queues = ['tower_broadcast_all', get_local_queuename()]
consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues,
AutoscalePool(min_workers=4))
consumer.run()
except KeyboardInterrupt:
logger.debug('Terminating Task Dispatcher')
if consumer:
consumer.stop()
def test_close(self):
cache = caches["default"]
cache.set("f", "1")
cache.close()
def test_close(self):
assert hasattr(cache, 'close')
cache.close()
def get_value(key):
pickled_value = cache.get(key)
cache.close()
if pickled_value is None:
return None
return pickle.loads(pickled_value)
def set_value(key, value, time_out=200):
# cache.set(key, pickle.dumps(value), time_out)
cache.set(key, pickle.dumps(value))
cache.close()
def test_close(self):
assert hasattr(cache, 'close')
cache.close()
def test_close(self):
cache = get_cache('default')
cache.close()
def test_close(self):
cache = get_cache("default")
cache.set("f", "1")
cache.close()
def tearDown(self):
super(DiskCacheTests, self).tearDown()
cache.close()
shutil.rmtree(self.dirname, ignore_errors=True)
def test_close(self):
cache = caches["default"]
cache.set("f", "1")
cache.close()
def wrapper_cleanup_new_process(*args, **kwargs):
django_connection.close()
django_cache.close()
return func(*args, **kwargs)
'ASKBOT_FILE_UPLOAD_DIR': {
'test_for_absence': True,
'message': 'Please replace setting ASKBOT_FILE_UPLOAD_DIR ',
'replace_hint': "with MEDIA_ROOT = '%s'"
},
'ASKBOT_UPLOADED_FILES_URL': {
'test_for_absence': True,
'message': 'Please replace setting ASKBOT_UPLOADED_FILES_URL ',
'replace_hint': "with MEDIA_URL = '/%s'"
},
'RECAPTCHA_USE_SSL': {
'value': True,
'message': 'Please add: RECAPTCHA_USE_SSL = True'
},
})
settings_tester.run()
if 'manage.py test' in ' '.join(sys.argv):
test_settings_for_test_runner()
def run():
try:
if getattr(django_settings, 'ASKBOT_SELF_TEST', True):
run_startup_tests()
except AskbotConfigError, error:
print error
sys.exit(1)
# close DB and cache connections to prevent issues in prefork mode
connection.close()
if hasattr(cache, 'close'):
cache.close()
def _commit_and_clear_log(self):
cache.close()
cache._logger.reset()
self.assertEquals(len(cache._logger.log), 0)
def close_connection(cls):
cache.close()
def test_close(self):
self.assertTrue(hasattr(cache, 'close'))
cache.close()
def close(self):
self.channel.close()
if self.connection:
self.connection.close()
cache.close()
_x().debug("Q listener stopped")
def tearDown(self):
super(DiskCacheTests, self).tearDown()
cache.close()
shutil.rmtree(self.dirname)
def test_cache_close(self, mock_close):
from django.core.cache import cache
cache.close()
mock_close.assert_not_called()
def test_close(self):
self.assertTrue(hasattr(cache, "close"))
cache.close()
def test_close(self):
assert hasattr(cache, "close")
cache.close()
def test_close(self):
cache = caches['default']
cache.set('f', '1')
cache.close()
def test_close(self):
cache = get_cache('default')
cache.set("f", "1")
cache.close()
def close_shared_handlers():
connection.close()
cache.close()
