def init_workers(self, use_workers=True): def shutdown_handler(active_workers): def _handler(signum, frame): try: for active_worker in active_workers: active_worker.terminate() signal.signal(signum, signal.SIG_DFL) os.kill(os.getpid(), signum ) # Rethrow signal, this time without catching it except Exception: logger.exception('Error in shutdown_handler') return _handler if use_workers: django_connection.close() django_cache.close() for idx in range(settings.JOB_EVENT_WORKERS): queue_actual = MPQueue(settings.JOB_EVENT_MAX_QUEUE_SIZE) w = Process(target=self.callback_worker, args=( queue_actual, idx, )) w.start() if settings.DEBUG: logger.info('Started worker %s' % str(idx)) self.worker_queues.append([0, queue_actual, w]) elif settings.DEBUG: logger.warn('Started callback receiver (no workers)') signal.signal(signal.SIGINT, shutdown_handler([p[2] for p in self.worker_queues])) signal.signal(signal.SIGTERM, shutdown_handler([p[2] for p in self.worker_queues]))
def test_bulk_sms_send_multiproc(self): # close database connections manually to make sure they're not passed # to subprocesses for conn in django_db_connections.all(): conn.close() # send the messages in parallel using 10 processes. use # multiprocessing rather than threads so we don't have to clean # up database connections that the threads might leave open. cache.close() pool = multiprocessing.Pool(10) try: for i in range(30): BulkMessageFactory(batch=self.batch, sms=None) for j in range(10): BulkMessageFactory(batch=self.batch, sms=None, deleted=True) # 40 is the first multiple of 10 greater than or equal to 31 num_sent = send_messages(self.batch, num_to_send=40, map_=pool.map) batch = Batch.objects.get(pk=self.batch.pk) self.assertEqual(batch.errors, 0) self.assertEqual(batch.status, Batch.COMPLETED) # assert that we report the right number sent too self.assertEqual(num_sent, 30) self.assertEqual(len(self.outbound), 30) finally: pool.terminate() pool.join()
def connect(self): self.accept() self.service_ids = self.scope['url_route']['kwargs'][ 'service_ids'].split(',') locations = get_vehicle_locations( journey__service__in=self.service_ids) self.send_locations(locations) if not locations: self.send_json([]) icarus = not any(location.journey.source_id != 75 for location in locations) for service_id in self.service_ids: async_to_sync(self.channel_layer.group_add)(f'service{service_id}', self.channel_name) if icarus: codes = ServiceCode.objects.filter(scheme__in=schemes, service=service_id) codes = codes.annotate( source_name=Replace('scheme', Value(' SIRI'))) siri_sources = SIRISource.objects.filter( name=OuterRef('source_name')) codes = codes.filter(Exists(siri_sources)) for code in codes: try: siri_one_shot(code, timezone.now(), bool(locations)) break except Poorly: pass cache.close()
def handle(self, *arg, **options): if options.get('status'): print(Control('dispatcher').status()) return if options.get('running'): print(Control('dispatcher').running()) return if options.get('reload'): return Control('dispatcher').control({'control': 'reload'}) # It's important to close these because we're _about_ to fork, and we # don't want the forked processes to inherit the open sockets # for the DB and cache connections (that way lies race conditions) django_connection.close() django_cache.close() # spawn a daemon thread to periodically enqueues scheduled tasks # (like the node heartbeat) periodic.run_continuously() consumer = None try: queues = ['tower_broadcast_all', get_local_queuename()] consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4)) consumer.run() except KeyboardInterrupt: logger.debug('Terminating Task Dispatcher') if consumer: consumer.stop()
def size(self): val = cache.get(self.package, -1) cache.close() if val == -1: val = os.path.getsize(self.apk.file.path) cache.set(self.package, val, 30) return val
def wrapper_cleanup_new_process(*args, **kwargs): from awx.conf.settings import SettingsWrapper # noqa django_connection.close() django_cache.close() SettingsWrapper.initialize() return func(*args, **kwargs)
def show(request, id): x = cache.get('poll_id__' + id) if x: poll = x else: poll = Polls.objects.filter(id=id) cache.set('poll_id__' + id, poll) cache.close() if poll.count() < 1: return HttpResponse('Not found') else: x = cache.get('polls_objects_get_id_%s' % id) if x: poll = x else: poll = Polls.objects.get(id=id) cache.set('polls_objects_get_id_%s' % id, poll) y = cache.get('options_objects_filter_poll_id_%s_order_by_-count' % id) if y: options = y else: options = Options.objects.filter(poll_id=id).order_by('-count') cache.set('options_objects_filter_poll_id_%s_order_by_-count' % id, options) c_cache = cache.get('comments_poll_id__' + id) if c_cache: comments = c_cache else: comments = Comments.objects.filter(poll_id=id).order_by('-id') cache.set('comments_poll_id__' + id, comments) paginator = Paginator(comments, 10) page = request.GET.get('p') try: comments = paginator.page(page) except PageNotAnInteger: comments = paginator.page(1) except EmptyPage: comments = paginator.page(paginator.num_pages) u = cache.get('optionsvotes_objects_filter_poll_poll.id_%s' % poll.id) if u: options_c = u else: options_c = OptionsVotes.objects.filter(poll=poll.id).count() cache.set('optionsvotes_objects_filter_poll_poll.id_%s' % poll.id, options_c) c = Context({ 'poll': poll, 'options': options, 'options_c': options_c, 'comments': comments, 'user': request.user }) c.update(csrf(request)) return render_to_response('show.html', c)
def clear_cache(request): try: del request.session['search-questions-post'] cache.clear() cache.close() except KeyError: pass return redirect('/')
def test_set_many(self): if not self._run_test(): return cache.set_many({'hello1':'world1','hello2':'world2'},30) self.assertEquals(len(cache._logger.log), 0) self.assertEquals(cache.get("hello1"), "world1") self.assertEquals(len(cache._logger.log), 0) cache.close() self.assertEquals(len(cache._logger.log), 1)
def test_set(self): if not self._run_test(): return cache.set("hello","world",30) self.assertEquals(len(cache._logger.log), 0) self.assertEquals(cache.get("hello"), "world") self.assertEquals(len(cache._logger.log), 0) cache.close() self.assertEquals(len(cache._logger.log), 1)
def finish(self, chunk = None): super(BaseHandler, self).finish(chunk = chunk) # Clean up django ORM connections django.db.connection.close() # Clean up after python-memcached from django.core.cache import cache if hasattr(cache, 'close'): cache.close()
def delete_session(request): try: del request.session['search-questions-post'] del request.session['start-time'] del request.session['count'] cache.clear() cache.close() except KeyError: pass return redirect('/')
def handle(self, *arg, **options): if options.get('status'): print(Control('dispatcher').status()) return if options.get('running'): print(Control('dispatcher').running()) return if options.get('reload'): return Control('dispatcher').control({'control': 'reload'}) # It's important to close these because we're _about_ to fork, and we # don't want the forked processes to inherit the open sockets # for the DB and memcached connections (that way lies race conditions) django_connection.close() django_cache.close() # spawn a daemon thread to periodically enqueues scheduled tasks # (like the node heartbeat) periodic.run_continuously() reaper.reap() consumer = None # don't ship external logs inside the dispatcher's parent process # this exists to work around a race condition + deadlock bug on fork # in cpython itself: # https://bugs.python.org/issue37429 AWXProxyHandler.disable() with Connection(settings.BROKER_URL) as conn: try: bcast = 'tower_broadcast_all' queues = [ Queue(q, Exchange(q), routing_key=q) for q in (settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()]) ] queues.append( Queue( construct_bcast_queue_name(bcast), exchange=Exchange(bcast, type='fanout'), routing_key=bcast, reply=True ) ) consumer = AWXConsumer( 'dispatcher', conn, TaskWorker(), queues, AutoscalePool(min_workers=4) ) consumer.run() except KeyboardInterrupt: logger.debug('Terminating Task Dispatcher') if consumer: consumer.stop()
def run(): try: if django_settings.ASKBOT_SELF_TEST: run_startup_tests() except AskbotConfigError as error: print(error) sys.exit(1) # close DB and cache connections to prevent issues in prefork mode connection.close() if hasattr(cache, 'close'): cache.close()
def index(request): x = cache.get('all_polls') if x: polls = x else: polls = Polls.objects.all().order_by('-id') cache.set('all_polls', polls, 60) cache.close() c = Context({'polls': polls, 'user': request.user}) return render_to_response('index.html', c)
def test_add(self): if not self._run_test(): return self.assertEquals(local.storage.get(version_key("hello")), None) cache.add("hello","world") self.assertEquals(len(cache._logger.log), 1) self.assertEquals(local.storage.get(version_key("hello")), "world") cache.add("hello","world1") self.assertEquals(local.storage.get(version_key("hello")), "world") self.assertEquals(len(cache._logger.log), 1) cache.close() self.assertEquals(len(cache._logger.log), 1)
def to_cache(article_id, schema_id, hash_value, result): article_cache = cache.get("highlighting-{0}".format(article_id)) if article_cache is None: article_cache = {} article_cache[schema_id] = {} article_cache[schema_id]['hash'] = hash_value article_cache[schema_id]['value'] = result cache.set("highlighting-{0}".format(article_id), article_cache) # print(article_cache) article_cache = cache.get("highlighting-{0}".format(article_id)) # print(article_cache) cache.close()
def finish(self, chunk = None): super(BaseHandler, self).finish(chunk = chunk) # Clean up django ORM connections dj_db.connection.close() logging.info('%d sql queries' % len(dj_db.connection.queries)) for query in dj_db.connection.queries: logging.debug('%s [%s seconds]' % (query['sql'], query['time'])) # Clean up after python-memcached from django.core.cache import cache if hasattr(cache, 'close'): cache.close()
def test_delete(self): if not self._run_test(): return cache.set("hello","world",30) self._commit_and_clear_log() self.assertEquals(cache.get("hello"),"world") self.assertEquals(len(cache._logger.log), 1) cache.delete("hello") self.assertEquals(len(cache._logger.log), 1) self.assertEquals(cache.get("hello"),None) self.assertEquals(len(cache._logger.log), 1) cache.close() self.assertEquals(len(cache._logger.log), 2)
def finish(self, chunk=None): super(BaseRedisWebSocketHandler, self).finish(chunk=chunk) connection.close() if False: info('%d sql queries' % len(connection.queries)) for query in connection.queries: debug('%s [%s seconds]' % (query['sql'], query['time'])) # Clean up cache from django.core.cache import cache if hasattr(cache, 'close'): cache.close()
def finish(self, chunk = None): super(BaseWebSocketHandler, self).finish(chunk = chunk) # Clean up django ORM connections django.db.connection.close() if False: logging.info('%d sql queries' % len(django.db.connection.queries)) for query in django.db.connection.queries: logging.debug('%s [%s seconds]' % (query['sql'], query['time'])) # Clean up after python-memcached from django.core.cache import cache if hasattr(cache, 'close'): cache.close()
def test_delete_many(self): if not self._run_test(): return cache.set_many({'hello1':'world1','hello2':'world2'},30) self._commit_and_clear_log() cache.delete_many(['hello1','hello2']) self.assertEquals(len(cache._logger.log), 0) cache.delete_many(['hello1','hello2']) self.assertEquals(len(cache._logger.log), 0) self.assertEquals(cache.get_many(['hello1','hello2']),{version_key('hello1'):None,version_key('hello2'):None}) self.assertEquals(len(cache._logger.log), 0) cache.close() self.assertEquals(len(cache._logger.log), 1)
def __clean_on_fork__(self): pid = os.getpid() # if the current pid does *not* match the value on self, it means # that value was copied on fork, and we're now in a *forked* process; # the *first* time we enter this code path (on setting access), # forcibly close DB/cache sockets and set a marker so we don't run # this code again _in this process_ # if pid != self.__dict__['pid'] and pid not in self.__dict__['__forks__']: self.__dict__['__forks__'][pid] = True # It's important to close these post-fork, because we # don't want the forked processes to inherit the open sockets # for the DB and cache connections (that way lies race conditions) connection.close() django_cache.close()
def handle(self, *arg, **options): if options.get('status'): print Control('dispatcher').status() return if options.get('running'): print Control('dispatcher').running() return if options.get('reload'): return Control('dispatcher').control({'control': 'reload'}) # It's important to close these because we're _about_ to fork, and we # don't want the forked processes to inherit the open sockets # for the DB and memcached connections (that way lies race conditions) django_connection.close() django_cache.close() beat = Process(target=self.beat) beat.daemon = True beat.start() reaper.reap() consumer = None with Connection(settings.BROKER_URL) as conn: try: bcast = 'tower_broadcast_all' queues = [ Queue(q, Exchange(q), routing_key=q) for q in (settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()]) ] queues.append( Queue( construct_bcast_queue_name(bcast), exchange=Exchange(bcast, type='fanout'), routing_key=bcast, reply=True ) ) consumer = AWXConsumer( 'dispatcher', conn, TaskWorker(), queues, AutoscalePool(min_workers=4) ) consumer.run() except KeyboardInterrupt: logger.debug('Terminating Task Dispatcher') if consumer: consumer.stop()
def up(self): idx = len(self.workers) # It's important to close these because we're _about_ to fork, and we # don't want the forked processes to inherit the open sockets # for the DB and cache connections (that way lies race conditions) django_connection.close() django_cache.close() worker = self.pool_cls(self.queue_size, self.target, (idx,) + self.target_args) self.workers.append(worker) try: worker.start() except Exception: logger.exception('could not fork') else: logger.debug('scaling up worker pid:{}'.format(worker.pid)) return idx, worker
def _create_pool(self): """ Create a multiprocessing worker pool for sending bulk messages. """ # close database connections manually to make sure they're not passed # to subprocesses logger.debug('Closing database connections before starting worker pool.') for conn in django_db_connections.all(): conn.close() cache.close() # send the messages in parallel using ``concurrent_workers`` processes. use # multiprocessing rather than threads so we don't have to clean # up database connections that the threads might leave open. logger.debug('Starting worker pool with {0} processes.'.format(self.concurrent_workers)) return multiprocessing.Pool(processes=self.concurrent_workers, maxtasksperchild=MAX_MESSAGES_PER_CHILD)
def finish(self, chunk=None): super(BaseWebSocketHandler, self).finish(chunk=chunk) # Clean up django ORM connections connection.close() if False: info("%d sql queries" % len(connection.queries)) for query in connection.queries: debug("%s [%s seconds]" % (query["sql"], query["time"])) # Clean up after python-memcached from django.core.cache import cache if hasattr(cache, "close"): cache.close()
def clean(self): """ :return: """ # Clean up django ORM connections django.db.connection.close() if settings.DEBUG: logger.info('%d sql queries' % len(django.db.connection.queries)) for query in django.db.connection.queries: logger.debug('%s [%s seconds]' % (query['sql'], query['time'])) # Clean up after python-memcached from django.core.cache import cache if hasattr(cache, 'close'): cache.close()
def process_message(self, msg, retry_count=0): """Process this message and return tuple (do_ack, reply_msg). do_ack must be boolean. reply_msg, if not None, must have the property ``next`` in the format: exchange/routing_key """ fn = None hdrs = None try: amqp_call_count = cache.get('amqp_call_count', 0) if amqp_call_count % 20 == 0: connection.close() # reset database connection after every 20 calls amqp_call_count = 0 cache.close() cache.set('amqp_call_count', amqp_call_count+1, 300000) except Exception, e: _x().exception(e)
def jail(task_id, func, args, kwargs): """Wraps the task in a jail, which catches all exceptions, and saves the status and result of the task execution to the task meta backend. If the call was successful, it saves the result to the task result backend, and sets the task status to ``"DONE"``. If the call results in an exception, it saves the exception as the task result, and sets the task status to ``"FAILURE"``. :param task_id: The id of the task. :param func: Callable object to execute. :param args: List of positional args to pass on to the function. :param kwargs: Keyword arguments mapping to pass on to the function. :returns: the function return value on success, or the exception instance on failure. """ # See: http://groups.google.com/group/django-users/browse_thread/ # thread/78200863d0c07c6d/38402e76cf3233e8?hl=en&lnk=gst& # q=multiprocessing#38402e76cf3233e8 from django.db import connection connection.close() # Reset cache connection from django.core.cache import cache cache.close() # Backend process cleanup default_backend.process_cleanup() # Convert any unicode keys in the keyword arguments to ascii. kwargs = dict([(k.encode("utf-8"), v) for k, v in kwargs.items()]) try: result = func(*args, **kwargs) except Exception, exc: default_backend.mark_as_failure(task_id, exc) return ExceptionInfo(sys.exc_info())
def handle(self, *arg, **options): if options.get('status'): print(Control('dispatcher').status()) return if options.get('running'): print(Control('dispatcher').running()) return if options.get('reload'): return Control('dispatcher').control({'control': 'reload'}) # It's important to close these because we're _about_ to fork, and we # don't want the forked processes to inherit the open sockets # for the DB and memcached connections (that way lies race conditions) django_connection.close() django_cache.close() # spawn a daemon thread to periodically enqueues scheduled tasks # (like the node heartbeat) periodic.run_continuously() reaper.reap() consumer = None # don't ship external logs inside the dispatcher's parent process # this exists to work around a race condition + deadlock bug on fork # in cpython itself: # https://bugs.python.org/issue37429 AWXProxyHandler.disable() try: queues = ['tower_broadcast_all', get_local_queuename()] consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4)) consumer.run() except KeyboardInterrupt: logger.debug('Terminating Task Dispatcher') if consumer: consumer.stop()
def test_close(self): cache = caches["default"] cache.set("f", "1") cache.close()
def test_close(self): assert hasattr(cache, 'close') cache.close()
def get_value(key): pickled_value = cache.get(key) cache.close() if pickled_value is None: return None return pickle.loads(pickled_value)
def set_value(key, value, time_out=200): # cache.set(key, pickle.dumps(value), time_out) cache.set(key, pickle.dumps(value)) cache.close()
def test_close(self): cache = get_cache('default') cache.close()
def test_close(self): cache = get_cache("default") cache.set("f", "1") cache.close()
def tearDown(self): super(DiskCacheTests, self).tearDown() cache.close() shutil.rmtree(self.dirname, ignore_errors=True)
def wrapper_cleanup_new_process(*args, **kwargs): django_connection.close() django_cache.close() return func(*args, **kwargs)
'ASKBOT_FILE_UPLOAD_DIR': { 'test_for_absence': True, 'message': 'Please replace setting ASKBOT_FILE_UPLOAD_DIR ', 'replace_hint': "with MEDIA_ROOT = '%s'" }, 'ASKBOT_UPLOADED_FILES_URL': { 'test_for_absence': True, 'message': 'Please replace setting ASKBOT_UPLOADED_FILES_URL ', 'replace_hint': "with MEDIA_URL = '/%s'" }, 'RECAPTCHA_USE_SSL': { 'value': True, 'message': 'Please add: RECAPTCHA_USE_SSL = True' }, }) settings_tester.run() if 'manage.py test' in ' '.join(sys.argv): test_settings_for_test_runner() def run(): try: if getattr(django_settings, 'ASKBOT_SELF_TEST', True): run_startup_tests() except AskbotConfigError, error: print error sys.exit(1) # close DB and cache connections to prevent issues in prefork mode connection.close() if hasattr(cache, 'close'): cache.close()
def _commit_and_clear_log(self): cache.close() cache._logger.reset() self.assertEquals(len(cache._logger.log), 0)
def close_connection(cls): cache.close()
def test_close(self): self.assertTrue(hasattr(cache, 'close')) cache.close()
def close(self): self.channel.close() if self.connection: self.connection.close() cache.close() _x().debug("Q listener stopped")
def tearDown(self): super(DiskCacheTests, self).tearDown() cache.close() shutil.rmtree(self.dirname)
def test_cache_close(self, mock_close): from django.core.cache import cache cache.close() mock_close.assert_not_called()
def test_close(self): self.assertTrue(hasattr(cache, "close")) cache.close()
def test_close(self): assert hasattr(cache, "close") cache.close()
def test_close(self): cache = caches['default'] cache.set('f', '1') cache.close()
def test_close(self): cache = get_cache('default') cache.set("f", "1") cache.close()
def close_shared_handlers(): connection.close() cache.close()