def _handle_consume_exception(self, events: List[Dict[str, Any]], exception: Exception) -> None: with configure_scope() as scope: scope.set_context("events", { "data": events, "queue_name": self.queue_name, }) if isinstance(exception, WorkerTimeoutException): with sentry_sdk.push_scope() as scope: scope.fingerprint = ['worker-timeout', self.queue_name] logging.exception("%s in queue %s", str(exception), self.queue_name, stack_info=True) else: logging.exception("Problem handling data on queue %s", self.queue_name, stack_info=True) if not os.path.exists(settings.QUEUE_ERROR_DIR): os.mkdir(settings.QUEUE_ERROR_DIR) # nocoverage # Use 'mark_sanitized' to prevent Pysa from detecting this false positive # flow. 'queue_name' is always a constant string. fname = mark_sanitized(f'{self.queue_name}.errors') fn = os.path.join(settings.QUEUE_ERROR_DIR, fname) line = f'{time.asctime()}\t{orjson.dumps(events).decode()}\n' lock_fn = fn + '.lock' with lockfile(lock_fn): with open(fn, 'a') as f: f.write(line) check_and_send_restart_signal()
def handle(self, *args, **options): # type: (*Any, **Any) -> None # TODO: this only acquires a lock on the system, not on the DB: # be careful not to run this on multiple systems. # In the meantime, we have an option to prevent this job from # running on >1 machine if settings.EMAIL_DELIVERER_DISABLED: return with lockfile("/tmp/zulip_email_deliver.lockfile"): while True: email_jobs_to_deliver = ScheduledEmail.objects.filter( scheduled_timestamp__lte=timezone_now()) if email_jobs_to_deliver: for job in email_jobs_to_deliver: try: send_email(**loads(job.data)) job.delete() except EmailNotDeliveredException: logger.warn("%r not delivered" % (job, )) time.sleep(10) else: # Less load on the db during times of activity, and more responsiveness when the load is low time.sleep(2)
def update_statistics(self, remaining_local_queue_size: int) -> None: total_seconds = sum(seconds for _, seconds in self.recent_consume_times) total_events = sum(events_number for events_number, _ in self.recent_consume_times) if total_events == 0: recent_average_consume_time = None else: recent_average_consume_time = total_seconds / total_events stats_dict = dict( update_time=time.time(), recent_average_consume_time=recent_average_consume_time, current_queue_size=remaining_local_queue_size, queue_last_emptied_timestamp=self.queue_last_emptied_timestamp, consumed_since_last_emptied=self.consumed_since_last_emptied, ) os.makedirs(settings.QUEUE_STATS_DIR, exist_ok=True) fname = f"{self.queue_name}.stats" fn = os.path.join(settings.QUEUE_STATS_DIR, fname) with lockfile(fn + ".lock"): tmp_fn = fn + ".tmp" with open(tmp_fn, "wb") as f: f.write( orjson.dumps(stats_dict, option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_INDENT_2)) os.rename(tmp_fn, fn) self.last_statistics_update_time = time.time()
def _handle_consume_exception(self, events: List[Dict[str, Any]], exception: Exception) -> None: if isinstance(exception, InterruptConsumeException): # The exception signals that no further error handling # is needed and the worker can proceed. return with configure_scope() as scope: scope.set_context( "events", { "data": events, "queue_name": self.queue_name, }, ) if isinstance(exception, WorkerTimeoutException): with sentry_sdk.push_scope() as scope: scope.fingerprint = ["worker-timeout", self.queue_name] logging.exception(exception, stack_info=True) else: logging.exception("Problem handling data on queue %s", self.queue_name, stack_info=True) if not os.path.exists(settings.QUEUE_ERROR_DIR): os.mkdir(settings.QUEUE_ERROR_DIR) # nocoverage # Use 'mark_sanitized' to prevent Pysa from detecting this false positive # flow. 'queue_name' is always a constant string. fname = mark_sanitized(f"{self.queue_name}.errors") fn = os.path.join(settings.QUEUE_ERROR_DIR, fname) line = f"{time.asctime()}\t{orjson.dumps(events).decode()}\n" lock_fn = fn + ".lock" with lockfile(lock_fn): with open(fn, "a") as f: f.write(line) check_and_send_restart_signal()
def handle(self, *args: Any, **options: Any) -> None: if settings.EMAIL_DELIVERER_DISABLED: sleep_forever() with lockfile("/tmp/zulip_email_deliver.lockfile"): while True: email_jobs_to_deliver = ScheduledEmail.objects.filter( scheduled_timestamp__lte=timezone_now()) if email_jobs_to_deliver: for job in email_jobs_to_deliver: # Reformat any jobs that used the old to_email # and to_user_ids argument formats. if 'to_email' in job: job['to_emails'] = [job['to_email']] del job['to_email'] if 'to_user_ids' in job: job['to_user_ids'] = [job['to_user_id']] del job['to_user_ids'] try: send_email(**loads(job.data)) job.delete() except EmailNotDeliveredException: logger.warning("%r not delivered" % (job, )) time.sleep(10) else: # Less load on the db during times of activity, # and more responsiveness when the load is low time.sleep(2)
def handle(self, *args: Any, **options: Any) -> None: if settings.EMAIL_DELIVERER_DISABLED: # Here doing a check and sleeping indefinitely on this setting might # not sound right. Actually we do this check to avoid running this # process on every server that might be in service to a realm. See # the comment in zproject/settings.py file about renaming this setting. while True: time.sleep(10 * 9) with lockfile("/tmp/zulip_scheduled_message_deliverer.lockfile"): while True: messages_to_deliver = ScheduledMessage.objects.filter( scheduled_timestamp__lte=timezone_now(), delivered=False) if messages_to_deliver: for message in messages_to_deliver: with transaction.atomic(): do_send_messages([self.construct_message(message)]) message.delivered = True message.save(update_fields=['delivered']) cur_time = timezone_now() time_next_min = (cur_time + timedelta(minutes=1)).replace( second=0, microsecond=0) sleep_time = (time_next_min - cur_time).total_seconds() time.sleep(sleep_time)
def update_statistics(self, remaining_queue_size: int) -> None: total_seconds = sum( [seconds for _, seconds in self.recent_consume_times]) total_events = sum( [events_number for events_number, _ in self.recent_consume_times]) if total_events == 0: recent_average_consume_time = None else: recent_average_consume_time = total_seconds / total_events stats_dict = dict( update_time=time.time(), recent_average_consume_time=recent_average_consume_time, current_queue_size=remaining_queue_size, queue_last_emptied_timestamp=self.queue_last_emptied_timestamp, consumed_since_last_emptied=self.consumed_since_last_emptied, ) os.makedirs(settings.QUEUE_STATS_DIR, exist_ok=True) fname = '%s.stats' % (self.queue_name, ) fn = os.path.join(settings.QUEUE_STATS_DIR, fname) with lockfile(fn + '.lock'): tmp_fn = fn + '.tmp' with open(tmp_fn, 'w') as f: serialized_dict = ujson.dumps(stats_dict, indent=2) serialized_dict += '\n' f.write(serialized_dict) os.rename(tmp_fn, fn)
def handle(self, *args, **options): # type: (*Any, **Any) -> None # TODO: this only acquires a lock on the system, not on the DB: # be careful not to run this on multiple systems. # In the meantime, we have an option to prevent this job from # running on >1 machine if settings.EMAIL_DELIVERER_DISABLED: return with lockfile("/tmp/zulip_email_deliver.lockfile"): while True: # make sure to use utcnow, otherwise it gets confused when you set the time with utcnow(), # and select with now() email_jobs_to_deliver = ScheduledJob.objects.filter( type=ScheduledJob.EMAIL, scheduled_timestamp__lte=datetime.utcnow()) if email_jobs_to_deliver: for job in email_jobs_to_deliver: if not send_email_job(job): logger.warn( "No exception raised, but %r sent as 0 bytes" % (job, )) else: job.delete() time.sleep(10) else: # Less load on the db during times of activity, and more responsiveness when the load is low time.sleep(2)
def handle(self, *args: Any, **options: Any) -> None: if settings.EMAIL_DELIVERER_DISABLED: # Here doing a check and sleeping indefinitely on this setting might # not sound right. Actually we do this check to avoid running this # process on every server that might be in service to a realm. See # the comment in zproject/settings.py file about renaming this setting. sleep_forever() with lockfile("/tmp/zulip_scheduled_message_deliverer.lockfile"): while True: messages_to_deliver = ScheduledMessage.objects.filter( scheduled_timestamp__lte=timezone_now(), delivered=False) if messages_to_deliver: for message in messages_to_deliver: with transaction.atomic(): do_send_messages([self.construct_message(message)]) message.delivered = True message.save(update_fields=['delivered']) cur_time = timezone_now() time_next_min = (cur_time + timedelta(minutes=1)).replace(second=0, microsecond=0) sleep_time = (time_next_min - cur_time).total_seconds() time.sleep(sleep_time)
def handle(self, *args, **options): # type: (*Any, **Any) -> None # TODO: this only acquires a lock on the system, not on the DB: # be careful not to run this on multiple systems. # In the meantime, we have an option to prevent this job from # running on >1 machine if settings.EMAIL_DELIVERER_DISABLED: return with lockfile("/tmp/zulip_email_deliver.lockfile"): while True: # make sure to use utcnow, otherwise it gets confused when you set the time with utcnow(), # and select with now() email_jobs_to_deliver = ScheduledJob.objects.filter(type=ScheduledJob.EMAIL, scheduled_timestamp__lte=datetime.utcnow()) if email_jobs_to_deliver: for job in email_jobs_to_deliver: if not send_email_job(job): logger.warn("No exception raised, but %r sent as 0 bytes" % (job,)) else: job.delete() time.sleep(10) else: # Less load on the db during times of activity, and more responsiveness when the load is low time.sleep(2)
def _handle_consume_exception(self, events: List[Dict[str, Any]]) -> None: self._log_problem() if not os.path.exists(settings.QUEUE_ERROR_DIR): os.mkdir(settings.QUEUE_ERROR_DIR) # nocoverage fname = '%s.errors' % (self.queue_name, ) fn = os.path.join(settings.QUEUE_ERROR_DIR, fname) line = '%s\t%s\n' % (time.asctime(), ujson.dumps(events)) lock_fn = fn + '.lock' with lockfile(lock_fn): with open(fn, 'ab') as f: f.write(line.encode('utf-8')) check_and_send_restart_signal()
def _handle_consume_exception(self, events: List[Dict[str, Any]]) -> None: self._log_problem() if not os.path.exists(settings.QUEUE_ERROR_DIR): os.mkdir(settings.QUEUE_ERROR_DIR) # nocoverage # Use 'mark_sanitized' to prevent Pysa from detecting this false positive # flow. 'queue_name' is always a constant string. fname = mark_sanitized(f'{self.queue_name}.errors') fn = os.path.join(settings.QUEUE_ERROR_DIR, fname) line = f'{time.asctime()}\t{orjson.dumps(events).decode()}\n' lock_fn = fn + '.lock' with lockfile(lock_fn): with open(fn, 'ab') as f: f.write(line.encode('utf-8')) check_and_send_restart_signal()
def consume_wrapper(self, data): try: self.consume(data) except Exception: self._log_problem() if not os.path.exists(settings.QUEUE_ERROR_DIR): os.mkdir(settings.QUEUE_ERROR_DIR) fname = '%s.errors' % (self.queue_name, ) fn = os.path.join(settings.QUEUE_ERROR_DIR, fname) line = '%s\t%s\n' % (time.asctime(), ujson.dumps(data)) lock_fn = fn + '.lock' with lockfile(lock_fn): with open(fn, 'a') as f: f.write(line) reset_queries()
def consume_wrapper(self, data): try: self.consume(data) except Exception: self._log_problem() if not os.path.exists(settings.QUEUE_ERROR_DIR): os.mkdir(settings.QUEUE_ERROR_DIR) fname = '%s.errors' % (self.queue_name,) fn = os.path.join(settings.QUEUE_ERROR_DIR, fname) line = u'%s\t%s\n' % (time.asctime(), ujson.dumps(data)) lock_fn = fn + '.lock' with lockfile(lock_fn): with open(fn, 'ab') as f: f.write(line.encode('utf-8')) reset_queries()
def consume_wrapper(self, data: Dict[str, Any]) -> None: try: self.consume(data) except Exception: self._log_problem() if not os.path.exists(settings.QUEUE_ERROR_DIR): os.mkdir(settings.QUEUE_ERROR_DIR) # nocoverage fname = '%s.errors' % (self.queue_name,) fn = os.path.join(settings.QUEUE_ERROR_DIR, fname) line = '%s\t%s\n' % (time.asctime(), ujson.dumps(data)) lock_fn = fn + '.lock' with lockfile(lock_fn): with open(fn, 'ab') as f: f.write(line.encode('utf-8')) check_and_send_restart_signal() finally: reset_queries()
def consume_wrapper(self, data: Dict[str, Any]) -> None: try: self.consume(data) except Exception: self._log_problem() if not os.path.exists(settings.QUEUE_ERROR_DIR): os.mkdir(settings.QUEUE_ERROR_DIR) # nocoverage fname = '%s.errors' % (self.queue_name, ) fn = os.path.join(settings.QUEUE_ERROR_DIR, fname) line = '%s\t%s\n' % (time.asctime(), ujson.dumps(data)) lock_fn = fn + '.lock' with lockfile(lock_fn): with open(fn, 'ab') as f: f.write(line.encode('utf-8')) check_and_send_restart_signal() finally: reset_queries()
def handle(self, *args: Any, **options: Any) -> None: with lockfile("/tmp/zulip_scheduled_message_deliverer.lockfile"): while True: messages_to_deliver = ScheduledMessage.objects.filter( scheduled_timestamp__lte=timezone_now(), delivered=False) if messages_to_deliver: for message in messages_to_deliver: with transaction.atomic(): do_send_messages([self.construct_message(message)]) message.delivered = True message.save(update_fields=['delivered']) cur_time = timezone_now() time_next_min = (cur_time + timedelta(minutes=1)).replace( second=0, microsecond=0) sleep_time = (time_next_min - cur_time).total_seconds() time.sleep(sleep_time)
def handle(self, *args: Any, **options: Any) -> None: if not settings.BILLING_PROCESSOR_ENABLED: sleep_forever() with lockfile("/tmp/zulip_billing_processor.lockfile"): while True: for processor in BillingProcessor.objects.exclude( state=BillingProcessor.STALLED): try: entry_processed = run_billing_processor_one_step(processor) except StripeConnectionError: time.sleep(5*60) # Less load on the db during times of activity # and more responsiveness when the load is low if entry_processed: time.sleep(10) else: time.sleep(2)
def handle(self, *args: Any, **options: Any) -> None: if settings.EMAIL_DELIVERER_DISABLED: while True: time.sleep(10*9) with lockfile("/tmp/zulip_email_deliver.lockfile"): while True: email_jobs_to_deliver = ScheduledEmail.objects.filter(scheduled_timestamp__lte=timezone_now()) if email_jobs_to_deliver: for job in email_jobs_to_deliver: try: send_email(**loads(job.data)) job.delete() except EmailNotDeliveredException: logger.warning("%r not delivered" % (job,)) time.sleep(10) else: # Less load on the db during times of activity, and more responsiveness when the load is low time.sleep(2)
def handle(self, *args: Any, **options: Any) -> None: if settings.EMAIL_DELIVERER_DISABLED: time.sleep(10**9) # "infinity seconds" with lockfile("/tmp/zulip_email_deliver.lockfile"): while True: email_jobs_to_deliver = ScheduledEmail.objects.filter( scheduled_timestamp__lte=timezone_now()) if email_jobs_to_deliver: for job in email_jobs_to_deliver: try: send_email(**loads(job.data)) job.delete() except EmailNotDeliveredException: logger.warning("%r not delivered" % (job, )) time.sleep(10) else: # Less load on the db during times of activity, # and more responsiveness when the load is low time.sleep(2)