def find_worker(self, identifier, headers, mailstatus_class, not_before=None, reply=None): record_performance = settings.STATSD_ENABLED FirstPolicy = import_string('munch_mailsend.policies.mx.First') if record_performance: from statsd.defaults.django import statsd total_timer = statsd.timer('munch_mailsend.policies.mx.all') total_timer.start() timer = statsd.timer('munch_mailsend.policies.mx.First') timer.start() workers = FirstPolicy().apply(headers, not_before) if record_performance: timer.stop() for path in settings.MAILSEND.get('WORKER_POLICIES'): try: policy = import_string(path) except ImportError: raise ImportError( '{} points to inexistant worker policy'.format(path)) except TypeError: raise TypeError('{} is not a valid WorkerPolicy'.format(path)) reply_code, reply_message = None, None if reply: reply_code, reply_message = reply.code, reply.message if record_performance: timer = statsd.timer(path) timer.start() workers = policy(identifier, headers, mailstatus_class, reply_code, reply_message, not_before).apply(workers) if record_performance: timer.stop() LastPolicy = import_string('munch_mailsend.policies.mx.Last') if record_performance: timer = statsd.timer('munch_mailsend.policies.mx.Last') timer.start() result = LastPolicy().apply(workers) if record_performance: timer.stop() total_timer.stop() return result
def _sub_model_func(model_cls): model_statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, model_cls.__name__) with statsd.timer(model_statsd_key.format('steps.get_insert_batch')): # NOTE I don't use generator pattern here, as it move all time into insert. # That makes hard to understand where real problem is in monitoring batch = tuple(model_cls.get_insert_batch(import_objects)) return model_cls, batch
def signup(request): template = get_template("polls/signup.html") stdlogger.info("In signup page") statsd.incr('fitcycle.signup',1) foo_timer = statsd.timer('signupTimer') if request.method == 'POST': form=PostForm(request.POST) if form.is_valid(): firstname=request.POST.get('firstname','') lastname=request.POST.get('lastname','') email=request.POST.get('email','') stdlogger.info("creating object for saving to db") prospect_obj=prospect(firstname=firstname, lastname=lastname, email=email) try: stdlogger.info("About to save") foo_timer.start() prospect_obj.save() foo_timer.stop() except Exception, e: stdlogger.error("Error in saving: %s" % e) return HttpResponseRedirect(reverse('index'))
def get_insert_batch(self, model_cls: Type[ClickHouseModel], objects: List[DjangoModel]) -> Iterable[tuple]: """ Gets a list of model_cls instances to insert into database :param model_cls: ClickHouseModel subclass to import :param objects: A list of django Model instances to sync :return: A list of model_cls objects """ defaults = {self.sign_col: 1} if self.version_col: defaults[self.version_col] = 1 serializer = model_cls.get_django_model_serializer(writable=True, defaults=defaults) new_objs = [serializer.serialize(obj) for obj in objects] statsd_key = "%s.sync.%s.steps.get_final_versions" % (config.STATSD_PREFIX, model_cls.__name__) with statsd.timer(statsd_key): # NOTE I don't use generator pattern here, as it move all time into insert. # That makes hard to understand where real problem is in monitoring old_objs = tuple(self.get_final_versions(model_cls, new_objs)) # -1 sign has been set get_final_versions() old_objs_versions = {} for obj in old_objs: pk = getattr(obj, self.pk_column) if self.version_col: old_objs_versions[pk] = getattr(obj, self.version_col) yield obj # 1 sign is set by default in serializer for obj in new_objs: pk = getattr(obj, self.pk_column) if self.version_col: obj = obj._replace(**{self.version_col: old_objs_versions.get(pk, 0) + 1}) yield obj
def register_operations_wrapped(self, import_key: str, operation: str, *pks: Any) -> int: """ This is a wrapper for register_operation method, checking main parameters. This method should be called from inner functions. :param import_key: A key, returned by ClickHouseModel.get_import_key() method :param operation: One of insert, update, delete :param pks: Primary keys to find records in main database. Should be string-serializable with str() method. :return: Number of registered operations """ if operation not in {'insert', 'update', 'delete'}: raise ValueError( 'operation must be one of [insert, update, delete]') statsd_key = "%s.sync.%s.register_operations" % (config.STATSD_PREFIX, import_key) statsd.incr(statsd_key + '.%s' % operation, len(pks)) with statsd.timer(statsd_key): ops_count = self.register_operations(import_key, operation, *pks) statsd_key = "%s.sync.%s.queue" % (config.STATSD_PREFIX, import_key) statsd.gauge(statsd_key, ops_count, delta=True) logger.debug( 'django-clickhouse: registered %s on %d items (%s) to storage' % (operation, len(pks), import_key)) return ops_count
def sync_batch_from_storage(cls): """ Gets one batch from storage and syncs it. :return: """ import_key = cls.get_import_key() storage = cls.get_storage() statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, import_key) try: with statsd.timer(statsd_key.format('total')): with statsd.timer(statsd_key.format('steps.pre_sync')): storage.pre_sync(import_key, lock_timeout=cls.get_lock_timeout()) with statsd.timer(statsd_key.format('steps.get_operations')): operations = storage.get_operations(import_key, cls.get_sync_batch_size()) statsd.incr(statsd_key.format('operations'), len(operations)) if operations: with statsd.timer(statsd_key.format('steps.get_sync_objects')): import_objects = cls.get_sync_objects(operations) else: import_objects = [] statsd.incr(statsd_key.format('import_objects'), len(import_objects)) if import_objects: batches = {} with statsd.timer(statsd_key.format('steps.get_insert_batch')): def _sub_model_func(model_cls): model_statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, model_cls.__name__) with statsd.timer(model_statsd_key.format('steps.get_insert_batch')): # NOTE I don't use generator pattern here, as it move all time into insert. # That makes hard to understand where real problem is in monitoring batch = tuple(model_cls.get_insert_batch(import_objects)) return model_cls, batch res = exec_multi_arg_func(_sub_model_func, cls.sub_models, threads_count=len(cls.sub_models)) batches = dict(res) with statsd.timer(statsd_key.format('steps.insert')): def _sub_model_func(model_cls): model_statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, model_cls.__name__) with statsd.timer(model_statsd_key.format('steps.insert')): model_cls.insert_batch(batches[model_cls]) exec_multi_arg_func(_sub_model_func, cls.sub_models, threads_count=len(cls.sub_models)) with statsd.timer(statsd_key.format('steps.post_sync')): storage.post_sync(import_key) except RedisLockTimeoutError: pass # skip this sync round if lock is acquired by another thread except Exception as ex: with statsd.timer(statsd_key.format('steps.post_sync')): storage.post_sync_failed(import_key) raise ex
def wrapper(*args, **kwargs): if settings.STATSD_ENABLED: from statsd.defaults.django import statsd timer = statsd.timer(name) timer.start() result = f(*args, **kwargs) if settings.STATSD_ENABLED: timer.stop() return result
def request_start(request): metric_id = f"platform.request.{re.sub('[^a-zA-Z]+', '', request.path)}.{request.method}" timer = statsd.timer(metric_id, rate=1) timer.start() return { 'Request-Timer': timer, 'Request-Metric-ID': metric_id, 'Request-Metric-Start': int(round(timer._start_time * 1000)) }
def request_start(request): metric_id = "platform.request.{}.{}".format( re.sub("[^a-zA-Z]+", "", request.path), request.method) timer = statsd.timer(metric_id, rate=1) timer.start() return { 'Request-Timer': timer, 'Request-Metric-ID': metric_id, 'Request-Metric-Start': int(round(timer._start_time * 1000)) }
def publish_to_device(message, user): """ :param message: :param user: """ timer = statsd.timer('task.publish_to_device') timer.start() devices = GCMDevice.objects.filter(user=user) if devices: devices.send_message(message) timer.stop()
def post(self, request): """ Create a new alert (POST). This endpoint accepts Common Alerting Protocal (CAP) 1.1 and 1.2, but does NOT accept ATOM/RSS feeds. In general, simply POST the entire XML message as the content of your request. """ statsd.incr('api.AlertListAPI.post') timer = statsd.timer('api.AlertListAPI.post') timer.start() data = request.data try: for item in data: item['contributor'] = request.user.pk except Exception, e: logging.error(e)
def post(self, request): """ Create a new alert (POST). This endpoint accepts Common Alerting Protocal (CAP) 1.1 and 1.2, but does NOT accept ATOM/RSS feeds. In general, simply POST the entire XML message as the content of your request. """ statsd.incr('api.AlertListAPI.post') timer = statsd.timer('api.AlertListAPI.post') timer.start() data = request.DATA try: for item in data: item['contributor'] = request.user.pk except Exception, e: logging.error(e)
def sync_batch_from_storage(cls): """ Gets one batch from storage and syncs it. :return: """ try: statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, cls.__name__) with statsd.timer(statsd_key.format('total')): storage = cls.get_storage() import_key = cls.get_import_key() with statsd.timer(statsd_key.format('pre_sync')): storage.pre_sync(import_key, lock_timeout=cls.get_lock_timeout()) with statsd.timer(statsd_key.format('get_operations')): operations = storage.get_operations(import_key, cls.get_sync_batch_size()) if operations: with statsd.timer(statsd_key.format('get_sync_objects')): import_objects = cls.get_sync_objects(operations) else: import_objects = [] if import_objects: batches = {} with statsd.timer(statsd_key.format('get_insert_batch')): for model_cls in cls.sub_models: batches[model_cls] = model_cls.get_insert_batch(import_objects) with statsd.timer(statsd_key.format('insert')): for model_cls, batch in batches.items(): model_cls.insert_batch(batch) with statsd.timer(statsd_key.format('post_sync')): storage.post_sync(import_key) storage.set_last_sync_time(import_key, now()) except RedisLockTimeoutError: pass # skip this sync round if lock is acquired by another thread
def get_insert_batch(self, model_cls, objects): # type: (Type[T], List[DjangoModel]) -> List[T] """ Gets a list of model_cls instances to insert into database :param model_cls: ClickHouseModel subclass to import :param objects: A list of django Model instances to sync :return: A list of model_cls objects """ new_objs = super(CollapsingMergeTree, self).get_insert_batch(model_cls, objects) statsd_key = "%s.sync.%s.get_final_versions" % (config.STATSD_PREFIX, model_cls.__name__) with statsd.timer(statsd_key): old_objs = self.get_final_versions(model_cls, new_objs) for obj in old_objs: self.set_obj_sign(obj, -1) for obj in new_objs: self.set_obj_sign(obj, 1) return old_objs + new_objs
def _sub_model_func(model_cls): model_statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, model_cls.__name__) with statsd.timer(model_statsd_key.format('steps.insert')): model_cls.insert_batch(batches[model_cls])
def route_envelope(identifier, headers, attempts, mailstatus_class_path, record_status_task_path, build_envelope_task_path, not_before=None, reply=None): """ This envelope routing task take initiate attempt to free Slimta Edge from SMTP connection """ record_performance = settings.STATSD_ENABLED lock = None mailstatus_class = import_string(mailstatus_class_path) latest_status = mailstatus_class.objects.filter( status__in=[AbstractMailStatus.DELETED] + list(AbstractMailStatus.FINAL_STATES), mail__identifier=identifier) if latest_status: log.debug("[{}] Envelope ignored because it has already been " "{} at {}".format(identifier, latest_status[0].status, latest_status[0].creation_date)) return # Ensure we close Django database connection because we don't # want to have opened connections while waiting for lock. connection.close() destination_domain = extract_domain(headers.get('To', '')) pool = headers.get(settings.MAILSEND['X_POOL_HEADER'], 'default') lock_name = '{}:lock:routing:{}:{}'.format( settings.MAILSEND['CACHE_PREFIX'], destination_domain, pool) lock_timeout = settings.MAILSEND['ROUTER_LOCK_TIMEOUT'] lock_blocking_timeout = settings.MAILSEND['ROUTER_LOCK_WAITING'] log.debug('[{}] Waiting for lock "{}" (timeout:{} ' 'second(s) and blocking for {} second(s))...'.format( identifier, lock_name, lock_timeout, lock_blocking_timeout)) if record_performance: from statsd.defaults.django import statsd lock_timer = statsd.timer('mailsend.tasks.lock_waiting') lock_timer.start() lock = acquire_lock(lock_name, timeout=lock_timeout, blocking_timeout=lock_blocking_timeout) if record_performance: lock_timer.stop() if lock: try: log.debug('[{}] Routing envelope (attempts={})...'.format( identifier, attempts)) worker, next_available, score, others = Worker.objects.find_worker( identifier, headers, mailstatus_class, not_before=not_before, reply=reply) if worker: log.debug( '[{}] Choosen worker is available at {} ' 'with a {} score. Full workers ranking: {}'.format( identifier, next_available.astimezone(), score, { w.get('ip'): { 'score': w.get('score'), 'next_available': str(w.get('next_available').astimezone()) } for w in others })) mail_status_kwargs = { 'source_ip': worker.ip, 'status': AbstractMailStatus.SENDING, 'destination_domain': extract_domain(headers.get('To')) } if not attempts: routing_key = worker.get_queue_name() else: routing_key = worker.get_queue_name(retry=True) attempt = send_email.s(identifier, headers, attempts, mailstatus_class_path, record_status_task_path, build_envelope_task_path, token=set_envelope_token(identifier)) now = timezone.now() if next_available: countdown = max(0, (next_available - now).total_seconds()) # And apply countdown to task if > 0 if countdown: mail_status_kwargs.update( {'creation_date': now + timedelta(seconds=countdown)}) attempt.set(countdown=countdown) log.info( '[{}] Queued with "{}" routing key in {} seconds'.format( identifier, routing_key, int(countdown))) mailstatus = mailstatus_class(**mail_status_kwargs) record_status_task = import_string(record_status_task_path) try: record_status_task(mailstatus, identifier, attempts + 1) except SoftFailure as exc: log.info('SoftFailure during "route_envelope" task (' 'discarding this task): {}'.format(str(exc)), exc_info=True) release_lock(lock_name) return release_lock(lock_name) return attempt.apply_async(routing_key=routing_key).id else: log.debug('[{}] No worker available. Re-route envelope ' 'in 5 minutes'.format(identifier)) release_lock(lock_name) return route_envelope.apply_async( (identifier, headers, attempts, mailstatus_class_path, record_status_task_path, build_envelope_task_path), { 'not_before': not_before, 'reply': reply }, countdown=60 * 5).id except Exception: release_lock(lock_name) raise else: log.debug('[{}] Failed to acquire lock after waiting {} second(s). ' 'Re-route task in 1-6 seconds.'.format( identifier, lock_blocking_timeout)) return route_envelope.apply_async( (identifier, headers, attempts), { 'mailstatus_class_path': mailstatus_class_path, 'record_status_task_path': record_status_task_path, 'build_envelope_task_path': build_envelope_task_path, 'not_before': not_before, 'reply': reply }, countdown=randint(1, 6)).id if lock: release_lock(lock_name)
def insert_tuples(self, model_class: Type['ClickHouseModel'], model_tuples: Iterable[tuple], batch_size: Optional[int] = None, formatted: bool = False) -> None: """ Inserts model_class namedtuples :param model_class: ClickHouse model, namedtuples are made from :param model_tuples: An iterable of tuples to insert :param batch_size: Size of batch :param formatted: If flag is set, tuples are expected to be ready to insert without calling field.to_db_string :return: None """ tuples_iterator = iter(model_tuples) try: first_tuple = next(tuples_iterator) except StopIteration: return # model_instances is empty if model_class.is_read_only() or model_class.is_system_model(): raise DatabaseException( "You can't insert into read only and system tables") fields_list = ','.join('`%s`' % name for name in first_tuple._fields) fields_dict = model_class.fields(writable=True) statsd_key = "%s.inserted_tuples.%s" % (config.STATSD_PREFIX, model_class.__name__) query = 'INSERT INTO `%s`.`%s` (%s) FORMAT TabSeparated\n' \ % (self.db_name, model_class.table_name(), fields_list) query_enc = query.encode('utf-8') def tuple_to_csv(tup): if formatted: str_gen = (getattr(tup, field_name) for field_name in first_tuple._fields) else: str_gen = (fields_dict[field_name].to_db_string(getattr( tup, field_name), quote=False) for field_name in first_tuple._fields) return '%s\n' % '\t'.join(str_gen) def gen(): buf = BytesIO() buf.write(query_enc) buf.write(tuple_to_csv(first_tuple).encode('utf-8')) # Collect lines in batches of batch_size lines = 1 for t in tuples_iterator: buf.write(tuple_to_csv(t).encode('utf-8')) lines += 1 if batch_size is not None and lines >= batch_size: # Return the current batch of lines statsd.incr(statsd_key, lines) yield buf.getvalue() # Start a new batch buf = BytesIO() buf.write(query_enc) lines = 0 # Return any remaining lines in partial batch if lines: statsd.incr(statsd_key, lines) yield buf.getvalue() # For testing purposes for data in gen(): with statsd.timer(statsd_key): logger.debug('django-clickhouse: insert tuple: %s' % data) self._send(data)