def find_worker(self,
identifier,
headers,
mailstatus_class,
not_before=None,
reply=None):
record_performance = settings.STATSD_ENABLED
FirstPolicy = import_string('munch_mailsend.policies.mx.First')
if record_performance:
from statsd.defaults.django import statsd
total_timer = statsd.timer('munch_mailsend.policies.mx.all')
total_timer.start()
timer = statsd.timer('munch_mailsend.policies.mx.First')
timer.start()
workers = FirstPolicy().apply(headers, not_before)
if record_performance:
timer.stop()
for path in settings.MAILSEND.get('WORKER_POLICIES'):
try:
policy = import_string(path)
except ImportError:
raise ImportError(
'{} points to inexistant worker policy'.format(path))
except TypeError:
raise TypeError('{} is not a valid WorkerPolicy'.format(path))
reply_code, reply_message = None, None
if reply:
reply_code, reply_message = reply.code, reply.message
if record_performance:
timer = statsd.timer(path)
timer.start()
workers = policy(identifier, headers, mailstatus_class, reply_code,
reply_message, not_before).apply(workers)
if record_performance:
timer.stop()
LastPolicy = import_string('munch_mailsend.policies.mx.Last')
if record_performance:
timer = statsd.timer('munch_mailsend.policies.mx.Last')
timer.start()
result = LastPolicy().apply(workers)
if record_performance:
timer.stop()
total_timer.stop()
return result
def _sub_model_func(model_cls):
model_statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, model_cls.__name__)
with statsd.timer(model_statsd_key.format('steps.get_insert_batch')):
# NOTE I don't use generator pattern here, as it move all time into insert.
# That makes hard to understand where real problem is in monitoring
batch = tuple(model_cls.get_insert_batch(import_objects))
return model_cls, batch
def signup(request):
template = get_template("polls/signup.html")
stdlogger.info("In signup page")
statsd.incr('fitcycle.signup',1)
foo_timer = statsd.timer('signupTimer')
if request.method == 'POST':
form=PostForm(request.POST)
if form.is_valid():
firstname=request.POST.get('firstname','')
lastname=request.POST.get('lastname','')
email=request.POST.get('email','')
stdlogger.info("creating object for saving to db")
prospect_obj=prospect(firstname=firstname, lastname=lastname, email=email)
try:
stdlogger.info("About to save")
foo_timer.start()
prospect_obj.save()
foo_timer.stop()
except Exception, e:
stdlogger.error("Error in saving: %s" % e)
return HttpResponseRedirect(reverse('index'))
def get_insert_batch(self, model_cls: Type[ClickHouseModel], objects: List[DjangoModel]) -> Iterable[tuple]:
"""
Gets a list of model_cls instances to insert into database
:param model_cls: ClickHouseModel subclass to import
:param objects: A list of django Model instances to sync
:return: A list of model_cls objects
"""
defaults = {self.sign_col: 1}
if self.version_col:
defaults[self.version_col] = 1
serializer = model_cls.get_django_model_serializer(writable=True, defaults=defaults)
new_objs = [serializer.serialize(obj) for obj in objects]
statsd_key = "%s.sync.%s.steps.get_final_versions" % (config.STATSD_PREFIX, model_cls.__name__)
with statsd.timer(statsd_key):
# NOTE I don't use generator pattern here, as it move all time into insert.
# That makes hard to understand where real problem is in monitoring
old_objs = tuple(self.get_final_versions(model_cls, new_objs))
# -1 sign has been set get_final_versions()
old_objs_versions = {}
for obj in old_objs:
pk = getattr(obj, self.pk_column)
if self.version_col:
old_objs_versions[pk] = getattr(obj, self.version_col)
yield obj
# 1 sign is set by default in serializer
for obj in new_objs:
pk = getattr(obj, self.pk_column)
if self.version_col:
obj = obj._replace(**{self.version_col: old_objs_versions.get(pk, 0) + 1})
yield obj
def register_operations_wrapped(self, import_key: str, operation: str,
*pks: Any) -> int:
"""
This is a wrapper for register_operation method, checking main parameters.
This method should be called from inner functions.
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
:param operation: One of insert, update, delete
:param pks: Primary keys to find records in main database. Should be string-serializable with str() method.
:return: Number of registered operations
"""
if operation not in {'insert', 'update', 'delete'}:
raise ValueError(
'operation must be one of [insert, update, delete]')
statsd_key = "%s.sync.%s.register_operations" % (config.STATSD_PREFIX,
import_key)
statsd.incr(statsd_key + '.%s' % operation, len(pks))
with statsd.timer(statsd_key):
ops_count = self.register_operations(import_key, operation, *pks)
statsd_key = "%s.sync.%s.queue" % (config.STATSD_PREFIX, import_key)
statsd.gauge(statsd_key, ops_count, delta=True)
logger.debug(
'django-clickhouse: registered %s on %d items (%s) to storage' %
(operation, len(pks), import_key))
return ops_count
def sync_batch_from_storage(cls):
"""
Gets one batch from storage and syncs it.
:return:
"""
import_key = cls.get_import_key()
storage = cls.get_storage()
statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, import_key)
try:
with statsd.timer(statsd_key.format('total')):
with statsd.timer(statsd_key.format('steps.pre_sync')):
storage.pre_sync(import_key, lock_timeout=cls.get_lock_timeout())
with statsd.timer(statsd_key.format('steps.get_operations')):
operations = storage.get_operations(import_key, cls.get_sync_batch_size())
statsd.incr(statsd_key.format('operations'), len(operations))
if operations:
with statsd.timer(statsd_key.format('steps.get_sync_objects')):
import_objects = cls.get_sync_objects(operations)
else:
import_objects = []
statsd.incr(statsd_key.format('import_objects'), len(import_objects))
if import_objects:
batches = {}
with statsd.timer(statsd_key.format('steps.get_insert_batch')):
def _sub_model_func(model_cls):
model_statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, model_cls.__name__)
with statsd.timer(model_statsd_key.format('steps.get_insert_batch')):
# NOTE I don't use generator pattern here, as it move all time into insert.
# That makes hard to understand where real problem is in monitoring
batch = tuple(model_cls.get_insert_batch(import_objects))
return model_cls, batch
res = exec_multi_arg_func(_sub_model_func, cls.sub_models, threads_count=len(cls.sub_models))
batches = dict(res)
with statsd.timer(statsd_key.format('steps.insert')):
def _sub_model_func(model_cls):
model_statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, model_cls.__name__)
with statsd.timer(model_statsd_key.format('steps.insert')):
model_cls.insert_batch(batches[model_cls])
exec_multi_arg_func(_sub_model_func, cls.sub_models, threads_count=len(cls.sub_models))
with statsd.timer(statsd_key.format('steps.post_sync')):
storage.post_sync(import_key)
except RedisLockTimeoutError:
pass # skip this sync round if lock is acquired by another thread
except Exception as ex:
with statsd.timer(statsd_key.format('steps.post_sync')):
storage.post_sync_failed(import_key)
raise ex
def wrapper(*args, **kwargs):
if settings.STATSD_ENABLED:
from statsd.defaults.django import statsd
timer = statsd.timer(name)
timer.start()
result = f(*args, **kwargs)
if settings.STATSD_ENABLED:
timer.stop()
return result
def request_start(request):
metric_id = f"platform.request.{re.sub('[^a-zA-Z]+', '', request.path)}.{request.method}"
timer = statsd.timer(metric_id, rate=1)
timer.start()
return {
'Request-Timer': timer,
'Request-Metric-ID': metric_id,
'Request-Metric-Start': int(round(timer._start_time * 1000))
}
def request_start(request):
metric_id = "platform.request.{}.{}".format(
re.sub("[^a-zA-Z]+", "", request.path), request.method)
timer = statsd.timer(metric_id, rate=1)
timer.start()
return {
'Request-Timer': timer,
'Request-Metric-ID': metric_id,
'Request-Metric-Start': int(round(timer._start_time * 1000))
}
def publish_to_device(message, user):
"""
:param message:
:param user:
"""
timer = statsd.timer('task.publish_to_device')
timer.start()
devices = GCMDevice.objects.filter(user=user)
if devices:
devices.send_message(message)
timer.stop()
def post(self, request):
"""
Create a new alert (POST). This endpoint accepts Common Alerting Protocal (CAP) 1.1 and 1.2, but does NOT accept
ATOM/RSS feeds. In general, simply POST the entire XML message as the content of your request.
"""
statsd.incr('api.AlertListAPI.post')
timer = statsd.timer('api.AlertListAPI.post')
timer.start()
data = request.data
try:
for item in data:
item['contributor'] = request.user.pk
except Exception, e:
logging.error(e)
def post(self, request):
"""
Create a new alert (POST). This endpoint accepts Common Alerting Protocal (CAP) 1.1 and 1.2, but does NOT accept
ATOM/RSS feeds. In general, simply POST the entire XML message as the content of your request.
"""
statsd.incr('api.AlertListAPI.post')
timer = statsd.timer('api.AlertListAPI.post')
timer.start()
data = request.DATA
try:
for item in data:
item['contributor'] = request.user.pk
except Exception, e:
logging.error(e)
def sync_batch_from_storage(cls):
"""
Gets one batch from storage and syncs it.
:return:
"""
try:
statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, cls.__name__)
with statsd.timer(statsd_key.format('total')):
storage = cls.get_storage()
import_key = cls.get_import_key()
with statsd.timer(statsd_key.format('pre_sync')):
storage.pre_sync(import_key, lock_timeout=cls.get_lock_timeout())
with statsd.timer(statsd_key.format('get_operations')):
operations = storage.get_operations(import_key, cls.get_sync_batch_size())
if operations:
with statsd.timer(statsd_key.format('get_sync_objects')):
import_objects = cls.get_sync_objects(operations)
else:
import_objects = []
if import_objects:
batches = {}
with statsd.timer(statsd_key.format('get_insert_batch')):
for model_cls in cls.sub_models:
batches[model_cls] = model_cls.get_insert_batch(import_objects)
with statsd.timer(statsd_key.format('insert')):
for model_cls, batch in batches.items():
model_cls.insert_batch(batch)
with statsd.timer(statsd_key.format('post_sync')):
storage.post_sync(import_key)
storage.set_last_sync_time(import_key, now())
except RedisLockTimeoutError:
pass # skip this sync round if lock is acquired by another thread
def get_insert_batch(self, model_cls, objects):
# type: (Type[T], List[DjangoModel]) -> List[T]
"""
Gets a list of model_cls instances to insert into database
:param model_cls: ClickHouseModel subclass to import
:param objects: A list of django Model instances to sync
:return: A list of model_cls objects
"""
new_objs = super(CollapsingMergeTree,
self).get_insert_batch(model_cls, objects)
statsd_key = "%s.sync.%s.get_final_versions" % (config.STATSD_PREFIX,
model_cls.__name__)
with statsd.timer(statsd_key):
old_objs = self.get_final_versions(model_cls, new_objs)
for obj in old_objs:
self.set_obj_sign(obj, -1)
for obj in new_objs:
self.set_obj_sign(obj, 1)
return old_objs + new_objs
def _sub_model_func(model_cls):
model_statsd_key = "%s.sync.%s.{0}" % (config.STATSD_PREFIX, model_cls.__name__)
with statsd.timer(model_statsd_key.format('steps.insert')):
model_cls.insert_batch(batches[model_cls])
def route_envelope(identifier,
headers,
attempts,
mailstatus_class_path,
record_status_task_path,
build_envelope_task_path,
not_before=None,
reply=None):
"""
This envelope routing task take initiate attempt
to free Slimta Edge from SMTP connection
"""
record_performance = settings.STATSD_ENABLED
lock = None
mailstatus_class = import_string(mailstatus_class_path)
latest_status = mailstatus_class.objects.filter(
status__in=[AbstractMailStatus.DELETED] +
list(AbstractMailStatus.FINAL_STATES),
mail__identifier=identifier)
if latest_status:
log.debug("[{}] Envelope ignored because it has already been "
"{} at {}".format(identifier, latest_status[0].status,
latest_status[0].creation_date))
return
# Ensure we close Django database connection because we don't
# want to have opened connections while waiting for lock.
connection.close()
destination_domain = extract_domain(headers.get('To', ''))
pool = headers.get(settings.MAILSEND['X_POOL_HEADER'], 'default')
lock_name = '{}:lock:routing:{}:{}'.format(
settings.MAILSEND['CACHE_PREFIX'], destination_domain, pool)
lock_timeout = settings.MAILSEND['ROUTER_LOCK_TIMEOUT']
lock_blocking_timeout = settings.MAILSEND['ROUTER_LOCK_WAITING']
log.debug('[{}] Waiting for lock "{}" (timeout:{} '
'second(s) and blocking for {} second(s))...'.format(
identifier, lock_name, lock_timeout, lock_blocking_timeout))
if record_performance:
from statsd.defaults.django import statsd
lock_timer = statsd.timer('mailsend.tasks.lock_waiting')
lock_timer.start()
lock = acquire_lock(lock_name,
timeout=lock_timeout,
blocking_timeout=lock_blocking_timeout)
if record_performance:
lock_timer.stop()
if lock:
try:
log.debug('[{}] Routing envelope (attempts={})...'.format(
identifier, attempts))
worker, next_available, score, others = Worker.objects.find_worker(
identifier,
headers,
mailstatus_class,
not_before=not_before,
reply=reply)
if worker:
log.debug(
'[{}] Choosen worker is available at {} '
'with a {} score. Full workers ranking: {}'.format(
identifier, next_available.astimezone(), score, {
w.get('ip'): {
'score':
w.get('score'),
'next_available':
str(w.get('next_available').astimezone())
}
for w in others
}))
mail_status_kwargs = {
'source_ip': worker.ip,
'status': AbstractMailStatus.SENDING,
'destination_domain': extract_domain(headers.get('To'))
}
if not attempts:
routing_key = worker.get_queue_name()
else:
routing_key = worker.get_queue_name(retry=True)
attempt = send_email.s(identifier,
headers,
attempts,
mailstatus_class_path,
record_status_task_path,
build_envelope_task_path,
token=set_envelope_token(identifier))
now = timezone.now()
if next_available:
countdown = max(0, (next_available - now).total_seconds())
# And apply countdown to task if > 0
if countdown:
mail_status_kwargs.update(
{'creation_date': now + timedelta(seconds=countdown)})
attempt.set(countdown=countdown)
log.info(
'[{}] Queued with "{}" routing key in {} seconds'.format(
identifier, routing_key, int(countdown)))
mailstatus = mailstatus_class(**mail_status_kwargs)
record_status_task = import_string(record_status_task_path)
try:
record_status_task(mailstatus, identifier, attempts + 1)
except SoftFailure as exc:
log.info('SoftFailure during "route_envelope" task ('
'discarding this task): {}'.format(str(exc)),
exc_info=True)
release_lock(lock_name)
return
release_lock(lock_name)
return attempt.apply_async(routing_key=routing_key).id
else:
log.debug('[{}] No worker available. Re-route envelope '
'in 5 minutes'.format(identifier))
release_lock(lock_name)
return route_envelope.apply_async(
(identifier, headers, attempts, mailstatus_class_path,
record_status_task_path, build_envelope_task_path), {
'not_before': not_before,
'reply': reply
},
countdown=60 * 5).id
except Exception:
release_lock(lock_name)
raise
else:
log.debug('[{}] Failed to acquire lock after waiting {} second(s). '
'Re-route task in 1-6 seconds.'.format(
identifier, lock_blocking_timeout))
return route_envelope.apply_async(
(identifier, headers, attempts), {
'mailstatus_class_path': mailstatus_class_path,
'record_status_task_path': record_status_task_path,
'build_envelope_task_path': build_envelope_task_path,
'not_before': not_before,
'reply': reply
},
countdown=randint(1, 6)).id
if lock:
release_lock(lock_name)
def insert_tuples(self,
model_class: Type['ClickHouseModel'],
model_tuples: Iterable[tuple],
batch_size: Optional[int] = None,
formatted: bool = False) -> None:
"""
Inserts model_class namedtuples
:param model_class: ClickHouse model, namedtuples are made from
:param model_tuples: An iterable of tuples to insert
:param batch_size: Size of batch
:param formatted: If flag is set, tuples are expected to be ready to insert without calling field.to_db_string
:return: None
"""
tuples_iterator = iter(model_tuples)
try:
first_tuple = next(tuples_iterator)
except StopIteration:
return # model_instances is empty
if model_class.is_read_only() or model_class.is_system_model():
raise DatabaseException(
"You can't insert into read only and system tables")
fields_list = ','.join('`%s`' % name for name in first_tuple._fields)
fields_dict = model_class.fields(writable=True)
statsd_key = "%s.inserted_tuples.%s" % (config.STATSD_PREFIX,
model_class.__name__)
query = 'INSERT INTO `%s`.`%s` (%s) FORMAT TabSeparated\n' \
% (self.db_name, model_class.table_name(), fields_list)
query_enc = query.encode('utf-8')
def tuple_to_csv(tup):
if formatted:
str_gen = (getattr(tup, field_name)
for field_name in first_tuple._fields)
else:
str_gen = (fields_dict[field_name].to_db_string(getattr(
tup, field_name),
quote=False)
for field_name in first_tuple._fields)
return '%s\n' % '\t'.join(str_gen)
def gen():
buf = BytesIO()
buf.write(query_enc)
buf.write(tuple_to_csv(first_tuple).encode('utf-8'))
# Collect lines in batches of batch_size
lines = 1
for t in tuples_iterator:
buf.write(tuple_to_csv(t).encode('utf-8'))
lines += 1
if batch_size is not None and lines >= batch_size:
# Return the current batch of lines
statsd.incr(statsd_key, lines)
yield buf.getvalue()
# Start a new batch
buf = BytesIO()
buf.write(query_enc)
lines = 0
# Return any remaining lines in partial batch
if lines:
statsd.incr(statsd_key, lines)
yield buf.getvalue()
# For testing purposes
for data in gen():
with statsd.timer(statsd_key):
logger.debug('django-clickhouse: insert tuple: %s' % data)
self._send(data)
