def flush_import_key(self, import_key):
keys = [
self.REDIS_KEY_RANK_TEMPLATE.format(import_key=import_key),
self.REDIS_KEY_OPS_TEMPLATE.format(import_key=import_key),
self.REDIS_KEY_LOCK.format(import_key=import_key),
self.REDIS_KEY_LAST_SYNC_TS.format(import_key=import_key)
]
self._redis.delete(*keys)
statsd.gauge("%s.sync.%s.queue" % (config.STATSD_PREFIX, import_key), 0)
def post_batch_removed(self, import_key: str, batch_size: int) -> None:
"""
This method marks that batch has been removed in statsd
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
:param batch_size: Batch size to subtract from queue counter
:return: None
"""
key = "%s.sync.%s.queue" % (config.STATSD_PREFIX, import_key)
statsd.gauge(key, self.operations_count(import_key))
def _handle_load_error(self, e, clusters, start_processing_time):
self.logger.error(
'Loading error: {} - rendered vcl-s not used'.format(e))
if settings.STATSD_ENABLE:
statsd.gauge('successful_reload_vcl', 0)
for cluster in clusters:
cluster.error_timestamp = start_processing_time
cluster.last_error_info = str(e)[:400]
cluster.save()
raise e
def refresh_giom_data():
ds = GiomDataReader(settings.GIOM_URL)
ds.read()
d = {}
m = []
tmp_dic = ds.dict()
for k in tmp_dic:
d["giom.%s" % k] = tmp_dic[k]
m.append("%s: %s" % (k, tmp_dic[k]))
statsd.gauge("meteo.giom.%s" % (k), tmp_dic[k])
logger_fd.info(d)
logger.debug("GIOM data: %s" % (",".join(m)))
d = ds.dict()
d['timestamp'] = time()
cache.set("meteo-giom", d)
def flush(self):
key_tpls = [
self.REDIS_KEY_RANK_TEMPLATE.format(import_key='*'),
self.REDIS_KEY_OPS_TEMPLATE.format(import_key='*'),
self.REDIS_KEY_LOCK.format(import_key='*'),
self.REDIS_KEY_LAST_SYNC_TS.format(import_key='*')
]
for tpl in key_tpls:
keys = self._redis.keys(tpl)
if keys:
self._redis.delete(*keys)
from .clickhouse_models import ClickHouseModel
for model in get_subclasses(ClickHouseModel):
if isinstance(model.get_storage(), self.__class__):
key = "%s.sync.%s.queue" % (config.STATSD_PREFIX, model.get_import_key())
statsd.gauge(key, 0)
def register_operations_wrapped(self, import_key, operation, *pks):
# type: (str, str, *Any) -> None
"""
This is a wrapper for register_operation method, checking main parameters.
This method should be called from inner functions.
:param import_key: A key, returned by ClickHouseModel.get_import_key() method
:param operation: One of insert, update, delete
:param pk: Primary key to find records in main database. Should be string-serializable with str() method.
:return: None
"""
if operation not in {'insert', 'update', 'delete'}:
raise ValueError(
'operation must be one of [insert, update, delete]')
statsd_key = "%s.sync.%s.queue" % (config.STATSD_PREFIX, import_key)
statsd.gauge(statsd_key, len(pks), delta=True)
logger.debug(
'django-clickhouse: registered %s on %d items (%s) to storage' %
(operation, len(pks), import_key))
return self.register_operations(import_key, operation, *pks)
def load_vcl(self, start_processing_time, clusters):
processing_stats = init_processing()
servers = ServerExtractor().extract_servers_by_clusters(clusters)
vcl_list = ParallelRenderer(
self.renderer_max_workers).render_vcl_for_servers(
start_processing_time.strftime("%Y%m%d_%H_%M_%S"), servers)
parallel_loader = ParallelLoader(self.loader_max_workers)
try:
loaded_vcl_list = parallel_loader.load_vcl_list(vcl_list)
self._update_vcl_versions(clusters, start_processing_time,
vcl_list)
except VclLoadException as e:
self._handle_load_error(e, clusters, start_processing_time)
else:
result = parallel_loader.use_vcl_list(start_processing_time,
loaded_vcl_list)
if result is False:
if settings.STATSD_ENABLE:
statsd.gauge('successful_reload_vcl', 0)
else:
if settings.STATSD_ENABLE:
statsd.gauge('successful_reload_vcl', 1)
return result
finally:
for phase, processing in processing_stats.items():
self.logger.info(
"vcl reload phase {}; calls: {}. time: {}".format(
phase, processing['calls'], processing['time']))
if settings.STATSD_ENABLE:
if phase in [
'render_vcl_for_servers', 'use_vcl_list',
'_discard_unused_vcls', '_append_vcl',
'extract_servers_by_clusters', 'fetch_render_data'
]:
statsd.timing(phase, processing['time'])
def push_translations(self):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
if self.gengo_watch_balance:
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
# statsd the balance so we can track it with graphite.
statsd.gauge('translation.gengo.balance', balance)
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
# Create language buckets for the jobs for this translator.
# We bucket by language because this makes it easier for a
# single Gengo translator to translate all the jobs in an
# order.
jobs = GengoJob.objects.filter(
tier=self.gengo_tier, status=STATUS_CREATED)
lang_buckets = {}
for job in jobs:
lang_buckets.setdefault(job.src_lang, []).append(job)
# For each bucket, assemble an order and post it.
for lang, jobs in lang_buckets.items():
batch = []
for job in jobs:
batch.append({
'id': job.id,
'lc_src': job.src_lang,
'lc_dst': job.dst_lang,
'tier': self.gengo_tier,
'text': getattr(job.content_object, job.src_field),
'unique_id': job.unique_id
})
try:
resp = gengo_api.translate_bulk(batch)
except GengoError as exc:
self.log_error(
instance=None, action='push-translations',
msg=unicode(exc),
metadata={
'batch': batch
})
continue
# We should have an `order_id` at this point, so we create a
# GengoOrder with it.
order = GengoOrder(order_id=resp['order_id'])
order.save()
order.log('created', metadata={'response': resp})
# Update all the jobs in the order.
for job in jobs:
job.assign_to_order(order)
if self.gengo_watch_balance:
# Update the balance and see if we're below the threshold.
balance = balance - float(resp['credits_used'])
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
def push_translations(self):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
if self.gengo_watch_balance:
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
# statsd the balance so we can track it with graphite.
statsd.gauge('translation.gengo.balance', balance)
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
# Create language buckets for the jobs for this translator.
# We bucket by language because this makes it easier for a
# single Gengo translator to translate all the jobs in an
# order.
jobs = GengoJob.objects.filter(tier=self.gengo_tier,
status=STATUS_CREATED)
lang_buckets = {}
for job in jobs:
lang_buckets.setdefault(job.src_lang, []).append(job)
# For each bucket, assemble an order and post it.
for lang, jobs in lang_buckets.items():
batch = []
for job in jobs:
batch.append({
'id': job.id,
'lc_src': job.src_lang,
'lc_dst': job.dst_lang,
'tier': self.gengo_tier,
'text': getattr(job.content_object, job.src_field),
'unique_id': job.unique_id
})
try:
resp = gengo_api.translate_bulk(batch)
except GengoError as exc:
self.log_error(instance=None,
action='push-translations',
msg=unicode(exc),
metadata={'batch': batch})
continue
# We should have an `order_id` at this point, so we create a
# GengoOrder with it.
order = GengoOrder(order_id=resp['order_id'])
order.save()
order.log('created', metadata={'response': resp})
# Update all the jobs in the order.
for job in jobs:
job.assign_to_order(order)
if self.gengo_watch_balance:
# Update the balance and see if we're below the threshold.
balance = balance - float(resp['credits_used'])
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
def on_shutter(self, state):
if not state.event_count:
# No new events since last snapshot.
return
# print('Workers: {0}'.format(pformat(state.workers, indent=4)))
# print('Tasks: {0}'.format(pformat(state.tasks, indent=4)))
# print('Total: {0.event_count} events, {0.task_count} tasks'.format(state))
proceed_success, proceed_failure = defaultdict(int), defaultdict(int)
for _, task in state.itertasks():
if task.state == states.SUCCESS:
proceed_success[task.worker.hostname] += 1
if task.state == states.FAILURE:
proceed_failure[task.worker.hostname] += 1
for worker in state.workers:
print(worker)
if worker not in proceed_success:
proceed_success[worker] = 0
if worker not in proceed_failure:
proceed_failure[worker] = 0
from statsd.defaults.django import statsd as statsd_client
for hostname, worker in state.workers.items():
worker_name = worker.hostname
statsd_client.gauge(f"celery.workers.{worker_name}.active", worker.active)
statsd_client.gauge(f"celery.workers.{worker_name}.alive", worker.alive)
statsd_client.gauge(f"celery.workers.{worker_name}.processed", worker.processed)
statsd_client.gauge(
f"celery.workers.{worker_name}.processed_succeeded", proceed_success[worker_name],
)
statsd_client.gauge(
f"celery.workers.{worker_name}.processed_failed", proceed_failure[worker_name],
)
# statsd_client.gauge(
# f"celery.workers.{worker_name}.status_string", worker.status_string
# )
from blueapps.core.celery.celery import app as celery_app
try:
broker = Broker(
celery_app.connection().as_uri(include_password=True),
http_api="http://*****:*****@localhost:15672/api/",
)
except NotImplementedError:
return
# inspect active queue names
inspect = celery_app.control.inspect()
queue_names = set()
for worker_name, queues in inspect.active_queues().items():
for q in queues:
queue_names.add(q["name"])
# query broker: rq/redis...
data = defaultdict(int)
for queue in broker.queues(queue_names):
data[queue["name"]] = queue.get("messages", 0)
statsd_client.gauge(f"celery.queues.{queue['name']}.tasks", queue.get("messages", 0))