def __record_change_metric_in_datadog(self, metric, change, processor=None, processing_time=None,
add_case_type_tag=False):
if change.metadata is not None:
common_tags = {
'datasource': change.metadata.data_source_name,
'is_deletion': change.metadata.is_deletion,
'pillow_name': self.get_name(),
'processor': processor.__class__.__name__ if processor else "all_processors",
}
metric_tags = common_tags.copy()
if add_case_type_tag and settings.ENTERPRISE_MODE and change.metadata.document_type == 'CommCareCase':
metric_tags['case_type'] = change.metadata.document_subtype
metrics_counter(metric, tags=metric_tags)
change_lag = (datetime.utcnow() - change.metadata.publish_timestamp).total_seconds()
metrics_gauge('commcare.change_feed.change_lag', change_lag, tags={
'pillow_name': self.get_name(),
'topic': _topic_for_ddog(
TopicPartition(change.topic, change.partition)
if change.partition is not None else change.topic
),
})
if processing_time:
metrics_counter('commcare.change_feed.processing_time.total', processing_time, tags=common_tags)
metrics_counter('commcare.change_feed.processing_time.count', tags=common_tags)
def send_unknown_user_type_stats():
metrics_gauge('commcare.fix_user_types.unknown_user_count',
_get_unknown_user_type_user_ids_approx_count(),
multiprocess_mode=MPM_MAX)
metrics_gauge('commcare.fix_user_types.unknown_user_form_count',
FormES().user_type(UNKNOWN_USER_TYPE).count(),
multiprocess_mode=MPM_MAX)
def _send_form_to_hubspot(form_id, webuser, hubspot_cookie, meta, extra_fields=None, email=False):
"""
This sends hubspot the user's first and last names and tracks everything they did
up until the point they signed up.
"""
if ((webuser and not hubspot_enabled_for_user(webuser))
or (not webuser and not hubspot_enabled_for_email(email))):
# This user has analytics disabled
metrics_gauge(
'commcare.hubspot_data.rejected.send_form_to_hubspot',
1,
tags={
'username': webuser.username if webuser else email,
}
)
return
hubspot_id = settings.ANALYTICS_IDS.get('HUBSPOT_API_ID')
if hubspot_id and hubspot_cookie:
data = {
'email': email if email else webuser.username,
'hs_context': json.dumps({"hutk": hubspot_cookie, "ipAddress": _get_client_ip(meta)}),
}
if webuser:
data.update({
'firstname': webuser.first_name,
'lastname': webuser.last_name,
})
if extra_fields:
data.update(extra_fields)
response = _send_hubspot_form_request(hubspot_id, form_id, data)
_log_response('HS', data, response)
response.raise_for_status()
def gauge_pending_user_confirmations():
metric_name = 'commcare.pending_user_confirmations'
from corehq.apps.users.models import Invitation
for stats in (Invitation.objects.filter(is_accepted=False).all()
.values('domain').annotate(Count('domain'))):
metrics_gauge(
metric_name, stats['domain__count'], tags={
'domain': stats['domain'],
'user_type': 'web',
},
multiprocess_mode=MPM_MAX
)
from corehq.apps.users.analytics import get_inactive_commcare_users_in_domain
for doc in Domain.get_all(include_docs=False):
domain_name = doc['key']
users = get_inactive_commcare_users_in_domain(domain_name)
num_unconfirmed = sum(1 for u in users if not u.is_account_confirmed)
if num_unconfirmed:
metrics_gauge(
metric_name, num_unconfirmed, tags={
'domain': domain_name,
'user_type': 'mobile',
},
multiprocess_mode=MPM_MAX
)
def reprocess_archive_stubs():
# Check for archive stubs
from corehq.form_processor.models import XFormInstance
from couchforms.models import UnfinishedArchiveStub
stubs = UnfinishedArchiveStub.objects.filter(attempts__lt=3)
metrics_gauge('commcare.unfinished_archive_stubs', len(stubs),
multiprocess_mode=MPM_MAX)
start = time.time()
cutoff = start + timedelta(minutes=4).total_seconds()
for stub in stubs:
# Exit this task after 4 minutes so that tasks remain short
if time.time() > cutoff:
return
try:
xform = XFormInstance.objects.get_form(stub.xform_id, stub.domain)
# If the history wasn't updated the first time around, run the whole thing again.
if not stub.history_updated:
XFormInstance.objects.do_archive(xform, stub.archive, stub.user_id, trigger_signals=True)
# If the history was updated the first time around, just send the update to kafka
else:
XFormInstance.objects.publish_archive_action_to_kafka(xform, stub.user_id, stub.archive)
except Exception:
# Errors should not prevent processing other stubs
notify_exception(None, "Error processing UnfinishedArchiveStub")
def get_valid_recipients(recipients, domain=None):
"""
This filters out any emails that have reported hard bounces or complaints to
Amazon SES
:param recipients: list of recipient emails
:return: list of recipient emails not marked as bounced
"""
from corehq.toggles import BLOCKED_DOMAIN_EMAIL_SENDERS
if domain and BLOCKED_DOMAIN_EMAIL_SENDERS.enabled(domain):
# don't sent email if domain is blocked
metrics_gauge('commcare.bounced_email', len(recipients), tags={
'email_domain': domain,
}, multiprocess_mode=MPM_LIVESUM)
return []
from corehq.util.models import BouncedEmail
bounced_emails = BouncedEmail.get_hard_bounced_emails(recipients)
for bounced_email in bounced_emails:
try:
email_domain = bounced_email.split('@')[1]
except IndexError:
email_domain = bounced_email
metrics_gauge('commcare.bounced_email', 1, tags={
'email_domain': email_domain,
}, multiprocess_mode=MPM_LIVESUM)
return [recipient for recipient in recipients if recipient not in bounced_emails]
def _record_datadog_metrics(self, changes_chunk, processing_time):
tags = {"pillow_name": self.get_name(), "mode": "chunked"}
change_count = len(changes_chunk)
if settings.ENTERPRISE_MODE:
type_counter = Counter([
change.metadata.document_subtype
for change in changes_chunk if change.metadata.document_type == 'CommCareCase'
])
for case_type, type_count in type_counter.items():
metrics_counter('commcare.change_feed.changes.count', type_count,
tags={**tags, 'case_type': case_type})
remainder = change_count - sum(type_counter.values())
if remainder:
metrics_counter('commcare.change_feed.changes.count', remainder, tags=tags)
else:
metrics_counter('commcare.change_feed.changes.count', change_count, tags=tags)
max_change_lag = (datetime.utcnow() - changes_chunk[0].metadata.publish_timestamp).total_seconds()
min_change_lag = (datetime.utcnow() - changes_chunk[-1].metadata.publish_timestamp).total_seconds()
metrics_gauge('commcare.change_feed.chunked.min_change_lag', min_change_lag, tags=tags)
metrics_gauge('commcare.change_feed.chunked.max_change_lag', max_change_lag, tags=tags)
# processing_time per change
metrics_counter('commcare.change_feed.processing_time.total', processing_time / change_count, tags=tags)
metrics_counter('commcare.change_feed.processing_time.count', tags=tags)
def datadog_report_user_stats(metric_name, commcare_users_by_domain):
commcare_users_by_domain = summarize_user_counts(commcare_users_by_domain,
n=50)
for domain, user_count in commcare_users_by_domain.items():
metrics_gauge(metric_name,
user_count,
tags={'domain': '_other' if domain is () else domain})
def _get_contact_ids_for_email_domain(email_domain):
"""
Searches Hubspot for an email domain and returns the list of matching
contact IDs for that email domain.
:param email_domain:
:return: list of matching contact IDs
"""
api_key = settings.ANALYTICS_IDS.get('HUBSPOT_API_KEY', None)
if api_key:
req = requests.get(
"https://api.hubapi.com/contacts/v1/search/query",
params={
'hapikey': api_key,
'q': f'@{email_domain}',
},
)
if req.status_code == 200:
return [
contact.get('vid') for contact in req.json().get('contacts')
]
if req.status_code == 429:
metrics_gauge(
'commcare.hubspot_data.rate_limited.get_contact_ids_for_email_domain',
1)
return []
def _report_current_global_two_factor_setup_rate_limiter():
for window, value, threshold in global_two_factor_setup_rate_limiter.iter_rates():
metrics_gauge('commcare.two_factor.global_two_factor_setup_threshold', threshold, tags={
'window': window
}, multiprocess_mode=MPM_MAX)
metrics_gauge('commcare.two_factor.global_two_factor_setup_usage', value, tags={
'window': window
}, multiprocess_mode=MPM_MAX)
def _report_current_global_submission_thresholds():
for window, value, threshold in global_submission_rate_limiter.iter_rates(
):
metrics_gauge('commcare.xform_submissions.global_threshold',
threshold,
tags={'window': window})
metrics_gauge('commcare.xform_submissions.global_usage',
value,
tags={'window': window})
def cleanup_blocked_hubspot_contacts():
"""
Remove any data stored about users from blocked domains and email domains
from Hubspot in case it somehow got there.
:return:
"""
if not HUBSPOT_ENABLED:
return
# First delete any user information from users that are members of
# blocked domains
blocked_domains = get_blocked_hubspot_domains()
for domain in blocked_domains:
user_query = UserES().domain(domain).source(['email', 'username'])
total_users = user_query.count()
chunk_size = 30 # Hubspot recommends fewer than 100 emails per request
num_chunks = int(math.ceil(float(total_users) / float(chunk_size)))
for chunk in range(num_chunks):
blocked_users = (user_query
.size(chunk_size)
.start(chunk * chunk_size)
.run()
.hits)
blocked_emails = []
for user in blocked_users:
username = user.get('username')
user_email = user.get('email')
blocked_emails.append(username)
if user_email and user_email != username:
blocked_emails.append(user_email)
ids_to_delete = _get_contact_ids_for_emails(set(blocked_emails))
num_deleted = sum(_delete_hubspot_contact(vid) for vid in ids_to_delete)
metrics_gauge(
'commcare.hubspot_data.deleted_user.blocked_domain',
num_deleted,
tags={
'domain': domain,
'ids_deleted': ids_to_delete,
}
)
# Next delete any user info from users that have emails or usernames ending
# in blocked email-domains
blocked_email_domains = get_blocked_hubspot_email_domains()
for email_domain in blocked_email_domains:
ids_to_delete = _get_contact_ids_for_email_domain(email_domain)
num_deleted = sum(_delete_hubspot_contact(vid) for vid in ids_to_delete)
metrics_gauge(
'commcare.hubspot_data.deleted_user.blocked_email_domain',
num_deleted,
tags={
'email_domain': email_domain,
'ids_deleted': ids_to_delete,
}
)
def _record_checkpoint_in_datadog(self):
metrics_counter('commcare.change_feed.change_feed.checkpoint', tags={
'pillow_name': self.get_name(),
})
checkpoint_sequence = self._normalize_checkpoint_sequence()
for topic, value in checkpoint_sequence.items():
metrics_gauge('commcare.change_feed.checkpoint_offsets', value, tags={
'pillow_name': self.get_name(),
'topic': _topic_for_ddog(topic),
})
def record_pillow_error_queue_size():
data = PillowError.objects.values('pillow').annotate(
num_errors=Count('id'))
for row in data:
metrics_gauge('commcare.pillowtop.error_queue',
row['num_errors'],
tags={
'pillow_name': row['pillow'],
'host': 'celery',
'group': 'celery'
})
def get_and_report_blockage_duration(self):
blockage_duration = self.get_blockage_duration()
metrics_gauge('commcare.celery.heartbeat.blockage_duration',
blockage_duration.total_seconds(),
tags={'celery_queue': self.queue})
if self.threshold:
metrics_gauge(
'commcare.celery.heartbeat.blockage_ok',
1
if blockage_duration.total_seconds() <= self.threshold else 0,
tags={'celery_queue': self.queue})
return blockage_duration
def track_es_doc_counts():
es = get_es_new()
stats = es.indices.stats(level='shards', metric='docs')
for name, data in stats['indices'].items():
for number, shard in data['shards'].items():
for i in shard:
if i['routing']['primary']:
tags = {
'index': name,
'shard': f'{name}_{number}',
}
metrics_gauge('commcare.elasticsearch.shards.docs.count', i['docs']['count'], tags)
metrics_gauge('commcare.elasticsearch.shards.docs.deleted', i['docs']['deleted'], tags)
def server_up(req):
"""
Health check view which can be hooked into server monitoring tools like 'pingdom'
Returns:
HttpResponse("success", status_code=200)
HttpResponse(error_message, status_code=500)
Hit serverup.txt to check all the default enabled services (always_check=True)
Hit serverup.txt?only={check_name} to only check a specific service
Hit serverup.txt?{check_name} to include a non-default check (currently only ``heartbeat``)
"""
only = req.GET.get('only', None)
if only and only in CHECKS:
checks_to_do = [only]
else:
checks_to_do = [
check for check, check_info in CHECKS.items() if
check_info['always_check'] or req.GET.get(check, None) is not None
]
statuses = run_checks(checks_to_do)
failed_checks = [(check, status) for check, status in statuses
if not status.success]
for check_name, status in statuses:
tags = {
'status': 'failed' if not status.success else 'ok',
'check': check_name
}
metrics_gauge('commcare.serverup.check',
status.duration,
tags=tags,
multiprocess_mode=MPM_MAX)
if failed_checks and not is_deploy_in_progress():
status_messages = [
html.linebreaks('<strong>{}</strong>: {}'.format(
check, html.escape(status.msg)).strip())
for check, status in failed_checks
]
create_metrics_event(
'Serverup check failed',
'\n'.join(status_messages),
alert_type='error',
aggregation_key='serverup',
)
status_messages.insert(0, 'Failed Checks (%s):' % os.uname()[1])
return HttpResponse(''.join(status_messages), status=500)
else:
return HttpResponse("success")
def _record_datadog_metrics(self, changes_chunk, processing_time):
change_count = len(changes_chunk)
by_data_source = defaultdict(list)
for change in changes_chunk:
by_data_source[change.metadata.data_source_name].append(change)
for data_source, changes in by_data_source.items():
tags = {"pillow_name": self.get_name(), 'datasource': data_source}
if settings.ENTERPRISE_MODE:
type_counter = Counter([
change.metadata.document_subtype
for change in changes_chunk
if change.metadata.document_type == 'CommCareCase'
])
for case_type, type_count in type_counter.items():
metrics_counter('commcare.change_feed.changes.count',
type_count,
tags={
**tags, 'case_type': case_type
})
remainder = change_count - sum(type_counter.values())
if remainder:
metrics_counter('commcare.change_feed.changes.count',
remainder,
tags={
**tags, 'case_type': 'NA'
})
else:
metrics_counter('commcare.change_feed.changes.count',
change_count,
tags={
**tags, 'case_type': 'NA'
})
tags = {"pillow_name": self.get_name()}
max_change_lag = (
datetime.utcnow() -
changes_chunk[0].metadata.publish_timestamp).total_seconds()
metrics_gauge('commcare.change_feed.chunked.max_change_lag',
max_change_lag,
tags=tags,
multiprocess_mode=MPM_MAX)
# processing_time per change
metrics_counter('commcare.change_feed.processing_time.total',
processing_time / change_count,
tags=tags)
metrics_counter('commcare.change_feed.processing_time.count',
tags=tags)
def track_pg_limits():
for db in settings.DATABASES:
with connections[db].cursor() as cursor:
query = """
select tab.relname, seq.relname
from pg_class seq
join pg_depend as dep on seq.oid=dep.objid
join pg_class as tab on dep.refobjid = tab.oid
join pg_attribute as att on att.attrelid=tab.oid and att.attnum=dep.refobjsubid
where seq.relkind='S' and att.attlen=4
"""
cursor.execute(query)
results = cursor.fetchall()
for table, sequence in results:
cursor.execute(f'select last_value from "{sequence}"')
current_value = cursor.fetchone()[0]
metrics_gauge('commcare.postgres.sequence.current_value', current_value, {'table': table, 'database': db})
def pillow_datadog_metrics():
def _is_couch(pillow):
# text is couch, json is kafka
return pillow['seq_format'] == 'text'
pillow_meta = get_all_pillows_json()
for pillow in pillow_meta:
# The host and group tags are added here to ensure they remain constant
# regardless of which celery worker the task get's executed on.
# Without this the sum of the metrics get's inflated.
tags = {
'pillow_name': pillow['name'],
'feed_type': 'couch' if _is_couch(pillow) else 'kafka',
'host': 'celery',
'group': 'celery'
}
metrics_gauge('commcare.change_feed.seconds_since_last_update',
pillow['seconds_since_last'],
tags=tags,
multiprocess_mode=MPM_MIN)
for topic_name, offset in pillow['offsets'].items():
if _is_couch(pillow):
tags_with_topic = {**tags, 'topic': topic_name}
processed_offset = pillow['seq']
else:
if not pillow['seq']:
# this pillow has never been initialized.
# (custom pillows on most environments)
continue
topic, partition = topic_name.split(',')
tags_with_topic = {
**tags, 'topic': '{}-{}'.format(topic, partition)
}
processed_offset = pillow['seq'][topic_name]
if processed_offset == 0:
# assume if nothing has been processed that this pillow is not
# supposed to be running
continue
metrics_gauge('commcare.change_feed.current_offsets',
offset,
tags=tags_with_topic,
multiprocess_mode=MPM_MAX)
metrics_gauge('commcare.change_feed.processed_offsets',
processed_offset,
tags=tags_with_topic,
multiprocess_mode=MPM_MAX)
needs_processing = offset - processed_offset
metrics_gauge('commcare.change_feed.need_processing',
needs_processing,
tags=tags_with_topic,
multiprocess_mode=MPM_MAX)
def cleanup_blocked_hubspot_contacts():
"""
Remove any data stored about users from blocked domains and email domains
from Hubspot in case it somehow got there.
:return:
"""
if not HUBSPOT_ENABLED:
return
time_started = datetime.utcnow()
remove_blocked_domain_contacts_from_hubspot()
remove_blocked_domain_invited_users_from_hubspot()
task_time = datetime.utcnow() - time_started
metrics_gauge('commcare.hubspot.runtimes.cleanup_blocked_hubspot_contacts',
task_time.seconds,
multiprocess_mode=MPM_LIVESUM)
def cleanup_stale_es_on_couch_domains(start_date=None,
end_date=None,
domains=None,
stdout=None):
"""
This is the response to https://dimagi-dev.atlassian.net/browse/SAAS-11489
and basically makes sure that there are no stale docs in the most active
domains still using the couch db backend until we can get them migrated.
"""
end = end_date or datetime.datetime.utcnow()
start = start_date or (end - datetime.timedelta(days=2))
couch_domains = domains or ACTIVE_COUCH_DOMAINS.get_enabled_domains()
for domain in couch_domains:
form_ids, has_discrepancies = _get_all_form_ids(domain, start, end)
if stdout:
stdout.write(f"Found {len(form_ids)} in {domain} for between "
f"{start.isoformat()} and {end.isoformat()}.")
if has_discrepancies:
metrics_gauge(
'commcare.es.couch_domain.couch_discrepancy_detected',
1,
tags={
'domain': domain,
})
if stdout:
stdout.write(
f"\tFound discrepancies in form counts for domain {domain}"
)
forms_not_in_es = _get_forms_not_in_es(form_ids)
if forms_not_in_es:
metrics_gauge('commcare.es.couch_domain.stale_docs_in_es',
len(forms_not_in_es),
tags={
'domain': domain,
})
if stdout:
stdout.write(f"\tFound {len(forms_not_in_es)} forms not in es "
f"for {domain}")
changes = _get_changes(domain, forms_not_in_es)
form_es_processor = get_xform_pillow().processors[0]
for change in changes:
form_es_processor.process_change(change)
def _get_user_hubspot_id(web_user):
api_key = settings.ANALYTICS_IDS.get('HUBSPOT_API_KEY', None)
if api_key and hubspot_enabled_for_user(web_user):
req = requests.get(
"https://api.hubapi.com/contacts/v1/contact/email/{}/profile".
format(six.moves.urllib.parse.quote(web_user.username)),
params={'hapikey': api_key},
)
if req.status_code == 404:
return None
req.raise_for_status()
return req.json().get("vid", None)
elif api_key:
metrics_gauge('commcare.hubspot_data.rejected.get_user_hubspot_id',
1,
tags={
'username': web_user.username,
})
return None
def generate_partner_reports():
"""
Generates analytics reports for partners that have requested tracking on
specific data points.
:return:
"""
time_started = datetime.utcnow()
last_month = add_months_to_date(datetime.today(), -1)
year = last_month.year
month = last_month.month
generate_monthly_mobile_worker_statistics(year, month)
generate_monthly_web_user_statistics(year, month)
generate_monthly_submissions_statistics(year, month)
send_partner_emails(year, month)
task_time = datetime.utcnow() - time_started
metrics_gauge('commcare.analytics.runtimes.generate_partner_reports',
task_time.seconds,
multiprocess_mode=MPM_LIVESUM)
def _delete_hubspot_contact(vid):
"""
Permanently deletes a Hubspot contact.
:param vid: (the contact ID)
:return: boolean if contact was deleted
"""
api_key = settings.ANALYTICS_IDS.get('HUBSPOT_API_KEY', None)
if api_key:
req = requests.delete(
f'https://api.hubapi.com/contacts/v1/contact/vid/{vid}',
params={
'hapikey': api_key,
})
if req.status_code == 200:
return True
if req.status_code == 429:
metrics_gauge(
'commcare.hubspot_data.rate_limited.delete_hubspot_contact', 1)
return False
def __record_change_metric_in_datadog(self,
metric,
change,
processing_time=None,
add_case_type_tag=False):
if change.metadata is not None:
metric_tags = {
'datasource': change.metadata.data_source_name,
'pillow_name': self.get_name(),
}
if add_case_type_tag:
metric_tags['case_type'] = 'NA'
if settings.ENTERPRISE_MODE and change.metadata.document_type == 'CommCareCase':
metric_tags['case_type'] = change.metadata.document_subtype
metrics_counter(metric, tags=metric_tags)
change_lag = (datetime.utcnow() -
change.metadata.publish_timestamp).total_seconds()
metrics_gauge(
'commcare.change_feed.change_lag',
change_lag,
tags={
'pillow_name':
self.get_name(),
'topic':
_topic_for_ddog(
TopicPartition(change.topic, change.partition)
if change.partition is not None else change.topic),
},
multiprocess_mode=MPM_MAX)
if processing_time:
tags = {'pillow_name': self.get_name()}
metrics_counter('commcare.change_feed.processing_time.total',
processing_time,
tags=tags)
metrics_counter('commcare.change_feed.processing_time.count',
tags=tags)
def delete_old_images_on_db(db_name, cutoff):
if isinstance(cutoff, str):
cutoff = parse_date(cutoff, default_timezone=None)
max_age = cutoff - timedelta(days=90)
db = get_blob_db()
def _get_query(db_name, max_age=max_age):
return BlobMeta.objects.using(db_name).filter(
type_code=CODES.form_attachment,
domain='icds-cas',
created_on__lt=max_age).order_by('created_on')
bytes_deleted = 0
query = _get_query(db_name)
metas = list(query[:1000])
run_again = len(metas) == 1000
if metas:
for meta in metas:
bytes_deleted += meta.content_length or 0
db.bulk_delete(metas=metas)
tags = {'database': db_name}
age = datetime.utcnow() - metas[-1].created_on
metrics_gauge('commcare.icds_images.max_age',
value=age.total_seconds(),
tags=tags)
row_estimate = estimate_row_count(query, db_name)
metrics_gauge('commcare.icds_images.count_estimate',
value=row_estimate,
tags=tags)
metrics_counter('commcare.icds_images.bytes_deleted',
value=bytes_deleted)
metrics_counter('commcare.icds_images.count_deleted', value=len(metas))
runtime = datetime.utcnow() - cutoff
if run_again and runtime.total_seconds() < MAX_RUNTIME:
delete_old_images_on_db.delay(db_name, cutoff)
def celery_record_time_to_start(task_id=None, task=None, **kwargs):
from corehq.util.metrics import metrics_counter, metrics_gauge
from corehq.util.metrics.const import MPM_MAX
tags = {
'celery_task_name': task.name,
'celery_queue': task.queue,
}
timer = TimeToStartTimer(task_id)
try:
time_to_start = timer.stop_and_pop_timing()
except TimingNotAvailable:
metrics_counter('commcare.celery.task.time_to_start_unavailable',
tags=tags)
else:
metrics_gauge('commcare.celery.task.time_to_start',
time_to_start.total_seconds(),
tags=tags,
multiprocess_mode=MPM_MAX)
get_task_time_to_start.set_cached_value(task_id).to(time_to_start)
TimeToRunTimer(task_id).start_timing()
def report_build_time(domain, app_id, build_type):
start = time.time()
# Histogram of all app builds
name = {
"new_release": 'commcare.app_build.new_release',
"live_preview": 'commcare.app_build.live_preview',
}[build_type]
buckets = (1, 10, 30, 60, 120, 240)
with metrics_histogram_timer(name, timing_buckets=buckets):
yield
# Detailed information for all apps that take longer than 30s to build
end = time.time()
duration = end - start
if duration > 30:
metrics_gauge('commcare.app_build.duration',
duration,
tags={
"domain": domain,
"app_id": app_id,
"build_type": build_type,
})
def prune_synclogs():
"""
Drops all partition tables containing data that's older than 63 days (9 weeks)
"""
db = router.db_for_write(SyncLogSQL)
oldest_date = SyncLogSQL.objects.aggregate(Min('date'))['date__min']
while oldest_date and (datetime.today() - oldest_date).days > SYNCLOG_RETENTION_DAYS:
year, week, _ = oldest_date.isocalendar()
table_name = "{base_name}_y{year}w{week}".format(
base_name=SyncLogSQL._meta.db_table,
year=year,
week="%02d" % week
)
drop_query = "DROP TABLE IF EXISTS {}".format(table_name)
with connections[db].cursor() as cursor:
cursor.execute(drop_query)
oldest_date += timedelta(weeks=1)
# find and log synclogs for which the trigger did not function properly
with connections[db].cursor() as cursor:
cursor.execute("select count(*) from only phone_synclogsql")
orphaned_synclogs = cursor.fetchone()[0]
metrics_gauge('commcare.orphaned_synclogs', orphaned_synclogs)