def change_from_kafka_message(message): change_meta = change_meta_from_kafka_message(message.value) try: document_store = get_document_store( data_source_type=change_meta.data_source_type, data_source_name=change_meta.data_source_name, domain=change_meta.domain, load_source="change_feed", ) except UnknownDocumentStore: document_store = None notify_error("Unknown document store: {}".format( change_meta.data_source_type)) except UnexpectedBackend: document_store = None notify_error("Change from incorrect backend", details=change_meta.to_json()) return Change( id=change_meta.document_id, sequence_id=message.offset, document=None, deleted=change_meta.is_deletion, metadata=change_meta, document_store=document_store, topic=message.topic, partition=message.partition, )
def excel_config(request, domain): """ Step one of three. This is the initial post when the user uploads the Excel file """ if request.method != 'POST': return HttpResponseRedirect(base.ImportCases.get_url(domain=domain)) if not request.FILES: return render_error(request, domain, 'Please choose an Excel file to import.') uploaded_file_handle = request.FILES['file'] try: case_upload, context = _process_file_and_get_upload( uploaded_file_handle, request, domain, max_columns=MAX_CASE_IMPORTER_COLUMNS) except ImporterFileNotFound as e: notify_error(f"Import file not found after initial upload: {str(e)}") return render_error(request, domain, get_importer_error_message(e)) except ImporterError as e: return render_error(request, domain, get_importer_error_message(e)) except SpreadsheetFileExtError: return render_error( request, domain, _("Please upload file with extension .xls or .xlsx")) context.update(_case_importer_breadcrumb_context(_('Case Options'), domain)) return render(request, "case_importer/excel_config.html", context)
def run_case_update_rules_for_domain(domain, now=None): now = now or datetime.utcnow() start_run = datetime.utcnow() all_rules = AutomaticUpdateRule.by_domain(domain) rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type(all_rules) for case_type, rules in rules_by_case_type.iteritems(): boundary_date = AutomaticUpdateRule.get_boundary_date(rules, now) case_id_chunks = AutomaticUpdateRule.get_case_ids(domain, case_type, boundary_date) for case_ids in case_id_chunks: for case in CaseAccessors(domain).iter_cases(case_ids): time_elapsed = datetime.utcnow() - start_run if time_elapsed.seconds > HALT_AFTER: notify_error( "Halting rule run for domain %s as it's been running for more than a day." % domain ) return for rule in rules: stop_processing = rule.apply_rule(case, now) if stop_processing: break for rule in rules: rule.last_run = now rule.save()
def process_pillow_retry(error_doc, producer=None): producer = producer or kafka_producer pillow_name_or_class = error_doc.pillow try: pillow = _get_pillow(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into.") % pillow_name_or_class) try: error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1 error_doc.save() finally: return try: if isinstance(pillow.get_change_feed(), CouchChangeFeed): _process_couch_change(pillow, error_doc) else: _process_kafka_change(producer, error_doc) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save()
def get_inbound_phone_entry(msg): if msg.backend_id: backend = SQLMobileBackend.load(msg.backend_id, is_couch_id=True) if toggles.INBOUND_SMS_LENIENCY.enabled(backend.domain): p = None if toggles.ONE_PHONE_NUMBER_MULTIPLE_CONTACTS.enabled( backend.domain): running_session_info = XFormsSessionSynchronization.get_running_session_info_for_channel( SMSChannel(backend_id=msg.backend_id, phone_number=msg.phone_number)) contact_id = running_session_info.contact_id if contact_id: p = PhoneNumber.get_phone_number_for_owner( contact_id, msg.phone_number) if p is not None: return (p, True) elif running_session_info.session_id: # This would be very unusual, as it would mean the supposedly running form session # is linked to a phone number, contact pair that doesn't exist in the PhoneNumber table notify_error( "Contact from running session has no match in PhoneNumber table. " "Only known way for this to happen is if you " "unregister a phone number for a contact " "while they are in an active session.", details={'running_session_info': running_session_info}) if not backend.is_global: p = PhoneNumber.get_two_way_number_with_domain_scope( msg.phone_number, backend.domains_with_access) return (p, p is not None) return (PhoneNumber.get_reserved_number(msg.phone_number), False)
def process_pillow_retry(error_doc_id): # Redis error logged in get_redis_client try: client = cache_core.get_redis_client() except cache_core.RedisClientError: return # Prevent more than one task from processing this error, just in case # it got enqueued twice. lock = client.lock( "pillow-retry-processing-%s" % error_doc_id, timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT*60 ) if lock.acquire(blocking=False): try: error_doc = PillowError.objects.get(id=error_doc_id) except PillowError.DoesNotExist: release_lock(lock, True) return pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into." ) % pillow_name_or_class) try: error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1 error_doc.save() finally: release_lock(lock, True) return change = error_doc.change_object try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain ) change.document_store = document_store pillow.process_change(change) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.queued = False error_doc.save() else: error_doc.delete() finally: release_lock(lock, True)
def check_non_dimagi_superusers(): non_dimagis_superuser = '******'.join((get_user_model().objects.filter( (Q(is_staff=True) | Q(is_superuser=True)) & ~Q(username__endswith='@dimagi.com') ).values_list('username', flat=True))) if non_dimagis_superuser: message = "{non_dimagis} have superuser privileges".format(non_dimagis=non_dimagis_superuser) _soft_assert_superusers(False, message) notify_error(message=message)
def form_session_handler(v, text, msg): """ The form session handler will use the inbound text to answer the next question in the open SQLXformsSession for the associated contact. If no session is open, the handler passes. If multiple sessions are open, they are all closed and an error message is displayed to the user. """ with critical_section_for_smsforms_sessions(v.owner_id): if toggles.ONE_PHONE_NUMBER_MULTIPLE_CONTACTS.enabled(v.domain): channel = get_channel_for_contact(v.owner_id, v.phone_number) running_session_info = XFormsSessionSynchronization.get_running_session_info_for_channel( channel) if running_session_info.session_id: session = SQLXFormsSession.by_session_id( running_session_info.session_id) if not session.session_is_open: # This should never happen. But if it does we should set the channel free # and act like there was no available session notify_error( "The supposedly running session was not open and was released. " 'No known way for this to happen, so worth investigating.' ) XFormsSessionSynchronization.clear_stale_channel_claim( channel) session = None else: session = None else: multiple, session = get_single_open_session_or_close_multiple( v.domain, v.owner_id) if multiple: send_sms_to_verified_number( v, get_message(MSG_MULTIPLE_SESSIONS, v)) return True if session: session.phone_number = v.phone_number session.modified_time = datetime.utcnow() session.save() # Metadata to be applied to the inbound message inbound_metadata = MessageMetadata( workflow=session.workflow, reminder_id=session.reminder_id, xforms_session_couch_id=session._id, ) add_msg_tags(msg, inbound_metadata) try: answer_next_question(v, text, msg, session) except Exception: # Catch any touchforms errors log_sms_exception(msg) send_sms_to_verified_number( v, get_message(MSG_TOUCHFORMS_DOWN, v)) return True else: return False
def process_pillow_retry(error_doc_id): # Redis error logged in get_redis_client try: client = cache_core.get_redis_client() except cache_core.RedisClientError: return # Prevent more than one task from processing this error, just in case # it got enqueued twice. lock = client.lock("pillow-retry-processing-%s" % error_doc_id, timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT * 60) if lock.acquire(blocking=False): try: error_doc = PillowError.objects.get(id=error_doc_id) except PillowError.DoesNotExist: release_lock(lock, True) return pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into.") % pillow_name_or_class) try: error_doc.total_attempts = PillowError.multi_attempts_cutoff( ) + 1 error_doc.save() finally: release_lock(lock, True) return change = error_doc.change_object try: try: from corehq.apps.userreports.pillow import ConfigurableReportKafkaPillow if isinstance(pillow, ConfigurableReportKafkaPillow): raise Exception('this is temporarily not supported!') except ImportError: pass pillow.process_change(change) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.queued = False error_doc.save() else: error_doc.delete() finally: release_lock(lock, True)
def process_pillow_retry(error_doc): pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into." ) % pillow_name_or_class) try: error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1 error_doc.save() finally: return change = error_doc.change_object delete_all_for_doc = False try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain, load_source="pillow_retry", ) change.document_store = document_store if isinstance(pillow.get_change_feed(), CouchChangeFeed): pillow.process_change(change) else: if change_metadata.data_source_type in ('couch', 'sql'): data_source_type = change_metadata.data_source_type else: # legacy metadata will have other values for non-sql # can remove this once all legacy errors have been processed data_source_type = 'sql' producer.send_change( get_topic_for_doc_type( change_metadata.document_type, data_source_type ), change_metadata ) delete_all_for_doc = True except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save() else: if delete_all_for_doc: PillowError.objects.filter(doc_id=error_doc.doc_id).delete() else: error_doc.delete()
def iter_changes(self, since, forever): """ Since can either be an integer (for single topic change feeds) or a dict of topics to integers (for multiple topic change feeds) """ # a special value of since=None will start from the end of the change stream # in milliseconds, -1 means wait forever for changes timeout = -1 if forever else MIN_TIMEOUT start_from_latest = since is None reset = 'largest' if start_from_latest else 'smallest' consumer = self._get_consumer(timeout, auto_offset_reset=reset) if not start_from_latest: if isinstance(since, dict): if not since: since = {topic: 0 for topic in self._topics} self._processed_topic_offsets = copy(since) else: # single topic single_topic = self._get_single_topic_or_fail() try: offset = int(since) # coerce sequence IDs to ints except ValueError: notify_error( "kafka pillow {} couldn't parse sequence ID {}. rewinding..." .format(self._group_id, since)) # since kafka only keeps 7 days of data this isn't a big deal. Hopefully we will only see # these once when each pillow moves over. offset = 0 self._processed_topic_offsets = {single_topic: offset} def _make_offset_tuple(topic): if topic in self._processed_topic_offsets: return (topic, self._partition, self._processed_topic_offsets[topic]) else: return (topic, self._partition) offsets = [_make_offset_tuple(topic) for topic in self._topics] if self.strict: self._validate_offsets(offsets) # this is how you tell the consumer to start from a certain point in the sequence consumer.set_topic_partitions(*offsets) try: for message in consumer: self._processed_topic_offsets[message.topic] = message.offset yield change_from_kafka_message(message) except ConsumerTimeout: assert not forever, 'Kafka pillow should not timeout when waiting forever!'
def iter_changes(self, since, forever): """ Since can either be an integer (for single topic change feeds) or a dict of topics to integers (for multiple topic change feeds) """ # a special value of since=None will start from the end of the change stream # in milliseconds, -1 means wait forever for changes timeout = -1 if forever else MIN_TIMEOUT start_from_latest = since is None reset = 'largest' if start_from_latest else 'smallest' consumer = self._get_consumer(timeout, auto_offset_reset=reset) if not start_from_latest: if isinstance(since, dict): if not since: since = {topic: 0 for topic in self._topics} self._processed_topic_offsets = copy(since) else: # single topic single_topic = self._get_single_topic_or_fail() try: offset = int(since) # coerce sequence IDs to ints except ValueError: notify_error("kafka pillow {} couldn't parse sequence ID {}. rewinding...".format( self._group_id, since )) # since kafka only keeps 7 days of data this isn't a big deal. Hopefully we will only see # these once when each pillow moves over. offset = 0 self._processed_topic_offsets = {single_topic: offset} def _make_offset_tuple(topic): if topic in self._processed_topic_offsets: return (topic, self._partition, self._processed_topic_offsets[topic]) else: return (topic, self._partition) offsets = [_make_offset_tuple(topic) for topic in self._topics] if self.strict: self._validate_offsets(offsets) # this is how you tell the consumer to start from a certain point in the sequence consumer.set_topic_partitions(*offsets) try: for message in consumer: self._processed_topic_offsets[message.topic] = message.offset yield change_from_kafka_message(message) except ConsumerTimeout: assert not forever, 'Kafka pillow should not timeout when waiting forever!'
def process_pillow_retry(error_doc): pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into.") % pillow_name_or_class) try: error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1 error_doc.save() finally: return change = error_doc.change_object delete_all_for_doc = False try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain, load_source="pillow_retry", ) change.document_store = document_store if isinstance(pillow.get_change_feed(), CouchChangeFeed): pillow.process_change(change) else: if change_metadata.data_source_type in ('couch', 'sql'): data_source_type = change_metadata.data_source_type else: # legacy metadata will have other values for non-sql # can remove this once all legacy errors have been processed data_source_type = 'sql' producer.send_change( get_topic_for_doc_type(change_metadata.document_type, data_source_type), change_metadata) delete_all_for_doc = True except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save() else: if delete_all_for_doc: PillowError.objects.filter(doc_id=error_doc.doc_id).delete() else: error_doc.delete()
def get_session_by_session_id(id): """ Utility method to first try and get a session in sql, then failing that get it in couch and log an error. """ sql_session = SQLXFormsSession.by_session_id(id) if sql_session: return sql_session couch_session = XFormsSession.by_session_id(id) if couch_session: notify_error('session {} could not be found in sql.'.format(couch_session._id)) return couch_session
def check_pillows_for_rewind(): for pillow in get_couch_pillow_instances(): checkpoint = pillow.checkpoint has_rewound, historical_seq = check_for_rewind(checkpoint) if has_rewound: notify_error( message='Found seq number lower than previous for {}. ' 'This could mean we are in a rewind state'.format(checkpoint.checkpoint_id), details={ 'pillow checkpoint seq': checkpoint.get_current_sequence_id(), 'stored seq': historical_seq } )
def run_case_update_rules_for_domain_and_db(domain, now, run_id, db=None): domain_obj = Domain.get_by_name(domain) max_allowed_updates = domain_obj.auto_case_update_limit or settings.MAX_RULE_UPDATES_IN_ONE_RUN start_run = datetime.utcnow() last_migration_check_time = None cases_checked = 0 case_update_result = CaseRuleActionResult() all_rules = list( AutomaticUpdateRule.by_domain( domain, AutomaticUpdateRule.WORKFLOW_CASE_UPDATE)) rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type( all_rules) for case_type, rules in six.iteritems(rules_by_case_type): boundary_date = AutomaticUpdateRule.get_boundary_date(rules, now) case_ids = list( AutomaticUpdateRule.get_case_ids(domain, case_type, boundary_date, db=db)) for case in CaseAccessors(domain).iter_cases(case_ids): migration_in_progress, last_migration_check_time = check_data_migration_in_progress( domain, last_migration_check_time) time_elapsed = datetime.utcnow() - start_run if (time_elapsed.seconds > HALT_AFTER or case_update_result.total_updates >= max_allowed_updates or migration_in_progress): DomainCaseRuleRun.done(run_id, DomainCaseRuleRun.STATUS_HALTED, cases_checked, case_update_result, db=db) notify_error("Halting rule run for domain %s." % domain) return case_update_result.add_result(run_rules_for_case(case, rules, now)) cases_checked += 1 run = DomainCaseRuleRun.done(run_id, DomainCaseRuleRun.STATUS_FINISHED, cases_checked, case_update_result, db=db) if run.status == DomainCaseRuleRun.STATUS_FINISHED: for rule in all_rules: AutomaticUpdateRule.objects.filter(pk=rule.pk).update(last_run=now)
def _check_es_rev(es_alias, doc_id, couch_revs): """ Specific docid and rev checker. es_alias: Elasticsearch alias doc_id: id to query in ES couch_rev: target couch_rev that you want to match """ es_interface = ElasticsearchInterface(get_es_new()) doc_id_query = { "filter": { "ids": { "values": [doc_id] } }, "fields": ["_id", "_rev"] } try: res = es_interface.search(es_alias, body=doc_id_query) status = False message = "Not in sync" if 'hits' in res: if res['hits'].get('total', 0) == 0: status = False # if doc doesn't exist it's def. not in sync message = "Not in sync %s" % es_alias elif 'hits' in res['hits']: fields = res['hits']['hits'][0]['fields'] if fields['_rev'] in couch_revs: status = True message = "%s OK" % es_alias else: status = False # less likely, but if it's there but the rev is off message = "Not in sync - %s stale" % es_alias else: status = False message = "Not in sync - query failed" notify_error("%s: %s" % (message, str(res))) except Exception as ex: message = "ES Error: %s" % ex status = False return { es_alias: { "es_alias": es_alias, "status": status, "message": message } }
def sync_couch_session_from_sql_session(sql_session): if sql_session.do_not_sync: return if not sql_session.couch_id: notify_error('Only existing sessions can be synced for now. sql session id is {}'.format(sql_session.pk)) return couch_doc = XFormsSession.get(sql_session.couch_id) for attr in SESSION_PROPERTIES_TO_SYNC: setattr(couch_doc, attr, getattr(sql_session, attr)) # don't call .save() since that will create a recursive loop of syncing XFormsSession.get_db().save_doc(couch_doc._doc)
def rebuild_tables_if_necessary(self): table_map = {t.get_table().name: t for t in self.tables} engine = self.get_sql_engine() with engine.begin() as connection: migration_context = get_migration_context(connection, table_map.keys()) diffs = compare_metadata(migration_context, metadata) tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys()) for table_name in tables_to_rebuild: table = table_map[table_name] try: self.rebuild_table(table) except TableRebuildError, e: notify_error(unicode(e))
def jserror(request): agent = request.META.get('HTTP_USER_AGENT', None) os = browser_name = browser_version = bot = TAG_UNKNOWN if agent: parsed_agent = httpagentparser.detect(agent) bot = parsed_agent.get('bot', False) if 'os' in parsed_agent: os = parsed_agent['os'].get('name', TAG_UNKNOWN) if 'browser' in parsed_agent: browser_version = parsed_agent['browser'].get( 'version', TAG_UNKNOWN) browser_name = parsed_agent['browser'].get('name', TAG_UNKNOWN) url = request.POST.get('page', None) domain = None if url: path = urlparse(url).path if path: domain = get_domain_from_url(path) domain = domain or '_unknown' metrics_counter('commcare.jserror.count', tags={ 'os': os, 'browser_version': browser_version, 'browser_name': browser_name, 'url': sanitize_url(url), 'bot': bot, 'domain': domain, }) notify_error(message=f'[JS] {request.POST.get("message")}', details={ 'message': request.POST.get('message'), 'domain': domain, 'page': url, 'file': request.POST.get('file'), 'line': request.POST.get('line'), 'stack': request.POST.get('stack'), 'meta': { 'os': os, 'browser_version': browser_version, 'browser_name': browser_name, 'bot': bot, } }) return HttpResponse('')
def iter_cases_and_run_rules(domain, case_iterator, rules, now, run_id, case_type, db=None, progress_helper=None): from corehq.apps.data_interfaces.models import ( CaseRuleActionResult, DomainCaseRuleRun, ) HALT_AFTER = 23 * 60 * 60 domain_obj = Domain.get_by_name(domain) max_allowed_updates = domain_obj.auto_case_update_limit or settings.MAX_RULE_UPDATES_IN_ONE_RUN start_run = datetime.utcnow() case_update_result = CaseRuleActionResult() cases_checked = 0 last_migration_check_time = None for case in case_iterator: migration_in_progress, last_migration_check_time = _check_data_migration_in_progress( domain, last_migration_check_time) time_elapsed = datetime.utcnow() - start_run if (time_elapsed.seconds > HALT_AFTER or case_update_result.total_updates >= max_allowed_updates or migration_in_progress): notify_error("Halting rule run for domain %s and case type %s." % (domain, case_type)) return DomainCaseRuleRun.done(run_id, cases_checked, case_update_result, db=db, halted=True) case_update_result.add_result(run_rules_for_case(case, rules, now)) if progress_helper is not None: progress_helper.increment_current_case_count() cases_checked += 1 return DomainCaseRuleRun.done(run_id, cases_checked, case_update_result, db=db)
def run_case_update_rules_for_domain(domain, now=None): domain_obj = Domain.get_by_name(domain) max_allowed_updates = domain_obj.auto_case_update_limit or settings.MAX_RULE_UPDATES_IN_ONE_RUN now = now or datetime.utcnow() start_run = datetime.utcnow() last_migration_check_time = None run_record = DomainCaseRuleRun.objects.create( domain=domain, started_on=start_run, status=DomainCaseRuleRun.STATUS_RUNNING, ) cases_checked = 0 case_update_result = CaseRuleActionResult() all_rules = AutomaticUpdateRule.by_domain( domain, AutomaticUpdateRule.WORKFLOW_CASE_UPDATE) rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type( all_rules) for case_type, rules in six.iteritems(rules_by_case_type): boundary_date = AutomaticUpdateRule.get_boundary_date(rules, now) case_ids = list( AutomaticUpdateRule.get_case_ids(domain, case_type, boundary_date)) for case in CaseAccessors(domain).iter_cases(case_ids): migration_in_progress, last_migration_check_time = check_data_migration_in_progress( domain, last_migration_check_time) time_elapsed = datetime.utcnow() - start_run if (time_elapsed.seconds > HALT_AFTER or case_update_result.total_updates >= max_allowed_updates or migration_in_progress): run_record.done(DomainCaseRuleRun.STATUS_HALTED, cases_checked, case_update_result) notify_error("Halting rule run for domain %s." % domain) return case_update_result.add_result(run_rules_for_case(case, rules, now)) cases_checked += 1 for rule in rules: rule.last_run = now rule.save() run_record.done(DomainCaseRuleRun.STATUS_FINISHED, cases_checked, case_update_result)
def rebuild_tables_if_necessary(self): tables_by_engine = defaultdict(dict) for adapter in self.table_adapters: tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter for engine_id, table_map in tables_by_engine.items(): engine = connection_manager.get_engine(engine_id) with engine.begin() as connection: migration_context = get_migration_context(connection, table_map.keys()) diffs = compare_metadata(migration_context, metadata) tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys()) for table_name in tables_to_rebuild: table = table_map[table_name] try: self.rebuild_table(table) except TableRebuildError, e: notify_error(unicode(e))
def _check_es_rev(index, doc_id, couch_revs): """ Specific docid and rev checker. index: Elasticsearch index doc_id: id to query in ES couch_rev: target couch_rev that you want to match """ es = get_es_new() doc_id_query = { "filter": { "ids": { "values": [doc_id] } }, "fields": ["_id", "_rev"] } try: res = es.search(index, body=doc_id_query) status = False message = "Not in sync" if res.has_key('hits'): if res['hits'].get('total', 0) == 0: status = False # if doc doesn't exist it's def. not in sync message = "Not in sync %s" % index elif 'hits' in res['hits']: fields = res['hits']['hits'][0]['fields'] if fields['_rev'] in couch_revs: status = True message = "%s OK" % index else: status = False # less likely, but if it's there but the rev is off message = "Not in sync - %s stale" % index else: status = False message = "Not in sync - query failed" notify_error("%s: %s" % (message, str(res))) except Exception, ex: message = "ES Error: %s" % ex status = False
def change_from_kafka_message(message): change_meta = change_meta_from_kafka_message(message.value) try: document_store = get_document_store( data_source_type=change_meta.data_source_type, data_source_name=change_meta.data_source_name, domain=change_meta.domain) except UnknownDocumentStore: document_store = None notify_error("Unknown document store: {}".format( change_meta.data_source_type)) return Change( id=change_meta.document_id, sequence_id=message.offset, document=None, deleted=change_meta.is_deletion, metadata=change_meta, document_store=document_store, )
def change_from_kafka_message(message): change_meta = change_meta_from_kafka_message(message.value) try: document_store = get_document_store( data_source_type=change_meta.data_source_type, data_source_name=change_meta.data_source_name, domain=change_meta.domain ) except UnknownDocumentStore: document_store = None notify_error("Unknown document store: {}".format(change_meta.data_source_type)) return Change( id=change_meta.document_id, sequence_id=message.offset, document=None, deleted=change_meta.is_deletion, metadata=change_meta, document_store=document_store, )
def _check_es_rev(index, doc_id, couch_revs): """ Specific docid and rev checker. index: rawes index doc_id: id to query in ES couch_rev: target couch_rev that you want to match """ es = get_es() doc_id_query = { "filter": { "ids": {"values": [doc_id]} }, "fields": ["_id", "_rev"] } try: res = es[index].get('_search', data=doc_id_query) status = False message = "Not in sync" if res.has_key('hits'): if res['hits'].get('total', 0) == 0: status = False # if doc doesn't exist it's def. not in sync message = "Not in sync %s" % index elif 'hits' in res['hits']: fields = res['hits']['hits'][0]['fields'] if fields['_rev'] in couch_revs: status = True message = "%s OK" % index else: status = False # less likely, but if it's there but the rev is off message = "Not in sync - %s stale" % index else: status = False message = "Not in sync - query failed" notify_error("%s: %s" % (message, str(res))) except Exception, ex: message = "ES Error: %s" % ex status = False
def rebuild_tables_if_necessary(self): tables_by_engine = defaultdict(dict) for adapter in self.table_adapters: tables_by_engine[adapter.engine_id][ adapter.get_table().name] = adapter for engine_id, table_map in tables_by_engine.items(): engine = connection_manager.get_engine(engine_id) with engine.begin() as connection: migration_context = get_migration_context( connection, table_map.keys()) diffs = compare_metadata(migration_context, metadata) tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys()) for table_name in tables_to_rebuild: table = table_map[table_name] try: self.rebuild_table(table) except TableRebuildError, e: notify_error(unicode(e))
def run_case_update_rules_for_domain_and_db(domain, now, run_id, db=None): domain_obj = Domain.get_by_name(domain) max_allowed_updates = domain_obj.auto_case_update_limit or settings.MAX_RULE_UPDATES_IN_ONE_RUN start_run = datetime.utcnow() last_migration_check_time = None cases_checked = 0 case_update_result = CaseRuleActionResult() all_rules = list(AutomaticUpdateRule.by_domain(domain, AutomaticUpdateRule.WORKFLOW_CASE_UPDATE)) rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type(all_rules) for case_type, rules in six.iteritems(rules_by_case_type): boundary_date = AutomaticUpdateRule.get_boundary_date(rules, now) case_ids = list(AutomaticUpdateRule.get_case_ids(domain, case_type, boundary_date, db=db)) for case in CaseAccessors(domain).iter_cases(case_ids): migration_in_progress, last_migration_check_time = check_data_migration_in_progress(domain, last_migration_check_time) time_elapsed = datetime.utcnow() - start_run if ( time_elapsed.seconds > HALT_AFTER or case_update_result.total_updates >= max_allowed_updates or migration_in_progress ): DomainCaseRuleRun.done(run_id, DomainCaseRuleRun.STATUS_HALTED, cases_checked, case_update_result, db=db) notify_error("Halting rule run for domain %s." % domain) return case_update_result.add_result(run_rules_for_case(case, rules, now)) cases_checked += 1 run = DomainCaseRuleRun.done(run_id, DomainCaseRuleRun.STATUS_FINISHED, cases_checked, case_update_result, db=db) if run.status == DomainCaseRuleRun.STATUS_FINISHED: for rule in all_rules: AutomaticUpdateRule.objects.filter(pk=rule.pk).update(last_run=now)
def _post_data(data, user_id): if not data.get("domain"): raise ValueError("Expected domain") if not user_id: notify_exception( None, "Making smsforms request w/o user_id. Will result in non-sticky session.", details={ 'session-id': data.get('session-id'), }) data = _get_formplayer_session_data(data) data_bytes = json.dumps(data).encode('utf-8') response = requests.post( url="{}/{}".format(get_formplayer_url(), data["action"]), data=data_bytes, headers={ "Content-Type": "application/json", "content-length": str(len(data_bytes)), "X-MAC-DIGEST": get_hmac_digest(settings.FORMPLAYER_INTERNAL_AUTH_KEY, data_bytes), "X-FORMPLAYER-SESSION": user_id, }, ) if response.status_code == 404: raise Http404(response.reason) if 500 <= response.status_code < 600: http_error_msg = '{} Server Error: {} for url: {}'.format( response.status_code, response.reason, response.url) notify_error(http_error_msg, details={ 'response': response, 'body': response.text }) raise HTTPError(http_error_msg, response=response) return response.json()
def create_tasks(self): survey_sessions_due_for_action = self.get_survey_sessions_due_for_action( ) all_open_session_ids = self.get_open_session_ids() for domain, connection_id, session_id, current_action_due, phone_number in survey_sessions_due_for_action: if skip_domain(domain): continue if toggles.ONE_PHONE_NUMBER_MULTIPLE_CONTACTS.enabled(domain): fake_session = SQLXFormsSession( session_id=session_id, connection_id=connection_id, phone_number=phone_number, ) if not XFormsSessionSynchronization.channel_is_available_for_session( fake_session): running_session_info = XFormsSessionSynchronization.get_running_session_info_for_channel( fake_session.get_channel()) # First confirm the supposedly running session is even open # and if it's not (should be exceedingly rare) release it and act like it wasn't there if running_session_info.session_id \ and running_session_info.session_id not in all_open_session_ids: notify_error( "The supposedly running session was not open and was released. " "No known way for this to happen, so worth investigating.", details={ 'running_session_info': running_session_info }) XFormsSessionSynchronization.clear_stale_channel_claim( fake_session.get_channel()) # This is the 99% case: there's a running session for the channel # so leave this session/action in the queue for later and move on to the next one else: continue enqueue_lock = self.get_enqueue_lock(session_id, current_action_due) if enqueue_lock.acquire(blocking=False): handle_due_survey_action.delay(domain, connection_id, session_id)
def report_formplayer_error(request, domain): data = json.loads(request.body) error_type = data.get('type') with sentry_sdk.configure_scope() as scope: scope.set_tag("cloudcare_error_type", error_type) if error_type == 'webformsession_request_failure': metrics_counter('commcare.formplayer.webformsession_request_failure', tags={ 'request': data.get('request'), 'statusText': data.get('statusText'), 'state': data.get('state'), 'status': data.get('status'), 'domain': domain, 'cloudcare_env': data.get('cloudcareEnv'), }) message = data.get("readableErrorMessage") or "request failure in web form session" thread_topic = _message_to_sentry_thread_topic(message) notify_error(message=f'[Cloudcare] {thread_topic}', details=data) elif error_type == 'show_error_notification': message = data.get('message') thread_topic = _message_to_sentry_thread_topic(message) metrics_counter('commcare.formplayer.show_error_notification', tags={ 'message': _message_to_tag_value(message or 'no_message'), 'domain': domain, 'cloudcare_env': data.get('cloudcareEnv'), }) notify_error(message=f'[Cloudcare] {thread_topic}', details=data) else: metrics_counter('commcare.formplayer.unknown_error_type', tags={ 'domain': domain, 'cloudcare_env': data.get('cloudcareEnv'), }) notify_error(message=f'[Cloudcare] unknown error type', details=data) return JsonResponse({'status': 'ok'})
def report_formplayer_error(request, domain): data = json.loads(request.body) error_type = data.get('type') if error_type == 'webformsession_request_failure': metrics_counter('commcare.formplayer.webformsession_request_failure', tags={ 'request': data.get('request'), 'statusText': data.get('statusText'), 'state': data.get('state'), 'status': data.get('status'), 'domain': domain, }) notify_error(message='Formplayer: request failure in web form session', details=data) elif error_type == 'show_error_notification': message = data.get('message') metrics_counter('commcare.formplayer.show_error_notification', tags={ 'message': _message_to_tag_value(message or 'no_message'), 'domain': domain, }) notify_error(message=f'Formplayer: showed error to user: {message}', details=data) else: metrics_counter('commcare.formplayer.unknown_error_type', tags={ 'domain': domain, }) notify_error(message=f'Formplayer: unknown error type', details=data) return JsonResponse({'status': 'ok'})
def process_pillow_retry(error_doc_id): try: error_doc = PillowError.objects.get(id=error_doc_id) except PillowError.DoesNotExist: return pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into.") % pillow_name_or_class) try: error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1 error_doc.save() finally: return change = error_doc.change_object try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain) change.document_store = document_store pillow.process_change(change) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save() else: error_doc.delete()
def send_first_message(domain, recipient, phone_entry_or_number, session, responses, logged_subevent, workflow): # This try/except section is just here (temporarily) to support future refactors # If any of these notify, they should be replaced with a comment as to why the two are different # so that someone refactoring in the future will know that this or that param is necessary. try: if session.workflow != workflow: # see if we can eliminate the workflow arg notify_error('Exploratory: session.workflow != workflow', details={ 'session.workflow': session.workflow, 'workflow': workflow}) if session.connection_id != recipient.get_id: # see if we can eliminate the recipient arg notify_error('Exploratory: session.connection_id != recipient.get_id', details={ 'session.connection_id': session.connection_id, 'recipient.get_id': recipient.get_id, 'recipient': recipient }) if session.related_subevent != logged_subevent: # see if we can eliminate the logged_subevent arg notify_error('Exploratory: session.related_subevent != logged_subevent', details={ 'session.connection_id': session.connection_id, 'logged_subevent': logged_subevent}) except Exception: # The above running is not mission critical, so if it errors just leave a message in the log # for us to follow up on. # Absence of the message below and messages above ever notifying # will indicate that we can remove these args. notify_exception(None, "Error in section of code that's just supposed help inform future refactors") if toggles.ONE_PHONE_NUMBER_MULTIPLE_CONTACTS.enabled(domain): if not XFormsSessionSynchronization.claim_channel_for_session(session): send_first_message.apply_async( args=(domain, recipient, phone_entry_or_number, session, responses, logged_subevent, workflow), countdown=60 ) return metrics_counter('commcare.smsforms.session_started', 1, tags={'domain': domain, 'workflow': workflow}) if len(responses) > 0: message = format_message_list(responses) metadata = MessageMetadata( workflow=workflow, xforms_session_couch_id=session.couch_id, ) if isinstance(phone_entry_or_number, PhoneNumber): send_sms_to_verified_number( phone_entry_or_number, message, metadata, logged_subevent=logged_subevent ) else: send_sms( domain, recipient, phone_entry_or_number, message, metadata ) logged_subevent.completed()
def run_case_update_rules_for_domain(domain, now=None): now = now or datetime.utcnow() start_run = datetime.utcnow() run_record = DomainCaseRuleRun.objects.create( domain=domain, started_on=start_run, status=DomainCaseRuleRun.STATUS_RUNNING, ) cases_checked = 0 case_update_result = CaseRuleActionResult() all_rules = AutomaticUpdateRule.by_domain(domain) rules_by_case_type = AutomaticUpdateRule.organize_rules_by_case_type(all_rules) for case_type, rules in rules_by_case_type.iteritems(): boundary_date = AutomaticUpdateRule.get_boundary_date(rules, now) case_id_chunks = AutomaticUpdateRule.get_case_ids(domain, case_type, boundary_date) for case_ids in case_id_chunks: for case in CaseAccessors(domain).iter_cases(case_ids): time_elapsed = datetime.utcnow() - start_run if ( time_elapsed.seconds > HALT_AFTER or case_update_result.total_updates >= settings.MAX_RULE_UPDATES_IN_ONE_RUN ): run_record.done(DomainCaseRuleRun.STATUS_HALTED, cases_checked, case_update_result) notify_error("Halting rule run for domain %s." % domain) return case_update_result.add_result(run_rules_for_case(case, rules, now)) cases_checked += 1 for rule in rules: rule.last_run = now rule.save() run_record.done(DomainCaseRuleRun.STATUS_FINISHED, cases_checked, case_update_result)
def iter_changes(self, since, forever): """ Since can either be an integer (for single topic change feeds) or a dict of topics to integers (for multiple topic change feeds) """ # a special value of since=None will start from the end of the change stream # in milliseconds, -1 means wait forever for changes timeout = -1 if forever else MIN_TIMEOUT reset = 'smallest' if since is not None else 'largest' consumer = self._get_consumer(timeout, auto_offset_reset=reset) if since is not None: if isinstance(since, dict): # multiple topics offsets = [(topic, self._partition, offset) for topic, offset in since.items()] else: # single topic topic = self._get_single_topic_or_fail() try: offset = int(since) # coerce sequence IDs to ints except ValueError: notify_error("kafka pillow {} couldn't parse sequence ID {}. rewinding...".format( self._group_id, since )) # since kafka only keeps 7 days of data this isn't a big deal. Hopefully we will only see # these once when each pillow moves over. offset = 0 offsets = [(topic, self._partition, offset)] # this is how you tell the consumer to start from a certain point in the sequence consumer.set_topic_partitions(*offsets) try: for message in consumer: yield change_from_kafka_message(message) except ConsumerTimeout: assert not forever, 'Kafka pillow should not timeout when waiting forever!'
def process_pillow_retry(error_doc_id): # Redis error logged in get_redis_client try: client = cache_core.get_redis_client() except cache_core.RedisClientError: return # Prevent more than one task from processing this error, just in case # it got enqueued twice. lock = client.lock( "pillow-retry-processing-%s" % error_doc_id, timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT*60 ) if lock.acquire(blocking=False): try: error_doc = PillowError.objects.get(id=error_doc_id) except PillowError.DoesNotExist: return pillow_class = error_doc.pillow try: pillow = get_pillow_instance(pillow_class) except ValueError: # all fluff pillows have module path of 'fluff' so can't be imported directly _, pillow_class_name = pillow_class.rsplit('.', 1) try: pillow = get_pillow_by_name(pillow_class_name) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into." ) % pillow_class) try: error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1 error_doc.save() finally: release_lock(lock, True) return change = error_doc.change_object if pillow.include_docs: try: change.set_document(pillow.get_couch_db().open_doc(change.id)) except ResourceNotFound: change.deleted = True try: try: from corehq.apps.userreports.pillow import ConfigurableIndicatorPillow if isinstance(pillow, ConfigurableIndicatorPillow): raise Exception('this is temporarily not supported!') except ImportError: pass pillow.process_change(change, is_retry_attempt=True) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.queued = False error_doc.save() else: error_doc.delete() finally: release_lock(lock, True)
def send_HTML_email(subject, recipient, html_content, text_content=None, cc=None, email_from=settings.DEFAULT_FROM_EMAIL, file_attachments=None, bcc=None): recipient = list(recipient) if not isinstance(recipient, basestring) else [ recipient ] if not text_content: text_content = getattr(settings, 'NO_HTML_EMAIL_MESSAGE', NO_HTML_EMAIL_MESSAGE) # this is a temporary spam-catcher, to be removed after fb#178059 is resolved if '*****@*****.**' in recipient: notify_error( "Found an email causing spammy emails to " "[email protected]. Here's the HTML content of email" "\n {}".format(html_content)) from_header = {'From': email_from} # From-header connection = get_connection() msg = EmailMultiAlternatives(subject, text_content, email_from, recipient, headers=from_header, connection=connection, cc=cc, bcc=bcc) for file in (file_attachments or []): if file: msg.attach(file["title"], file["file_obj"].getvalue(), file["mimetype"]) msg.attach_alternative(html_content, "text/html") try: msg.send() except SMTPSenderRefused as e: error_subject = _('ERROR: Could not send "%(subject)s"') % { 'subject': subject, } if e.smtp_code == 552: error_text = _('Could not send email: file size is too large.') else: error_text = e.smtp_error error_text = '%s\n\n%s' % ( error_text, _('Please contact %(support_email)s for assistance.') % { 'support_email': settings.SUPPORT_EMAIL, }, ) error_msg = EmailMultiAlternatives( error_subject, error_text, email_from, recipient, headers=from_header, connection=connection, cc=cc, bcc=bcc, ) error_msg.send()
def send_HTML_email(subject, recipient, html_content, text_content=None, cc=None, email_from=settings.DEFAULT_FROM_EMAIL, file_attachments=None, bcc=None, ga_track=False, ga_tracking_info=None): recipient = list(recipient) if not isinstance(recipient, basestring) else [recipient] if not text_content: text_content = getattr(settings, 'NO_HTML_EMAIL_MESSAGE', NO_HTML_EMAIL_MESSAGE) # this is a temporary spam-catcher, to be removed after fb#178059 is resolved if '*****@*****.**' in recipient: notify_error("Found an email causing spammy emails to " "[email protected]. Here's the HTML content of email" "\n {}".format(html_content) ) if ga_track and settings.ANALYTICS_IDS.get('GOOGLE_ANALYTICS_API_ID'): ga_data = { 'v': 1, 'tid': settings.ANALYTICS_IDS.get('GOOGLE_ANALYTICS_API_ID'), 'cid': uuid.uuid4().hex, 'dt': subject.encode('utf-8'), 't': 'event', 'ec': 'email' } extra_data = ga_tracking_info if ga_tracking_info else {} ga_data.update(extra_data) post_data = urlencode(ga_data) url = "https://www.google-analytics.com/collect?" + post_data new_content = '<img src="{url}&ea=open"/>\n</body>'.format(url=url) html_content, count = re.subn(r'(.*)</body>', r'\1'+new_content, html_content) assert count != 0, 'Attempted to add tracking to HTML Email with no closing body tag' from_header = {'From': email_from} # From-header connection = get_connection() msg = EmailMultiAlternatives(subject, text_content, email_from, recipient, headers=from_header, connection=connection, cc=cc, bcc=bcc) for file in (file_attachments or []): if file: msg.attach(file["title"], file["file_obj"].getvalue(), file["mimetype"]) msg.attach_alternative(html_content, "text/html") try: msg.send() except SMTPSenderRefused as e: error_subject = _('ERROR: Could not send "%(subject)s"') % { 'subject': subject, } if e.smtp_code == 552: error_text = _('Could not send email: file size is too large.') else: error_text = e.smtp_error error_text = '%s\n\n%s' % ( error_text, _('Please contact %(support_email)s for assistance.') % { 'support_email': settings.SUPPORT_EMAIL, }, ) error_msg = EmailMultiAlternatives( error_subject, error_text, email_from, recipient, headers=from_header, connection=connection, cc=cc, bcc=bcc, ) error_msg.send() if ga_track and settings.ANALYTICS_IDS.get('GOOGLE_ANALYTICS_API_ID'): try: try: requests.get(url + "&ea=send") except SSLError: # if we get an ssl error try without verification requests.get(url + "&ea=send", verify=False) except Exception as e: # never fail hard on analytics logging.exception(u'Unable to send google analytics request for tracked email: {}'.format(e))
def process_pillow_retry(error_doc_id): # Redis error logged in get_redis_client try: client = cache_core.get_redis_client() except cache_core.RedisClientError: return # Prevent more than one task from processing this error, just in case # it got enqueued twice. lock = client.lock( "pillow-retry-processing-%s" % error_doc_id, timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT*60 ) if lock.acquire(blocking=False): try: error_doc = PillowError.objects.get(id=error_doc_id) except PillowError.DoesNotExist: release_lock(lock, True) return pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: if not settings.UNIT_TESTING: _assert = soft_assert(to='@'.join(['czue', 'dimagi.com'])) _assert(False, 'Pillow retry {} is still using legacy class {}'.format( error_doc.pk, pillow_name_or_class )) pillow = _try_legacy_import(pillow_name_or_class) if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into." ) % pillow_name_or_class) try: error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1 error_doc.save() finally: release_lock(lock, True) return change = error_doc.change_object if getattr(pillow, 'include_docs', False): try: change.set_document(pillow.get_couch_db().open_doc(change.id)) except ResourceNotFound: change.deleted = True try: try: from corehq.apps.userreports.pillow import ConfigurableReportKafkaPillow if isinstance(pillow, ConfigurableReportKafkaPillow): raise Exception('this is temporarily not supported!') except ImportError: pass pillow.process_change(change, is_retry_attempt=True) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.queued = False error_doc.save() else: error_doc.delete() finally: release_lock(lock, True)
def send_HTML_email(subject, recipient, html_content, text_content=None, cc=None, email_from=settings.DEFAULT_FROM_EMAIL, file_attachments=None, bcc=None, ga_track=False, ga_tracking_info=None): recipient = list(recipient) if not isinstance(recipient, basestring) else [ recipient ] if not text_content: text_content = getattr(settings, 'NO_HTML_EMAIL_MESSAGE', NO_HTML_EMAIL_MESSAGE) # this is a temporary spam-catcher, to be removed after fb#178059 is resolved if '*****@*****.**' in recipient: notify_error( "Found an email causing spammy emails to " "[email protected]. Here's the HTML content of email" "\n {}".format(html_content)) if ga_track and settings.ANALYTICS_IDS.get('GOOGLE_ANALYTICS_API_ID'): ga_data = { 'v': 1, 'tid': settings.ANALYTICS_IDS.get('GOOGLE_ANALYTICS_API_ID'), 'cid': uuid.uuid4().hex, 'dt': subject.encode('utf-8'), 't': 'event', 'ec': 'email' } extra_data = ga_tracking_info if ga_tracking_info else {} ga_data.update(extra_data) post_data = urlencode(ga_data) url = "https://www.google-analytics.com/collect?" + post_data new_content = '<img src="{url}&ea=open"/>\n</body>'.format(url=url) html_content, count = re.subn(r'(.*)</body>', r'\1' + new_content, html_content) assert count != 0, 'Attempted to add tracking to HTML Email with no closing body tag' from_header = {'From': email_from} # From-header connection = get_connection() msg = EmailMultiAlternatives(subject, text_content, email_from, recipient, headers=from_header, connection=connection, cc=cc, bcc=bcc) for file in (file_attachments or []): if file: msg.attach(file["title"], file["file_obj"].getvalue(), file["mimetype"]) msg.attach_alternative(html_content, "text/html") try: msg.send() except SMTPSenderRefused as e: error_subject = _('ERROR: Could not send "%(subject)s"') % { 'subject': subject, } if e.smtp_code == 552: error_text = _('Could not send email: file size is too large.') else: error_text = e.smtp_error error_text = '%s\n\n%s' % ( error_text, _('Please contact %(support_email)s for assistance.') % { 'support_email': settings.SUPPORT_EMAIL, }, ) error_msg = EmailMultiAlternatives( error_subject, error_text, email_from, recipient, headers=from_header, connection=connection, cc=cc, bcc=bcc, ) error_msg.send() if ga_track and settings.ANALYTICS_IDS.get('GOOGLE_ANALYTICS_API_ID'): try: try: requests.get(url + "&ea=send") except SSLError: # if we get an ssl error try without verification requests.get(url + "&ea=send", verify=False) except Exception as e: # never fail hard on analytics logging.exception( u'Unable to send google analytics request for tracked email: {}' .format(e))