def testSMSLogSync(self):
prev_couch_count = self.getSMSLogCount()
prev_sql_count = self.getSMSCount()
# Test Create
smslog = SMSLog()
self.setRandomSMSLogValues(smslog)
smslog.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), prev_couch_count + 1)
self.assertEqual(self.getSMSCount(), prev_sql_count + 1)
sms = SMS.objects.get(couch_id=smslog._id)
self.checkFieldValues(smslog, sms, SMSLog._migration_get_fields())
self.assertTrue(SMSLog.get_db().get_rev(smslog._id).startswith('1-'))
# Test Update
self.setRandomSMSLogValues(smslog)
smslog.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), prev_couch_count + 1)
self.assertEqual(self.getSMSCount(), prev_sql_count + 1)
sms = SMS.objects.get(couch_id=smslog._id)
self.checkFieldValues(smslog, sms, SMSLog._migration_get_fields())
self.assertTrue(SMSLog.get_db().get_rev(smslog._id).startswith('2-'))
def post(request, domain):
# TODO: Figure out if this is being used anywhere and remove it if not
"""
We assume sms sent to HQ will come in the form
http://hqurl.com?username=%(username)s&password=%(password)s&id=%(phone_number)s&text=%(message)s
"""
text = request.REQUEST.get('text', '')
to = request.REQUEST.get('id', '')
username = request.REQUEST.get('username', '')
# ah, plaintext passwords....
# this seems to be the most common API that a lot of SMS gateways expose
password = request.REQUEST.get('password', '')
if not text or not to or not username or not password:
error_msg = 'ERROR missing parameters. Received: %(1)s, %(2)s, %(3)s, %(4)s' % \
( text, to, username, password )
logging.error(error_msg)
return HttpResponseBadRequest(error_msg)
user = authenticate(username=username, password=password)
if user is None or not user.is_active:
return HttpResponseBadRequest("Authentication fail")
msg = SMSLog(
domain=domain,
# TODO: how to map phone numbers to recipients, when phone numbers are shared?
#couch_recipient=id,
phone_number=to,
direction=INCOMING,
date=datetime.now(),
text=text)
msg.save()
return HttpResponse('OK')
def send_sms(domain, contact, phone_number, text, metadata=None):
"""
Sends an outbound SMS. Returns false if it fails.
"""
if phone_number is None:
return False
if isinstance(phone_number, int) or isinstance(phone_number, long):
phone_number = str(phone_number)
phone_number = clean_phone_number(phone_number)
msg = SMSLog(
domain=domain,
phone_number=phone_number,
direction=OUTGOING,
date = datetime.utcnow(),
backend_id=None,
location_id=get_location_id_by_contact(domain, contact),
text = text
)
if contact:
msg.couch_recipient = contact._id
msg.couch_recipient_doc_type = contact.doc_type
add_msg_tags(msg, metadata)
return queue_outgoing_sms(msg)
def message_log_report(request):
show_dates = True
datespan = request.datespan
domains = Domain.get_all()
for dom in domains:
dom.sms_incoming = SMSLog.count_incoming_by_domain(dom.name, datespan.startdate_param, datespan.enddate_param)
dom.sms_outgoing = SMSLog.count_outgoing_by_domain(dom.name, datespan.startdate_param, datespan.enddate_param)
dom.sms_total = SMSLog.count_by_domain(dom.name, datespan.startdate_param, datespan.enddate_param)
context = get_hqadmin_base_context(request)
headers = DataTablesHeader(
DataTablesColumn("Domain"),
DataTablesColumn("Incoming Messages", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Outgoing Messages", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Total Messages", sort_type=DTSortType.NUMERIC),
)
context["headers"] = headers
context["aoColumns"] = headers.render_aoColumns
context.update({"domains": domains, "show_dates": show_dates, "datespan": datespan})
context["layout_flush_content"] = True
return render(request, "hqadmin/message_log_report.html", context)
def send_sms_to_verified_number(verified_number, text):
"""
Sends an sms using the given verified phone number entry.
verified_number The VerifiedNumber entry to use when sending.
text The text of the message to send.
return True on success, False on failure
"""
try:
backend = verified_number.backend
module = __import__(backend.outbound_module, fromlist=["send"])
kwargs = backend.outbound_params
msg = SMSLog(
couch_recipient_doc_type = verified_number.owner_doc_type,
couch_recipient = verified_number.owner_id,
phone_number = "+" + str(verified_number.phone_number),
direction = OUTGOING,
date = datetime.utcnow(),
domain = verified_number.domain,
text = text
)
try:
msg.backend_api = module.API_ID
except Exception:
pass
module.send(msg, **kwargs)
msg.save()
return True
except Exception as e:
logging.exception("Exception while sending SMS to VerifiedNumber id " + verified_number._id)
return False
def send_sms(domain, id, phone_number, text):
"""
Sends an outbound SMS. Returns false if it fails.
"""
if phone_number is None:
return False
if isinstance(phone_number, int) or isinstance(phone_number, long):
phone_number = str(phone_number)
logging.debug('Sending message: %s' % text)
phone_number = clean_phone_number(phone_number)
msg = SMSLog(domain=domain,
couch_recipient=id,
couch_recipient_doc_type="CouchUser",
phone_number=phone_number,
direction=OUTGOING,
date = datetime.utcnow(),
text = text)
try:
api = get_backend_api(msg)
try:
msg.backend_api = api.API_ID
except Exception:
pass
api.send(msg)
msg.save()
return True
except Exception:
logging.exception("Problem sending SMS to %s" % phone_number)
return False
def post(request, domain):
# TODO: Figure out if this is being used anywhere and remove it if not
"""
We assume sms sent to HQ will come in the form
http://hqurl.com?username=%(username)s&password=%(password)s&id=%(phone_number)s&text=%(message)s
"""
text = request.REQUEST.get('text', '')
to = request.REQUEST.get('id', '')
username = request.REQUEST.get('username', '')
# ah, plaintext passwords....
# this seems to be the most common API that a lot of SMS gateways expose
password = request.REQUEST.get('password', '')
if not text or not to or not username or not password:
error_msg = 'ERROR missing parameters. Received: %(1)s, %(2)s, %(3)s, %(4)s' % \
( text, to, username, password )
logging.error(error_msg)
return HttpResponseBadRequest(error_msg)
user = authenticate(username=username, password=password)
if user is None or not user.is_active:
return HttpResponseBadRequest("Authentication fail")
msg = SMSLog(domain=domain,
# TODO: how to map phone numbers to recipients, when phone numbers are shared?
#couch_recipient=id,
phone_number=to,
direction=INCOMING,
date = datetime.now(),
text = text)
msg.save()
return HttpResponse('OK')
def message_log_report(request):
show_dates = True
datespan = request.datespan
domains = Domain.get_all()
for dom in domains:
dom.sms_incoming = SMSLog.count_incoming_by_domain(
dom.name, datespan.startdate_param, datespan.enddate_param)
dom.sms_outgoing = SMSLog.count_outgoing_by_domain(
dom.name, datespan.startdate_param, datespan.enddate_param)
dom.sms_total = SMSLog.count_by_domain(dom.name,
datespan.startdate_param,
datespan.enddate_param)
context = get_hqadmin_base_context(request)
headers = DataTablesHeader(
DataTablesColumn("Domain"),
DataTablesColumn("Incoming Messages", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Outgoing Messages", sort_type=DTSortType.NUMERIC),
DataTablesColumn("Total Messages", sort_type=DTSortType.NUMERIC))
context["headers"] = headers
context["aoColumns"] = headers.render_aoColumns
context.update({
"domains": domains,
"show_dates": show_dates,
"datespan": datespan
})
context['layout_flush_content'] = True
return render(request, "hqadmin/message_log_report.html", context)
def send_sms(domain, contact, phone_number, text, metadata=None):
"""
Sends an outbound SMS. Returns false if it fails.
"""
if phone_number is None:
return False
if isinstance(phone_number, int) or isinstance(phone_number, long):
phone_number = str(phone_number)
phone_number = clean_phone_number(phone_number)
msg = SMSLog(
domain=domain,
phone_number=phone_number,
direction=OUTGOING,
date = datetime.utcnow(),
backend_id=None,
text = text
)
if contact:
msg.couch_recipient = contact._id
msg.couch_recipient_doc_type = contact.doc_type
add_msg_tags(msg, metadata)
def onerror():
logging.exception("Problem sending SMS to %s" % phone_number)
return queue_outgoing_sms(msg, onerror=onerror)
def send_sms(domain, contact, phone_number, text, metadata=None):
"""
Sends an outbound SMS. Returns false if it fails.
"""
if phone_number is None:
return False
if isinstance(phone_number, int) or isinstance(phone_number, long):
phone_number = str(phone_number)
phone_number = clean_phone_number(phone_number)
msg = SMSLog(domain=domain,
phone_number=phone_number,
direction=OUTGOING,
date=datetime.utcnow(),
backend_id=None,
text=text)
if contact:
msg.couch_recipient = contact._id
msg.couch_recipient_doc_type = contact.doc_type
add_msg_tags(msg, metadata)
def onerror():
logging.exception("Problem sending SMS to %s" % phone_number)
return queue_outgoing_sms(msg, onerror=onerror)
def handle(self, *args, **options):
num_sms = 0
start_datetime = datetime.datetime(*str_to_int_tuple(args[0:6]))
end_datetime = datetime.datetime(*str_to_int_tuple(args[6:12]))
for domain in Domain.get_all():
key = [domain.name, 'SMSLog']
sms_docs = SMSLog.get_db().view('sms/by_domain',
reduce=False,
startkey=key + [start_datetime.isoformat()],
endkey=key + [end_datetime.isoformat(), {}],
)
for sms_doc in sms_docs:
sms_log = SMSLog.get(sms_doc['id'])
if options.get('create', False):
SmsBillable.create(sms_log)
print 'Created billable for SMSLog %s in domain %s from %s' \
% (sms_doc['id'], domain.name, sms_log.date)
else:
print 'Found SMSLog %s in domain %s from %s' \
% (sms_doc['id'], domain.name, sms_log.date)
num_sms += 1
print 'Number of SMSs in datetime range: %d' % num_sms
def handle(self, *args, **options):
billables_created = 0
for domain in Domain.get_all():
key = [domain.name, 'SMSLog']
start_date = [datetime.datetime(2014, 1, 1).isoformat()]
end_date = [datetime.datetime(2014, 1, 24).isoformat()]
sms_docs = SMSLog.get_db().view('sms/by_domain',
reduce=False,
startkey=key + start_date,
endkey=key + end_date + [{}])
for sms_doc in sms_docs:
sms_log = SMSLog.get(sms_doc['id'])
try:
if sms_log.phone_number is not None:
parse_phone_number(sms_log.phone_number)
except PhoneNumberParseException:
billables = SmsBillable.objects.filter(log_id=sms_log._id)
if len(billables) == 0:
SmsBillable.create(sms_log)
billables_created += 1
print 'created SmsBillable for invalid number %s in domain %s, id=%s'\
% (sms_log.phone_number, domain.name, sms_log._id)
elif len(billables) > 1:
print "Warning: >1 SmsBillable exists for SMSLog with id=%" % sms_log._id
print 'Number of SmsBillables created: %d' % billables_created
print 'Completed retrobilling.'
def _get_fake_sms(self, text):
msg = SMSLog(domain=self.domain,
phone_number='+16175555454',
direction=OUTGOING,
date=datetime.utcnow(),
backend_id=self.mobile_backend.get_id,
text=text)
msg.save()
return msg
def incoming(phone_number, text, backend_api, timestamp=None, domain_scope=None, backend_message_id=None, delay=True):
"""
entry point for incoming sms
phone_number - originating phone number
text - message content
backend_api - backend API ID of receiving sms backend
timestamp - message received timestamp; defaults to now (UTC)
domain_scope - if present, only messages from phone numbers that can be
definitively linked to this domain will be processed; others will be
dropped (useful to provide security when simulating incoming sms)
"""
# Log message in message log
phone_number = clean_phone_number(phone_number)
msg = SMSLog(
phone_number = phone_number,
direction = INCOMING,
date = timestamp or datetime.utcnow(),
text = text,
domain_scope = domain_scope,
backend_api = backend_api,
backend_message_id = backend_message_id,
)
if settings.SMS_QUEUE_ENABLED:
msg.processed = False
msg.datetime_to_process = datetime.utcnow()
msg.queued_timestamp = msg.datetime_to_process
msg.save()
enqueue_directly(msg)
else:
msg.processed = True
msg.save()
process_incoming(msg, delay=delay)
return msg
def testSMSLogSync(self):
self.deleteAllLogs()
self.assertEqual(self.getSMSLogCount(), 0)
self.assertEqual(self.getSMSCount(), 0)
# Test Create
smslog = SMSLog()
self.setRandomSMSLogValues(smslog)
smslog.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), 1)
self.assertEqual(self.getSMSCount(), 1)
sms = SMS.objects.get(couch_id=smslog._id)
self.checkFieldValues(smslog, sms, SMSLog._migration_get_fields())
self.assertTrue(SMSLog.get_db().get_rev(smslog._id).startswith('1-'))
# Test Update
self.setRandomSMSLogValues(smslog)
smslog.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), 1)
self.assertEqual(self.getSMSCount(), 1)
sms = SMS.objects.get(couch_id=smslog._id)
self.checkFieldValues(smslog, sms, SMSLog._migration_get_fields())
self.assertTrue(SMSLog.get_db().get_rev(smslog._id).startswith('2-'))
def create_outgoing_sms(self, backend):
sms = SMSLog(
domain=self.domain,
date=datetime.utcnow(),
direction='O',
phone_number='9991234567',
text='message',
backend_id=backend.couch_id
)
sms.save()
return sms
def _get_fake_sms(self, text):
msg = SMSLog(
domain=self.domain,
phone_number='+16175555454',
direction=OUTGOING,
date=datetime.utcnow(),
backend_id=self.mobile_backend.get_id,
text=text
)
msg.save()
return msg
def main_context(self):
contacts = CommCareUser.by_domain(self.domain, reduce=True)
web_users = WebUser.by_domain(self.domain)
web_users_admins = web_users_read_only = 0
facilities = SQLLocation.objects.filter(domain=self.domain, location_type__name__iexact='FACILITY')
admin_role_list = UserRole.by_domain_and_name(self.domain, 'Administrator')
if admin_role_list:
admin_role = admin_role_list[0]
else:
admin_role = None
for web_user in web_users:
dm = web_user.get_domain_membership(self.domain)
if admin_role and dm.role_id == admin_role.get_id:
web_users_admins += 1
else:
web_users_read_only += 1
main_context = super(GlobalStats, self).main_context
entities_reported_stock = SQLLocation.objects.filter(
domain=self.domain,
location_type__administrative=False
).count()
context = {
'root_name': self.root_name,
'country': SQLLocation.objects.filter(domain=self.domain,
location_type__name__iexact=self.root_name).count(),
'region': SQLLocation.objects.filter(domain=self.domain, location_type__name__iexact='region').count(),
'district': SQLLocation.objects.filter(
domain=self.domain,
location_type__name__iexact='district'
).count(),
'entities_reported_stock': entities_reported_stock,
'facilities': len(facilities),
'contacts': contacts[0]['value'] if contacts else 0,
'web_users': len(web_users),
'web_users_admins': web_users_admins,
'web_users_read_only': web_users_read_only,
'products': SQLProduct.objects.filter(domain=self.domain, is_archived=False).count(),
'product_stocks': StockState.objects.filter(sql_product__domain=self.domain).count(),
'stock_transactions': StockTransaction.objects.filter(report__domain=self.domain).count(),
'inbound_messages': SMSLog.count_incoming_by_domain(self.domain),
'outbound_messages': SMSLog.count_outgoing_by_domain(self.domain)
}
if self.show_supply_point_types:
counts = SQLLocation.objects.values('location_type__name').filter(domain=self.domain).annotate(
Count('location_type')
).order_by('location_type__name')
context['location_types'] = counts
main_context.update(context)
return main_context
def main_context(self):
contacts = CommCareUser.by_domain(self.domain, reduce=True)
web_users = WebUser.by_domain(self.domain)
web_users_admins = web_users_read_only = 0
facilities = SQLLocation.objects.filter(domain=self.domain, location_type__iexact='FACILITY')
for web_user in web_users:
role = web_user.get_domain_membership(self.domain).role
if role and role.name.lower().startswith('admin'):
web_users_admins += 1
else:
web_users_read_only += 1
main_context = super(GlobalStats, self).main_context
location_types = Domain.get_by_name(self.domain).location_types
administrative_types = [
location_type.name
for location_type in location_types
if not location_type.administrative
]
entities_reported_stock = SQLLocation.objects.filter(
domain=self.domain,
location_type__in=administrative_types
).count()
context = {
'root_name': self.root_name,
'country': SQLLocation.objects.filter(domain=self.domain,
location_type__iexact=self.root_name).count(),
'region': SQLLocation.objects.filter(domain=self.domain, location_type__iexact='region').count(),
'district': SQLLocation.objects.filter(domain=self.domain, location_type__iexact='district').count(),
'entities_reported_stock': entities_reported_stock,
'facilities': len(facilities),
'contacts': contacts[0]['value'] if contacts else 0,
'web_users': len(web_users),
'web_users_admins': web_users_admins,
'web_users_read_only': web_users_read_only,
'products': SQLProduct.objects.filter(domain=self.domain).count(),
'product_stocks': StockState.objects.filter(sql_product__domain=self.domain).count(),
'stock_transactions': StockTransaction.objects.filter(report__domain=self.domain).count(),
'inbound_messages': SMSLog.count_incoming_by_domain(self.domain),
'outbound_messages': SMSLog.count_outgoing_by_domain(self.domain)
}
if self.show_supply_point_types:
counts = SQLLocation.objects.values('location_type').filter(domain=self.domain).annotate(
Count('location_type')
).order_by('location_type')
context['location_types'] = counts
main_context.update(context)
return main_context
def send_sms_with_backend(domain, phone_number, text, backend_id):
msg = SMSLog(domain = domain,
phone_number = phone_number,
direction = OUTGOING,
date = datetime.utcnow(),
text = text)
if backend_id == "MOBILE_BACKEND_MACH":
try:
try:
msg.backend_api = mach_api.API_ID
except Exception:
pass
mach_api.send(msg)
msg.save()
return True
except Exception:
logging.exception("Exception while sending SMS to %s with backend %s" % (phone_number, backend_id))
return False
elif backend_id == "MOBILE_BACKEND_UNICEL":
try:
try:
msg.backend_api = unicel_api.API_ID
except Exception:
pass
unicel_api.send(msg)
msg.save()
return True
except Exception:
logging.exception("Exception while sending SMS to %s with backend %s" % (phone_number, backend_id))
return False
else:
try:
backend = MobileBackend.get(backend_id)
except Exception:
backend = None
if backend is None:
return False
try:
module = __import__(backend.outbound_module, fromlist=["send"])
try:
msg.backend_api = module.API_ID
except Exception:
pass
kwargs = backend.outbound_params
module.send(msg, **kwargs)
msg.save()
return True
except Exception as e:
logging.exception("Exception while sending SMS to %s with backend %s" % (phone_number, backend_id))
return False
def testFRISMSLogSync(self):
prev_couch_count = self.getSMSLogCount()
prev_sql_count = self.getSMSCount()
# Test Create
smslog = SMSLog()
self.setRandomSMSLogValues(smslog)
smslog.save()
sleep(1)
smslog = FRISMSLog.get(smslog._id)
self.setRandomFRISMSLogValues(smslog)
smslog.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), prev_couch_count + 1)
self.assertEqual(self.getSMSCount(), prev_sql_count + 1)
sms = SMS.objects.get(couch_id=smslog._id)
self.checkFieldValues(smslog, sms, FRISMSLog._migration_get_fields())
self.assertTrue(FRISMSLog.get_db().get_rev(smslog._id).startswith('2-'))
# Test Update
self.setRandomSMSLogValues(smslog)
self.setRandomFRISMSLogValues(smslog)
smslog.save()
sleep(1)
self.assertEqual(self.getSMSLogCount(), prev_couch_count + 1)
self.assertEqual(self.getSMSCount(), prev_sql_count + 1)
sms = SMS.objects.get(couch_id=smslog._id)
self.checkFieldValues(smslog, sms, FRISMSLog._migration_get_fields())
self.assertTrue(SMSLog.get_db().get_rev(smslog._id).startswith('3-'))
def send_test_message(verified_number, text, metadata=None):
msg = SMSLog(couch_recipient_doc_type=verified_number.owner_doc_type,
couch_recipient=verified_number.owner_id,
phone_number="+" + str(verified_number.phone_number),
direction=OUTGOING,
date=datetime.utcnow(),
domain=verified_number.domain,
text=text,
processed=True,
datetime_to_process=datetime.utcnow(),
queued_timestamp=datetime.utcnow())
msg.save()
add_msg_tags(msg, metadata)
return True
def tearDown(self):
SmsBillable.objects.all().delete()
SmsGatewayFee.objects.all().delete()
SmsGatewayFeeCriteria.objects.all().delete()
SmsUsageFee.objects.all().delete()
SmsUsageFeeCriteria.objects.all().delete()
self.currency_usd.delete()
self.other_currency.delete()
SMSLog.get_db().delete_docs(
SMSLog.by_domain_asc(generator.TEST_DOMAIN).all())
SMSBackend.get_db().delete_docs(SMSBackend.get_db().all_docs(
keys=self.backend_ids.values(), include_docs=True).all())
def testPostToIncomingAscii(self):
fake_post = {InboundParams.SENDER: str(self.number),
InboundParams.MESSAGE: self.message_ascii,
InboundParams.TIMESTAMP: datetime.now().strftime(DATE_FORMAT),
InboundParams.DCS: self.dcs,
InboundParams.UDHI: '0'}
client = Client()
response = client.post('/unicel/in/', fake_post)
self.assertEqual(200, response.status_code)
self.assertEqual(1, SMSLog.count_by_domain(self.domain))
log = SMSLog.by_domain_dsc(self.domain).all()[0]
self.assertEqual(self.message_ascii, log.text)
self.assertEqual(INCOMING, log.direction)
self.assertEqual(log.date.strftime(DATE_FORMAT),
fake_post[InboundParams.TIMESTAMP])
def tearDown(self):
SmsBillable.objects.all().delete()
SmsGatewayFee.objects.all().delete()
SmsGatewayFeeCriteria.objects.all().delete()
SmsUsageFee.objects.all().delete()
SmsUsageFeeCriteria.objects.all().delete()
self.currency_usd.delete()
self.other_currency.delete()
SMSLog.get_db().delete_docs(
SMSLog.by_domain_asc(generator.TEST_DOMAIN).all()
)
for api_id, backend_id in self.backend_ids.iteritems():
SQLMobileBackend.load(backend_id, is_couch_id=True).delete()
def incoming(phone_number,
text,
backend_api,
timestamp=None,
domain_scope=None,
backend_message_id=None,
delay=True,
backend_attributes=None,
raw_text=None):
"""
entry point for incoming sms
phone_number - originating phone number
text - message content
backend_api - backend API ID of receiving sms backend
timestamp - message received timestamp; defaults to now (UTC)
domain_scope - if present, only messages from phone numbers that can be
definitively linked to this domain will be processed; others will be
dropped (useful to provide security when simulating incoming sms)
"""
# Log message in message log
if text is None:
text = ""
phone_number = clean_phone_number(phone_number)
msg = SMSLog(
phone_number=phone_number,
direction=INCOMING,
date=timestamp or datetime.utcnow(),
text=text,
domain_scope=domain_scope,
backend_api=backend_api,
backend_message_id=backend_message_id,
raw_text=raw_text,
)
if backend_attributes:
for k, v in backend_attributes.items():
setattr(msg, k, v)
if settings.SMS_QUEUE_ENABLED:
msg.processed = False
msg.datetime_to_process = datetime.utcnow()
msg.queued_timestamp = msg.datetime_to_process
msg.save()
enqueue_directly(msg)
else:
msg.processed = True
msg.save()
process_incoming(msg, delay=delay)
return msg
def send_sms_to_verified_number(verified_number, text, metadata=None):
"""
Sends an sms using the given verified phone number entry.
verified_number The VerifiedNumber entry to use when sending.
text The text of the message to send.
return True on success, False on failure
"""
backend = verified_number.backend
msg = SMSLog(couch_recipient_doc_type=verified_number.owner_doc_type,
couch_recipient=verified_number.owner_id,
phone_number="+" + str(verified_number.phone_number),
direction=OUTGOING,
date=datetime.utcnow(),
domain=verified_number.domain,
backend_id=backend._id,
text=text)
add_msg_tags(msg, metadata)
def onerror():
logging.exception("Exception while sending SMS to VerifiedNumber id " +
verified_number._id)
return queue_outgoing_sms(msg, onerror=onerror)
def _sms_count(user, startdate, enddate, message_type="SMSLog"):
"""
Returns a dictionary of messages seen for a given type, user, and date
range of the format:
{
I: inbound_count,
O: outbound_count
}
"""
# utilizable if we want to stick it somewhere else
start_timestamp = json_format_datetime(startdate)
end_timestamp = json_format_datetime(enddate)
ret = {}
for direction in [INCOMING, OUTGOING]:
results = (
SMSLog.get_db()
.view(
"sms/by_recipient",
startkey=[user.doc_type, user._id, message_type, direction, start_timestamp],
endkey=[user.doc_type, user._id, message_type, direction, end_timestamp],
reduce=True,
)
.all()
)
ret[direction] = results[0]["value"] if results else 0
return ret
def send_test_message(verified_number, text, metadata=None):
msg = SMSLog(
couch_recipient_doc_type=verified_number.owner_doc_type,
couch_recipient=verified_number.owner_id,
phone_number="+" + str(verified_number.phone_number),
direction=OUTGOING,
date=datetime.utcnow(),
domain=verified_number.domain,
text=text,
processed=True,
datetime_to_process=datetime.utcnow(),
queued_timestamp=datetime.utcnow(),
)
msg.save()
add_msg_tags(msg, metadata)
return True
def handle_domain_specific_delays(msg, domain_object, utcnow):
"""
Checks whether or not we need to hold off on sending an outbound message
due to any restrictions set on the domain, and delays processing of the
message if necessary.
Returns True if a delay was made, False if not.
"""
domain_now = ServerTime(utcnow).user_time(domain_object.get_default_timezone()).done()
if len(domain_object.restricted_sms_times) > 0:
if not time_within_windows(domain_now, domain_object.restricted_sms_times):
delay_processing(msg, settings.SMS_QUEUE_DOMAIN_RESTRICTED_RETRY_INTERVAL)
return True
if msg.chat_user_id is None and len(domain_object.sms_conversation_times) > 0:
if time_within_windows(domain_now, domain_object.sms_conversation_times):
sms_conversation_length = domain_object.sms_conversation_length
conversation_start_timestamp = utcnow - timedelta(minutes=sms_conversation_length)
if SMSLog.inbound_entry_exists(msg.couch_recipient_doc_type,
msg.couch_recipient,
conversation_start_timestamp,
utcnow):
delay_processing(msg, 1)
return True
return False
def handle(self, *args, **options):
if len(args) == 0:
raise CommandError(
"Usage: python manage.py fix_smslog_recipient_doc_type <domain1 domain2 ...>"
)
for domain in args:
print "*** Processing Domain %s ***" % domain
user_cache = {}
for msg in SMSLog.by_domain_asc(domain).all():
if msg.couch_recipient:
if msg.couch_recipient_doc_type != "CommCareCase":
user = None
if msg.couch_recipient in user_cache:
user = user_cache[msg.couch_recipient]
else:
try:
user = CouchUser.get_by_user_id(
msg.couch_recipient)
except Exception:
user = None
if user is None:
print "Could not find user %s" % msg.couch_recipient
user_cache[msg.couch_recipient] = user
if user and msg.couch_recipient_doc_type != user.doc_type:
msg.couch_recipient_doc_type = user.doc_type
msg.save()
else:
if msg.couch_recipient_doc_type is not None or msg.couch_recipient is not None:
msg.couch_recipient = None
msg.couch_recipient_doc_type = None
msg.save()
def get_sms_couch_ids(self):
result = SMSLog.view(
'sms/by_domain',
include_docs=False,
reduce=False,
).all()
return [row['id'] for row in result if row['key'][1] == 'SMSLog']
def handle(self, *args, **options):
if len(args) == 0:
raise CommandError("Usage: python manage.py fix_smslog_recipient_doc_type <domain1 domain2 ...>")
for domain in args:
print "*** Processing Domain %s ***" % domain
user_cache = {}
for msg in SMSLog.by_domain_asc(domain).all():
if msg.couch_recipient:
if msg.couch_recipient_doc_type != "CommCareCase":
user = None
if msg.couch_recipient in user_cache:
user = user_cache[msg.couch_recipient]
else:
try:
user = CouchUser.get_by_user_id(msg.couch_recipient)
except Exception:
user = None
if user is None:
print "Could not find user %s" % msg.couch_recipient
user_cache[msg.couch_recipient] = user
if user and msg.couch_recipient_doc_type != user.doc_type:
msg.couch_recipient_doc_type = user.doc_type
msg.save()
else:
if msg.couch_recipient_doc_type is not None or msg.couch_recipient is not None:
msg.couch_recipient = None
msg.couch_recipient_doc_type = None
msg.save()
def message_test(request, domain, phone_number):
if request.method == "POST":
message = request.POST.get("message", "")
domain_scope = None if request.couch_user.is_superuser else domain
try:
incoming(phone_number, message, "TEST", domain_scope=domain_scope)
except DomainScopeValidationError:
messages.error(
request,
_("Invalid phone number being simulated. You may only " \
"simulate SMS from verified numbers belonging to contacts " \
"in this domain.")
)
except Exception:
notify_exception(request)
messages.error(
request,
_("An error has occurred. Please try again in a few minutes " \
"and if the issue persists, please contact CommCareHQ " \
"Support.")
)
context = get_sms_autocomplete_context(request, domain)
context['domain'] = domain
context['messagelog'] = SMSLog.by_domain_dsc(domain)
context['now'] = datetime.utcnow()
tz = report_utils.get_timezone(request.couch_user.user_id, domain)
context['timezone'] = tz
context['timezone_now'] = datetime.now(tz=tz)
context['layout_flush_content'] = True
context['phone_number'] = phone_number
return render(request, "sms/message_tester.html", context)
def send_sms(self, msg, delay=True, *args, **kwargs):
phone_number = strip_plus(msg.phone_number)
if not phone_number.startswith("63"):
raise MegamobileException("Only Filipino phone numbers are supported")
phone_number = phone_number[2:]
text = msg.text.encode("utf-8")
pid = None
if msg.in_reply_to:
original_msg = SMSLog.get(msg.in_reply_to)
pid = getattr(original_msg, "megamobile_pid", None)
pid = pid or DEFAULT_PID
setattr(msg, "megamobile_pid", pid)
msg.save()
params = urlencode({
"pid" : pid,
"cel" : phone_number,
"msg" : text,
"src" : self.source_identifier,
})
api_account_name = quote(self.api_account_name)
url = "http://api.mymegamobile.com/%s?%s" % (api_account_name, params)
response = urlopen(url, timeout=settings.SMS_GATEWAY_TIMEOUT).read()
def send(self, msg, delay=True, *args, **kwargs):
phone_number = strip_plus(msg.phone_number)
if not phone_number.startswith("63"):
raise MegamobileException(
"Only Filipino phone numbers are supported")
phone_number = phone_number[2:]
text = msg.text.encode("utf-8")
pid = None
if msg.in_reply_to:
original_msg = SMSLog.get(msg.in_reply_to)
pid = getattr(original_msg, "megamobile_pid", None)
pid = pid or DEFAULT_PID
setattr(msg, "megamobile_pid", pid)
msg.save()
params = urlencode({
"pid": pid,
"cel": phone_number,
"msg": text,
"src": self.source_identifier,
})
api_account_name = quote(self.api_account_name)
url = "http://api.mymegamobile.com/%s?%s" % (api_account_name, params)
response = urlopen(url, timeout=settings.SMS_GATEWAY_TIMEOUT).read()
def get_sms_couch_ids(self):
result = SMSLog.view(
'sms/by_domain',
include_docs=False,
reduce=False,
).all()
return [row['id'] for row in result if row['key'][1] == 'SMSLog']
def handle_domain_specific_delays(msg, domain_object, utcnow):
"""
Checks whether or not we need to hold off on sending an outbound message
due to any restrictions set on the domain, and delays processing of the
message if necessary.
Returns True if a delay was made, False if not.
"""
domain_now = tz_utils.adjust_datetime_to_timezone(utcnow, pytz.utc.zone,
domain_object.default_timezone)
if len(domain_object.restricted_sms_times) > 0:
if not time_within_windows(domain_now, domain_object.restricted_sms_times):
delay_processing(msg, settings.SMS_QUEUE_DOMAIN_RESTRICTED_RETRY_INTERVAL)
return True
if msg.chat_user_id is None and len(domain_object.sms_conversation_times) > 0:
if time_within_windows(domain_now, domain_object.sms_conversation_times):
sms_conversation_length = domain_object.sms_conversation_length
conversation_start_timestamp = utcnow - timedelta(minutes=sms_conversation_length)
if SMSLog.inbound_entry_exists(msg.couch_recipient_doc_type,
msg.couch_recipient,
conversation_start_timestamp,
utcnow):
delay_processing(msg, 1)
return True
return False
def _sms_count(user, startdate, enddate, message_type='SMSLog'):
"""
Returns a dictionary of messages seen for a given type, user, and date
range of the format:
{
I: inbound_count,
O: outbound_count
}
"""
# utilizable if we want to stick it somewhere else
start_timestamp = json_format_datetime(startdate)
end_timestamp = json_format_datetime(enddate)
ret = {}
for direction in [INCOMING, OUTGOING]:
results = SMSLog.get_db().view("sms/by_recipient",
startkey=[
user.doc_type, user._id,
message_type, direction,
start_timestamp
],
endkey=[
user.doc_type, user._id,
message_type, direction,
end_timestamp
],
reduce=True).all()
ret[direction] = results[0]['value'] if results else 0
return ret
def handle(self, *labels, **options):
db = SMSLog.get_db()
# active_domains = db.view(
# "sms/by_domain",
# reduce=True,
# group_level=1,
# ).all()
# active_domains = [d['key'][0] for d in active_domains]
active_domains = ['pathfinder']
startkey = lambda d: [d, "SMSLog", "2013-08-01"]
endkey = lambda d: [d, "SMSLog", "2013-10-15"]
for domain in active_domains:
data = db.view(
"sms/by_domain",
reduce=False,
startkey=startkey(domain),
endkey=endkey(domain),
).all()
sms_ids = [d['id'] for d in data]
for doc in iter_docs(db, sms_ids):
sms_log = SMSLog.wrap(doc)
if not sms_log.billed and sms_log.backend_api in [
'MACH',
# 'TROPO',
# 'UNICEL',
]:
# we're going to assume the SMS messages were sent successfully
# at the time they were actually sent
successful_responses = {
'MACH': "MACH RESPONSE +OK 01 message queued (dest=%s)" % sms_log.phone_number,
'TROPO': "<success>true</success>",
'UNICEL': "success",
}
print "Retroactively billing SMLog %s in domain %s" % (sms_log._id, sms_log.domain)
try:
create_billable_for_sms(
sms_log,
sms_log.backend_api,
delay=False,
response=successful_responses[sms_log.backend_api],
)
except Exception as e:
print "Retroactive bill was not successful due to error: %s" % e
logging.exception(e)
def process_sms(message_id):
"""
message_id - _id of an SMSLog entry
"""
# Note that Redis error/exception notifications go out from the
# run_sms_queue command, so no need to send them out here
# otherwise we'd get too many emails.
rcache = cache_core.get_redis_default_cache()
if not isinstance(rcache, RedisCache):
return
try:
client = rcache.raw_client
except NotImplementedError:
return
utcnow = datetime.utcnow()
# Prevent more than one task from processing this SMS, just in case
# the message got enqueued twice.
message_lock = get_lock(client, "sms-queue-processing-%s" % message_id)
if message_lock.acquire(blocking=False):
msg = SMSLog.get(message_id)
if message_is_stale(msg, utcnow):
msg.set_system_error(ERROR_MESSAGE_IS_STALE)
message_lock.release()
return
if msg.direction == OUTGOING:
domain_object = Domain.get_by_name(msg.domain, strict=True)
if handle_domain_specific_delays(msg, domain_object, utcnow):
message_lock.release()
return
requeue = False
# Process inbound SMS from a single contact one at a time
recipient_block = msg.direction == INCOMING
if (isinstance(msg.processed, bool)
and not msg.processed
and not msg.error
and msg.datetime_to_process < utcnow):
if recipient_block:
recipient_lock = get_lock(client,
"sms-queue-recipient-phone-%s" % msg.phone_number)
recipient_lock.acquire(blocking=True)
if msg.direction == OUTGOING:
requeue = handle_outgoing(msg)
elif msg.direction == INCOMING:
handle_incoming(msg)
else:
msg.set_system_error(ERROR_INVALID_DIRECTION)
if recipient_block:
recipient_lock.release()
message_lock.release()
if requeue:
process_sms.delay(message_id)
def process_sms(message_id):
"""
message_id - _id of an SMSLog entry
"""
# Note that Redis error/exception notifications go out from the
# run_sms_queue command, so no need to send them out here
# otherwise we'd get too many emails.
rcache = cache_core.get_redis_default_cache()
if not isinstance(rcache, RedisCache):
return
try:
client = rcache.raw_client
except NotImplementedError:
return
utcnow = datetime.utcnow()
# Prevent more than one task from processing this SMS, just in case
# the message got enqueued twice.
message_lock = get_lock(client, "sms-queue-processing-%s" % message_id)
if message_lock.acquire(blocking=False):
msg = SMSLog.get(message_id)
if message_is_stale(msg, utcnow):
set_error(msg, ERROR_MESSAGE_IS_STALE)
message_lock.release()
return
if msg.direction == OUTGOING:
domain_object = Domain.get_by_name(msg.domain, strict=True)
if handle_domain_specific_delays(msg, domain_object, utcnow):
message_lock.release()
return
requeue = False
# Process inbound SMS from a single contact one at a time
recipient_block = msg.direction == INCOMING
if (isinstance(msg.processed, bool)
and not msg.processed
and not msg.error
and msg.datetime_to_process < utcnow):
if recipient_block:
recipient_lock = get_lock(client,
"sms-queue-recipient-phone-%s" % msg.phone_number)
recipient_lock.acquire(blocking=True)
if msg.direction == OUTGOING:
requeue = handle_outgoing(msg)
elif msg.direction == INCOMING:
handle_incoming(msg)
else:
set_error(msg, ERROR_INVALID_DIRECTION)
if recipient_block:
recipient_lock.release()
message_lock.release()
if requeue:
process_sms.delay(message_id)
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = SMSLog.by_domain_date(self.domain, startdate, enddate)
result = []
username_map = {
} # Store the results of username lookups for faster loading
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get(
"abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain
in abbreviated_phone_number_domains)
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
recipient_id = message.couch_recipient
if recipient_id in [None, ""]:
username = "******"
elif recipient_id in username_map:
username = username_map.get(recipient_id)
else:
username = "******"
try:
if message.couch_recipient_doc_type == "CommCareCase":
username = CommCareCase.get(recipient_id).name
else:
username = CouchUser.get_by_user_id(
recipient_id).username
except Exception:
pass
username_map[recipient_id] = username
phone_number = message.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[
0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(
message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt_timestamp(timestamp),
self._fmt(username),
self._fmt(phone_number),
self._fmt(direction_map.get(message.direction, "-")),
self._fmt(message.text),
])
return result
def get_items_to_be_processed(self, utcnow):
# We're just querying for ids here, so no need to limit
entries = SMSLog.view(
"sms/queued_sms",
startkey="1970-01-01T00:00:00Z",
endkey=json_format_datetime(utcnow),
include_docs=False
).all()
return entries
def main_context(self):
contacts = CommCareUser.by_domain(self.domain, reduce=True)
web_users = WebUser.by_domain(self.domain, reduce=True)
main_context = super(GlobalStats, self).main_context
context = {
'supply_points': SQLLocation.objects.filter(domain=self.domain).count(),
'facilities': SQLLocation.objects.filter(domain=self.domain, location_type__iexact='FACILITY').count(),
'contacts': contacts[0]['value'] if contacts else 0,
'web_users': web_users[0]['value'] if web_users else 0,
'products': SQLProduct.objects.filter(domain=self.domain).count(),
'product_stocks': StockState.objects.filter(sql_product__domain=self.domain).count(),
'stock_transactions': StockTransaction.objects.filter(report__domain=self.domain).count(),
'inbound_messages': SMSLog.count_incoming_by_domain(self.domain),
'outbound_messages': SMSLog.count_outgoing_by_domain(self.domain)
}
main_context.update(context)
return main_context
def get_last_outbound_sms(self, doc_type, contact_id):
sms = SMSLog.view("sms/by_recipient",
startkey=[doc_type, contact_id, "SMSLog", "O", {}],
endkey=[doc_type, contact_id, "SMSLog", "O"],
descending=True,
include_docs=True,
reduce=False,
).first()
return sms
def get_sms_and_call_couch_count(self):
result = SMSLog.view(
'sms/by_domain',
include_docs=False,
reduce=True,
).all()
if result:
return result[0]['value']
return 0
def tearDown(self):
SmsBillable.objects.all().delete()
SmsGatewayFee.objects.all().delete()
SmsGatewayFeeCriteria.objects.all().delete()
SmsUsageFee.objects.all().delete()
SmsUsageFeeCriteria.objects.all().delete()
self.currency_usd.delete()
for log in SMSLog.by_domain_asc(generator.TEST_DOMAIN):
log.delete()
def rows(self):
startdate = json_format_datetime(self.datespan.startdate_utc)
enddate = json_format_datetime(self.datespan.enddate_utc)
data = SMSLog.by_domain_date(self.domain, startdate, enddate)
result = []
direction_map = {
INCOMING: _("Incoming"),
OUTGOING: _("Outgoing"),
}
# Retrieve message log options
message_log_options = getattr(settings, "MESSAGE_LOG_OPTIONS", {})
abbreviated_phone_number_domains = message_log_options.get(
"abbreviated_phone_number_domains", [])
abbreviate_phone_number = (self.domain
in abbreviated_phone_number_domains)
contact_cache = {}
for message in data:
if message.direction == OUTGOING and not message.processed:
continue
recipient_id = message.couch_recipient
doc = None
if recipient_id not in [None, ""]:
try:
if message.couch_recipient_doc_type == "CommCareCase":
doc = CommCareCase.get(recipient_id)
else:
doc = CouchUser.get_by_user_id(recipient_id)
except Exception:
pass
if doc:
doc_info = get_doc_info(doc.to_json(), self.domain,
contact_cache)
else:
doc_info = None
phone_number = message.phone_number
if abbreviate_phone_number and phone_number is not None:
phone_number = phone_number[0:7] if phone_number[
0:1] == "+" else phone_number[0:6]
timestamp = tz_utils.adjust_datetime_to_timezone(
message.date, pytz.utc.zone, self.timezone.zone)
result.append([
self._fmt_timestamp(timestamp),
self._fmt_contact_link(message, doc_info),
self._fmt(phone_number),
self._fmt(direction_map.get(message.direction, "-")),
self._fmt(message.text),
])
return result
def arbitrary_messages_by_backend_and_direction(backend_ids,
phone_number=None,
domain=None,
directions=DIRECTIONS):
phone_number = phone_number or TEST_NUMBER
domain = domain or TEST_DOMAIN
messages = []
for api_id, instance_id in backend_ids.items():
for direction in directions:
sms_log = SMSLog(direction=direction,
phone_number=phone_number,
domain=domain,
backend_api=api_id,
backend_id=instance_id,
text=arbitrary_message(),
date=datetime.datetime.utcnow())
sms_log.save()
messages.append(sms_log)
return messages
def messaging(request, domain, template="sms/default.html"):
context = get_sms_autocomplete_context(request, domain)
context['domain'] = domain
context['messagelog'] = SMSLog.by_domain_dsc(domain)
context['now'] = datetime.utcnow()
tz = report_utils.get_timezone(request.couch_user.user_id, domain)
context['timezone'] = tz
context['timezone_now'] = datetime.now(tz=tz)
context['layout_flush_content'] = True
return render(request, template, context)
def getSMSLogCount(self):
result = SMSLog.view(
'sms/by_domain',
startkey=[self.domain, 'SMSLog'],
endkey=[self.domain, 'SMSLog', {}],
include_docs=False,
reduce=True,
).all()
if result:
return result[0]['value']
return 0
def tearDown(self):
for smslog in SMSLog.view(
'sms/by_domain',
startkey=[self.domain, 'SMSLog'],
endkey=[self.domain, 'SMSLog', {}],
include_docs=True,
reduce=False,
).all():
smslog.delete()
SMS.objects.filter(domain=self.domain).delete()
def handle(self, *labels, **options):
messages = SMSLog.view("sms/migrate_smslog_2012_04", include_docs=True)
print "Migrating MessageLog to SMSLog"
for message in messages:
try:
message.doc_type = "SMSLog"
message.base_doc = "MessageLog"
message.couch_recipient_doc_type = "CouchUser"
message.save()
except Exception as e:
print "There was an error migrating message %s." % (
message._id)
def handle(self, *args, **options):
missing_domain_billables = SmsBillable.objects.filter(
Q(domain=u'') | Q(domain=None))
for billable in missing_domain_billables:
msg_log = SMSLog.get(billable.log_id)
billable.domain = msg_log.domain
if billable.domain:
billable.save()
print "Updated Billable from %s for domain %s" % (
billable.date_created, billable.domain)
else:
print "could not find a domain in SMSLog %s." % billable.log_id
def process_sms(message_id):
"""
message_id - _id of an SMSLog entry
"""
client = get_redis_client()
utcnow = datetime.utcnow()
# Prevent more than one task from processing this SMS, just in case
# the message got enqueued twice.
message_lock = get_lock(client, "sms-queue-processing-%s" % message_id)
if message_lock.acquire(blocking=False):
msg = SMSLog.get(message_id)
if message_is_stale(msg, utcnow):
msg.set_system_error(SMS.ERROR_MESSAGE_IS_STALE)
release_lock(message_lock, True)
return
if msg.direction == OUTGOING:
if msg.domain:
domain_object = Domain.get_by_name(msg.domain, strict=True)
else:
domain_object = None
if domain_object and handle_domain_specific_delays(msg, domain_object, utcnow):
release_lock(message_lock, True)
return
requeue = False
# Process inbound SMS from a single contact one at a time
recipient_block = msg.direction == INCOMING
if (isinstance(msg.processed, bool)
and not msg.processed
and not msg.error
and msg.datetime_to_process < utcnow):
if recipient_block:
recipient_lock = get_lock(client,
"sms-queue-recipient-phone-%s" % msg.phone_number)
recipient_lock.acquire(blocking=True)
if msg.direction == OUTGOING:
requeue = handle_outgoing(msg)
elif msg.direction == INCOMING:
handle_incoming(msg)
else:
msg.set_system_error(SMS.ERROR_INVALID_DIRECTION)
if recipient_block:
release_lock(recipient_lock, True)
release_lock(message_lock, True)
if requeue:
process_sms.delay(message_id)
def setUp(self):
self.domain = 'mockdomain'
all_logs = SMSLog.by_domain_asc(self.domain).all()
for log in all_logs:
log.delete()
self.user = '******'
self.password = '******'
self.number = 5555551234
self.couch_user = WebUser.create(self.domain, self.user, self.password)
self.couch_user.add_phone_number(self.number)
self.couch_user.save()
self.dcs = '8'
self.message_ascii = 'It Works'
self.message_utf_hex = '0939093F0928094D092609400020091509300924093E00200939094800200907093800200938092E092F00200915093E092E002009390948003F'