def contact_resolve(self, org_id: int, channel_id: int, urn: str):
org = Org.objects.get(id=org_id)
user = get_anonymous_user()
contact_urn = ContactURN.lookup(org, urn)
if contact_urn:
contact = contact_urn.contact
else:
contact = create_contact_locally(org,
user,
name="",
language="",
urns=[urn],
fields={},
group_uuids=[])
contact_urn = ContactURN.lookup(org, urn)
return {
"contact": {
"id": contact.id,
"uuid": str(contact.uuid),
"name": contact.name
},
"urn": {
"id": contact_urn.id,
"identity": contact_urn.identity
},
}
def _create_contact_batch(self, batch):
"""
Bulk creates a batch of contacts from flat representations
"""
for c in batch:
c["object"] = Contact(
org=c["org"],
name=c["name"],
language=c["language"],
is_stopped=c["is_stopped"],
is_blocked=c["is_blocked"],
is_active=c["is_active"],
created_by=c["user"],
created_on=c["created_on"],
modified_by=c["user"],
modified_on=c["modified_on"],
fields=c["fields_as_json"],
)
Contact.objects.bulk_create([c["object"] for c in batch])
# now that contacts have pks, bulk create the actual URN, value and group membership objects
batch_urns = []
batch_memberships = []
for c in batch:
org = c["org"]
c["urns"] = []
if c["tel"]:
c["urns"].append(
ContactURN(
org=org,
contact=c["object"],
priority=50,
scheme=TEL_SCHEME,
path=c["tel"],
identity=URN.from_tel(c["tel"]),
)
)
if c["twitter"]:
c["urns"].append(
ContactURN(
org=org,
contact=c["object"],
priority=50,
scheme=TWITTER_SCHEME,
path=c["twitter"],
identity=URN.from_twitter(c["twitter"]),
)
)
for g in c["groups"]:
batch_memberships.append(ContactGroup.contacts.through(contact=c["object"], contactgroup=g))
batch_urns += c["urns"]
ContactURN.objects.bulk_create(batch_urns)
ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
def _create_contact_batch(self, batch):
"""
Bulk creates a batch of contacts from flat representations
"""
for c in batch:
c['object'] = Contact(org=c['org'], name=c['name'], language=c['language'],
is_stopped=c['is_stopped'], is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=c['user'], created_on=c['created_on'],
modified_by=c['user'], modified_on=c['modified_on'])
Contact.objects.bulk_create([c['object'] for c in batch])
# now that contacts have pks, bulk create the actual URN, value and group membership objects
batch_urns = []
batch_values = []
batch_memberships = []
for c in batch:
org = c['org']
c['urns'] = []
if c['tel']:
c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TEL_SCHEME,
path=c['tel'], urn=URN.from_tel(c['tel'])))
if c['twitter']:
c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TWITTER_SCHEME,
path=c['twitter'], urn=URN.from_twitter(c['twitter'])))
if c['gender']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['gender'],
string_value=c['gender']))
if c['age']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['age'],
string_value=str(c['age']), decimal_value=c['age']))
if c['joined']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['joined'],
string_value=datetime_to_str(c['joined']), datetime_value=c['joined']))
if c['ward']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['ward'],
string_value=c['ward'].name, location_value=c['ward']))
if c['district']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['district'],
string_value=c['district'].name, location_value=c['district']))
if c['state']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['state'],
string_value=c['state'].name, location_value=c['state']))
for g in c['groups']:
batch_memberships.append(ContactGroup.contacts.through(contact=c['object'], contactgroup=g))
batch_urns += c['urns']
ContactURN.objects.bulk_create(batch_urns)
Value.objects.bulk_create(batch_values)
ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
def _create_contact_batch(self, batch):
"""
Bulk creates a batch of contacts from flat representations
"""
for c in batch:
c['object'] = Contact(org=c['org'],
name=c['name'],
language=c['language'],
is_stopped=c['is_stopped'],
is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=c['user'],
created_on=c['created_on'],
modified_by=c['user'],
modified_on=c['modified_on'],
fields=c['fields_as_json'])
Contact.objects.bulk_create([c['object'] for c in batch])
# now that contacts have pks, bulk create the actual URN, value and group membership objects
batch_urns = []
batch_memberships = []
for c in batch:
org = c['org']
c['urns'] = []
if c['tel']:
c['urns'].append(
ContactURN(org=org,
contact=c['object'],
priority=50,
scheme=TEL_SCHEME,
path=c['tel'],
identity=URN.from_tel(c['tel'])))
if c['twitter']:
c['urns'].append(
ContactURN(org=org,
contact=c['object'],
priority=50,
scheme=TWITTER_SCHEME,
path=c['twitter'],
identity=URN.from_twitter(c['twitter'])))
for g in c['groups']:
batch_memberships.append(
ContactGroup.contacts.through(contact=c['object'],
contactgroup=g))
batch_urns += c['urns']
ContactURN.objects.bulk_create(batch_urns)
ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
def validate_urns(self, attrs, source):
# if we have tel URNs, we may need a country to normalize by
tel_sender = self.org.get_send_channel(TEL_SCHEME)
country = tel_sender.country if tel_sender else None
urns = []
for urn in attrs.get(source, []):
try:
parsed = ContactURN.parse_urn(urn)
except ValueError, e:
raise ValidationError(e.message)
norm_scheme, norm_path = ContactURN.normalize_urn(parsed.scheme, parsed.path, country)
if not ContactURN.validate_urn(norm_scheme, norm_path):
raise ValidationError("Invalid URN: '%s'" % urn)
urns.append((norm_scheme, norm_path))
def create_group_contacts(self, spec, org, user):
self._log(f"Generating group contacts...")
for g in spec["groups"]:
size = int(g.get("size", 0))
if size > 0:
group = ContactGroup.user_groups.get(org=org, name=g["name"])
contacts = []
for i in range(size):
urn = f"tel:+250788{i:06}"
contact = ContactURN.lookup(org, urn)
if not contact:
contact = Contact.create(org,
user,
name="",
language="",
urns=[urn],
fields={},
groups=[])
contacts.append(contact)
Contact.bulk_change_group(user, contacts, group, add=True)
self._log(self.style.SUCCESS("OK") + "\n")
def create_contact_locally(org,
user,
name,
language,
urns,
fields,
group_uuids,
status=Contact.STATUS_ACTIVE,
last_seen_on=None):
orphaned_urns = {}
for urn in urns:
existing = ContactURN.lookup(org, urn)
if existing:
if existing.contact_id:
raise MailroomException(
"contact/create", None,
{"error": "URNs in use by other contacts"})
else:
orphaned_urns[urn] = existing
contact = Contact.objects.create(
org=org,
name=name,
language=language,
created_by=user,
modified_by=user,
created_on=timezone.now(),
status=status,
last_seen_on=last_seen_on,
)
update_urns_locally(contact, urns)
update_fields_locally(user, contact, fields)
update_groups_locally(contact, group_uuids, add=True)
return contact
def form_valid(self, form):
org = self.request.user.get_org()
if not org: # pragma: no cover
raise Exception(_("No org for this user, cannot claim"))
self.object = Channel.objects.filter(
claim_code=self.form.cleaned_data["claim_code"]).first()
country = self.object.country
phone_country = ContactURN.derive_country_from_tel(
self.form.cleaned_data["phone_number"], str(self.object.country))
# always prefer the country of the phone number they are entering if we have one
if phone_country and phone_country != country: # pragma: needs cover
self.object.country = phone_country
analytics.track(self.request.user.username, "temba.channel_create")
self.object.claim(org, self.request.user,
self.form.cleaned_data["phone_number"])
self.object.save()
# trigger a sync
self.object.trigger_sync()
return super().form_valid(form)
def validate_urn(self, attrs, source):
urns = []
if 'channel' in attrs and attrs['channel']:
country = attrs['channel'].country
for urn in attrs.get(source, []):
parsed = ContactURN.parse_urn(urn)
norm_scheme, norm_path = ContactURN.normalize_urn(parsed.scheme, parsed.path, country)
if not ContactURN.validate_urn(norm_scheme, norm_path):
raise ValidationError("Invalid URN: '%s'" % urn)
urns.append((norm_scheme, norm_path))
else:
raise ValidationError("You must specify a valid channel")
attrs['urn'] = urns
return attrs
def get_or_create_contact(self, urn):
if ':' in urn:
parsed = ContactURN.parse_urn(urn)
urn = (parsed.scheme, parsed.path)
else:
urn = (TEL_SCHEME, urn) # assume phone number
return Contact.get_or_create(self.org, self.user, name=None, urns=[urn])
def clean(self):
country = self.org.get_country_code()
# validate URN fields
for field_key, value in self.cleaned_data.iteritems():
if field_key.startswith('__urn__') and value:
scheme = field_key[7:field_key.rfind('__')]
norm_scheme, norm_path = ContactURN.normalize_urn(scheme, value, country)
existing = Contact.from_urn(self.org, norm_scheme, norm_path)
if existing and existing != self.instance:
self._errors[field_key] = _("Used by another contact")
elif not ContactURN.validate_urn(norm_scheme, norm_path):
self._errors[field_key] = _("Invalid format")
return self.cleaned_data
def validate_urn(self, attrs, source):
urns = []
if 'channel' in attrs and attrs['channel']:
country = attrs['channel'].country
for urn in attrs.get(source, []):
parsed = ContactURN.parse_urn(urn)
norm_scheme, norm_path = ContactURN.normalize_urn(parsed.scheme, parsed.path, country)
if not ContactURN.validate_urn(norm_scheme, norm_path):
raise ValidationError("Invalid URN: '%s'" % urn)
urns.append((norm_scheme, norm_path))
else:
raise ValidationError("You must specify a valid channel")
attrs['urn'] = urns
return attrs
def clean(self):
channel = self.org.get_receive_channel(TEL_SCHEME)
country = channel.country if channel else None
# validate URN fields
for field_key, value in self.cleaned_data.iteritems():
if field_key.startswith('__urn__') and value:
scheme = field_key[7:]
norm_scheme, norm_path = ContactURN.normalize_urn(scheme, value, country)
existing = Contact.from_urn(self.org, norm_scheme, norm_path)
if existing and existing != self.instance:
self._errors[field_key] = _("Used by another contact")
elif not ContactURN.validate_urn(norm_scheme, norm_path):
self._errors[field_key] = _("Invalid format")
return self.cleaned_data
def clean(self):
channel = self.org.get_receive_channel(TEL_SCHEME)
country = channel.country if channel else None
# validate URN fields
for field_key, value in self.cleaned_data.iteritems():
if field_key.startswith('__urn__') and value:
scheme = field_key[7:]
norm_scheme, norm_path = ContactURN.normalize_urn(scheme, value, country)
existing = Contact.from_urn(self.org, norm_scheme, norm_path)
if existing and existing != self.instance:
self._errors[field_key] = _("Used by another contact")
elif not ContactURN.validate_urn(norm_scheme, norm_path):
self._errors[field_key] = _("Invalid format")
return self.cleaned_data
def validate_urns(self, attrs, source):
urns = []
# get a channel
channel = self.org.get_send_channel(TEL_SCHEME)
if channel:
for urn in attrs.get(source, []):
parsed = ContactURN.parse_urn(urn)
norm_scheme, norm_path = ContactURN.normalize_urn(parsed.scheme, parsed.path, channel.country)
if not ContactURN.validate_urn(norm_scheme, norm_path):
raise ValidationError("Invalid URN: '%s'" % urn)
urns.append((norm_scheme, norm_path))
else:
raise ValidationError("You cannot start flows without at least one phone number configured")
attrs['urns'] = urns
return attrs
def get_or_create_contact(self, urn):
if ':' in urn:
parsed = ContactURN.parse_urn(urn)
urn = (parsed.scheme, parsed.path)
else:
urn = (TEL_SCHEME, urn) # assume phone number
return Contact.get_or_create(self.org,
self.user,
name=None,
urns=[urn])
def validate_urns(self, attrs, source):
urns = None
request_urns = attrs.get(source, None)
if request_urns is not None:
urns = []
for urn in request_urns:
try:
parsed = ContactURN.parse_urn(urn)
except ValueError:
raise ValidationError("Unable to parse URN: '%s'" % urn)
norm_scheme, norm_path = ContactURN.normalize_urn(parsed.scheme, parsed.path)
if not ContactURN.validate_urn(norm_scheme, norm_path):
raise ValidationError("Invalid URN: '%s'" % urn)
urns.append((norm_scheme, norm_path))
attrs['urns'] = urns
return attrs
def validate_urns(self, attrs, source):
urns = None
request_urns = attrs.get(source, None)
if request_urns is not None:
urns = []
for urn in request_urns:
try:
parsed = ContactURN.parse_urn(urn)
except ValueError:
raise ValidationError("Unable to parse URN: '%s'" % urn)
norm_scheme, norm_path = ContactURN.normalize_urn(parsed.scheme, parsed.path)
if not ContactURN.validate_urn(norm_scheme, norm_path):
raise ValidationError("Invalid URN: '%s'" % urn)
urns.append((norm_scheme, norm_path))
attrs['urns'] = urns
return attrs
def update_urns_locally(contact, urns: list[str]):
country = contact.org.default_country_code
priority = ContactURN.PRIORITY_HIGHEST
urns_created = [] # new URNs created
urns_attached = [] # existing orphan URNs attached
urns_retained = [] # existing URNs retained
for urn_as_string in urns:
normalized = URN.normalize(urn_as_string, country)
urn = ContactURN.lookup(contact.org, normalized)
if not urn:
urn = ContactURN.create(contact.org,
contact,
normalized,
priority=priority)
urns_created.append(urn)
# unassigned URN or different contact
elif not urn.contact or urn.contact != contact:
urn.contact = contact
urn.priority = priority
urn.save()
urns_attached.append(urn)
else:
if urn.priority != priority:
urn.priority = priority
urn.save()
urns_retained.append(urn)
# step down our priority
priority -= 1
# detach any existing URNs that weren't included
urn_ids = [u.pk for u in (urns_created + urns_attached + urns_retained)]
urns_detached = ContactURN.objects.filter(contact=contact).exclude(
id__in=urn_ids)
urns_detached.update(contact=None)
def contact_resolve(self, org_id: int, channel_id: int, urn: str):
org = Org.objects.get(id=org_id)
user = get_anonymous_user()
try:
urn = URN.normalize(urn, org.default_country_code)
if not URN.validate(urn, org.default_country_code):
raise ValueError()
except ValueError:
raise MailroomException("contact/resolve", None,
{"error": "invalid URN"})
contact_urn = ContactURN.lookup(org, urn)
if contact_urn:
contact = contact_urn.contact
else:
contact = create_contact_locally(org,
user,
name="",
language="",
urns=[urn],
fields={},
group_uuids=[])
contact_urn = ContactURN.lookup(org, urn)
return {
"contact": {
"id": contact.id,
"uuid": str(contact.uuid),
"name": contact.name
},
"urn": {
"id": contact_urn.id,
"identity": contact_urn.identity
},
}
def form_valid(self, form):
org = self.request.user.get_org()
self.object = Channel.objects.filter(
claim_code=self.form.cleaned_data["claim_code"]).first()
country = self.object.country
phone_country = ContactURN.derive_country_from_tel(
self.form.cleaned_data["phone_number"], str(self.object.country))
# always prefer the country of the phone number they are entering if we have one
if phone_country and phone_country != country: # pragma: needs cover
self.object.country = phone_country
self.object.claim(org, self.request.user,
self.form.cleaned_data["phone_number"])
self.object.save()
# trigger a sync
self.object.trigger_sync()
return super().form_valid(form)
def create_channel_event(self,
channel,
urn,
event_type,
occurred_on=None,
extra=None):
urn_obj = ContactURN.lookup(channel.org,
urn,
country_code=channel.country)
if urn_obj:
contact = urn_obj.contact
else:
contact = self.create_contact(urns=[urn])
urn_obj = contact.urns.get()
return ChannelEvent.objects.create(
org=channel.org,
channel=channel,
contact=contact,
contact_urn=urn_obj,
occurred_on=occurred_on or timezone.now(),
event_type=event_type,
extra=extra,
)
def send(self, channel, msg, text):
# build our payload
payload = {'message': {'text': text}}
metadata = msg.metadata if hasattr(msg, 'metadata') else {}
quick_replies = metadata.get('quick_replies', [])
formatted_replies = [dict(title=item[:self.quick_reply_text_size], payload=item[:self.quick_reply_text_size],
content_type='text') for item in quick_replies]
if quick_replies:
payload['message']['quick_replies'] = formatted_replies
# this is a ref facebook id, temporary just for this message
if URN.is_path_fb_ref(msg.urn_path):
payload['recipient'] = dict(user_ref=URN.fb_ref_from_path(msg.urn_path))
else:
payload['recipient'] = dict(id=msg.urn_path)
url = "https://graph.facebook.com/v2.5/me/messages"
params = {'access_token': channel.config[Channel.CONFIG_AUTH_TOKEN]}
headers = {'Content-Type': 'application/json'}
start = time.time()
payload = json.dumps(payload)
event = HttpEvent('POST', url, json.dumps(payload))
try:
response = requests.post(url, payload, params=params, headers=headers, timeout=15)
event.status_code = response.status_code
event.response_body = response.text
except Exception as e:
raise SendException(six.text_type(e), event=event, start=start)
# for now we only support sending one attachment per message but this could change in future
attachments = Attachment.parse_all(msg.attachments)
attachment = attachments[0] if attachments else None
if attachment:
category = attachment.content_type.split('/')[0]
payload = json.loads(payload)
payload['message'] = {'attachment': {'type': category, 'payload': {'url': attachment.url}}}
payload = json.dumps(payload)
event = HttpEvent('POST', url, payload)
try:
response = requests.post(url, payload, params=params, headers=headers, timeout=15)
event.status_code = response.status_code
event.response_body = response.text
except Exception as e:
raise SendException(six.text_type(e), event=event, start=start)
if response.status_code != 200: # pragma: no cover
raise SendException("Got non-200 response [%d] from Facebook" % response.status_code,
event=event, start=start)
# grab our external id out, Facebook response is in format:
# "{"recipient_id":"997011467086879","message_id":"mid.1459532331848:2534ddacc3993a4b78"}"
external_id = None
try:
external_id = response.json()['message_id']
except Exception as e: # pragma: no cover
# if we can't pull out our message id, that's ok, we still sent
pass
# if we sent Facebook a user_ref, look up the real Facebook id for this contact, should be in 'recipient_id'
if URN.is_path_fb_ref(msg.urn_path):
contact_obj = Contact.objects.get(id=msg.contact)
org_obj = Org.objects.get(id=channel.org)
channel_obj = Channel.objects.get(id=channel.id)
try:
real_fb_id = response.json()['recipient_id']
# associate this contact with our real FB id
ContactURN.get_or_create(org_obj, contact_obj, URN.from_facebook(real_fb_id), channel=channel_obj)
# save our ref_id as an external URN on this contact
ContactURN.get_or_create(org_obj, contact_obj, URN.from_external(URN.fb_ref_from_path(msg.urn_path)))
# finally, disassociate our temp ref URN with this contact
ContactURN.objects.filter(id=msg.contact_urn).update(contact=None)
except Exception as e: # pragma: no cover
# if we can't pull out the recipient id, that's ok, msg was sent
pass
Channel.success(channel, msg, WIRED, start, event=event, external_id=external_id)
class USSDSession(ChannelSession):
USSD_PULL = INCOMING = 'I'
USSD_PUSH = OUTGOING = 'O'
objects = USSDQuerySet.as_manager()
class Meta:
proxy = True
@property
def should_end(self):
return self.status == self.ENDING
def mark_ending(self): # session to be ended
if self.status != self.ENDING:
self.status = self.ENDING
self.save(update_fields=['status'])
def close(self): # session has successfully ended
if self.status == self.ENDING:
self.status = self.COMPLETED
else:
self.status = self.INTERRUPTED
self.ended_on = timezone.now()
self.save(update_fields=['status', 'ended_on'])
def start_async(self, flow, date, message_id):
from temba.msgs.models import Msg, USSD
message = Msg.objects.create(channel=self.channel,
contact=self.contact,
contact_urn=self.contact_urn,
sent_on=date,
connection=self,
msg_type=USSD,
external_id=message_id,
created_on=timezone.now(),
modified_on=timezone.now(),
org=self.channel.org,
direction=self.INCOMING)
flow.start([], [self.contact],
start_msg=message,
restart_participants=True,
connection=self)
def handle_async(self, urn, content, date, message_id):
from temba.msgs.models import Msg, USSD
Msg.create_incoming(channel=self.channel,
org=self.org,
urn=urn,
text=content or '',
date=date,
connection=self,
msg_type=USSD,
external_id=message_id)
def handle_sync(self): # pragma: needs cover
# TODO: implement for InfoBip and other sync APIs
pass
@classmethod
def handle_incoming(cls,
channel,
urn,
date,
external_id,
contact=None,
message_id=None,
status=None,
content=None,
starcode=None,
org=None,
async=True):
trigger = None
contact_urn = None
# handle contact with channel
urn = URN.from_tel(urn)
if not contact:
contact, contact_urn = Contact.get_or_create(
channel.org, urn, channel)
elif urn:
contact_urn = ContactURN.get_or_create(org,
contact,
urn,
channel=channel)
contact.set_preferred_channel(channel)
if contact_urn:
contact_urn.update_affinity(channel)
# setup session
defaults = dict(channel=channel,
contact=contact,
contact_urn=contact_urn,
org=channel.org if channel else contact.org)
if status == cls.TRIGGERED:
trigger = Trigger.find_trigger_for_ussd_session(contact, starcode)
if not trigger:
return False
defaults.update(
dict(started_on=date, direction=cls.USSD_PULL, status=status))
elif status == cls.INTERRUPTED:
defaults.update(dict(ended_on=date, status=status))
else:
defaults.update(dict(status=cls.IN_PROGRESS))
# check if there's an initiated PUSH connection
connection = cls.objects.get_initiated_push(contact)
created = False
if not connection:
try:
connection = cls.objects.select_for_update().exclude(status__in=ChannelSession.DONE)\
.get(external_id=external_id)
created = False
for k, v in six.iteritems(defaults):
setattr(connection, k, v() if callable(v) else v)
connection.save()
except cls.DoesNotExist:
defaults['external_id'] = external_id
connection = cls.objects.create(**defaults)
FlowSession.create(contact, connection=connection)
created = True
else:
defaults.update(dict(external_id=external_id))
for key, value in six.iteritems(defaults):
setattr(connection, key, value)
connection.save()
created = None
# start session
if created and async and trigger:
connection.start_async(trigger.flow, date, message_id)
def resolve_twitter_ids():
r = get_redis_connection()
# TODO: we can't use our non-overlapping task decorator as it creates a loop in the celery resolver when registering
if r.get("resolve_twitter_ids_task"): # pragma: no cover
return
with r.lock("resolve_twitter_ids_task", 1800):
# look up all 'twitter' URNs, limiting to 30k since that's the most our API would allow anyways
twitter_urns = ContactURN.objects.filter(
scheme=TWITTER_SCHEME, contact__is_stopped=False, contact__is_blocked=False
).exclude(contact=None)
twitter_urns = twitter_urns[:30000].only("id", "org", "contact", "path")
api_key = settings.TWITTER_API_KEY
api_secret = settings.TWITTER_API_SECRET
client = Twython(api_key, api_secret)
updated = 0
print("found %d twitter urns to resolve" % len(twitter_urns))
# contacts we will stop
stop_contacts = []
# we try to look these up 100 at a time
for urn_batch in chunk_list(twitter_urns, 100):
screen_names = [u.path for u in urn_batch]
screen_map = {u.path: u for u in urn_batch}
# try to fetch our users by screen name
try:
resp = client.lookup_user(screen_name=",".join(screen_names))
for twitter_user in resp:
screen_name = twitter_user["screen_name"].lower()
twitter_id = twitter_user["id"]
if screen_name in screen_map and twitter_user["id"]:
twitterid_urn = URN.normalize(URN.from_twitterid(twitter_id, screen_name))
old_urn = screen_map[screen_name]
# create our new contact URN
new_urn = ContactURN.get_or_create(old_urn.org, old_urn.contact, twitterid_urn)
# if our new URN already existed for another contact and it is newer
# than our old contact, reassign it to the old contact
if (
new_urn.contact != old_urn.contact
and new_urn.contact.created_on > old_urn.contact.created_on
):
new_urn.contact = old_urn.contact
new_urn.save(update_fields=["contact"])
# get rid of our old URN
ContactURN.objects.filter(id=old_urn.id).update(contact=None)
del screen_map[screen_name]
updated += 1
except Exception as e:
# if this wasn't an exception caused by not finding any of the users, then break
if str(e).find("No user matches") < 0:
# exit, we'll try again later
print("exiting resolve_twitter_ids due to exception: %s" % e)
break
# add all remaining contacts to the contacts we will stop
for contact in screen_map.values():
stop_contacts.append(contact)
# stop all the contacts we couldn't resolve that have only a twitter URN
stopped = 0
for contact_urn in stop_contacts:
contact = contact_urn.contact
if len(contact.urns.all()) == 1:
contact.stop(contact.created_by)
stopped += 1
if len(twitter_urns) > 0:
print("updated %d twitter urns, %d stopped" % (updated, len(stop_contacts)))
def handle_incoming(
cls,
channel,
urn,
date,
external_id,
contact=None,
message_id=None,
status=None,
content=None,
starcode=None,
org=None,
do_async=True,
):
trigger = None
contact_urn = None
# handle contact with channel
urn = URN.from_tel(urn)
if not contact:
contact, contact_urn = Contact.get_or_create(
channel.org, urn, channel)
elif urn:
contact_urn = ContactURN.get_or_create(org,
contact,
urn,
channel=channel)
contact.set_preferred_channel(channel)
if contact_urn:
contact_urn.update_affinity(channel)
# setup session
defaults = dict(channel=channel,
contact=contact,
contact_urn=contact_urn,
org=channel.org if channel else contact.org)
if status == cls.TRIGGERED:
trigger = Trigger.find_trigger_for_ussd_session(contact, starcode)
if not trigger:
return False
defaults.update(
dict(started_on=date, direction=cls.USSD_PULL, status=status))
elif status == cls.INTERRUPTED:
defaults.update(dict(ended_on=date, status=status))
else:
defaults.update(dict(status=cls.IN_PROGRESS))
# check if there's an initiated PUSH connection
connection = cls.objects.get_initiated_push(contact)
created = False
if not connection:
try:
connection = (cls.objects.select_for_update().exclude(
status__in=ChannelSession.DONE).get(
external_id=external_id))
created = False
for k, v in defaults.items():
setattr(connection, k, v() if callable(v) else v)
connection.save()
except cls.DoesNotExist:
defaults["external_id"] = external_id
connection = cls.objects.create(**defaults)
FlowSession.create(contact, connection=connection)
created = True
else:
defaults.update(dict(external_id=external_id))
for key, value in defaults.items():
setattr(connection, key, value)
connection.save()
created = None
# start session
if created and do_async and trigger:
connection.start_async(trigger.flow, date, message_id)
# resume session, deal with incoming content and all the other states
else:
connection.handle_async(urn, content, date, message_id)
return connection
def test_claim(self):
# remove our explicit country so it needs to be derived from channels
self.org.country = None
self.org.save()
Channel.objects.all().delete()
reg_data = dict(cmds=[
dict(cmd="fcm", fcm_id="FCM111", uuid="uuid"),
dict(cmd="status", cc="RW", dev="Nexus")
])
# must be a post
response = self.client.get(reverse("register"),
content_type="application/json")
self.assertEqual(500, response.status_code)
# try a legit register
response = self.client.post(reverse("register"),
json.dumps(reg_data),
content_type="application/json")
self.assertEqual(200, response.status_code)
android1 = Channel.objects.get()
self.assertIsNone(android1.org)
self.assertIsNone(android1.address)
self.assertIsNone(android1.alert_email)
self.assertEqual(android1.country, "RW")
self.assertEqual(android1.device, "Nexus")
self.assertEqual(android1.config["FCM_ID"], "FCM111")
self.assertEqual(android1.uuid, "uuid")
self.assertTrue(android1.secret)
self.assertTrue(android1.claim_code)
self.assertEqual(android1.created_by, get_anonymous_user())
# check channel JSON in response
response_json = response.json()
self.assertEqual(
response_json,
dict(cmds=[
dict(
cmd="reg",
relayer_claim_code=android1.claim_code,
relayer_secret=android1.secret,
relayer_id=android1.id,
)
]),
)
# try registering again with same details
response = self.client.post(reverse("register"),
json.dumps(reg_data),
content_type="application/json")
self.assertEqual(response.status_code, 200)
android1 = Channel.objects.get()
response_json = response.json()
self.assertEqual(
response_json,
dict(cmds=[
dict(
cmd="reg",
relayer_claim_code=android1.claim_code,
relayer_secret=android1.secret,
relayer_id=android1.id,
)
]),
)
# view claim page
self.login(self.admin)
response = self.client.get(reverse("channels.types.android.claim"))
self.assertContains(response, "https://app.rapidpro.io/android/")
# try to claim as non-admin
self.login(self.user)
response = self.client.post(
reverse("channels.types.android.claim"),
dict(claim_code=android1.claim_code, phone_number="0788123123"))
self.assertLoginRedirect(response)
# try to claim with an invalid phone number
self.login(self.admin)
response = self.client.post(
reverse("channels.types.android.claim"),
dict(claim_code=android1.claim_code, phone_number="078123"))
self.assertEqual(response.status_code, 200)
self.assertFormError(response, "form", "phone_number",
"Invalid phone number, try again.")
# claim our channel
response = self.client.post(
reverse("channels.types.android.claim"),
dict(claim_code=android1.claim_code, phone_number="0788123123"))
# redirect to welcome page
self.assertIn("success", response.get("Location", None))
self.assertRedirect(response, reverse("public.public_welcome"))
# channel is updated with org details and claim code is now blank
android1.refresh_from_db()
secret = android1.secret
self.assertEqual(android1.org, self.org)
self.assertEqual(android1.address, "+250788123123") # normalized
self.assertEqual(android1.alert_email,
self.admin.email) # the logged-in user
self.assertEqual(android1.config["FCM_ID"], "FCM111")
self.assertEqual(android1.uuid, "uuid")
self.assertFalse(android1.claim_code)
# try having a device register again
response = self.client.post(reverse("register"),
json.dumps(reg_data),
content_type="application/json")
self.assertEqual(response.status_code, 200)
# should return same channel but with a new claim code and secret
android1.refresh_from_db()
self.assertEqual(android1.org, self.org)
self.assertEqual(android1.address, "+250788123123")
self.assertEqual(android1.alert_email, self.admin.email)
self.assertEqual(android1.config["FCM_ID"], "FCM111")
self.assertEqual(android1.uuid, "uuid")
self.assertEqual(android1.is_active, True)
self.assertTrue(android1.claim_code)
self.assertNotEqual(android1.secret, secret)
# should be able to claim again
response = self.client.post(
reverse("channels.types.android.claim"),
dict(claim_code=android1.claim_code, phone_number="0788123123"))
self.assertRedirect(response, reverse("public.public_welcome"))
# try having a device register yet again with new FCM ID
reg_data["cmds"][0]["fcm_id"] = "FCM222"
response = self.client.post(reverse("register"),
json.dumps(reg_data),
content_type="application/json")
self.assertEqual(response.status_code, 200)
# should return same channel but with FCM updated
android1.refresh_from_db()
self.assertEqual(android1.org, self.org)
self.assertEqual(android1.address, "+250788123123")
self.assertEqual(android1.alert_email, self.admin.email)
self.assertEqual(android1.config["FCM_ID"], "FCM222")
self.assertEqual(android1.uuid, "uuid")
self.assertEqual(android1.is_active, True)
# we can claim again with new phone number
response = self.client.post(
reverse("channels.types.android.claim"),
dict(claim_code=android1.claim_code, phone_number="+250788123124"))
self.assertRedirect(response, reverse("public.public_welcome"))
android1.refresh_from_db()
self.assertEqual(android1.org, self.org)
self.assertEqual(android1.address, "+250788123124")
self.assertEqual(android1.alert_email, self.admin.email)
self.assertEqual(android1.config["FCM_ID"], "FCM222")
self.assertEqual(android1.uuid, "uuid")
self.assertEqual(android1.is_active, True)
# release and then register with same details and claim again
old_uuid = android1.uuid
android1.release()
response = self.client.post(reverse("register"),
json.dumps(reg_data),
content_type="application/json")
claim_code = response.json()["cmds"][0]["relayer_claim_code"]
self.assertEqual(response.status_code, 200)
response = self.client.post(
reverse("channels.types.android.claim"),
dict(claim_code=claim_code, phone_number="+250788123124"))
self.assertRedirect(response, reverse("public.public_welcome"))
android1.refresh_from_db()
self.assertNotEqual(android1.uuid,
old_uuid) # inactive channel now has new UUID
# and we have a new Android channel with our UUID
android2 = Channel.objects.get(is_active=True)
self.assertNotEqual(android2, android1)
self.assertEqual(android2.uuid, "uuid")
# try to claim a bogus channel
response = self.client.post(reverse("channels.types.android.claim"),
dict(claim_code="Your Mom"))
self.assertEqual(response.status_code, 200)
self.assertFormError(
response, "form", "claim_code",
"Invalid claim code, please check and try again.")
# check our primary tel channel is the same as our outgoing
default_sender = self.org.get_send_channel(TEL_SCHEME)
self.assertEqual(default_sender, android2)
self.assertEqual(default_sender,
self.org.get_receive_channel(TEL_SCHEME))
self.assertFalse(default_sender.is_delegate_sender())
response = self.client.get(
reverse("channels.channel_bulk_sender_options"))
self.assertEqual(response.status_code, 200)
response = self.client.post(
reverse("channels.channel_create_bulk_sender") + "?connection=NX",
dict(connection="NX"))
self.assertFormError(response, "form", "channel",
"Can't add sender for that number")
# try to claim a bulk Nexmo sender (without adding Nexmo account to org)
claim_nexmo_url = reverse("channels.channel_create_bulk_sender"
) + "?connection=NX&channel=%d" % android2.pk
response = self.client.post(claim_nexmo_url,
dict(connection="NX", channel=android2.pk))
self.assertFormError(response, "form", "connection",
"A connection to a Nexmo account is required")
# send channel is still our Android device
self.assertEqual(self.org.get_send_channel(TEL_SCHEME), android2)
self.assertFalse(self.org.is_connected_to_nexmo())
# now connect to nexmo
self.org.connect_nexmo("123", "456", self.admin)
self.assertTrue(self.org.is_connected_to_nexmo())
# now adding Nexmo bulk sender should work
response = self.client.post(claim_nexmo_url,
dict(connection="NX", channel=android2.pk))
self.assertRedirect(response, reverse("orgs.org_home"))
# new Nexmo channel created for delegated sending
nexmo = self.org.get_send_channel(TEL_SCHEME)
self.assertEqual(nexmo.channel_type, "NX")
self.assertEqual(nexmo.parent, android2)
self.assertTrue(nexmo.is_delegate_sender())
self.assertEqual(nexmo.tps, 1)
channel_config = nexmo.config
self.assertEqual(channel_config[Channel.CONFIG_NEXMO_API_KEY], "123")
self.assertEqual(channel_config[Channel.CONFIG_NEXMO_API_SECRET],
"456")
# reading our nexmo channel should now offer a disconnect option
nexmo = self.org.channels.filter(channel_type="NX").first()
response = self.client.get(
reverse("channels.channel_read", args=[nexmo.uuid]))
self.assertContains(response, "Disable Bulk Sending")
# receiving still job of our Android device
self.assertEqual(self.org.get_receive_channel(TEL_SCHEME), android2)
# re-register device with country as US
reg_data = dict(cmds=[
dict(cmd="fcm", fcm_id="FCM222", uuid="uuid"),
dict(cmd="status", cc="US", dev="Nexus 5X")
])
response = self.client.post(reverse("register"),
json.dumps(reg_data),
content_type="application/json")
self.assertEqual(response.status_code, 200)
# channel country and device updated
android2.refresh_from_db()
self.assertEqual(android2.country, "US")
self.assertEqual(android2.device, "Nexus 5X")
self.assertEqual(android2.org, self.org)
self.assertEqual(android2.config["FCM_ID"], "FCM222")
self.assertEqual(android2.uuid, "uuid")
self.assertTrue(android2.is_active)
# set back to RW...
android2.country = "RW"
android2.save()
# our country is RW
self.assertEqual(self.org.get_country_code(), "RW")
# remove nexmo
nexmo.release()
self.assertEqual(self.org.get_country_code(), "RW")
# register another device with country as US
reg_data = dict(cmds=[
dict(cmd="fcm", fcm_id="FCM444", uuid="uuid4"),
dict(cmd="status", cc="US", dev="Nexus 6P")
])
response = self.client.post(reverse("register"),
json.dumps(reg_data),
content_type="application/json")
claim_code = response.json()["cmds"][0]["relayer_claim_code"]
# try to claim it...
self.client.post(
reverse("channels.types.android.claim"),
dict(claim_code=claim_code, phone_number="12065551212"))
# should work, can have two channels in different countries
channel = Channel.objects.get(country="US")
self.assertEqual(channel.address, "+12065551212")
self.assertEqual(
Channel.objects.filter(org=self.org, is_active=True).count(), 2)
# normalize a URN with a fully qualified number
number, valid = URN.normalize_number("+12061112222", None)
self.assertTrue(valid)
# not international format
number, valid = URN.normalize_number("0788383383", None)
self.assertFalse(valid)
# get our send channel without a URN, should just default to last
default_channel = self.org.get_send_channel(TEL_SCHEME)
self.assertEqual(default_channel, channel)
# get our send channel for a Rwandan URN
rwanda_channel = self.org.get_send_channel(
TEL_SCHEME, ContactURN.create(self.org, None, "tel:+250788383383"))
self.assertEqual(rwanda_channel, android2)
# and a US one
us_channel = self.org.get_send_channel(
TEL_SCHEME, ContactURN.create(self.org, None, "tel:+12065555353"))
self.assertEqual(us_channel, channel)
# a different country altogether should just give us the default
us_channel = self.org.get_send_channel(
TEL_SCHEME, ContactURN.create(self.org, None, "tel:+593997290044"))
self.assertEqual(us_channel, channel)
self.org = Org.objects.get(id=self.org.id)
self.assertIsNone(self.org.get_country_code())
# yet another registration in rwanda
reg_data = dict(cmds=[
dict(cmd="fcm", fcm_id="FCM555", uuid="uuid5"),
dict(cmd="status", cc="RW", dev="Nexus 5")
])
response = self.client.post(reverse("register"),
json.dumps(reg_data),
content_type="application/json")
claim_code = response.json()["cmds"][0]["relayer_claim_code"]
# try to claim it with number taken by other Android channel
response = self.client.post(
reverse("channels.types.android.claim"),
dict(claim_code=claim_code, phone_number="+250788123124"))
self.assertFormError(
response, "form", "phone_number",
"Another channel has this number. Please remove that channel first."
)
# create channel in another org
Channel.create(self.org2, self.admin2, "RW", "A", "", "+250788382382")
# can claim it with this number, and because it's a fully qualified RW number, doesn't matter that channel is US
response = self.client.post(
reverse("channels.types.android.claim"),
dict(claim_code=claim_code, phone_number="+250788382382"))
self.assertRedirect(response, reverse("public.public_welcome"))
# should be added with RW as the country
self.assertTrue(
Channel.objects.get(address="+250788382382",
country="RW",
org=self.org))
def create_contacts(self, orgs, locations, num_total):
batch_size = 5000
num_test_contacts = len(orgs) * len(USERS)
group_membership_model = ContactGroup.contacts.through
group_counts = defaultdict(int)
self._log("Creating %d test contacts...\n" % num_test_contacts)
for org in orgs:
for user in org.cache['users']:
Contact.get_test_contact(user)
self._log("Creating %d regular contacts...\n" % (num_total - num_test_contacts))
base_contact_id = self.get_current_id(Contact) + 1
# Disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, Value, group_membership_model):
names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch = 1
for index_batch in chunk_list(range(num_total - num_test_contacts), batch_size):
contacts = []
urns = []
values = []
memberships = []
def add_to_group(g):
group_counts[g] += 1
memberships.append(group_membership_model(contact_id=c['id'], contactgroup=g))
for c_index in index_batch: # pragma: no cover
org = orgs[c_index] if c_index < len(orgs) else self.random_org(orgs) # at least 1 contact per org
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(float(num_test_contacts + c_index) / num_total)
c = {
'id': base_contact_id + c_index, # database id this contact will have when created
'org': org,
'user': org.cache['users'][0],
'name': name,
'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None,
'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'ward': location[0] if location else None,
'district': location[1] if location else None,
'state': location[2] if location else None,
'language': self.random_choice(CONTACT_LANGS),
'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB),
'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB),
'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB),
'created_on': created_on,
'modified_on': self.random_date(created_on, self.db_ends_on),
}
if c['is_active']:
if not c['is_blocked'] and not c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_ALL])
if c['is_blocked']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED])
if c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_STOPPED])
contacts.append(Contact(org=org, name=c['name'], language=c['language'],
is_stopped=c['is_stopped'], is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=user, created_on=c['created_on'],
modified_by=user, modified_on=c['modified_on']))
if c['tel']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TEL_SCHEME,
path=c['tel'], urn=URN.from_tel(c['tel'])))
if c['twitter']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TWITTER_SCHEME,
path=c['twitter'], urn=URN.from_twitter(c['twitter'])))
if c['gender']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['gender'],
string_value=c['gender']))
if c['age']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['age'],
string_value=str(c['age']), decimal_value=c['age']))
if c['joined']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['joined'],
string_value=datetime_to_str(c['joined']), datetime_value=c['joined']))
if location:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['ward'],
string_value=c['ward'].name, location_value=c['ward']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['district'],
string_value=c['district'].name, location_value=c['district']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['state'],
string_value=c['state'].name, location_value=c['state']))
# let each group decide if it is taking this contact
for g in org.cache['groups']:
if g.member(c) if callable(g.member) else self.probability(g.member):
add_to_group(g)
Contact.objects.bulk_create(contacts)
ContactURN.objects.bulk_create(urns)
Value.objects.bulk_create(values)
group_membership_model.objects.bulk_create(memberships)
self._log(" > Created batch %d of %d\n" % (batch, max(num_total // batch_size, 1)))
batch += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
ContactGroupCount.objects.bulk_create(counts)
# for sanity check that our presumed last contact id matches the last actual contact id
assert c['id'] == Contact.objects.order_by('-id').first().id
def handle_incoming(
cls,
channel,
urn,
date,
external_id,
contact=None,
message_id=None,
status=None,
content=None,
starcode=None,
org=None,
do_async=True,
):
trigger = None
contact_urn = None
# handle contact with channel
urn = URN.from_tel(urn)
if not contact:
contact, contact_urn = Contact.get_or_create(channel.org, urn, channel)
elif urn:
contact_urn = ContactURN.get_or_create(org, contact, urn, channel=channel)
contact.set_preferred_channel(channel)
if contact_urn:
contact_urn.update_affinity(channel)
# setup session
defaults = dict(
channel=channel, contact=contact, contact_urn=contact_urn, org=channel.org if channel else contact.org
)
if status == cls.TRIGGERED:
trigger = Trigger.find_trigger_for_ussd_session(contact, starcode)
if not trigger:
return False
defaults.update(dict(started_on=date, direction=cls.USSD_PULL, status=status))
elif status == cls.INTERRUPTED:
defaults.update(dict(ended_on=date, status=status))
else:
defaults.update(dict(status=cls.IN_PROGRESS))
# check if there's an initiated PUSH connection
connection = cls.objects.get_initiated_push(contact)
created = False
if not connection:
try:
connection = (
cls.objects.select_for_update()
.exclude(status__in=ChannelConnection.DONE)
.get(external_id=external_id)
)
created = False
for k, v in defaults.items():
setattr(connection, k, v() if callable(v) else v)
connection.save()
except cls.DoesNotExist:
defaults["external_id"] = external_id
connection = cls.objects.create(**defaults)
FlowSession.create(contact, connection=connection)
created = True
else:
defaults.update(dict(external_id=external_id))
for key, value in defaults.items():
setattr(connection, key, value)
connection.save()
created = None
# start session
if created and do_async and trigger:
connection.start_async(trigger.flow, date, message_id)
# resume session, deal with incoming content and all the other states
else:
connection.handle_async(urn, content, date, message_id)
return connection
