def create_contact(self,
name=None,
number=None,
twitter=None,
twitterid=None,
urn=None,
**kwargs):
"""
Create a contact in the master test org
"""
urns = []
if number:
urns.append(URN.from_tel(number))
if twitter:
urns.append(URN.from_twitter(twitter))
if twitterid:
urns.append(URN.from_twitterid(twitterid))
if urn:
urns.append(urn)
if not name and not urns: # pragma: no cover
raise ValueError("Need a name or URN to create a contact")
kwargs["name"] = name
kwargs["urns"] = urns
if "org" not in kwargs:
kwargs["org"] = self.org
if "user" not in kwargs:
kwargs["user"] = self.user
return Contact.get_or_create_by_urns(**kwargs)
def create_contact(self,
name=None,
number=None,
twitter=None,
urn=None,
is_test=False,
**kwargs):
"""
Create a contact in the master test org
"""
urns = []
if number:
urns.append(URN.from_tel(number))
if twitter:
urns.append(URN.from_twitter(twitter))
if urn:
urns.append(urn)
if not name and not urns: # pragma: no cover
raise ValueError("Need a name or URN to create a contact")
kwargs['name'] = name
kwargs['urns'] = urns
kwargs['is_test'] = is_test
if 'org' not in kwargs:
kwargs['org'] = self.org
if 'user' not in kwargs:
kwargs['user'] = self.user
return Contact.get_or_create(**kwargs)
def create_contact(self,
name=None,
number=None,
twitter=None,
urn=None,
fields=None,
**kwargs):
"""
Create a contact in the master test org
"""
org = kwargs.pop("org", None) or self.org
user = kwargs.pop("user", None) or self.user
urns = []
if number:
urns.append(URN.from_tel(number))
if twitter:
urns.append(URN.from_twitter(twitter))
if urn:
urns.append(urn)
assert name or urns, "contact should have a name or a contact"
kwargs["name"] = name
kwargs["urns"] = urns
contact = Contact.get_or_create_by_urns(org, user, **kwargs)
if fields:
update_fields_locally(user, contact, fields)
return contact
def create_contact(self, name=None, number=None, twitter=None, urn=None, is_test=False, **kwargs):
"""
Create a contact in the master test org
"""
urns = []
if number:
urns.append(URN.from_tel(number))
if twitter:
urns.append(URN.from_twitter(twitter))
if urn:
urns.append(urn)
if not name and not urns: # pragma: no cover
raise ValueError("Need a name or URN to create a contact")
kwargs['name'] = name
kwargs['urns'] = urns
kwargs['is_test'] = is_test
if 'org' not in kwargs:
kwargs['org'] = self.org
if 'user' not in kwargs:
kwargs['user'] = self.user
return Contact.get_or_create(**kwargs)
def create_contact(self, name=None, number=None, twitter=None, twitterid=None, urn=None, is_test=False, **kwargs):
"""
Create a contact in the master test org
"""
urns = []
if number:
urns.append(URN.from_tel(number))
if twitter:
urns.append(URN.from_twitter(twitter))
if twitterid:
urns.append(URN.from_twitterid(twitterid))
if urn:
urns.append(urn)
if not name and not urns: # pragma: no cover
raise ValueError("Need a name or URN to create a contact")
kwargs["name"] = name
kwargs["urns"] = urns
kwargs["is_test"] = is_test
if "org" not in kwargs:
kwargs["org"] = self.org
if "user" not in kwargs:
kwargs["user"] = self.user
return Contact.get_or_create_by_urns(**kwargs)
def send(self, channel, msg, text):
twitter = TembaTwython.from_channel(channel)
start = time.time()
try:
urn = getattr(msg, "urn", URN.from_twitter(msg.urn_path))
(scheme, path, query, display) = URN.to_parts(urn)
# this is a legacy URN (no display), the path is our screen name
if scheme == TWITTER_SCHEME:
dm = twitter.send_direct_message(screen_name=path, text=text)
external_id = dm["id"]
# this is a new twitterid URN, our path is our user id
else:
metadata = msg.metadata if hasattr(msg, "metadata") else {}
quick_replies = metadata.get("quick_replies", [])
formatted_replies = [dict(label=item[: self.quick_reply_text_size]) for item in quick_replies]
if quick_replies:
params = {
"event": {
"type": "message_create",
"message_create": {
"target": {"recipient_id": path},
"message_data": {
"text": text,
"quick_reply": {"type": "options", "options": formatted_replies},
},
},
}
}
dm = twitter.post("direct_messages/events/new", params=params)
external_id = dm["event"]["id"]
else:
dm = twitter.send_direct_message(user_id=path, text=text)
external_id = dm["id"]
except Exception as e:
error_code = getattr(e, "error_code", 400)
fatal = False
if error_code == 404: # handle doesn't exist
fatal = True
elif error_code == 403:
for err in self.FATAL_403S:
if str(e).find(err) >= 0:
fatal = True
break
# if message can never be sent, stop them contact
if fatal:
contact = Contact.objects.get(id=msg.contact)
contact.stop(contact.modified_by)
raise SendException(str(e), events=twitter.events, fatal=fatal, start=start)
Channel.success(channel, msg, WIRED, start, events=twitter.events, external_id=external_id)
def test_resolve(self, mock_lookup_user):
self.joe = self.create_contact("joe", twitter="therealjoe")
urn = self.joe.get_urns()[0]
# test no return value, should cause joe to be stopped
mock_lookup_user.return_value = []
resolve_twitter_ids()
self.joe.refresh_from_db()
urn.refresh_from_db()
self.assertTrue(self.joe.is_stopped)
self.assertIsNone(urn.display)
self.assertEqual("twitter:therealjoe", urn.identity)
self.assertEqual("therealjoe", urn.path)
self.joe.unstop(self.admin)
# test a real return value
mock_lookup_user.return_value = [dict(screen_name="TheRealJoe", id="123456")]
resolve_twitter_ids()
urn.refresh_from_db()
self.assertIsNone(urn.contact)
new_urn = self.joe.get_urns()[0]
self.assertEqual("twitterid:123456", new_urn.identity)
self.assertEqual("123456", new_urn.path)
self.assertEqual("therealjoe", new_urn.display)
self.assertEqual("twitterid:123456#therealjoe", new_urn.urn)
old_fred = self.create_contact("old fred", urn=URN.from_twitter("fred"))
new_fred = self.create_contact("new fred", urn=URN.from_twitterid("12345", screen_name="fred"))
mock_lookup_user.return_value = [dict(screen_name="fred", id="12345")]
resolve_twitter_ids()
# new fred shouldn't have any URNs anymore as he really is old_fred
self.assertEqual(0, len(new_fred.urns.all()))
# old fred should be unchanged
self.assertEqual("twitterid:12345", old_fred.urns.all()[0].identity)
self.jane = self.create_contact("jane", twitter="jane10")
mock_lookup_user.side_effect = Exception(
"Twitter API returned a 404 (Not Found), No user matches for specified terms."
)
resolve_twitter_ids()
self.jane.refresh_from_db()
self.assertTrue(self.jane.is_stopped)
self.sarah = self.create_contact("sarah", twitter="sarah20")
mock_lookup_user.side_effect = Exception("Unable to reach API")
resolve_twitter_ids()
self.sarah.refresh_from_db()
self.assertFalse(self.sarah.is_stopped)
def test_resolve(self, mock_lookup_user):
self.joe = self.create_contact("joe", twitter="therealjoe")
urn = self.joe.get_urns()[0]
# test no return value, should cause joe to be stopped
mock_lookup_user.return_value = []
resolve_twitter_ids()
self.joe.refresh_from_db()
urn.refresh_from_db()
self.assertTrue(self.joe.is_stopped)
self.assertIsNone(urn.display)
self.assertEqual("twitter:therealjoe", urn.identity)
self.assertEqual("therealjoe", urn.path)
self.joe.unstop(self.admin)
# test a real return value
mock_lookup_user.return_value = [dict(screen_name="TheRealJoe", id="123456")]
resolve_twitter_ids()
urn.refresh_from_db()
self.assertIsNone(urn.contact)
new_urn = self.joe.get_urns()[0]
self.assertEqual("twitterid:123456", new_urn.identity)
self.assertEqual("123456", new_urn.path)
self.assertEqual("therealjoe", new_urn.display)
self.assertEqual("twitterid:123456#therealjoe", new_urn.urn)
old_fred = self.create_contact("old fred", urn=URN.from_twitter("fred"))
new_fred = self.create_contact("new fred", urn=URN.from_twitterid("12345", screen_name="fred"))
mock_lookup_user.return_value = [dict(screen_name="fred", id="12345")]
resolve_twitter_ids()
# new fred shouldn't have any URNs anymore as he really is old_fred
self.assertEqual(0, len(new_fred.urns.all()))
# old fred should be unchanged
self.assertEqual("twitterid:12345", old_fred.urns.all()[0].identity)
self.jane = self.create_contact("jane", twitter="jane10")
mock_lookup_user.side_effect = Exception(
"Twitter API returned a 404 (Not Found), No user matches for specified terms."
)
resolve_twitter_ids()
self.jane.refresh_from_db()
self.assertTrue(self.jane.is_stopped)
self.sarah = self.create_contact("sarah", twitter="sarah20")
mock_lookup_user.side_effect = Exception("Unable to reach API")
resolve_twitter_ids()
self.sarah.refresh_from_db()
self.assertFalse(self.sarah.is_stopped)
def _create_contact_batch(self, batch):
"""
Bulk creates a batch of contacts from flat representations
"""
for c in batch:
c["object"] = Contact(
org=c["org"],
name=c["name"],
language=c["language"],
is_stopped=c["is_stopped"],
is_blocked=c["is_blocked"],
is_active=c["is_active"],
created_by=c["user"],
created_on=c["created_on"],
modified_by=c["user"],
modified_on=c["modified_on"],
fields=c["fields_as_json"],
)
Contact.objects.bulk_create([c["object"] for c in batch])
# now that contacts have pks, bulk create the actual URN, value and group membership objects
batch_urns = []
batch_memberships = []
for c in batch:
org = c["org"]
c["urns"] = []
if c["tel"]:
c["urns"].append(
ContactURN(
org=org,
contact=c["object"],
priority=50,
scheme=TEL_SCHEME,
path=c["tel"],
identity=URN.from_tel(c["tel"]),
)
)
if c["twitter"]:
c["urns"].append(
ContactURN(
org=org,
contact=c["object"],
priority=50,
scheme=TWITTER_SCHEME,
path=c["twitter"],
identity=URN.from_twitter(c["twitter"]),
)
)
for g in c["groups"]:
batch_memberships.append(ContactGroup.contacts.through(contact=c["object"], contactgroup=g))
batch_urns += c["urns"]
ContactURN.objects.bulk_create(batch_urns)
ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
def _create_contact_batch(self, batch):
"""
Bulk creates a batch of contacts from flat representations
"""
for c in batch:
c["object"] = Contact(
org=c["org"],
name=c["name"],
language=c["language"],
is_stopped=c["is_stopped"],
is_blocked=c["is_blocked"],
is_active=c["is_active"],
created_by=c["user"],
created_on=c["created_on"],
modified_by=c["user"],
modified_on=c["modified_on"],
fields=c["fields_as_json"],
)
Contact.objects.bulk_create([c["object"] for c in batch])
# now that contacts have pks, bulk create the actual URN, value and group membership objects
batch_urns = []
batch_memberships = []
for c in batch:
org = c["org"]
c["urns"] = []
if c["tel"]:
c["urns"].append(
ContactURN(
org=org,
contact=c["object"],
priority=50,
scheme=TEL_SCHEME,
path=c["tel"],
identity=URN.from_tel(c["tel"]),
)
)
if c["twitter"]:
c["urns"].append(
ContactURN(
org=org,
contact=c["object"],
priority=50,
scheme=TWITTER_SCHEME,
path=c["twitter"],
identity=URN.from_twitter(c["twitter"]),
)
)
for g in c["groups"]:
batch_memberships.append(ContactGroup.contacts.through(contact=c["object"], contactgroup=g))
batch_urns += c["urns"]
ContactURN.objects.bulk_create(batch_urns)
ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
def _create_contact_batch(self, batch):
"""
Bulk creates a batch of contacts from flat representations
"""
for c in batch:
c['object'] = Contact(org=c['org'], name=c['name'], language=c['language'],
is_stopped=c['is_stopped'], is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=c['user'], created_on=c['created_on'],
modified_by=c['user'], modified_on=c['modified_on'])
Contact.objects.bulk_create([c['object'] for c in batch])
# now that contacts have pks, bulk create the actual URN, value and group membership objects
batch_urns = []
batch_values = []
batch_memberships = []
for c in batch:
org = c['org']
c['urns'] = []
if c['tel']:
c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TEL_SCHEME,
path=c['tel'], urn=URN.from_tel(c['tel'])))
if c['twitter']:
c['urns'].append(ContactURN(org=org, contact=c['object'], priority=50, scheme=TWITTER_SCHEME,
path=c['twitter'], urn=URN.from_twitter(c['twitter'])))
if c['gender']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['gender'],
string_value=c['gender']))
if c['age']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['age'],
string_value=str(c['age']), decimal_value=c['age']))
if c['joined']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['joined'],
string_value=datetime_to_str(c['joined']), datetime_value=c['joined']))
if c['ward']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['ward'],
string_value=c['ward'].name, location_value=c['ward']))
if c['district']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['district'],
string_value=c['district'].name, location_value=c['district']))
if c['state']:
batch_values.append(Value(org=org, contact=c['object'], contact_field=org.cache['fields']['state'],
string_value=c['state'].name, location_value=c['state']))
for g in c['groups']:
batch_memberships.append(ContactGroup.contacts.through(contact=c['object'], contactgroup=g))
batch_urns += c['urns']
ContactURN.objects.bulk_create(batch_urns)
Value.objects.bulk_create(batch_values)
ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
def _create_contact_batch(self, batch):
"""
Bulk creates a batch of contacts from flat representations
"""
for c in batch:
c['object'] = Contact(org=c['org'],
name=c['name'],
language=c['language'],
is_stopped=c['is_stopped'],
is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=c['user'],
created_on=c['created_on'],
modified_by=c['user'],
modified_on=c['modified_on'],
fields=c['fields_as_json'])
Contact.objects.bulk_create([c['object'] for c in batch])
# now that contacts have pks, bulk create the actual URN, value and group membership objects
batch_urns = []
batch_memberships = []
for c in batch:
org = c['org']
c['urns'] = []
if c['tel']:
c['urns'].append(
ContactURN(org=org,
contact=c['object'],
priority=50,
scheme=TEL_SCHEME,
path=c['tel'],
identity=URN.from_tel(c['tel'])))
if c['twitter']:
c['urns'].append(
ContactURN(org=org,
contact=c['object'],
priority=50,
scheme=TWITTER_SCHEME,
path=c['twitter'],
identity=URN.from_twitter(c['twitter'])))
for g in c['groups']:
batch_memberships.append(
ContactGroup.contacts.through(contact=c['object'],
contactgroup=g))
batch_urns += c['urns']
ContactURN.objects.bulk_create(batch_urns)
ContactGroup.contacts.through.objects.bulk_create(batch_memberships)
def send(self, channel, msg, text):
twitter = TembaTwython.from_channel(channel)
start = time.time()
try:
urn = getattr(msg, 'urn', URN.from_twitter(msg.urn_path))
(scheme, path, display) = URN.to_parts(urn)
# this is a legacy URN (no display), the path is our screen name
if scheme == TWITTER_SCHEME:
dm = twitter.send_direct_message(screen_name=path, text=text)
# this is a new twitterid URN, our path is our user id
else:
dm = twitter.send_direct_message(user_id=path, text=text)
except Exception as e:
error_code = getattr(e, 'error_code', 400)
fatal = False
if error_code == 404: # handle doesn't exist
fatal = True
elif error_code == 403:
for err in self.FATAL_403S:
if six.text_type(e).find(err) >= 0:
fatal = True
break
# if message can never be sent, stop them contact
if fatal:
contact = Contact.objects.get(id=msg.contact)
contact.stop(contact.modified_by)
raise SendException(str(e),
events=twitter.events,
fatal=fatal,
start=start)
external_id = dm['id']
Channel.success(channel,
msg,
WIRED,
start,
events=twitter.events,
external_id=external_id)
def send(self, channel, msg, text):
twitter = TembaTwython.from_channel(channel)
start = time.time()
try:
urn = getattr(msg, 'urn', URN.from_twitter(msg.urn_path))
(scheme, path, display) = URN.to_parts(urn)
print("scheme %s" % (scheme))
print("path %s" % (path))
print("display %s" % (display))
# this is a legacy URN (no display), the path is our screen name
if scheme == TWITTER_SCHEME:
dm = twitter.send_direct_message(screen_name=path, text=text)
external_id = dm['id']
# this is a new twitterid URN, our path is our user id
else:
metadata = msg.metadata if hasattr(msg, 'metadata') else {}
quick_replies = metadata.get('quick_replies', [])
formatted_replies = [
dict(label=item[:self.quick_reply_text_size])
for item in quick_replies
]
if quick_replies:
params = {
'event': {
'type': 'message_create',
'message_create': {
'target': {
'recipient_id': path
},
'message_data': {
'text': text,
'quick_reply': {
'type': 'options',
'options': formatted_replies
}
}
}
}
}
dm = twitter.post('direct_messages/events/new',
params=params)
external_id = dm['event']['id']
else:
print(path)
print(text)
dm = twitter.send_direct_message(user_id=path, text=text)
external_id = dm['id']
except Exception as e:
error_code = getattr(e, 'error_code', 400)
fatal = False
print("Twitter type")
print(e)
if error_code == 404: # handle doesn't exist
fatal = True
elif error_code == 403:
for err in self.FATAL_403S:
if six.text_type(e).find(err) >= 0:
fatal = True
break
# if message can never be sent, stop them contact
if fatal:
contact = Contact.objects.get(id=msg.contact)
contact.stop(contact.modified_by)
raise SendException(str(e),
events=twitter.events,
fatal=fatal,
start=start)
Channel.success(channel,
msg,
WIRED,
start,
events=twitter.events,
external_id=external_id)
def create_contacts(self, orgs, locations, num_total):
batch_size = 5000
num_test_contacts = len(orgs) * len(USERS)
group_membership_model = ContactGroup.contacts.through
group_counts = defaultdict(int)
self._log("Creating %d test contacts...\n" % num_test_contacts)
for org in orgs:
for user in org.cache['users']:
Contact.get_test_contact(user)
self._log("Creating %d regular contacts...\n" % (num_total - num_test_contacts))
base_contact_id = self.get_current_id(Contact) + 1
# Disable table triggers to speed up insertion and in the case of contact group m2m, avoid having an unsquashed
# count row for every contact
with DisableTriggersOn(Contact, ContactURN, Value, group_membership_model):
names = [('%s %s' % (c1, c2)).strip() for c2 in CONTACT_NAMES[1] for c1 in CONTACT_NAMES[0]]
names = [n if n else None for n in names]
batch = 1
for index_batch in chunk_list(range(num_total - num_test_contacts), batch_size):
contacts = []
urns = []
values = []
memberships = []
def add_to_group(g):
group_counts[g] += 1
memberships.append(group_membership_model(contact_id=c['id'], contactgroup=g))
for c_index in index_batch: # pragma: no cover
org = orgs[c_index] if c_index < len(orgs) else self.random_org(orgs) # at least 1 contact per org
name = self.random_choice(names)
location = self.random_choice(locations) if self.probability(CONTACT_HAS_FIELD_PROB) else None
created_on = self.timeline_date(float(num_test_contacts + c_index) / num_total)
c = {
'id': base_contact_id + c_index, # database id this contact will have when created
'org': org,
'user': org.cache['users'][0],
'name': name,
'tel': '+2507%08d' % c_index if self.probability(CONTACT_HAS_TEL_PROB) else None,
'twitter': '%s%d' % (name.replace(' ', '_').lower() if name else 'tweep', c_index) if self.probability(CONTACT_HAS_TWITTER_PROB) else None,
'gender': self.random_choice(('M', 'F')) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'age': self.random.randint(16, 80) if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'joined': self.random_date() if self.probability(CONTACT_HAS_FIELD_PROB) else None,
'ward': location[0] if location else None,
'district': location[1] if location else None,
'state': location[2] if location else None,
'language': self.random_choice(CONTACT_LANGS),
'is_stopped': self.probability(CONTACT_IS_STOPPED_PROB),
'is_blocked': self.probability(CONTACT_IS_BLOCKED_PROB),
'is_active': self.probability(1 - CONTACT_IS_DELETED_PROB),
'created_on': created_on,
'modified_on': self.random_date(created_on, self.db_ends_on),
}
if c['is_active']:
if not c['is_blocked'] and not c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_ALL])
if c['is_blocked']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_BLOCKED])
if c['is_stopped']:
add_to_group(org.cache['system_groups'][ContactGroup.TYPE_STOPPED])
contacts.append(Contact(org=org, name=c['name'], language=c['language'],
is_stopped=c['is_stopped'], is_blocked=c['is_blocked'],
is_active=c['is_active'],
created_by=user, created_on=c['created_on'],
modified_by=user, modified_on=c['modified_on']))
if c['tel']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TEL_SCHEME,
path=c['tel'], urn=URN.from_tel(c['tel'])))
if c['twitter']:
urns.append(ContactURN(org=org, contact_id=c['id'], priority=50, scheme=TWITTER_SCHEME,
path=c['twitter'], urn=URN.from_twitter(c['twitter'])))
if c['gender']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['gender'],
string_value=c['gender']))
if c['age']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['age'],
string_value=str(c['age']), decimal_value=c['age']))
if c['joined']:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['joined'],
string_value=datetime_to_str(c['joined']), datetime_value=c['joined']))
if location:
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['ward'],
string_value=c['ward'].name, location_value=c['ward']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['district'],
string_value=c['district'].name, location_value=c['district']))
values.append(Value(org=org, contact_id=c['id'], contact_field=org.cache['fields']['state'],
string_value=c['state'].name, location_value=c['state']))
# let each group decide if it is taking this contact
for g in org.cache['groups']:
if g.member(c) if callable(g.member) else self.probability(g.member):
add_to_group(g)
Contact.objects.bulk_create(contacts)
ContactURN.objects.bulk_create(urns)
Value.objects.bulk_create(values)
group_membership_model.objects.bulk_create(memberships)
self._log(" > Created batch %d of %d\n" % (batch, max(num_total // batch_size, 1)))
batch += 1
# create group count records manually
counts = []
for group, count in group_counts.items():
counts.append(ContactGroupCount(group=group, count=count, is_squashed=True))
ContactGroupCount.objects.bulk_create(counts)
# for sanity check that our presumed last contact id matches the last actual contact id
assert c['id'] == Contact.objects.order_by('-id').first().id
