def setUp(self):
self.clear_cache()
self.user = self.create_user("tito")
self.admin = self.create_user("Administrator")
self.org = Org.objects.create(name="Nyaruka Ltd.", timezone="Africa/Kigali",
created_by=self.user, modified_by=self.user)
self.org.initialize()
self.org.administrators.add(self.admin)
self.admin.set_org(self.org)
self.org.administrators.add(self.user)
self.user.set_org(self.org)
self.tel_mtn = Channel.objects.create(org=self.org, name="MTN", channel_type="A", role="SR",
address="+250780000000", secret="12345", gcm_id="123",
created_by=self.user, modified_by=self.user)
self.tel_tigo = Channel.objects.create(org=self.org, name="Tigo", channel_type="A", role="SR",
address="+250720000000", secret="23456", gcm_id="234",
created_by=self.user, modified_by=self.user)
self.tel_bulk = Channel.objects.create(org=self.org, name="Nexmo", channel_type="NX", role="S",
parent=self.tel_tigo, secret="34567",
created_by=self.user, modified_by=self.user)
self.twitter = Channel.objects.create(org=self.org, name="Twitter", channel_type="TT", role="SR",
created_by=self.user, modified_by=self.user)
# for generating tuples of scheme, path and channel
generate_tel_mtn = lambda num: (TEL_SCHEME, "+25078%07d" % (num + 1), self.tel_mtn)
generate_tel_tigo = lambda num: (TEL_SCHEME, "+25072%07d" % (num + 1), self.tel_tigo)
generate_twitter = lambda num: (TWITTER_SCHEME, "tweep_%d" % (num + 1), self.twitter)
self.urn_generators = (generate_tel_mtn, generate_tel_tigo, generate_twitter)
self.field_nick = ContactField.get_or_create(self.org, 'nick', 'Nickname', show_in_table=True, value_type=TEXT)
self.field_age = ContactField.get_or_create(self.org, 'age', 'Age', show_in_table=True, value_type=DECIMAL)
def setUp(self):
self.clear_cache()
self.user = self.create_user("tito")
self.admin = self.create_user("Administrator")
self.org = Org.objects.create(name="Nyaruka Ltd.", timezone="Africa/Kigali",
created_by=self.user, modified_by=self.user)
self.org.initialize()
self.org.administrators.add(self.admin)
self.admin.set_org(self.org)
self.org.administrators.add(self.user)
self.user.set_org(self.org)
self.tel_mtn = Channel.create(self.org, self.user, 'RW', 'A', name="MTN", address="+250780000000",
secret="12345", gcm_id="123")
self.tel_tigo = Channel.create(self.org, self.user, 'RW', 'A', name="Tigo", address="+250720000000",
secret="23456", gcm_id="234")
self.tel_bulk = Channel.create(self.org, self.user, 'RW', 'NX', name="Nexmo", parent=self.tel_tigo)
self.twitter = Channel.create(self.org, self.user, None, 'TT', name="Twitter", address="billy_bob")
# for generating tuples of scheme, path and channel
def generate_tel_mtn(num):
return TEL_SCHEME, "+25078%07d" % (num + 1), self.tel_mtn
def generate_tel_tigo(num):
return TEL_SCHEME, "+25072%07d" % (num + 1), self.tel_tigo
def generate_twitter(num):
return TWITTER_SCHEME, "tweep_%d" % (num + 1), self.twitter
self.urn_generators = (generate_tel_mtn, generate_tel_tigo, generate_twitter)
self.field_nick = ContactField.get_or_create(self.org, self.admin, 'nick', 'Nickname', show_in_table=True, value_type=Value.TYPE_TEXT)
self.field_age = ContactField.get_or_create(self.org, self.admin, 'age', 'Age', show_in_table=True, value_type=Value.TYPE_DECIMAL)
def test_campaign_events(self):
url = reverse('api.v2.campaign_events')
self.assertEndpointAccess(url)
flow = self.create_flow()
reporters = self.create_group("Reporters", [self.joe, self.frank])
registration = ContactField.get_or_create(self.org, self.admin, 'registration', "Registration")
campaign1 = Campaign.create(self.org, self.admin, "Reminders", reporters)
event1 = CampaignEvent.create_message_event(self.org, self.admin, campaign1, registration,
1, CampaignEvent.UNIT_DAYS, "Don't forget to brush your teeth")
campaign2 = Campaign.create(self.org, self.admin, "Notifications", reporters)
event2 = CampaignEvent.create_flow_event(self.org, self.admin, campaign2, registration,
6, CampaignEvent.UNIT_HOURS, flow, delivery_hour=12)
# create event for another org
joined = ContactField.get_or_create(self.org2, self.admin2, 'joined', "Joined On")
spammers = ContactGroup.get_or_create(self.org2, self.admin2, "Spammers")
spam = Campaign.create(self.org2, self.admin2, "Cool stuff", spammers)
CampaignEvent.create_flow_event(self.org2, self.admin2, spam, joined,
6, CampaignEvent.UNIT_HOURS, flow, delivery_hour=12)
# no filtering
with self.assertNumQueries(NUM_BASE_REQUEST_QUERIES + 4):
response = self.fetchJSON(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json['next'], None)
self.assertResultsByUUID(response, [event2, event1])
self.assertEqual(response.json['results'][0], {
'uuid': event2.uuid,
'campaign': {'uuid': campaign2.uuid, 'name': "Notifications"},
'relative_to': {'key': "registration", 'label': "Registration"},
'offset': 6,
'unit': 'hours',
'delivery_hour': 12,
'flow': {'uuid': flow.uuid, 'name': "Color Flow"},
'message': None,
'created_on': format_datetime(event2.created_on)
})
# filter by UUID
response = self.fetchJSON(url, 'uuid=%s' % event1.uuid)
self.assertResultsByUUID(response, [event1])
# filter by campaign name
response = self.fetchJSON(url, 'campaign=Reminders')
self.assertResultsByUUID(response, [event1])
# filter by campaign UUID
response = self.fetchJSON(url, 'campaign=%s' % campaign1.uuid)
self.assertResultsByUUID(response, [event1])
# filter by invalid campaign
response = self.fetchJSON(url, 'campaign=invalid')
self.assertResultsByUUID(response, [])
def test_field_results(self):
(c1, c2, c3, c4) = (self.create_contact("Contact1", '0788111111'),
self.create_contact("Contact2", '0788222222'),
self.create_contact("Contact3", '0788333333'),
self.create_contact("Contact4", '0788444444'))
# create a gender field that uses strings
gender = ContactField.get_or_create(self.org, 'gender', label="Gender", value_type=TEXT)
c1.set_field('gender', "Male")
c2.set_field('gender', "Female")
c3.set_field('gender', "Female")
result = Value.get_value_summary(contact_field=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(2, result['unset']) # this is two as we have the default contact created by our unit tests
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Female", 2)
self.assertResult(result, 1, "Male", 1)
# create an born field that uses decimals
born = ContactField.get_or_create(self.org, 'born', label="Born", value_type=DECIMAL)
c1.set_field('born', 1977)
c2.set_field('born', 1990)
c3.set_field('born', 1977)
result = Value.get_value_summary(contact_field=born)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(2, result['unset'])
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "1977", 2)
self.assertResult(result, 1, "1990", 1)
# ok, state field!
state = ContactField.get_or_create(self.org, 'state', label="State", value_type=STATE)
c1.set_field('state', "Kigali City")
c2.set_field('state', "Kigali City")
result = Value.get_value_summary(contact_field=state)[0]
self.assertEquals(1, len(result['categories']))
self.assertEquals(2, result['set'])
self.assertEquals(3, result['unset'])
self.assertResult(result, 0, "1708283", 2)
reg_date = ContactField.get_or_create(self.org, 'reg_date', label="Registration Date", value_type=DATETIME)
now = timezone.now()
c1.set_field('reg_date', now.replace(hour=9))
c2.set_field('reg_date', now.replace(hour=4))
c3.set_field('reg_date', now - timedelta(days=1))
result = Value.get_value_summary(contact_field=reg_date)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(2, result['unset'])
self.assertResult(result, 0, (now - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0), 1)
self.assertResult(result, 1, now.replace(hour=0, minute=0, second=0, microsecond=0), 2)
def validate_label(self, value):
if not ContactField.is_valid_label(value):
raise serializers.ValidationError("Can only contain letters, numbers and hypens.")
key = ContactField.make_key(value)
if not ContactField.is_valid_key(key):
raise serializers.ValidationError("Generated key \"%s\" is invalid or a reserved name." % key)
return value
def save(self):
label = self.validated_data.get('label')
value_type = self.validated_data.get('value_type')
if self.instance:
key = self.instance.key
else:
key = ContactField.make_key(label)
return ContactField.get_or_create(self.context['org'], self.context['user'], key, label, value_type=value_type)
def save(self):
label = self.validated_data.get("label")
value_type = self.validated_data.get("value_type")
if self.instance:
key = self.instance.key
else:
key = ContactField.make_key(label)
return ContactField.get_or_create(self.context["org"], self.context["user"], key, label, value_type=value_type)
def setUp(self):
super().setUp()
self.planting_date = ContactField.get_or_create(
self.org, self.admin, "planting_date", "Planting Date", value_type=Value.TYPE_DATETIME
)
self.contact = self.create_contact("Ben Haggerty", number="+12065552020")
self.contact.set_field(self.admin, "planting_date", "2018-06-23T13:48:12.123456Z")
# create a campaign with a message event 1 day after planting date
self.farmers = self.create_group("Farmers", [self.contact])
self.campaign = Campaign.create(self.org, self.admin, "Planting Reminders", self.farmers)
self.event = CampaignEvent.create_message_event(
self.org,
self.admin,
self.campaign,
relative_to=self.planting_date,
offset=1,
unit="D",
message={
"eng": "Hi @(upper(contact.name)) don't forget to plant on @(format_date(contact.planting_date))"
},
base_language="eng",
)
def contact_field(contact, arg):
field = ContactField.get_by_key(contact.org, arg.lower())
if field is None:
return MISSING_VALUE
value = contact.get_field_display(field)
return value or MISSING_VALUE
def restore_object(self, attrs, instance=None):
"""
Update our contact field
"""
if instance: # pragma: no cover
raise ValidationError("Invalid operation")
org = self.user.get_org()
key = attrs.get('key', None)
label = attrs.get('label')
value_type = attrs.get('value_type')
if not key:
key = ContactField.make_key(label)
return ContactField.get_or_create(org, key, label, value_type=value_type)
def pre_save(self, task):
extra_fields = []
cleaned_data = self.form.cleaned_data
# enumerate the columns which the user has chosen to include as fields
for column in self.column_controls:
if cleaned_data[column['include_field']]:
label = cleaned_data[column['label_field']]
label = label.strip()
value_type = cleaned_data[column['type_field']]
org = self.derive_org()
field_key = slugify_with(label)
existing_field = ContactField.get_by_label(org, label)
if existing_field:
field_key = existing_field.key
extra_fields.append(dict(key=field_key, header=column['header'], label=label, type=value_type))
# update the extra_fields in the task's params
params = json.loads(task.import_params)
params['extra_fields'] = extra_fields
task.import_params = json.dumps(params)
return task
def validate(self, data):
key = data.get("key")
label = data.get("label")
if not key:
key = ContactField.make_key(label)
if not ContactField.is_valid_key(key):
raise serializers.ValidationError(_("Generated key for '%s' is invalid or a reserved name") % label)
fields_count = ContactField.user_fields.filter(org=self.org).count()
if not self.instance and fields_count >= ContactField.MAX_ORG_CONTACTFIELDS:
raise serializers.ValidationError(
"This org has %s contact fields and the limit is %s. "
"You must delete existing ones before "
"you can create new ones." % (fields_count, ContactField.MAX_ORG_CONTACTFIELDS)
)
data["key"] = key
return data
def get_contact_field(cls, path):
parts = path.split(".")
if len(parts) > 1:
if parts[0] in ("parent", "child"):
parts = parts[1:]
if len(parts) < 2:
return None
if parts[0] == "contact":
field_name = parts[1]
if ContactField.is_valid_key(field_name):
return parts[1]
return None
def clean(self):
for key in self.cleaned_data:
if key.startswith('field_'):
idx = key[6:]
label = self.cleaned_data["label_%s" % idx]
if label:
if not ContactField.is_valid_label(label):
raise forms.ValidationError(_("Field names can only contain letters, numbers, spaces and hypens"))
elif label in RESERVED_CONTACT_FIELDS:
raise forms.ValidationError(_("Field name '%s' is a reserved word") % label)
return self.cleaned_data
def clean(self):
# don't allow users to specify field keys or labels
re_col_name_field = re.compile(r'column_\w+_label')
for key, value in self.data.items():
if re_col_name_field.match(key):
field_label = value
field_key = slugify_with(value)
if not ContactField.is_valid_label(field_label):
raise ValidationError(_("Field names can only contain letters, numbers, spaces and hypens"))
if field_key in RESERVED_CONTACT_FIELDS:
raise ValidationError(_("%s is a reserved name for contact fields") % value)
return self.cleaned_data
def create_event(event_spec, notification, campaign):
org = notification.org_dest
user = notification.created_by
relative_to = ContactField.get_or_create(
org,
user,
key=event_spec["relative_to"]["key"],
label=event_spec["relative_to"]["label"],
value_type="D",
)
# create our message flow for message events
if event_spec["event_type"] == CampaignEvent.TYPE_MESSAGE:
message = event_spec["message"]
base_language = event_spec.get("base_language")
if not isinstance(message, dict):
try:
message = json.loads(message)
except ValueError:
# if it's not a language dict, turn it into one
message = dict(base=message)
base_language = "base"
event = CampaignEvent.create_message_event(
org,
user,
campaign,
relative_to,
event_spec["offset"],
event_spec["unit"],
message,
event_spec["delivery_hour"],
base_language=base_language,
)
event.update_flow_name()
else:
flow = Flow.objects.filter(org=org, is_active=True, name=event_spec["flow"]["name"]).last()
if flow:
CampaignEvent.create_flow_event(
org,
user,
campaign,
relative_to,
event_spec["offset"],
event_spec["unit"],
flow,
event_spec["delivery_hour"],
)
def clean(self):
used_labels = []
for key in self.cleaned_data:
if key.startswith('field_'):
idx = key[6:]
label = self.cleaned_data["label_%s" % idx]
if label:
if not ContactField.is_valid_label(label):
raise forms.ValidationError(_("Field names can only contain letters, numbers and hypens"))
if label.lower() in used_labels:
raise ValidationError(_("Field names must be unique"))
elif label in Contact.RESERVED_FIELDS:
raise forms.ValidationError(_("Field name '%s' is a reserved word") % label)
used_labels.append(label.lower())
return self.cleaned_data
def setUp(self):
super(CampaignTest, self).setUp()
self.farmer1 = self.create_contact("Rob Jasper", "+250788111111")
self.farmer2 = self.create_contact("Mike Gordon", "+250788222222")
self.nonfarmer = self.create_contact("Trey Anastasio", "+250788333333")
self.farmers = self.create_group("Farmers", [self.farmer1, self.farmer2])
self.reminder_flow = self.create_flow()
self.reminder2_flow = self.create_flow()
# create a voice flow to make sure they work too, not a proper voice flow but
# sufficient for assuring these flow types show up where they should
self.voice_flow = self.create_flow()
self.voice_flow.name = 'IVR flow'
self.voice_flow.flow_type = 'V'
self.voice_flow.save()
# create a contact field for our planting date
self.planting_date = ContactField.get_or_create(self.org, self.admin, 'planting_date', "Planting Date")
def form_valid(self, form):
try:
cleaned_data = form.cleaned_data
org = self.request.user.get_org()
for key in cleaned_data:
if key.startswith('field_'):
idx = key[6:]
label = cleaned_data["label_%s" % idx]
field = cleaned_data[key]
show_in_table = cleaned_data["show_%s" % idx]
value_type = cleaned_data['type_%s' % idx]
if field == '__new_field':
if label:
analytics.track(self.request.user.username, 'temba.contactfield_created')
key = ContactField.make_key(label)
ContactField.get_or_create(org, key, label, show_in_table=show_in_table, value_type=value_type)
else:
if label:
ContactField.get_or_create(org, field.key, label, show_in_table=show_in_table, value_type=value_type)
else:
ContactField.hide_field(org, field.key)
if 'HTTP_X_PJAX' not in self.request.META:
return HttpResponseRedirect(self.get_success_url())
else: # pragma: no cover
return self.render_to_response(self.get_context_data(form=form,
success_url=self.get_success_url(),
success_script=getattr(self, 'success_script', None)))
except IntegrityError as e: # pragma: no cover
message = str(e).capitalize()
errors = self.form._errors.setdefault(forms.forms.NON_FIELD_ERRORS, forms.utils.ErrorList())
errors.append(message)
return self.render_to_response(self.get_context_data(form=form))
def test_fields(self):
url = reverse('api.v2.fields')
self.assertEndpointAccess(url)
ContactField.get_or_create(self.org, self.admin, 'nick_name', "Nick Name")
ContactField.get_or_create(self.org, self.admin, 'registered', "Registered On", value_type=Value.TYPE_DATETIME)
ContactField.get_or_create(self.org2, self.admin2, 'not_ours', "Something Else")
# no filtering
with self.assertNumQueries(NUM_BASE_REQUEST_QUERIES + 1):
response = self.fetchJSON(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json['next'], None)
self.assertEqual(response.json['results'], [
{'key': 'registered', 'label': "Registered On", 'value_type': "datetime"},
{'key': 'nick_name', 'label': "Nick Name", 'value_type': "text"}
])
# filter by key
response = self.fetchJSON(url, 'key=nick_name')
self.assertEqual(response.json['results'], [{'key': 'nick_name', 'label': "Nick Name", 'value_type': "text"}])
def import_campaigns(cls,
org,
user,
campaign_defs,
same_site=False) -> List:
"""
Import campaigns from a list of exported campaigns
"""
imported = []
for campaign_def in campaign_defs:
name = campaign_def[Campaign.EXPORT_NAME]
campaign = None
group = None
# first check if we have the objects by UUID
if same_site:
group = ContactGroup.user_groups.filter(
uuid=campaign_def[Campaign.EXPORT_GROUP]["uuid"],
org=org).first()
if group: # pragma: needs cover
group.name = campaign_def[Campaign.EXPORT_GROUP]["name"]
group.save()
campaign = Campaign.objects.filter(
org=org, uuid=campaign_def[Campaign.EXPORT_UUID]).first()
if campaign: # pragma: needs cover
campaign.name = Campaign.get_unique_name(org,
name,
ignore=campaign)
campaign.save()
# fall back to lookups by name
if not group:
group = ContactGroup.get_user_group_by_name(
org, campaign_def[Campaign.EXPORT_GROUP]["name"])
if not campaign:
campaign = Campaign.objects.filter(org=org, name=name).first()
# all else fails, create the objects from scratch
if not group:
group = ContactGroup.create_static(
org, user, campaign_def[Campaign.EXPORT_GROUP]["name"])
if not campaign:
campaign_name = Campaign.get_unique_name(org, name)
campaign = Campaign.create(org, user, campaign_name, group)
else:
campaign.group = group
campaign.save()
# deactivate all of our events, we'll recreate these
for event in campaign.events.all():
event.release()
# fill our campaign with events
for event_spec in campaign_def[Campaign.EXPORT_EVENTS]:
field_key = event_spec["relative_to"]["key"]
if field_key == "created_on":
relative_to = ContactField.system_fields.filter(
org=org, key=field_key).first()
else:
relative_to = ContactField.get_or_create(
org,
user,
key=field_key,
label=event_spec["relative_to"]["label"],
value_type="D")
start_mode = event_spec.get("start_mode",
CampaignEvent.MODE_INTERRUPT)
# create our message flow for message events
if event_spec["event_type"] == CampaignEvent.TYPE_MESSAGE:
message = event_spec["message"]
base_language = event_spec.get("base_language")
if not isinstance(message, dict):
try:
message = json.loads(message)
except ValueError:
# if it's not a language dict, turn it into one
message = dict(base=message)
base_language = "base"
event = CampaignEvent.create_message_event(
org,
user,
campaign,
relative_to,
event_spec["offset"],
event_spec["unit"],
message,
event_spec["delivery_hour"],
base_language=base_language,
start_mode=start_mode,
)
event.update_flow_name()
else:
flow = Flow.objects.filter(
org=org,
is_active=True,
is_system=False,
uuid=event_spec["flow"]["uuid"]).first()
if flow:
CampaignEvent.create_flow_event(
org,
user,
campaign,
relative_to,
event_spec["offset"],
event_spec["unit"],
flow,
event_spec["delivery_hour"],
start_mode=start_mode,
)
imported.append(campaign)
return imported
def import_campaigns(cls, exported_json, org, user, same_site=False):
"""
Import campaigns from our export file
"""
from temba.orgs.models import EARLIEST_IMPORT_VERSION
if exported_json.get('version', 0) < EARLIEST_IMPORT_VERSION:
raise ValueError(
_("Unknown version (%s)" % exported_json.get('version', 0)))
if 'campaigns' in exported_json:
for campaign_spec in exported_json['campaigns']:
name = campaign_spec['name']
campaign = None
group = None
# first check if we have the objects by id
if same_site:
group = ContactGroup.user_groups.filter(
id=campaign_spec['group']['id'],
org=org,
is_active=True).first()
if group:
group.name = campaign_spec['group']['name']
group.save()
campaign = Campaign.objects.filter(
org=org, id=campaign_spec['id']).first()
if campaign:
campaign.name = Campaign.get_unique_name(
org, name, ignore=campaign)
campaign.save()
# fall back to lookups by name
if not group:
group = ContactGroup.get_user_group(
org, campaign_spec['group']['name'])
if not campaign:
campaign = Campaign.objects.filter(org=org,
name=name).first()
# all else fails, create the objects from scratch
if not group:
group = ContactGroup.create(org, user,
campaign_spec['group']['name'])
if not campaign:
campaign_name = Campaign.get_unique_name(org, name)
campaign = Campaign.create(org, user, campaign_name, group)
else:
campaign.group = group
campaign.save()
# we want to nuke old single message flows
for event in campaign.events.all():
if event.flow.flow_type == Flow.MESSAGE:
event.flow.delete()
# and all of the events, we'll recreate these
campaign.events.all().delete()
# fill our campaign with events
for event_spec in campaign_spec['events']:
relative_to = ContactField.get_or_create(
org,
user,
key=event_spec['relative_to']['key'],
label=event_spec['relative_to']['label'])
# create our message flow for message events
if event_spec['event_type'] == MESSAGE_EVENT:
event = CampaignEvent.create_message_event(
org, user, campaign, relative_to,
event_spec['offset'], event_spec['unit'],
event_spec['message'], event_spec['delivery_hour'])
event.update_flow_name()
else:
flow = Flow.objects.filter(
org=org,
is_active=True,
id=event_spec['flow']['id']).first()
if flow:
CampaignEvent.create_flow_event(
org, user, campaign, relative_to,
event_spec['offset'], event_spec['unit'], flow,
event_spec['delivery_hour'])
# update our scheduled events for this campaign
EventFire.update_campaign_events(campaign)
def test_views(self):
# update the planting date for our contacts
self.farmer1.set_field(self.user, 'planting_date', '1/10/2020')
# get the resulting time (including minutes)
planting_date = self.farmer1.get_field_value(self.planting_date)
# don't log in, try to create a new campaign
response = self.client.get(reverse('campaigns.campaign_create'))
self.assertRedirect(response, reverse('users.user_login'))
# ok log in as an org
self.login(self.admin)
# go to to the creation page
response = self.client.get(reverse('campaigns.campaign_create'))
self.assertEqual(200, response.status_code)
# groups shouldn't include the group that isn't ready
self.assertEqual(
set(response.context['form'].fields['group'].queryset),
{self.farmers})
post_data = dict(name="Planting Reminders", group=self.farmers.pk)
response = self.client.post(reverse('campaigns.campaign_create'),
post_data)
# should redirect to read page for this campaign
campaign = Campaign.objects.get()
self.assertRedirect(
response, reverse('campaigns.campaign_read', args=[campaign.pk]))
# go to the list page, should be there as well
response = self.client.get(reverse('campaigns.campaign_list'))
self.assertContains(response, "Planting Reminders")
# try searching for the campaign by group name
response = self.client.get(
reverse('campaigns.campaign_list') + "?search=farmers")
self.assertContains(response, "Planting Reminders")
# test no match
response = self.client.get(
reverse('campaigns.campaign_list') + "?search=factory")
self.assertNotContains(response, "Planting Reminders")
# archive a campaign
post_data = dict(action='archive', objects=campaign.pk)
self.client.post(reverse('campaigns.campaign_list'), post_data)
response = self.client.get(reverse('campaigns.campaign_list'))
self.assertNotContains(response, "Planting Reminders")
# restore the campaign
response = self.client.get(reverse('campaigns.campaign_archived'))
self.assertContains(response, "Planting Reminders")
post_data = dict(action='restore', objects=campaign.pk)
self.client.post(reverse('campaigns.campaign_archived'), post_data)
response = self.client.get(reverse('campaigns.campaign_archived'))
self.assertNotContains(response, "Planting Reminders")
response = self.client.get(reverse('campaigns.campaign_list'))
self.assertContains(response, "Planting Reminders")
# test viewers cannot use action archive or restore
self.client.logout()
# create a viewer
self.viewer = self.create_user("Viewer")
self.org.viewers.add(self.viewer)
self.viewer.set_org(self.org)
self.login(self.viewer)
# go to the list page, should be there as well
response = self.client.get(reverse('campaigns.campaign_list'))
self.assertContains(response, "Planting Reminders")
# cannot archive a campaign
post_data = dict(action='archive', objects=campaign.pk)
self.client.post(reverse('campaigns.campaign_list'), post_data)
response = self.client.get(reverse('campaigns.campaign_list'))
self.assertContains(response, "Planting Reminders")
response = self.client.get(reverse('campaigns.campaign_archived'))
self.assertNotContains(response, "Planting Reminders")
self.client.logout()
self.login(self.admin)
# see if we can create a new event, should see both sms and voice flows
response = self.client.get(
reverse('campaigns.campaignevent_create') +
"?campaign=%d" % campaign.pk)
self.assertContains(response, self.reminder_flow.name)
self.assertContains(response, self.voice_flow.name)
self.assertEqual(200, response.status_code)
post_data = dict(relative_to=self.planting_date.pk,
delivery_hour=-1,
base='',
direction='A',
offset=2,
unit='D',
event_type='M',
flow_to_start=self.reminder_flow.pk)
response = self.client.post(
reverse('campaigns.campaignevent_create') +
"?campaign=%d" % campaign.pk, post_data)
self.assertTrue(response.context['form'].errors)
self.assertIn(
'A message is required',
six.text_type(response.context['form'].errors['__all__']))
post_data = dict(relative_to=self.planting_date.pk,
delivery_hour=-1,
base='allo!' * 500,
direction='A',
offset=2,
unit='D',
event_type='M',
flow_to_start=self.reminder_flow.pk)
response = self.client.post(
reverse('campaigns.campaignevent_create') +
"?campaign=%d" % campaign.pk, post_data)
self.assertTrue(response.context['form'].errors)
self.assertTrue(
"Translation for 'Default' exceeds the %d character limit."
% Msg.MAX_TEXT_LEN in six.text_type(
response.context['form'].errors['__all__']))
post_data = dict(relative_to=self.planting_date.pk,
delivery_hour=-1,
base='',
direction='A',
offset=2,
unit='D',
event_type='F')
response = self.client.post(
reverse('campaigns.campaignevent_create') +
"?campaign=%d" % campaign.pk, post_data)
self.assertTrue(response.context['form'].errors)
self.assertIn('Please select a flow',
response.context['form'].errors['flow_to_start'])
post_data = dict(relative_to=self.planting_date.pk,
delivery_hour=-1,
base='',
direction='A',
offset=2,
unit='D',
event_type='F',
flow_to_start=self.reminder_flow.pk)
response = self.client.post(
reverse('campaigns.campaignevent_create') +
"?campaign=%d" % campaign.pk, post_data)
# should be redirected back to our campaign read page
self.assertRedirect(
response, reverse('campaigns.campaign_read', args=[campaign.pk]))
# should now have a campaign event
event = CampaignEvent.objects.get()
self.assertEqual(self.reminder_flow, event.flow)
self.assertEqual(self.planting_date, event.relative_to)
self.assertEqual(2, event.offset)
# read the campaign read page
response = self.client.get(
reverse('campaigns.campaign_read', args=[campaign.pk]))
self.assertContains(response, "Reminder Flow")
self.assertContains(response, "1")
# convert our planting date to UTC and calculate with our offset
utc_planting_date = planting_date.astimezone(pytz.utc)
scheduled_date = utc_planting_date + timedelta(days=2)
# should also have event fires scheduled for our contacts
fire = EventFire.objects.get()
self.assertEqual(scheduled_date.hour, fire.scheduled.hour)
self.assertEqual(scheduled_date.minute, fire.scheduled.minute)
self.assertEqual(scheduled_date.day, fire.scheduled.day)
self.assertEqual(scheduled_date.month, fire.scheduled.month)
self.assertEqual(scheduled_date.year, fire.scheduled.year)
self.assertEqual(event, fire.event)
post_data = dict(relative_to=self.planting_date.pk,
delivery_hour=15,
base='',
direction='A',
offset=1,
unit='D',
event_type='F',
flow_to_start=self.reminder_flow.pk)
response = self.client.post(
reverse('campaigns.campaignevent_update', args=[event.pk]),
post_data)
# should be redirected back to our campaign event read page
self.assertRedirect(
response, reverse('campaigns.campaignevent_read', args=[event.pk]))
# should now have update the campaign event
event = CampaignEvent.objects.get()
self.assertEqual(self.reminder_flow, event.flow)
self.assertEqual(self.planting_date, event.relative_to)
self.assertEqual(1, event.offset)
# should also event fires rescheduled for our contacts
fire = EventFire.objects.get()
self.assertEqual(13, fire.scheduled.hour)
self.assertEqual(0, fire.scheduled.minute)
self.assertEqual(0, fire.scheduled.second)
self.assertEqual(0, fire.scheduled.microsecond)
self.assertEqual(2, fire.scheduled.day)
self.assertEqual(10, fire.scheduled.month)
self.assertEqual(2020, fire.scheduled.year)
self.assertEqual(event, fire.event)
post_data = dict(relative_to=self.planting_date.pk,
delivery_hour=15,
base='',
direction='A',
offset=2,
unit='D',
event_type='F',
flow_to_start=self.reminder2_flow.pk)
self.client.post(
reverse('campaigns.campaignevent_create') +
"?campaign=%d" % campaign.pk, post_data)
# trying to archive our flow should fail since it belongs to a campaign
post_data = dict(action='archive', objects=[self.reminder_flow.pk])
response = self.client.post(reverse('flows.flow_list'), post_data)
self.reminder_flow.refresh_from_db()
self.assertFalse(self.reminder_flow.is_archived)
self.assertEqual(
'Reminder Flow is used inside a campaign. To archive it, first remove it from your campaigns.',
response.get('Temba-Toast'))
post_data = dict(
action='archive',
objects=[self.reminder_flow.pk, self.reminder2_flow.pk])
response = self.client.post(reverse('flows.flow_list'), post_data)
self.assertEqual(
'Planting Reminder and Reminder Flow are used inside a campaign. To archive them, first remove them from your campaigns.',
response.get('Temba-Toast'))
CampaignEvent.objects.filter(flow=self.reminder2_flow.pk).delete()
# archive the campaign
post_data = dict(action='archive', objects=campaign.pk)
self.client.post(reverse('campaigns.campaign_list'), post_data)
response = self.client.get(reverse('campaigns.campaign_list'))
self.assertNotContains(response, "Planting Reminders")
# should have no event fires
self.assertFalse(EventFire.objects.all())
# restore the campaign
post_data = dict(action='restore', objects=campaign.pk)
self.client.post(reverse('campaigns.campaign_archived'), post_data)
# EventFire should be back
self.assertTrue(EventFire.objects.all())
# set a planting date on our other farmer
self.farmer2.set_field(self.user, 'planting_date', '1/6/2022')
# should have two fire events now
fires = EventFire.objects.all()
self.assertEqual(2, len(fires))
fire = fires[0]
self.assertEqual(2, fire.scheduled.day)
self.assertEqual(10, fire.scheduled.month)
self.assertEqual(2020, fire.scheduled.year)
self.assertEqual(event, fire.event)
fire = fires[1]
self.assertEqual(2, fire.scheduled.day)
self.assertEqual(6, fire.scheduled.month)
self.assertEqual(2022, fire.scheduled.year)
self.assertEqual(event, fire.event)
# setting a planting date on our outside contact has no effect
self.nonfarmer.set_field(self.user, 'planting_date', '1/7/2025')
self.assertEqual(2, EventFire.objects.all().count())
# remove one of the farmers from the group
response = self.client.post(
reverse('contacts.contact_read', args=[self.farmer1.uuid]),
dict(contact=self.farmer1.pk, group=self.farmers.pk))
self.assertEqual(200, response.status_code)
# should only be one event now (on farmer 2)
fire = EventFire.objects.get()
self.assertEqual(2, fire.scheduled.day)
self.assertEqual(6, fire.scheduled.month)
self.assertEqual(2022, fire.scheduled.year)
self.assertEqual(event, fire.event)
# but if we add him back in, should be updated
post_data = dict(name=self.farmer1.name,
groups=[self.farmers.id],
__urn__tel=self.farmer1.get_urn('tel').path)
planting_date_field = ContactField.get_by_key(self.org,
'planting_date')
self.client.post(
reverse('contacts.contact_update', args=[self.farmer1.id]),
post_data)
response = self.client.post(
reverse('contacts.contact_update_fields', args=[self.farmer1.id]),
dict(contact_field=planting_date_field.id, field_value='4/8/2020'))
self.assertRedirect(
response, reverse('contacts.contact_read',
args=[self.farmer1.uuid]))
fires = EventFire.objects.all()
self.assertEqual(2, len(fires))
fire = fires[0]
self.assertEqual(5, fire.scheduled.day)
self.assertEqual(8, fire.scheduled.month)
self.assertEqual(2020, fire.scheduled.year)
self.assertEqual(event, fire.event)
self.assertEqual(str(fire), "%s - %s" % (fire.event, fire.contact))
event = CampaignEvent.objects.get()
# get the detail page of the event
response = self.client.get(
reverse('campaigns.campaignevent_read', args=[event.pk]))
self.assertEqual(200, response.status_code)
self.assertEqual(response.context['scheduled_event_fires_count'], 0)
self.assertEqual(len(response.context['scheduled_event_fires']), 2)
# delete an event
self.client.post(
reverse('campaigns.campaignevent_delete', args=[event.pk]), dict())
self.assertFalse(CampaignEvent.objects.all()[0].is_active)
response = self.client.get(
reverse('campaigns.campaign_read', args=[campaign.pk]))
self.assertNotContains(response, "Color Flow")
def test_dst_scheduling(self):
# set our timezone to something that honors DST
eastern = pytz.timezone('US/Eastern')
self.org.timezone = eastern
self.org.save()
# create our campaign and event
campaign = Campaign.create(self.org, self.admin, "Planting Reminders",
self.farmers)
event = CampaignEvent.create_flow_event(self.org,
self.admin,
campaign,
relative_to=self.planting_date,
offset=2,
unit='D',
flow=self.reminder_flow)
# set the time to something pre-dst (fall back on November 4th at 2am to 1am)
self.farmer1.set_field(self.user, 'planting_date',
"03-11-2029 12:30:00")
EventFire.update_campaign_events(campaign)
# try changing our field type to something non-date, should throw
with self.assertRaises(ValueError):
ContactField.get_or_create(self.org,
self.admin,
'planting_date',
value_type=Value.TYPE_TEXT)
# we should be scheduled to go off on the 5th at 12:30:10 Eastern
fire = EventFire.objects.get()
self.assertEqual(5, fire.scheduled.day)
self.assertEqual(11, fire.scheduled.month)
self.assertEqual(2029, fire.scheduled.year)
self.assertEqual(12, fire.scheduled.astimezone(eastern).hour)
# assert our offsets are different (we crossed DST)
self.assertNotEqual(
fire.scheduled.utcoffset(),
self.farmer1.get_field_value(self.planting_date).utcoffset())
# the number of hours between these two events should be 49 (two days 1 hour)
delta = fire.scheduled - self.farmer1.get_field_value(
self.planting_date)
self.assertEqual(delta.days, 2)
self.assertEqual(delta.seconds, 3600)
# spring forward case, this will go across a DST jump forward scenario
self.farmer1.set_field(self.user, 'planting_date',
"10-03-2029 02:30:00")
EventFire.update_campaign_events(campaign)
fire = EventFire.objects.get()
self.assertEqual(12, fire.scheduled.day)
self.assertEqual(3, fire.scheduled.month)
self.assertEqual(2029, fire.scheduled.year)
self.assertEqual(2, fire.scheduled.astimezone(eastern).hour)
# assert our offsets changed (we crossed DST)
self.assertNotEqual(
fire.scheduled.utcoffset(),
self.farmer1.get_field_value(self.planting_date).utcoffset())
# delta should be 47 hours exactly
delta = fire.scheduled - self.farmer1.get_field_value(
self.planting_date)
self.assertEqual(delta.days, 1)
self.assertEqual(delta.seconds, 82800)
# release our campaign event
event.release()
# should be able to change our field type now
ContactField.get_or_create(self.org,
self.admin,
'planting_date',
value_type=Value.TYPE_TEXT)
def validate_label(self, value):
if value and not ContactField.is_valid_label(value):
raise serializers.ValidationError("Field can only contain letters, numbers and hypens")
return value
def restore_object(self, attrs, instance=None):
"""
Create or update our campaign
"""
if instance: # pragma: no cover
raise ValidationError("Invalid operation")
org = self.user.get_org()
# parse our arguments
message = attrs.get('message', None)
flow = attrs.get('flow', None)
if not message and not flow:
raise ValidationError("Must specify either a flow or a message for the event")
if message and flow:
raise ValidationError("You cannot set both a flow and a message on an event, it must be only one")
campaign_id = attrs.get('campaign', None)
event_id = attrs.get('event', None)
if not campaign_id and not event_id:
raise ValidationError("You must specify either a campaign to create a new event, or an event to update")
offset = attrs.get('offset')
unit = attrs.get('unit')
delivery_hour = attrs.get('delivery_hour')
relative_to = attrs.get('relative_to')
# load our contact field
existing_field = ContactField.objects.filter(label=relative_to, org=org, is_active=True)
if not existing_field:
relative_to_field = ContactField.get_or_create(org, ContactField.make_key(relative_to), relative_to)
else:
relative_to_field = existing_field[0]
if 'event' in attrs:
event = CampaignEvent.objects.get(pk=attrs['event'], is_active=True, campaign__org=org)
# we are being set to a flow
if 'flow' in attrs:
flow = Flow.objects.get(pk=attrs['flow'], is_active=True, org=org)
event.flow = flow
event.event_type = FLOW_EVENT
event.message = None
# we are being set to a message
else:
event.message = attrs['message']
# if we aren't currently a message event, we need to create our hidden message flow
if event.event_type != MESSAGE_EVENT:
event.flow = Flow.create_single_message(org, self.user, event.message)
event.event_type = MESSAGE_EVENT
# otherwise, we can just update that flow
else:
# set our single message on our flow
event.flow.update_single_message_flow(message=attrs['message'])
# update our other attributes
event.offset = offset
event.unit = unit
event.delivery_hour = delivery_hour
event.relative_to = relative_to_field
event.save()
event.update_flow_name()
else:
campaign = Campaign.objects.get(pk=attrs['campaign'], is_active=True, org=org)
event_type = MESSAGE_EVENT
if 'flow' in attrs:
flow = Flow.objects.get(pk=attrs['flow'], is_active=True, org=org)
event_type = FLOW_EVENT
else:
flow = Flow.create_single_message(org, self.user, message)
event = CampaignEvent.objects.create(campaign=campaign, relative_to=relative_to_field, offset=offset,
unit=unit, event_type=event_type, flow=flow, message=message,
created_by=self.user, modified_by=self.user)
event.update_flow_name()
return event
def test_scheduling(self):
campaign = Campaign.create(self.org, self.admin, "Planting Reminders", self.farmers)
self.assertEquals("Planting Reminders", unicode(campaign))
# create a reminder for our first planting event
planting_reminder = CampaignEvent.create_flow_event(self.org, self.admin, campaign, relative_to=self.planting_date,
offset=0, unit='D', flow=self.reminder_flow, delivery_hour=17)
self.assertEquals("Planting Date == 0 -> Color Flow", unicode(planting_reminder))
# schedule our reminders
EventFire.update_campaign_events(campaign)
# we should haven't any event fires created, since neither of our farmers have a planting date
self.assertEquals(0, EventFire.objects.all().count())
# ok, set a planting date on one of our contacts
self.farmer1.set_field(self.user, 'planting_date', "05-10-2020 12:30:10")
# update our campaign events
EventFire.update_campaign_events(campaign)
# should have one event now
fire = EventFire.objects.get()
self.assertEquals(5, fire.scheduled.day)
self.assertEquals(10, fire.scheduled.month)
self.assertEquals(2020, fire.scheduled.year)
# account for timezone difference, our org is in UTC+2
self.assertEquals(17 - 2, fire.scheduled.hour)
self.assertEquals(self.farmer1, fire.contact)
self.assertEquals(planting_reminder, fire.event)
self.assertIsNone(fire.fired)
# change the date of our date
self.farmer1.set_field(self.user, 'planting_date', "06-10-2020 12:30:10")
EventFire.update_campaign_events_for_contact(campaign, self.farmer1)
fire = EventFire.objects.get()
self.assertEquals(6, fire.scheduled.day)
self.assertEquals(10, fire.scheduled.month)
self.assertEquals(2020, fire.scheduled.year)
self.assertEquals(self.farmer1, fire.contact)
self.assertEquals(planting_reminder, fire.event)
# set it to something invalid
self.farmer1.set_field(self.user, 'planting_date', "what?")
EventFire.update_campaign_events_for_contact(campaign, self.farmer1)
self.assertFalse(EventFire.objects.all())
# now something valid again
self.farmer1.set_field(self.user, 'planting_date', "07-10-2020 12:30:10")
EventFire.update_campaign_events_for_contact(campaign, self.farmer1)
fire = EventFire.objects.get()
self.assertEquals(7, fire.scheduled.day)
self.assertEquals(10, fire.scheduled.month)
self.assertEquals(2020, fire.scheduled.year)
self.assertEquals(self.farmer1, fire.contact)
self.assertEquals(planting_reminder, fire.event)
# create another reminder
planting_reminder2 = CampaignEvent.create_flow_event(self.org, self.admin, campaign, relative_to=self.planting_date,
offset=1, unit='D', flow=self.reminder2_flow)
self.assertEquals(1, planting_reminder2.abs_offset())
# update the campaign
EventFire.update_campaign_events(campaign)
# should have two events now, ordered by date
events = EventFire.objects.all()
self.assertEquals(planting_reminder, events[0].event)
self.assertEquals(7, events[0].scheduled.day)
self.assertEquals(planting_reminder2, events[1].event)
self.assertEquals(8, events[1].scheduled.day)
# mark one of the events as inactive
planting_reminder2.is_active = False
planting_reminder2.save()
# update the campaign
EventFire.update_campaign_events(campaign)
# back to only one event
event = EventFire.objects.get()
self.assertEquals(planting_reminder, event.event)
self.assertEquals(7, event.scheduled.day)
# update our date
self.farmer1.set_field(self.user, 'planting_date', '09-10-2020 12:30')
# should have updated
event = EventFire.objects.get()
self.assertEquals(planting_reminder, event.event)
self.assertEquals(9, event.scheduled.day)
# let's remove our contact field
ContactField.hide_field(self.org, self.user, 'planting_date')
# shouldn't have anything scheduled
self.assertFalse(EventFire.objects.all())
# add it back in
ContactField.get_or_create(self.org, self.admin, 'planting_date', "planting Date")
# should be back!
event = EventFire.objects.get()
self.assertEquals(planting_reminder, event.event)
self.assertEquals(9, event.scheduled.day)
# change our fire date to sometimein the past so it gets triggered
event.scheduled = timezone.now() - timedelta(hours=1)
event.save()
# schedule our events to fire
check_campaigns_task()
# should have one flow run now
run = FlowRun.objects.get()
self.assertEquals(event.contact, run.contact)
def create_org(self, spec, superuser, country, locations):
self._log(f"\nCreating org {spec['name']}...\n")
org = Org.objects.create(
uuid=spec["uuid"],
name=spec["name"],
timezone=pytz.timezone("America/Los_Angeles"),
brand="rapidpro.io",
country=country,
created_on=timezone.now(),
created_by=superuser,
modified_by=superuser,
)
ContactGroup.create_system_groups(org)
ContactField.create_system_fields(org)
org.init_topups(100_000)
# set our sequences to make ids stable across orgs
with connection.cursor() as cursor:
cursor.execute(
"ALTER SEQUENCE contacts_contact_id_seq RESTART WITH %s",
[spec["sequence_start"]])
cursor.execute(
"ALTER SEQUENCE contacts_contacturn_id_seq RESTART WITH %s",
[spec["sequence_start"]])
cursor.execute(
"ALTER SEQUENCE contacts_contactgroup_id_seq RESTART WITH %s",
[spec["sequence_start"]])
cursor.execute("ALTER SEQUENCE flows_flow_id_seq RESTART WITH %s",
[spec["sequence_start"]])
cursor.execute(
"ALTER SEQUENCE channels_channel_id_seq RESTART WITH %s",
[spec["sequence_start"]])
cursor.execute(
"ALTER SEQUENCE campaigns_campaign_id_seq RESTART WITH %s",
[spec["sequence_start"]])
cursor.execute(
"ALTER SEQUENCE campaigns_campaignevent_id_seq RESTART WITH %s",
[spec["sequence_start"]])
cursor.execute("ALTER SEQUENCE msgs_label_id_seq RESTART WITH %s",
[spec["sequence_start"]])
cursor.execute(
"ALTER SEQUENCE templates_template_id_seq RESTART WITH %s",
[spec["sequence_start"]])
cursor.execute(
"ALTER SEQUENCE templates_templatetranslation_id_seq RESTART WITH %s",
[spec["sequence_start"]])
self.create_channels(spec, org, superuser)
self.create_fields(spec, org, superuser)
self.create_globals(spec, org, superuser)
self.create_labels(spec, org, superuser)
self.create_groups(spec, org, superuser)
self.create_flows(spec, org, superuser)
self.create_contacts(spec, org, superuser)
self.create_group_contacts(spec, org, superuser)
self.create_campaigns(spec, org, superuser)
self.create_templates(spec, org, superuser)
self.create_classifiers(spec, org, superuser)
self.create_ticketers(spec, org, superuser)
return org
def import_campaigns(cls, exported_json, org, user, same_site=False):
"""
Import campaigns from our export file
"""
from temba.orgs.models import EARLIEST_IMPORT_VERSION
if Flow.is_before_version(exported_json.get("version", "0"), EARLIEST_IMPORT_VERSION): # pragma: needs cover
raise ValueError(_("Unknown version (%s)" % exported_json.get("version", 0)))
if "campaigns" in exported_json:
for campaign_spec in exported_json["campaigns"]:
name = campaign_spec["name"]
campaign = None
group = None
# first check if we have the objects by id
if same_site:
group = ContactGroup.user_groups.filter(uuid=campaign_spec["group"]["uuid"], org=org).first()
if group: # pragma: needs cover
group.name = campaign_spec["group"]["name"]
group.save()
campaign = Campaign.objects.filter(org=org, uuid=campaign_spec["uuid"]).first()
if campaign: # pragma: needs cover
campaign.name = Campaign.get_unique_name(org, name, ignore=campaign)
campaign.save()
# fall back to lookups by name
if not group:
group = ContactGroup.get_user_group(org, campaign_spec["group"]["name"])
if not campaign:
campaign = Campaign.objects.filter(org=org, name=name).first()
# all else fails, create the objects from scratch
if not group:
group = ContactGroup.create_static(org, user, campaign_spec["group"]["name"])
if not campaign:
campaign_name = Campaign.get_unique_name(org, name)
campaign = Campaign.create(org, user, campaign_name, group)
else:
campaign.group = group
campaign.save()
# deactivate all of our events, we'll recreate these
for event in campaign.events.all():
event.release()
# fill our campaign with events
for event_spec in campaign_spec["events"]:
field_key = event_spec["relative_to"]["key"]
if field_key == "created_on":
relative_to = ContactField.system_fields.filter(org=org, key=field_key).first()
else:
relative_to = ContactField.get_or_create(
org, user, key=field_key, label=event_spec["relative_to"]["label"], value_type="D"
)
start_mode = event_spec.get("start_mode", CampaignEvent.MODE_INTERRUPT)
# create our message flow for message events
if event_spec["event_type"] == CampaignEvent.TYPE_MESSAGE:
message = event_spec["message"]
base_language = event_spec.get("base_language")
if not isinstance(message, dict):
try:
message = json.loads(message)
except ValueError:
# if it's not a language dict, turn it into one
message = dict(base=message)
base_language = "base"
event = CampaignEvent.create_message_event(
org,
user,
campaign,
relative_to,
event_spec["offset"],
event_spec["unit"],
message,
event_spec["delivery_hour"],
base_language=base_language,
start_mode=start_mode,
)
event.update_flow_name()
else:
flow = Flow.objects.filter(
org=org, is_active=True, is_system=False, uuid=event_spec["flow"]["uuid"]
).first()
if flow:
CampaignEvent.create_flow_event(
org,
user,
campaign,
relative_to,
event_spec["offset"],
event_spec["unit"],
flow,
event_spec["delivery_hour"],
start_mode=start_mode,
)
# update our scheduled events for this campaign
EventFire.update_campaign_events(campaign)
def test_field_results(self):
c1 = self.create_contact("Contact1", '0788111111')
c2 = self.create_contact("Contact2", '0788222222')
c3 = self.create_contact("Contact3", '0788333333')
self.create_contact("Contact4", '0788444444')
# create a gender field that uses strings
gender = ContactField.get_or_create(self.org, self.admin, 'gender', label="Gender", value_type=Value.TYPE_TEXT)
c1.set_field(self.user, 'gender', "Male")
c2.set_field(self.user, 'gender', "Female")
c3.set_field(self.user, 'gender', "Female")
result = Value.get_value_summary(contact_field=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(2, result['unset']) # this is two as we have the default contact created by our unit tests
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Female", 2)
self.assertResult(result, 1, "Male", 1)
# create an born field that uses decimals
born = ContactField.get_or_create(self.org, self.admin, 'born', label="Born", value_type=Value.TYPE_DECIMAL)
c1.set_field(self.user, 'born', 1977)
c2.set_field(self.user, 'born', 1990)
c3.set_field(self.user, 'born', 1977)
result = Value.get_value_summary(contact_field=born)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(2, result['unset'])
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "1977", 2)
self.assertResult(result, 1, "1990", 1)
# ok, state field!
state = ContactField.get_or_create(self.org, self.admin, 'state', label="State", value_type=Value.TYPE_STATE)
c1.set_field(self.user, 'state', "Kigali City")
c2.set_field(self.user, 'state', "Kigali City")
result = Value.get_value_summary(contact_field=state)[0]
self.assertEquals(1, len(result['categories']))
self.assertEquals(2, result['set'])
self.assertEquals(3, result['unset'])
self.assertResult(result, 0, "1708283", 2)
reg_date = ContactField.get_or_create(self.org, self.admin, 'reg_date', label="Registration Date", value_type=Value.TYPE_DATETIME)
now = timezone.now()
c1.set_field(self.user, 'reg_date', now.replace(hour=9))
c2.set_field(self.user, 'reg_date', now.replace(hour=4))
c3.set_field(self.user, 'reg_date', now - timedelta(days=1))
result = Value.get_value_summary(contact_field=reg_date)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(2, result['unset'])
self.assertResult(result, 0, now.replace(hour=0, minute=0, second=0, microsecond=0), 2)
self.assertResult(result, 1, (now - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0), 1)
# make sure categories returned are sorted by count, not name
c2.set_field(self.user, 'gender', "Male")
result = Value.get_value_summary(contact_field=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(2, result['unset']) # this is two as we have the default contact created by our unit tests
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Male", 2)
self.assertResult(result, 1, "Female", 1)
# check the modified date is tracked for fields
original_value = Value.objects.get(contact=c1, contact_field=gender)
c1.set_field(self.user, 'gender', 'unknown')
new_value = Value.objects.get(contact=c1, contact_field=gender)
self.assertTrue(new_value.modified_on > original_value.modified_on)
self.assertNotEqual(new_value.string_value, original_value.string_value)
def get_value_summary(cls, ruleset=None, contact_field=None, filters=None, segment=None):
"""
Returns the results for the passed in ruleset or contact field given the passed in filters and segments.
Filters are expected in the following formats:
{ field: rulesetId, categories: ["Red", "Blue", "Yellow"] }
Segments are expected in these formats instead:
{ ruleset: 1515, categories: ["Red", "Blue"] } // segmenting by another field, for those categories
{ groups: 124,151,151 } // segment by each each group in the passed in ids
{ location: "State", parent: null } // segment for each admin boundary within the parent
{ contact_field: "Country", values: ["US", "EN", "RW"] } // segment by a contact field for these values
"""
from temba.contacts.models import ContactGroup, ContactField
from temba.flows.models import TrueTest, RuleSet
start = time.time()
results = []
if (not ruleset and not contact_field) or (ruleset and contact_field):
raise ValueError("Must specify either a RuleSet or Contact field.")
org = ruleset.flow.org if ruleset else contact_field.org
open_ended = ruleset and ruleset.ruleset_type == RuleSet.TYPE_WAIT_MESSAGE and len(ruleset.get_rules()) == 1
# default our filters to an empty list if None are passed in
if filters is None:
filters = []
# build the kwargs for our subcall
kwargs = dict(ruleset=ruleset, contact_field=contact_field, filters=filters)
# this is our list of dependencies, that is things that will blow away our results
dependencies = set()
fingerprint_dict = dict(filters=filters, segment=segment)
if ruleset:
fingerprint_dict['ruleset'] = ruleset.id
dependencies.add(RULESET_KEY % ruleset.id)
if contact_field:
fingerprint_dict['contact_field'] = contact_field.id
dependencies.add(CONTACT_KEY % contact_field.id)
for contact_filter in filters:
if 'ruleset' in contact_filter:
dependencies.add(RULESET_KEY % contact_filter['ruleset'])
if 'groups' in contact_filter:
for group_id in contact_filter['groups']:
dependencies.add(GROUP_KEY % group_id)
if 'location' in contact_filter:
field = ContactField.get_by_label(org, contact_filter['location'])
dependencies.add(CONTACT_KEY % field.id)
if segment:
if 'ruleset' in segment:
dependencies.add(RULESET_KEY % segment['ruleset'])
if 'groups' in segment:
for group_id in segment['groups']:
dependencies.add(GROUP_KEY % group_id)
if 'location' in segment:
field = ContactField.get_by_label(org, segment['location'])
dependencies.add(CONTACT_KEY % field.id)
# our final redis key will contain each dependency as well as a HASH representing the fingerprint of the
# kwargs passed to this method, generate that hash
fingerprint = hash(dict_to_json(fingerprint_dict))
# generate our key
key = VALUE_SUMMARY_CACHE_KEY + ":" + str(org.id) + ":".join(sorted(list(dependencies))) + ":" + str(fingerprint)
# does our value exist?
r = get_redis_connection()
cached = r.get(key)
if cached is not None:
try:
return json_to_dict(cached)
except:
# failed decoding, oh well, go calculate it instead
pass
if segment:
# segmenting a result is the same as calculating the result with the addition of each
# category as a filter so we expand upon the passed in filters to do this
if 'ruleset' in segment and 'categories' in segment:
for category in segment['categories']:
category_filter = list(filters)
category_filter.append(dict(ruleset=segment['ruleset'], categories=[category]))
# calculate our results for this segment
kwargs['filters'] = category_filter
(set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs)
results.append(dict(label=category, open_ended=open_ended, set=set_count, unset=unset_count, categories=categories))
# segmenting by groups instead, same principle but we add group filters
elif 'groups' in segment:
for group_id in segment['groups']:
# load our group
group = ContactGroup.user_groups.get(is_active=True, org=org, pk=group_id)
category_filter = list(filters)
category_filter.append(dict(groups=[group_id]))
# calculate our results for this segment
kwargs['filters'] = category_filter
(set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs)
results.append(dict(label=group.name, open_ended=open_ended, set=set_count, unset_count=unset_count, categories=categories))
# segmenting by a contact field, only for passed in categories
elif 'contact_field' in segment and 'values' in segment:
# look up the contact field
field = ContactField.get_by_label(org, segment['contact_field'])
for value in segment['values']:
value_filter = list(filters)
value_filter.append(dict(contact_field=field.pk, values=[value]))
# calculate our results for this segment
kwargs['filters'] = value_filter
(set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs)
results.append(dict(label=value, open_ended=open_ended, set=set_count, unset=unset_count, categories=categories))
# segmenting by a location field
elif 'location' in segment:
# look up the contact field
field = ContactField.get_by_label(org, segment['location'])
# make sure they are segmenting on a location type that makes sense
if field.value_type not in [STATE, DISTRICT]:
raise ValueError(_("Cannot segment on location for field that is not a State or District type"))
# make sure our org has a country for location based responses
if not org.country:
raise ValueError(_("Cannot segment by location until country has been selected for organization"))
# the boundaries we will segment by
parent = org.country
# figure out our parent
parent_osm_id = segment.get('parent', None)
if parent_osm_id:
parent = AdminBoundary.objects.get(osm_id=parent_osm_id)
# get all the boundaries we are segmenting on
boundaries = list(AdminBoundary.objects.filter(parent=parent).order_by('name'))
# if the field is a district field, they need to specify the parent state
if not parent_osm_id and field.value_type == DISTRICT:
raise ValueError(_("You must specify a parent state to segment results by district"))
# if this is a district, we can speed things up by only including those districts in our parent, build
# the filter for that
if parent and field.value_type == DISTRICT:
location_filters = [filters, dict(location=field.pk, boundary=[b.osm_id for b in boundaries])]
else:
location_filters = filters
# get all the contacts segment by location first
(location_set_contacts, location_unset_contacts, location_results) = \
cls.get_filtered_value_summary(contact_field=field, filters=location_filters, return_contacts=True)
# now get the contacts for our primary query
kwargs['return_contacts'] = True
kwargs['filter_contacts'] = location_set_contacts
(primary_set_contacts, primary_unset_contacts, primary_results) = cls.get_filtered_value_summary(**kwargs)
# build a map of osm_id to location_result
osm_results = {lr['label']: lr for lr in location_results}
empty_result = dict(contacts=list())
for boundary in boundaries:
location_result = osm_results.get(boundary.osm_id, empty_result)
# clone our primary results
segmented_results = dict(label=boundary.name,
boundary=boundary.osm_id,
open_ended=open_ended)
location_categories = list()
location_contacts = set(location_result['contacts'])
for category in primary_results:
category_contacts = set(category['contacts'])
intersection = location_contacts & category_contacts
location_categories.append(dict(label=category['label'], count=len(intersection)))
segmented_results['set'] = len(location_contacts & primary_set_contacts)
segmented_results['unset'] = len(location_contacts & primary_unset_contacts)
segmented_results['categories'] = location_categories
results.append(segmented_results)
results = sorted(results, key=lambda r: r['label'])
else:
(set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs)
# Check we have and we have an OPEN ENDED ruleset
if ruleset and len(ruleset.get_rules()) == 1 and isinstance(ruleset.get_rules()[0].test, TrueTest):
cursor = connection.cursor()
custom_sql = """SELECT w.label, count(*) AS count FROM (
SELECT
regexp_split_to_table(LOWER(text), E'[^[:alnum:]_]') AS label
FROM msgs_msg INNER JOIN contacts_contact ON ( msgs_msg.contact_id = contacts_contact.id )
WHERE msgs_msg.id IN (
SELECT
msg_id
FROM flows_flowstep_messages, flows_flowstep
WHERE flowstep_id = flows_flowstep.id AND
flows_flowstep.step_uuid = '%s'
) AND contacts_contact.is_test = False
) w group by w.label order by count desc;""" % ruleset.uuid
cursor.execute(custom_sql)
unclean_categories = get_dict_from_cursor(cursor)
categories = []
ignore_words = get_stop_words('english')
for category in unclean_categories:
if len(category['label']) > 1 and category['label'] not in ignore_words and len(categories) < 100:
categories.append(dict(label=category['label'], count=int(category['count'])))
# sort by count, then alphabetically
categories = sorted(categories, key=lambda c: (-c['count'], c['label']))
results.append(dict(label=unicode(_("All")), open_ended=open_ended, set=set_count, unset=unset_count, categories=categories))
# for each of our dependencies, add our key as something that depends on it
pipe = r.pipeline()
for dependency in dependencies:
pipe.sadd(dependency, key)
pipe.expire(dependency, VALUE_SUMMARY_CACHE_TIME)
# and finally set our result
pipe.set(key, dict_to_json(results), VALUE_SUMMARY_CACHE_TIME)
pipe.execute()
# leave me: nice for profiling..
#from django.db import connection as db_connection, reset_queries
#print "=" * 80
#for query in db_connection.queries:
# print "%s - %s" % (query['time'], query['sql'][:1000])
#print "-" * 80
#print "took: %f" % (time.time() - start)
#print "=" * 80
#reset_queries()
return results
def test_category_results(self):
self.setup_color_gender_flow()
# create a state field:
# assign c1 and c2 to Kigali
state = ContactField.get_or_create(self.org, 'state', label="State", value_type=STATE)
district = ContactField.get_or_create(self.org, 'district', label="District", value_type=DISTRICT)
self.c1.set_field('state', "Kigali City")
self.c1.set_field('district', "Kigali")
self.c2.set_field('state', "Kigali City")
self.c2.set_field('district', "Kigali")
self.run_color_gender_flow(self.c1, "red", "male", "16")
self.run_color_gender_flow(self.c2, "blue", "female", "19")
self.run_color_gender_flow(self.c3, "green", "male", "75")
self.run_color_gender_flow(self.c4, "maroon", "female", "50")
# create a group of the women
ladies = self.create_group("Ladies", [self.c2, self.c4])
# get our rulesets
color = RuleSet.objects.get(flow=self.flow, label="Color")
gender = RuleSet.objects.get(flow=self.flow, label="Gender")
age = RuleSet.objects.get(flow=self.flow, label="Age")
# categories should be in the same order as our rules, should have correct counts
result = Value.get_value_summary(ruleset=color)[0]
self.assertEquals(3, len(result['categories']))
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Red", 2)
self.assertResult(result, 1, "Blue", 1)
self.assertResult(result, 2, "Green", 1)
# check our age category as well
result = Value.get_value_summary(ruleset=age)[0]
self.assertEquals(3, len(result['categories']))
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Child", 1)
self.assertResult(result, 1, "Adult", 2)
self.assertResult(result, 2, "Senior", 1)
# and our gender categories
result = Value.get_value_summary(ruleset=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Male", 2)
self.assertResult(result, 1, "Female", 2)
# now filter the results and only get responses by men
result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male"])])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 0)
self.assertResult(result, 2, "Green", 1)
# what about men that are adults?
result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male"]),
dict(ruleset=age.pk, categories=["Adult"])])[0]
self.assertResult(result, 0, "Red", 0)
self.assertResult(result, 1, "Blue", 0)
self.assertResult(result, 2, "Green", 0)
# union of all genders
result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male", "Female"]),
dict(ruleset=age.pk, categories=["Adult"])])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 1)
self.assertResult(result, 2, "Green", 0)
# just women adults by group
result = Value.get_value_summary(ruleset=color, filters=[dict(groups=[ladies.pk]), dict(ruleset=age.pk, categories="Adult")])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 1)
self.assertResult(result, 2, "Green", 0)
# remove one of the women from the group
ladies.update_contacts([self.c2], False)
# get a new summary
result = Value.get_value_summary(ruleset=color, filters=[dict(groups=[ladies.pk]), dict(ruleset=age.pk, categories="Adult")])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 0)
self.assertResult(result, 2, "Green", 0)
# ok, back in she goes
ladies.update_contacts([self.c2], True)
# do another run for contact 1
run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16")
# totals should reflect the new value, not the old
result = Value.get_value_summary(ruleset=color)[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 2)
self.assertResult(result, 2, "Green", 1)
# what if we do a partial run?
self.send_message(self.flow, "red", contact=self.c1, restart_participants=True)
# should change our male/female breakdown since c1 now no longer has a gender
result = Value.get_value_summary(ruleset=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertResult(result, 0, "Male", 1)
self.assertResult(result, 1, "Female", 2)
# back to a full flow
run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16")
# ok, now segment by gender
result = Value.get_value_summary(ruleset=color, filters=[], segment=dict(ruleset=gender.pk, categories=["Male", "Female"]))
male_result = result[0]
self.assertResult(male_result, 0, "Red", 0)
self.assertResult(male_result, 1, "Blue", 1)
self.assertResult(male_result, 2, "Green", 1)
female_result = result[1]
self.assertResult(female_result, 0, "Red", 1)
self.assertResult(female_result, 1, "Blue", 1)
self.assertResult(female_result, 2, "Green", 0)
# add in a filter at the same time
result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=color.pk, categories=["Blue"])],
segment=dict(ruleset=gender.pk, categories=["Male", "Female"]))
male_result = result[0]
self.assertResult(male_result, 0, "Red", 0)
self.assertResult(male_result, 1, "Blue", 1)
self.assertResult(male_result, 2, "Green", 0)
female_result = result[1]
self.assertResult(female_result, 0, "Red", 0)
self.assertResult(female_result, 1, "Blue", 1)
self.assertResult(female_result, 2, "Green", 0)
# ok, try segmenting by location instead
result = Value.get_value_summary(ruleset=color, segment=dict(location="State"))
eastern_result = result[0]
self.assertEquals('171591', eastern_result['boundary'])
self.assertEquals('Eastern Province', eastern_result['label'])
self.assertResult(eastern_result, 0, "Red", 0)
self.assertResult(eastern_result, 1, "Blue", 0)
self.assertResult(eastern_result, 2, "Green", 0)
kigali_result = result[1]
self.assertEquals('1708283', kigali_result['boundary'])
self.assertEquals('Kigali City', kigali_result['label'])
self.assertResult(kigali_result, 0, "Red", 0)
self.assertResult(kigali_result, 1, "Blue", 2)
self.assertResult(kigali_result, 2, "Green", 0)
# updating state location leads to updated data
self.c2.set_field('state', "Eastern Province")
result = Value.get_value_summary(ruleset=color, segment=dict(location="State"))
eastern_result = result[0]
self.assertEquals('171591', eastern_result['boundary'])
self.assertEquals('Eastern Province', eastern_result['label'])
self.assertResult(eastern_result, 0, "Red", 0)
self.assertResult(eastern_result, 1, "Blue", 1)
self.assertResult(eastern_result, 2, "Green", 0)
kigali_result = result[1]
self.assertEquals('1708283', kigali_result['boundary'])
self.assertEquals('Kigali City', kigali_result['label'])
self.assertResult(kigali_result, 0, "Red", 0)
self.assertResult(kigali_result, 1, "Blue", 1)
self.assertResult(kigali_result, 2, "Green", 0)
# segment by district instead
result = Value.get_value_summary(ruleset=color, segment=dict(parent="1708283", location="District"))
# only on district in kigali
self.assertEquals(1, len(result))
kigali_result = result[0]
self.assertEquals('60485579', kigali_result['boundary'])
self.assertEquals('Kigali', kigali_result['label'])
self.assertResult(kigali_result, 0, "Red", 0)
self.assertResult(kigali_result, 1, "Blue", 2)
self.assertResult(kigali_result, 2, "Green", 0)
def test_field_results(self):
(c1, c2, c3, c4) = (self.create_contact("Contact1", '0788111111'),
self.create_contact("Contact2", '0788222222'),
self.create_contact("Contact3", '0788333333'),
self.create_contact("Contact4", '0788444444'))
# create a gender field that uses strings
gender = ContactField.get_or_create(self.org,
'gender',
label="Gender",
value_type=TEXT)
c1.set_field('gender', "Male")
c2.set_field('gender', "Female")
c3.set_field('gender', "Female")
result = Value.get_value_summary(contact_field=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(
2, result['unset']
) # this is two as we have the default contact created by our unit tests
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Female", 2)
self.assertResult(result, 1, "Male", 1)
# create an born field that uses decimals
born = ContactField.get_or_create(self.org,
'born',
label="Born",
value_type=DECIMAL)
c1.set_field('born', 1977)
c2.set_field('born', 1990)
c3.set_field('born', 1977)
result = Value.get_value_summary(contact_field=born)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(2, result['unset'])
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "1977", 2)
self.assertResult(result, 1, "1990", 1)
# ok, state field!
state = ContactField.get_or_create(self.org,
'state',
label="State",
value_type=STATE)
c1.set_field('state', "Kigali City")
c2.set_field('state', "Kigali City")
result = Value.get_value_summary(contact_field=state)[0]
self.assertEquals(1, len(result['categories']))
self.assertEquals(2, result['set'])
self.assertEquals(3, result['unset'])
self.assertResult(result, 0, "1708283", 2)
reg_date = ContactField.get_or_create(self.org,
'reg_date',
label="Registration Date",
value_type=DATETIME)
now = timezone.now()
c1.set_field('reg_date', now.replace(hour=9))
c2.set_field('reg_date', now.replace(hour=4))
c3.set_field('reg_date', now - timedelta(days=1))
result = Value.get_value_summary(contact_field=reg_date)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(2, result['unset'])
self.assertResult(
result, 0, now.replace(hour=0, minute=0, second=0, microsecond=0),
2)
self.assertResult(result, 1,
(now - timedelta(days=1)).replace(hour=0,
minute=0,
second=0,
microsecond=0), 1)
# make sure categories returned are sorted by count, not name
c2.set_field('gender', "Male")
result = Value.get_value_summary(contact_field=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertEquals(3, result['set'])
self.assertEquals(
2, result['unset']
) # this is two as we have the default contact created by our unit tests
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Male", 2)
self.assertResult(result, 1, "Female", 1)
def test_category_results(self):
self.setup_color_gender_flow()
# create a state field:
# assign c1 and c2 to Kigali
state = ContactField.get_or_create(self.org, 'state', label="State", value_type=STATE)
district = ContactField.get_or_create(self.org, 'district', label="District", value_type=DISTRICT)
self.c1.set_field('state', "Kigali City")
self.c1.set_field('district', "Kigali")
self.c2.set_field('state', "Kigali City")
self.c2.set_field('district', "Kigali")
self.run_color_gender_flow(self.c1, "red", "male", "16")
self.run_color_gender_flow(self.c2, "blue", "female", "19")
self.run_color_gender_flow(self.c3, "green", "male", "75")
self.run_color_gender_flow(self.c4, "maroon", "female", "50")
# create a group of the women
ladies = self.create_group("Ladies", [self.c2, self.c4])
# get our rulesets
color = RuleSet.objects.get(flow=self.flow, label="Color")
gender = RuleSet.objects.get(flow=self.flow, label="Gender")
age = RuleSet.objects.get(flow=self.flow, label="Age")
# categories should be in the same order as our rules, should have correct counts
result = Value.get_value_summary(ruleset=color)[0]
self.assertEquals(3, len(result['categories']))
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Red", 2)
self.assertResult(result, 1, "Blue", 1)
self.assertResult(result, 2, "Green", 1)
# check our age category as well
result = Value.get_value_summary(ruleset=age)[0]
self.assertEquals(3, len(result['categories']))
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Child", 1)
self.assertResult(result, 1, "Adult", 2)
self.assertResult(result, 2, "Senior", 1)
# and our gender categories
result = Value.get_value_summary(ruleset=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Male", 2)
self.assertResult(result, 1, "Female", 2)
# now filter the results and only get responses by men
result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male"])])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 0)
self.assertResult(result, 2, "Green", 1)
# what about men that are adults?
result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male"]),
dict(ruleset=age.pk, categories=["Adult"])])[0]
self.assertResult(result, 0, "Red", 0)
self.assertResult(result, 1, "Blue", 0)
self.assertResult(result, 2, "Green", 0)
# union of all genders
result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male", "Female"]),
dict(ruleset=age.pk, categories=["Adult"])])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 1)
self.assertResult(result, 2, "Green", 0)
# just women adults by group
result = Value.get_value_summary(ruleset=color, filters=[dict(groups=[ladies.pk]), dict(ruleset=age.pk, categories="Adult")])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 1)
self.assertResult(result, 2, "Green", 0)
# remove one of the women from the group
ladies.update_contacts([self.c2], False)
# get a new summary
result = Value.get_value_summary(ruleset=color, filters=[dict(groups=[ladies.pk]), dict(ruleset=age.pk, categories="Adult")])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 0)
self.assertResult(result, 2, "Green", 0)
# ok, back in she goes
ladies.update_contacts([self.c2], True)
# do another run for contact 1
run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16")
# totals should reflect the new value, not the old
result = Value.get_value_summary(ruleset=color)[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 2)
self.assertResult(result, 2, "Green", 1)
# what if we do a partial run?
self.send_message(self.flow, "red", contact=self.c1, restart_participants=True)
# should change our male/female breakdown since c1 now no longer has a gender
result = Value.get_value_summary(ruleset=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertResult(result, 0, "Male", 1)
self.assertResult(result, 1, "Female", 2)
# back to a full flow
run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16")
# ok, now segment by gender
result = Value.get_value_summary(ruleset=color, filters=[], segment=dict(ruleset=gender.pk, categories=["Male", "Female"]))
male_result = result[0]
self.assertResult(male_result, 0, "Red", 0)
self.assertResult(male_result, 1, "Blue", 1)
self.assertResult(male_result, 2, "Green", 1)
female_result = result[1]
self.assertResult(female_result, 0, "Red", 1)
self.assertResult(female_result, 1, "Blue", 1)
self.assertResult(female_result, 2, "Green", 0)
# segment by gender again, but use the contact field to do so
result = Value.get_value_summary(ruleset=color, filters=[], segment=dict(contact_field="Gender", values=["MALE", "Female"]))
male_result = result[0]
self.assertResult(male_result, 0, "Red", 0)
self.assertResult(male_result, 1, "Blue", 1)
self.assertResult(male_result, 2, "Green", 1)
female_result = result[1]
self.assertResult(female_result, 0, "Red", 1)
self.assertResult(female_result, 1, "Blue", 1)
self.assertResult(female_result, 2, "Green", 0)
# add in a filter at the same time
result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=color.pk, categories=["Blue"])],
segment=dict(ruleset=gender.pk, categories=["Male", "Female"]))
male_result = result[0]
self.assertResult(male_result, 0, "Red", 0)
self.assertResult(male_result, 1, "Blue", 1)
self.assertResult(male_result, 2, "Green", 0)
female_result = result[1]
self.assertResult(female_result, 0, "Red", 0)
self.assertResult(female_result, 1, "Blue", 1)
self.assertResult(female_result, 2, "Green", 0)
# ok, try segmenting by location instead
result = Value.get_value_summary(ruleset=color, segment=dict(location="State"))
eastern_result = result[0]
self.assertEquals('171591', eastern_result['boundary'])
self.assertEquals('Eastern Province', eastern_result['label'])
self.assertResult(eastern_result, 0, "Red", 0)
self.assertResult(eastern_result, 1, "Blue", 0)
self.assertResult(eastern_result, 2, "Green", 0)
kigali_result = result[1]
self.assertEquals('1708283', kigali_result['boundary'])
self.assertEquals('Kigali City', kigali_result['label'])
self.assertResult(kigali_result, 0, "Red", 0)
self.assertResult(kigali_result, 1, "Blue", 2)
self.assertResult(kigali_result, 2, "Green", 0)
# updating state location leads to updated data
self.c2.set_field('state', "Eastern Province")
result = Value.get_value_summary(ruleset=color, segment=dict(location="State"))
eastern_result = result[0]
self.assertEquals('171591', eastern_result['boundary'])
self.assertEquals('Eastern Province', eastern_result['label'])
self.assertResult(eastern_result, 0, "Red", 0)
self.assertResult(eastern_result, 1, "Blue", 1)
self.assertResult(eastern_result, 2, "Green", 0)
kigali_result = result[1]
self.assertEquals('1708283', kigali_result['boundary'])
self.assertEquals('Kigali City', kigali_result['label'])
self.assertResult(kigali_result, 0, "Red", 0)
self.assertResult(kigali_result, 1, "Blue", 1)
self.assertResult(kigali_result, 2, "Green", 0)
# segment by district instead
result = Value.get_value_summary(ruleset=color, segment=dict(parent="1708283", location="District"))
# only on district in kigali
self.assertEquals(1, len(result))
kigali_result = result[0]
self.assertEquals('60485579', kigali_result['boundary'])
self.assertEquals('Kigali', kigali_result['label'])
self.assertResult(kigali_result, 0, "Red", 0)
self.assertResult(kigali_result, 1, "Blue", 2)
self.assertResult(kigali_result, 2, "Green", 0)
# do a sanity check on our choropleth view
self.login(self.admin)
response = self.client.get(reverse('flows.ruleset_choropleth', args=[color.pk]) +
"?_format=json&boundary=" + self.org.country.osm_id)
# response should be valid json
response = json.loads(response.content)
# should have breaks
self.assertTrue('breaks' in response)
# should have two categories, Blue and Others
self.assertEquals(2, len(response['categories']))
self.assertEquals("Blue", response['categories'][0])
self.assertEquals("Others", response['categories'][1])
# assert our kigali result
kigali_result = response['scores']['1708283']
self.assertEquals(1, kigali_result['score'])
self.assertEquals("Kigali City", kigali_result['name'])
self.assertEquals("Blue", kigali_result['results'][0]['label'])
self.assertEquals("Others", kigali_result['results'][1]['label'])
self.assertEquals(1, kigali_result['results'][0]['count'])
self.assertEquals(0, kigali_result['results'][1]['count'])
self.assertEquals(100, kigali_result['results'][0]['percentage'])
self.assertEquals(0, kigali_result['results'][1]['percentage'])
with patch('temba.values.models.Value.get_value_summary') as mock:
mock.return_value = []
response = self.client.get(reverse('flows.ruleset_choropleth', args=[color.pk]) +
"?_format=json&boundary=" + self.org.country.osm_id)
# response should be valid json
response = json.loads(response.content)
# should have two categories, Blue and Others
self.assertEquals(2, len(response['categories']))
self.assertEquals("", response['categories'][0])
self.assertEquals("", response['categories'][1])
# all counts and percentage are 0
self.assertEquals(0, response['totals']['count'])
self.assertEquals(0, response['totals']['results'][0]['count'])
self.assertEquals(0, response['totals']['results'][0]['percentage'])
self.assertEquals(0, response['totals']['results'][1]['count'])
self.assertEquals(0, response['totals']['results'][1]['percentage'])
# and empty string labels
self.assertEquals("", response['totals']['results'][0]['label'])
self.assertEquals("", response['totals']['results'][1]['label'])
# also check our analytics view
response = self.client.get(reverse('flows.ruleset_analytics'))
# make sure we have only one flow in it
flows = json.loads(response.context['flows'])
self.assertEquals(1, len(flows))
self.assertEquals(3, len(flows[0]['rules']))
def import_campaigns(cls, exported_json, org, user, same_site=False):
"""
Import campaigns from our export file
"""
from temba.orgs.models import EARLIEST_IMPORT_VERSION
if Flow.is_before_version(exported_json.get('version', "0"), EARLIEST_IMPORT_VERSION): # pragma: needs cover
raise ValueError(_("Unknown version (%s)" % exported_json.get('version', 0)))
if 'campaigns' in exported_json:
for campaign_spec in exported_json['campaigns']:
name = campaign_spec['name']
campaign = None
group = None
# first check if we have the objects by id
if same_site:
group = ContactGroup.user_groups.filter(uuid=campaign_spec['group']['uuid'], org=org).first()
if group: # pragma: needs cover
group.name = campaign_spec['group']['name']
group.save()
campaign = Campaign.objects.filter(org=org, uuid=campaign_spec['uuid']).first()
if campaign: # pragma: needs cover
campaign.name = Campaign.get_unique_name(org, name, ignore=campaign)
campaign.save()
# fall back to lookups by name
if not group:
group = ContactGroup.get_user_group(org, campaign_spec['group']['name'])
if not campaign:
campaign = Campaign.objects.filter(org=org, name=name).first()
# all else fails, create the objects from scratch
if not group:
group = ContactGroup.create_static(org, user, campaign_spec['group']['name'])
if not campaign:
campaign_name = Campaign.get_unique_name(org, name)
campaign = Campaign.create(org, user, campaign_name, group)
else:
campaign.group = group
campaign.save()
# we want to nuke old single message flows
for event in campaign.events.all():
if event.flow.flow_type == Flow.MESSAGE:
event.flow.release()
# and all of the events, we'll recreate these
campaign.events.all().delete()
# fill our campaign with events
for event_spec in campaign_spec['events']:
relative_to = ContactField.get_or_create(org, user,
key=event_spec['relative_to']['key'],
label=event_spec['relative_to']['label'])
# create our message flow for message events
if event_spec['event_type'] == CampaignEvent.TYPE_MESSAGE:
message = event_spec['message']
base_language = event_spec.get('base_language')
if not isinstance(message, dict):
try:
message = json.loads(message)
except ValueError:
# if it's not a language dict, turn it into one
message = dict(base=message)
base_language = 'base'
event = CampaignEvent.create_message_event(org, user, campaign, relative_to,
event_spec['offset'],
event_spec['unit'],
message,
event_spec['delivery_hour'],
base_language=base_language)
event.update_flow_name()
else:
flow = Flow.objects.filter(org=org, is_active=True, uuid=event_spec['flow']['uuid']).first()
if flow:
CampaignEvent.create_flow_event(org, user, campaign, relative_to,
event_spec['offset'],
event_spec['unit'],
flow,
event_spec['delivery_hour'])
# update our scheduled events for this campaign
EventFire.update_campaign_events(campaign)
def import_campaigns(cls, exported_json, org, user, same_site=False):
"""
Import campaigns from our export file
"""
from temba.orgs.models import EARLIEST_IMPORT_VERSION
if exported_json.get('version', 0) < EARLIEST_IMPORT_VERSION:
raise ValueError(_("Unknown version (%s)" % exported_json.get('version', 0)))
if 'campaigns' in exported_json:
for campaign_spec in exported_json['campaigns']:
name = campaign_spec['name']
campaign = None
group = None
# first check if we have the objects by id
if same_site:
group = ContactGroup.user_groups.filter(uuid=campaign_spec['group']['uuid'], org=org).first()
if group:
group.name = campaign_spec['group']['name']
group.save()
campaign = Campaign.objects.filter(org=org, uuid=campaign_spec['uuid']).first()
if campaign:
campaign.name = Campaign.get_unique_name(org, name, ignore=campaign)
campaign.save()
# fall back to lookups by name
if not group:
group = ContactGroup.get_user_group(org, campaign_spec['group']['name'])
if not campaign:
campaign = Campaign.objects.filter(org=org, name=name).first()
# all else fails, create the objects from scratch
if not group:
group = ContactGroup.create_static(org, user, campaign_spec['group']['name'])
if not campaign:
campaign_name = Campaign.get_unique_name(org, name)
campaign = Campaign.create(org, user, campaign_name, group)
else:
campaign.group = group
campaign.save()
# we want to nuke old single message flows
for event in campaign.events.all():
if event.flow.flow_type == Flow.MESSAGE:
event.flow.release()
# and all of the events, we'll recreate these
campaign.events.all().delete()
# fill our campaign with events
for event_spec in campaign_spec['events']:
relative_to = ContactField.get_or_create(org, user,
key=event_spec['relative_to']['key'],
label=event_spec['relative_to']['label'])
# create our message flow for message events
if event_spec['event_type'] == CampaignEvent.TYPE_MESSAGE:
event = CampaignEvent.create_message_event(org, user, campaign, relative_to,
event_spec['offset'],
event_spec['unit'],
event_spec['message'],
event_spec['delivery_hour'])
event.update_flow_name()
else:
flow = Flow.objects.filter(org=org, is_active=True, uuid=event_spec['flow']['uuid']).first()
if flow:
CampaignEvent.create_flow_event(org, user, campaign, relative_to,
event_spec['offset'],
event_spec['unit'],
flow,
event_spec['delivery_hour'])
# update our scheduled events for this campaign
EventFire.update_campaign_events(campaign)
def test_category_results(self):
self.setup_color_gender_flow()
# create a state field:
# assign c1 and c2 to Kigali
state = ContactField.get_or_create(self.org,
'state',
label="State",
value_type=STATE)
district = ContactField.get_or_create(self.org,
'district',
label="District",
value_type=DISTRICT)
self.c1.set_field('state', "Kigali City")
self.c1.set_field('district', "Kigali")
self.c2.set_field('state', "Kigali City")
self.c2.set_field('district', "Kigali")
self.run_color_gender_flow(self.c1, "red", "male", "16")
self.run_color_gender_flow(self.c2, "blue", "female", "19")
self.run_color_gender_flow(self.c3, "green", "male", "75")
self.run_color_gender_flow(self.c4, "maroon", "female", "50")
# create a group of the women
ladies = self.create_group("Ladies", [self.c2, self.c4])
# get our rulesets
color = RuleSet.objects.get(flow=self.flow, label="Color")
gender = RuleSet.objects.get(flow=self.flow, label="Gender")
age = RuleSet.objects.get(flow=self.flow, label="Age")
# categories should be in the same order as our rules, should have correct counts
result = Value.get_value_summary(ruleset=color)[0]
self.assertEquals(3, len(result['categories']))
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Red", 2)
self.assertResult(result, 1, "Blue", 1)
self.assertResult(result, 2, "Green", 1)
# check our age category as well
result = Value.get_value_summary(ruleset=age)[0]
self.assertEquals(3, len(result['categories']))
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Child", 1)
self.assertResult(result, 1, "Adult", 2)
self.assertResult(result, 2, "Senior", 1)
# and our gender categories
result = Value.get_value_summary(ruleset=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertFalse(result['open_ended'])
self.assertResult(result, 0, "Male", 2)
self.assertResult(result, 1, "Female", 2)
# now filter the results and only get responses by men
result = Value.get_value_summary(
ruleset=color,
filters=[dict(ruleset=gender.pk, categories=["Male"])])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 0)
self.assertResult(result, 2, "Green", 1)
# what about men that are adults?
result = Value.get_value_summary(ruleset=color,
filters=[
dict(ruleset=gender.pk,
categories=["Male"]),
dict(ruleset=age.pk,
categories=["Adult"])
])[0]
self.assertResult(result, 0, "Red", 0)
self.assertResult(result, 1, "Blue", 0)
self.assertResult(result, 2, "Green", 0)
# union of all genders
result = Value.get_value_summary(
ruleset=color,
filters=[
dict(ruleset=gender.pk, categories=["Male", "Female"]),
dict(ruleset=age.pk, categories=["Adult"])
])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 1)
self.assertResult(result, 2, "Green", 0)
# just women adults by group
result = Value.get_value_summary(ruleset=color,
filters=[
dict(groups=[ladies.pk]),
dict(ruleset=age.pk,
categories="Adult")
])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 1)
self.assertResult(result, 2, "Green", 0)
# remove one of the women from the group
ladies.update_contacts([self.c2], False)
# get a new summary
result = Value.get_value_summary(ruleset=color,
filters=[
dict(groups=[ladies.pk]),
dict(ruleset=age.pk,
categories="Adult")
])[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 0)
self.assertResult(result, 2, "Green", 0)
# ok, back in she goes
ladies.update_contacts([self.c2], True)
# do another run for contact 1
run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16")
# totals should reflect the new value, not the old
result = Value.get_value_summary(ruleset=color)[0]
self.assertResult(result, 0, "Red", 1)
self.assertResult(result, 1, "Blue", 2)
self.assertResult(result, 2, "Green", 1)
# what if we do a partial run?
self.send_message(self.flow,
"red",
contact=self.c1,
restart_participants=True)
# should change our male/female breakdown since c1 now no longer has a gender
result = Value.get_value_summary(ruleset=gender)[0]
self.assertEquals(2, len(result['categories']))
self.assertResult(result, 0, "Male", 1)
self.assertResult(result, 1, "Female", 2)
# back to a full flow
run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16")
# ok, now segment by gender
result = Value.get_value_summary(ruleset=color,
filters=[],
segment=dict(
ruleset=gender.pk,
categories=["Male", "Female"]))
male_result = result[0]
self.assertResult(male_result, 0, "Red", 0)
self.assertResult(male_result, 1, "Blue", 1)
self.assertResult(male_result, 2, "Green", 1)
female_result = result[1]
self.assertResult(female_result, 0, "Red", 1)
self.assertResult(female_result, 1, "Blue", 1)
self.assertResult(female_result, 2, "Green", 0)
# add in a filter at the same time
result = Value.get_value_summary(
ruleset=color,
filters=[dict(ruleset=color.pk, categories=["Blue"])],
segment=dict(ruleset=gender.pk, categories=["Male", "Female"]))
male_result = result[0]
self.assertResult(male_result, 0, "Red", 0)
self.assertResult(male_result, 1, "Blue", 1)
self.assertResult(male_result, 2, "Green", 0)
female_result = result[1]
self.assertResult(female_result, 0, "Red", 0)
self.assertResult(female_result, 1, "Blue", 1)
self.assertResult(female_result, 2, "Green", 0)
# ok, try segmenting by location instead
result = Value.get_value_summary(ruleset=color,
segment=dict(location="State"))
eastern_result = result[0]
self.assertEquals('171591', eastern_result['boundary'])
self.assertEquals('Eastern Province', eastern_result['label'])
self.assertResult(eastern_result, 0, "Red", 0)
self.assertResult(eastern_result, 1, "Blue", 0)
self.assertResult(eastern_result, 2, "Green", 0)
kigali_result = result[1]
self.assertEquals('1708283', kigali_result['boundary'])
self.assertEquals('Kigali City', kigali_result['label'])
self.assertResult(kigali_result, 0, "Red", 0)
self.assertResult(kigali_result, 1, "Blue", 2)
self.assertResult(kigali_result, 2, "Green", 0)
# updating state location leads to updated data
self.c2.set_field('state', "Eastern Province")
result = Value.get_value_summary(ruleset=color,
segment=dict(location="State"))
eastern_result = result[0]
self.assertEquals('171591', eastern_result['boundary'])
self.assertEquals('Eastern Province', eastern_result['label'])
self.assertResult(eastern_result, 0, "Red", 0)
self.assertResult(eastern_result, 1, "Blue", 1)
self.assertResult(eastern_result, 2, "Green", 0)
kigali_result = result[1]
self.assertEquals('1708283', kigali_result['boundary'])
self.assertEquals('Kigali City', kigali_result['label'])
self.assertResult(kigali_result, 0, "Red", 0)
self.assertResult(kigali_result, 1, "Blue", 1)
self.assertResult(kigali_result, 2, "Green", 0)
# segment by district instead
result = Value.get_value_summary(ruleset=color,
segment=dict(parent="1708283",
location="District"))
# only on district in kigali
self.assertEquals(1, len(result))
kigali_result = result[0]
self.assertEquals('60485579', kigali_result['boundary'])
self.assertEquals('Kigali', kigali_result['label'])
self.assertResult(kigali_result, 0, "Red", 0)
self.assertResult(kigali_result, 1, "Blue", 2)
self.assertResult(kigali_result, 2, "Green", 0)
# do a sanity check on our choropleth view
self.login(self.admin)
response = self.client.get(
reverse('flows.ruleset_choropleth', args=[color.pk]) +
"?_format=json&boundary=" + self.org.country.osm_id)
# response should be valid json
response = json.loads(response.content)
# should have breaks
self.assertTrue('breaks' in response)
# should have two categories, Blue and Others
self.assertEquals(2, len(response['categories']))
self.assertEquals("Blue", response['categories'][0])
self.assertEquals("Others", response['categories'][1])
# assert our kigali result
kigali_result = response['scores']['1708283']
self.assertEquals(1, kigali_result['score'])
self.assertEquals("Kigali City", kigali_result['name'])
self.assertEquals("Blue", kigali_result['results'][0]['label'])
self.assertEquals("Others", kigali_result['results'][1]['label'])
self.assertEquals(1, kigali_result['results'][0]['count'])
self.assertEquals(0, kigali_result['results'][1]['count'])
self.assertEquals(100, kigali_result['results'][0]['percentage'])
self.assertEquals(0, kigali_result['results'][1]['percentage'])
with patch('temba.values.models.Value.get_value_summary') as mock:
mock.return_value = []
response = self.client.get(
reverse('flows.ruleset_choropleth', args=[color.pk]) +
"?_format=json&boundary=" + self.org.country.osm_id)
# response should be valid json
response = json.loads(response.content)
# should have two categories, Blue and Others
self.assertEquals(2, len(response['categories']))
self.assertEquals("", response['categories'][0])
self.assertEquals("", response['categories'][1])
# all counts and percentage are 0
self.assertEquals(0, response['totals']['count'])
self.assertEquals(0, response['totals']['results'][0]['count'])
self.assertEquals(0,
response['totals']['results'][0]['percentage'])
self.assertEquals(0, response['totals']['results'][1]['count'])
self.assertEquals(0,
response['totals']['results'][1]['percentage'])
# and empty string labels
self.assertEquals("", response['totals']['results'][0]['label'])
self.assertEquals("", response['totals']['results'][1]['label'])
# also check our analytics view
response = self.client.get(reverse('flows.ruleset_analytics'))
# make sure we have only one flow in it
flows = json.loads(response.context['flows'])
self.assertEquals(1, len(flows))
self.assertEquals(3, len(flows[0]['rules']))
def save(self):
key = self.validated_data.get("key")
label = self.validated_data.get("label")
value_type = self.validated_data.get("value_type")
return ContactField.get_or_create(self.org, self.user, key, label, value_type=value_type)
def import_campaigns(cls, exported_json, org, user, same_site=False):
"""
Import campaigns from our export file
"""
from temba.orgs.models import EARLIEST_IMPORT_VERSION
if exported_json.get("version", 0) < EARLIEST_IMPORT_VERSION:
raise ValueError(_("Unknown version (%s)" % exported_json.get("version", 0)))
if "campaigns" in exported_json:
for campaign_spec in exported_json["campaigns"]:
name = campaign_spec["name"]
campaign = None
group = None
# first check if we have the objects by id
if same_site:
group = ContactGroup.user_groups.filter(
id=campaign_spec["group"]["id"], org=org, is_active=True
).first()
if group:
group.name = campaign_spec["group"]["name"]
group.save()
campaign = Campaign.objects.filter(org=org, id=campaign_spec["id"]).first()
if campaign:
campaign.name = Campaign.get_unique_name(org, name, ignore=campaign)
campaign.save()
# fall back to lookups by name
if not group:
group = ContactGroup.user_groups.filter(name=campaign_spec["group"]["name"], org=org).first()
if not campaign:
campaign = Campaign.objects.filter(org=org, name=name).first()
# all else fails, create the objects from scratch
if not group:
group = ContactGroup.create(org, user, campaign_spec["group"]["name"])
if not campaign:
campaign_name = Campaign.get_unique_name(org, name)
campaign = Campaign.create(org, user, campaign_name, group)
else:
campaign.group = group
campaign.save()
# we want to nuke old single message flows
for event in campaign.events.all():
if event.flow.flow_type == Flow.MESSAGE:
event.flow.delete()
# and all of the events, we'll recreate these
campaign.events.all().delete()
# fill our campaign with events
for event_spec in campaign_spec["events"]:
relative_to = ContactField.get_or_create(
org, key=event_spec["relative_to"]["key"], label=event_spec["relative_to"]["label"]
)
# create our message flow for message events
if event_spec["event_type"] == MESSAGE_EVENT:
event = CampaignEvent.create_message_event(
org,
user,
campaign,
relative_to,
event_spec["offset"],
event_spec["unit"],
event_spec["message"],
event_spec["delivery_hour"],
)
event.update_flow_name()
else:
flow = Flow.objects.filter(org=org, id=event_spec["flow"]["id"]).first()
if flow:
CampaignEvent.create_flow_event(
org,
user,
campaign,
relative_to,
event_spec["offset"],
event_spec["unit"],
flow,
event_spec["delivery_hour"],
)
# update our scheduled events for this campaign
EventFire.update_campaign_events(campaign)
def validate_key(self, value):
if value and not ContactField.is_valid_key(value):
raise serializers.ValidationError("Field is invalid or a reserved name")
return value
def import_campaigns(cls, exported_json, org, user, same_site=False):
"""
Import campaigns from our export file
"""
from temba.orgs.models import EARLIEST_IMPORT_VERSION
if Flow.is_before_version(exported_json.get("version", "0"), EARLIEST_IMPORT_VERSION): # pragma: needs cover
raise ValueError(_("Unknown version (%s)" % exported_json.get("version", 0)))
if "campaigns" in exported_json:
for campaign_spec in exported_json["campaigns"]:
name = campaign_spec["name"]
campaign = None
group = None
# first check if we have the objects by id
if same_site:
group = ContactGroup.user_groups.filter(uuid=campaign_spec["group"]["uuid"], org=org).first()
if group: # pragma: needs cover
group.name = campaign_spec["group"]["name"]
group.save()
campaign = Campaign.objects.filter(org=org, uuid=campaign_spec["uuid"]).first()
if campaign: # pragma: needs cover
campaign.name = Campaign.get_unique_name(org, name, ignore=campaign)
campaign.save()
# fall back to lookups by name
if not group:
group = ContactGroup.get_user_group(org, campaign_spec["group"]["name"])
if not campaign:
campaign = Campaign.objects.filter(org=org, name=name).first()
# all else fails, create the objects from scratch
if not group:
group = ContactGroup.create_static(org, user, campaign_spec["group"]["name"])
if not campaign:
campaign_name = Campaign.get_unique_name(org, name)
campaign = Campaign.create(org, user, campaign_name, group)
else:
campaign.group = group
campaign.save()
# deactivate all of our events, we'll recreate these
for event in campaign.events.all():
event.release()
# fill our campaign with events
for event_spec in campaign_spec["events"]:
field_key = event_spec["relative_to"]["key"]
if field_key == "created_on":
relative_to = ContactField.system_fields.filter(org=org, key=field_key).first()
else:
relative_to = ContactField.get_or_create(
org, user, key=field_key, label=event_spec["relative_to"]["label"], value_type="D"
)
start_mode = event_spec.get("start_mode", CampaignEvent.MODE_INTERRUPT)
# create our message flow for message events
if event_spec["event_type"] == CampaignEvent.TYPE_MESSAGE:
message = event_spec["message"]
base_language = event_spec.get("base_language")
if not isinstance(message, dict):
try:
message = json.loads(message)
except ValueError:
# if it's not a language dict, turn it into one
message = dict(base=message)
base_language = "base"
event = CampaignEvent.create_message_event(
org,
user,
campaign,
relative_to,
event_spec["offset"],
event_spec["unit"],
message,
event_spec["delivery_hour"],
base_language=base_language,
start_mode=start_mode,
)
event.update_flow_name()
else:
flow = Flow.objects.filter(
org=org, is_active=True, is_system=False, uuid=event_spec["flow"]["uuid"]
).first()
if flow:
CampaignEvent.create_flow_event(
org,
user,
campaign,
relative_to,
event_spec["offset"],
event_spec["unit"],
flow,
event_spec["delivery_hour"],
start_mode=start_mode,
)
# update our scheduled events for this campaign
EventFire.update_campaign_events(campaign)
def get_value_summary(cls, ruleset=None, contact_field=None, filters=None, segment=None):
"""
Returns the results for the passed in ruleset or contact field given the passed in filters and segments.
Filters are expected in the following formats:
{ field: rulesetId, categories: ["Red", "Blue", "Yellow"] }
Segments are expected in these formats instead:
{ ruleset: 1515, categories: ["Red", "Blue"] } // segmenting by another field, for those categories
{ groups: 124,151,151 } // segment by each each group in the passed in ids
{ location: "State", parent: null } // segment for each admin boundary within the parent
{ contact_field: "Country", values: ["US", "EN", "RW"] } // segment by a contact field for these values
"""
from temba.contacts.models import ContactGroup, ContactField
from temba.flows.models import TrueTest, RuleSet
start = time.time()
results = []
if (not ruleset and not contact_field) or (ruleset and contact_field):
raise ValueError("Must specify either a RuleSet or Contact field.")
org = ruleset.flow.org if ruleset else contact_field.org
open_ended = ruleset and ruleset.ruleset_type == RuleSet.TYPE_WAIT_MESSAGE and len(ruleset.get_rules()) == 1
# default our filters to an empty list if None are passed in
if filters is None:
filters = []
# build the kwargs for our subcall
kwargs = dict(ruleset=ruleset, contact_field=contact_field, filters=filters)
# this is our list of dependencies, that is things that will blow away our results
dependencies = set()
fingerprint_dict = dict(filters=filters, segment=segment)
if ruleset:
fingerprint_dict['ruleset'] = ruleset.id
dependencies.add(RULESET_KEY % ruleset.id)
if contact_field:
fingerprint_dict['contact_field'] = contact_field.id
dependencies.add(CONTACT_KEY % contact_field.id)
for contact_filter in filters:
if 'ruleset' in contact_filter:
dependencies.add(RULESET_KEY % contact_filter['ruleset'])
if 'groups' in contact_filter:
for group_id in contact_filter['groups']:
dependencies.add(GROUP_KEY % group_id)
if 'location' in contact_filter:
field = ContactField.get_by_label(org, contact_filter['location'])
dependencies.add(CONTACT_KEY % field.id)
if segment:
if 'ruleset' in segment:
dependencies.add(RULESET_KEY % segment['ruleset'])
if 'groups' in segment:
for group_id in segment['groups']:
dependencies.add(GROUP_KEY % group_id)
if 'location' in segment:
field = ContactField.get_by_label(org, segment['location'])
dependencies.add(CONTACT_KEY % field.id)
# our final redis key will contain each dependency as well as a HASH representing the fingerprint of the
# kwargs passed to this method, generate that hash
fingerprint = hash(dict_to_json(fingerprint_dict))
# generate our key
key = VALUE_SUMMARY_CACHE_KEY + ":" + str(org.id) + ":".join(sorted(list(dependencies))) + ":" + str(fingerprint)
# does our value exist?
r = get_redis_connection()
cached = r.get(key)
if cached is not None:
try:
return json_to_dict(cached)
except Exception:
# failed decoding, oh well, go calculate it instead
pass
if segment:
# segmenting a result is the same as calculating the result with the addition of each
# category as a filter so we expand upon the passed in filters to do this
if 'ruleset' in segment and 'categories' in segment:
for category in segment['categories']:
category_filter = list(filters)
category_filter.append(dict(ruleset=segment['ruleset'], categories=[category]))
# calculate our results for this segment
kwargs['filters'] = category_filter
(set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs)
results.append(dict(label=category, open_ended=open_ended, set=set_count, unset=unset_count, categories=categories))
# segmenting by groups instead, same principle but we add group filters
elif 'groups' in segment:
for group_id in segment['groups']:
# load our group
group = ContactGroup.user_groups.get(org=org, pk=group_id)
category_filter = list(filters)
category_filter.append(dict(groups=[group_id]))
# calculate our results for this segment
kwargs['filters'] = category_filter
(set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs)
results.append(dict(label=group.name, open_ended=open_ended, set=set_count, unset_count=unset_count, categories=categories))
# segmenting by a contact field, only for passed in categories
elif 'contact_field' in segment and 'values' in segment:
# look up the contact field
field = ContactField.get_by_label(org, segment['contact_field'])
for value in segment['values']:
value_filter = list(filters)
value_filter.append(dict(contact_field=field.pk, values=[value]))
# calculate our results for this segment
kwargs['filters'] = value_filter
(set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs)
results.append(dict(label=value, open_ended=open_ended, set=set_count, unset=unset_count, categories=categories))
# segmenting by a location field
elif 'location' in segment:
# look up the contact field
field = ContactField.get_by_label(org, segment['location'])
# make sure they are segmenting on a location type that makes sense
if field.value_type not in [Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD]:
raise ValueError(_("Cannot segment on location for field that is not a State or District type"))
# make sure our org has a country for location based responses
if not org.country:
raise ValueError(_("Cannot segment by location until country has been selected for organization"))
# the boundaries we will segment by
parent = org.country
# figure out our parent
parent_osm_id = segment.get('parent', None)
if parent_osm_id:
parent = AdminBoundary.objects.get(osm_id=parent_osm_id)
# get all the boundaries we are segmenting on
boundaries = list(AdminBoundary.objects.filter(parent=parent).order_by('name'))
# if the field is a district field, they need to specify the parent state
if not parent_osm_id and field.value_type == Value.TYPE_DISTRICT:
raise ValueError(_("You must specify a parent state to segment results by district"))
if not parent_osm_id and field.value_type == Value.TYPE_WARD:
raise ValueError(_("You must specify a parent state to segment results by ward"))
# if this is a district, we can speed things up by only including those districts in our parent, build
# the filter for that
if parent and field.value_type in [Value.TYPE_DISTRICT, Value.TYPE_WARD]:
location_filters = [filters, dict(location=field.pk, boundary=[b.osm_id for b in boundaries])]
else:
location_filters = filters
# get all the contacts segment by location first
(location_set_contacts, location_unset_contacts, location_results) = \
cls.get_filtered_value_summary(contact_field=field, filters=location_filters, return_contacts=True)
# now get the contacts for our primary query
kwargs['return_contacts'] = True
kwargs['filter_contacts'] = location_set_contacts
(primary_set_contacts, primary_unset_contacts, primary_results) = cls.get_filtered_value_summary(**kwargs)
# build a map of osm_id to location_result
osm_results = {lr['label']: lr for lr in location_results}
empty_result = dict(contacts=list())
for boundary in boundaries:
location_result = osm_results.get(boundary.osm_id, empty_result)
# clone our primary results
segmented_results = dict(label=boundary.name,
boundary=boundary.osm_id,
open_ended=open_ended)
location_categories = list()
location_contacts = set(location_result['contacts'])
for category in primary_results:
category_contacts = set(category['contacts'])
intersection = location_contacts & category_contacts
location_categories.append(dict(label=category['label'], count=len(intersection)))
segmented_results['set'] = len(location_contacts & primary_set_contacts)
segmented_results['unset'] = len(location_contacts & primary_unset_contacts)
segmented_results['categories'] = location_categories
results.append(segmented_results)
results = sorted(results, key=lambda r: r['label'])
else:
(set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs)
# Check we have and we have an OPEN ENDED ruleset
if ruleset and len(ruleset.get_rules()) == 1 and isinstance(ruleset.get_rules()[0].test, TrueTest):
cursor = connection.cursor()
custom_sql = """SELECT w.label, count(*) AS count FROM (
SELECT
regexp_split_to_table(LOWER(text), E'[^[:alnum:]_]') AS label
FROM msgs_msg INNER JOIN contacts_contact ON ( msgs_msg.contact_id = contacts_contact.id )
WHERE msgs_msg.id IN (
SELECT
msg_id
FROM flows_flowstep_messages, flows_flowstep
WHERE flowstep_id = flows_flowstep.id AND
flows_flowstep.step_uuid = '%s'
) AND contacts_contact.is_test = False
) w group by w.label order by count desc;""" % ruleset.uuid
cursor.execute(custom_sql)
unclean_categories = get_dict_from_cursor(cursor)
categories = []
org_languages = [lang.name.lower() for lang in org.languages.filter(orgs=None).distinct()]
if 'english' not in org_languages:
org_languages.append('english')
ignore_words = []
for lang in org_languages:
ignore_words += safe_get_stop_words(lang)
for category in unclean_categories:
if len(category['label']) > 1 and category['label'] not in ignore_words and len(categories) < 100:
categories.append(dict(label=category['label'], count=int(category['count'])))
# sort by count, then alphabetically
categories = sorted(categories, key=lambda c: (-c['count'], c['label']))
results.append(dict(label=unicode(_("All")), open_ended=open_ended, set=set_count, unset=unset_count, categories=categories))
# for each of our dependencies, add our key as something that depends on it
pipe = r.pipeline()
for dependency in dependencies:
pipe.sadd(dependency, key)
pipe.expire(dependency, VALUE_SUMMARY_CACHE_TIME)
# and finally set our result
pipe.set(key, dict_to_json(results), VALUE_SUMMARY_CACHE_TIME)
pipe.execute()
# leave me: nice for profiling..
#from django.db import connection as db_connection, reset_queries
#print "=" * 80
#for query in db_connection.queries:
# print "%s - %s" % (query['time'], query['sql'][:1000])
#print "-" * 80
#print "took: %f" % (time.time() - start)
#print "=" * 80
#reset_queries()
return results
def test_scheduling(self):
campaign = Campaign.objects.create(name="Planting Reminders", group=self.farmers, org=self.org,
created_by=self.admin, modified_by=self.admin)
self.assertEquals("Planting Reminders", unicode(campaign))
# create a reminder for our first planting event
planting_reminder = CampaignEvent.objects.create(campaign=campaign, relative_to=self.planting_date, offset=0,
flow=self.reminder_flow, delivery_hour=17,
created_by=self.admin, modified_by=self.admin)
self.assertEquals("Planting Date == 0 -> Color Flow", unicode(planting_reminder))
# schedule our reminders
EventFire.update_campaign_events(campaign)
# we should haven't any event fires created, since neither of our farmers have a planting date
self.assertEquals(0, EventFire.objects.all().count())
# ok, set a planting date on one of our contacts
self.farmer1.set_field('planting_date', "05-10-2020 12:30:10")
# update our campaign events
EventFire.update_campaign_events(campaign)
# should have one event now
fire = EventFire.objects.get()
self.assertEquals(5, fire.scheduled.day)
self.assertEquals(10, fire.scheduled.month)
self.assertEquals(2020, fire.scheduled.year)
# account for timezone difference, our org is in UTC+2
self.assertEquals(17 - 2, fire.scheduled.hour)
self.assertEquals(self.farmer1, fire.contact)
self.assertEquals(planting_reminder, fire.event)
self.assertIsNone(fire.fired)
# change the date of our date
self.farmer1.set_field('planting_date', "06-10-2020 12:30:10")
EventFire.update_campaign_events_for_contact(campaign, self.farmer1)
fire = EventFire.objects.get()
self.assertEquals(6, fire.scheduled.day)
self.assertEquals(10, fire.scheduled.month)
self.assertEquals(2020, fire.scheduled.year)
self.assertEquals(self.farmer1, fire.contact)
self.assertEquals(planting_reminder, fire.event)
# set it to something invalid
self.farmer1.set_field('planting_date', "what?")
EventFire.update_campaign_events_for_contact(campaign, self.farmer1)
self.assertFalse(EventFire.objects.all())
# now something valid again
self.farmer1.set_field('planting_date', "07-10-2020 12:30:10")
EventFire.update_campaign_events_for_contact(campaign, self.farmer1)
fire = EventFire.objects.get()
self.assertEquals(7, fire.scheduled.day)
self.assertEquals(10, fire.scheduled.month)
self.assertEquals(2020, fire.scheduled.year)
self.assertEquals(self.farmer1, fire.contact)
self.assertEquals(planting_reminder, fire.event)
# create another reminder
planting_reminder2 = CampaignEvent.objects.create(campaign=campaign, relative_to=self.planting_date, offset=1,
flow=self.reminder2_flow,
created_by=self.admin, modified_by=self.admin)
self.assertEquals(1, planting_reminder2.abs_offset())
# update the campaign
EventFire.update_campaign_events(campaign)
# should have two events now, ordered by date
events = EventFire.objects.all()
self.assertEquals(planting_reminder, events[0].event)
self.assertEquals(7, events[0].scheduled.day)
self.assertEquals(planting_reminder2, events[1].event)
self.assertEquals(8, events[1].scheduled.day)
# mark one of the events as inactive
planting_reminder2.is_active = False
planting_reminder2.save()
# update the campaign
EventFire.update_campaign_events(campaign)
# back to only one event
event = EventFire.objects.get()
self.assertEquals(planting_reminder, event.event)
self.assertEquals(7, event.scheduled.day)
# update our date
self.farmer1.set_field('planting_date', '09-10-2020 12:30')
# should have updated
event = EventFire.objects.get()
self.assertEquals(planting_reminder, event.event)
self.assertEquals(9, event.scheduled.day)
# let's remove our contact field
ContactField.hide_field(self.org, 'planting_date')
# shouldn't have anything scheduled
self.assertFalse(EventFire.objects.all())
# add it back in
ContactField.get_or_create(self.org, 'planting_date', "planting Date")
# should be back!
event = EventFire.objects.get()
self.assertEquals(planting_reminder, event.event)
self.assertEquals(9, event.scheduled.day)
# try firing the event
event.fire()
# should have one flow run now
run = FlowRun.objects.get()
self.assertEquals(event.contact, run.contact)
def save(self):
"""
Update our contact
"""
name = self.validated_data.get("name")
fields = self.validated_data.get("fields")
language = self.validated_data.get("language")
# treat empty names as None
if not name:
name = None
changed = []
if self.instance:
if self.parsed_urns is not None:
self.instance.update_urns(self.user, self.parsed_urns)
# update our name and language
if name != self.instance.name:
self.instance.name = name
changed.append("name")
else:
self.instance = Contact.get_or_create_by_urns(
self.org,
self.user,
name,
urns=self.parsed_urns,
language=language,
force_urn_update=True)
# Contact.get_or_create doesn't nullify language so do that here
if "language" in self.validated_data and language is None:
self.instance.language = language.lower() if language else None
changed.append("language")
# save our contact if it changed
if changed:
self.instance.save(update_fields=changed, handle_update=True)
# update our fields
if fields is not None:
for key, value in fields.items():
existing_by_key = ContactField.user_fields.filter(
org=self.org, key__iexact=key, is_active=True).first()
if existing_by_key:
self.instance.set_field(self.user, existing_by_key.key,
value)
continue
elif self.new_fields and key in self.new_fields:
new_field = ContactField.get_or_create(
org=self.org,
user=self.user,
key=regex.sub("[^A-Za-z0-9]+", "_", key).lower(),
label=key)
self.instance.set_field(self.user, new_field.key, value)
# TODO as above, need to get users to stop updating via label
existing_by_label = ContactField.get_by_label(self.org, key)
if existing_by_label:
self.instance.set_field(self.user, existing_by_label.key,
value)
# update our contact's groups
if self.group_objs is not None:
self.instance.update_static_groups(self.user, self.group_objs)
return self.instance
