def setUp(self): self.clear_cache() self.user = self.create_user("tito") self.admin = self.create_user("Administrator") self.org = Org.objects.create(name="Nyaruka Ltd.", timezone="Africa/Kigali", created_by=self.user, modified_by=self.user) self.org.initialize() self.org.administrators.add(self.admin) self.admin.set_org(self.org) self.org.administrators.add(self.user) self.user.set_org(self.org) self.tel_mtn = Channel.objects.create(org=self.org, name="MTN", channel_type="A", role="SR", address="+250780000000", secret="12345", gcm_id="123", created_by=self.user, modified_by=self.user) self.tel_tigo = Channel.objects.create(org=self.org, name="Tigo", channel_type="A", role="SR", address="+250720000000", secret="23456", gcm_id="234", created_by=self.user, modified_by=self.user) self.tel_bulk = Channel.objects.create(org=self.org, name="Nexmo", channel_type="NX", role="S", parent=self.tel_tigo, secret="34567", created_by=self.user, modified_by=self.user) self.twitter = Channel.objects.create(org=self.org, name="Twitter", channel_type="TT", role="SR", created_by=self.user, modified_by=self.user) # for generating tuples of scheme, path and channel generate_tel_mtn = lambda num: (TEL_SCHEME, "+25078%07d" % (num + 1), self.tel_mtn) generate_tel_tigo = lambda num: (TEL_SCHEME, "+25072%07d" % (num + 1), self.tel_tigo) generate_twitter = lambda num: (TWITTER_SCHEME, "tweep_%d" % (num + 1), self.twitter) self.urn_generators = (generate_tel_mtn, generate_tel_tigo, generate_twitter) self.field_nick = ContactField.get_or_create(self.org, 'nick', 'Nickname', show_in_table=True, value_type=TEXT) self.field_age = ContactField.get_or_create(self.org, 'age', 'Age', show_in_table=True, value_type=DECIMAL)
def setUp(self): self.clear_cache() self.user = self.create_user("tito") self.admin = self.create_user("Administrator") self.org = Org.objects.create(name="Nyaruka Ltd.", timezone="Africa/Kigali", created_by=self.user, modified_by=self.user) self.org.initialize() self.org.administrators.add(self.admin) self.admin.set_org(self.org) self.org.administrators.add(self.user) self.user.set_org(self.org) self.tel_mtn = Channel.create(self.org, self.user, 'RW', 'A', name="MTN", address="+250780000000", secret="12345", gcm_id="123") self.tel_tigo = Channel.create(self.org, self.user, 'RW', 'A', name="Tigo", address="+250720000000", secret="23456", gcm_id="234") self.tel_bulk = Channel.create(self.org, self.user, 'RW', 'NX', name="Nexmo", parent=self.tel_tigo) self.twitter = Channel.create(self.org, self.user, None, 'TT', name="Twitter", address="billy_bob") # for generating tuples of scheme, path and channel def generate_tel_mtn(num): return TEL_SCHEME, "+25078%07d" % (num + 1), self.tel_mtn def generate_tel_tigo(num): return TEL_SCHEME, "+25072%07d" % (num + 1), self.tel_tigo def generate_twitter(num): return TWITTER_SCHEME, "tweep_%d" % (num + 1), self.twitter self.urn_generators = (generate_tel_mtn, generate_tel_tigo, generate_twitter) self.field_nick = ContactField.get_or_create(self.org, self.admin, 'nick', 'Nickname', show_in_table=True, value_type=Value.TYPE_TEXT) self.field_age = ContactField.get_or_create(self.org, self.admin, 'age', 'Age', show_in_table=True, value_type=Value.TYPE_DECIMAL)
def test_campaign_events(self): url = reverse('api.v2.campaign_events') self.assertEndpointAccess(url) flow = self.create_flow() reporters = self.create_group("Reporters", [self.joe, self.frank]) registration = ContactField.get_or_create(self.org, self.admin, 'registration', "Registration") campaign1 = Campaign.create(self.org, self.admin, "Reminders", reporters) event1 = CampaignEvent.create_message_event(self.org, self.admin, campaign1, registration, 1, CampaignEvent.UNIT_DAYS, "Don't forget to brush your teeth") campaign2 = Campaign.create(self.org, self.admin, "Notifications", reporters) event2 = CampaignEvent.create_flow_event(self.org, self.admin, campaign2, registration, 6, CampaignEvent.UNIT_HOURS, flow, delivery_hour=12) # create event for another org joined = ContactField.get_or_create(self.org2, self.admin2, 'joined', "Joined On") spammers = ContactGroup.get_or_create(self.org2, self.admin2, "Spammers") spam = Campaign.create(self.org2, self.admin2, "Cool stuff", spammers) CampaignEvent.create_flow_event(self.org2, self.admin2, spam, joined, 6, CampaignEvent.UNIT_HOURS, flow, delivery_hour=12) # no filtering with self.assertNumQueries(NUM_BASE_REQUEST_QUERIES + 4): response = self.fetchJSON(url) self.assertEqual(response.status_code, 200) self.assertEqual(response.json['next'], None) self.assertResultsByUUID(response, [event2, event1]) self.assertEqual(response.json['results'][0], { 'uuid': event2.uuid, 'campaign': {'uuid': campaign2.uuid, 'name': "Notifications"}, 'relative_to': {'key': "registration", 'label': "Registration"}, 'offset': 6, 'unit': 'hours', 'delivery_hour': 12, 'flow': {'uuid': flow.uuid, 'name': "Color Flow"}, 'message': None, 'created_on': format_datetime(event2.created_on) }) # filter by UUID response = self.fetchJSON(url, 'uuid=%s' % event1.uuid) self.assertResultsByUUID(response, [event1]) # filter by campaign name response = self.fetchJSON(url, 'campaign=Reminders') self.assertResultsByUUID(response, [event1]) # filter by campaign UUID response = self.fetchJSON(url, 'campaign=%s' % campaign1.uuid) self.assertResultsByUUID(response, [event1]) # filter by invalid campaign response = self.fetchJSON(url, 'campaign=invalid') self.assertResultsByUUID(response, [])
def test_field_results(self): (c1, c2, c3, c4) = (self.create_contact("Contact1", '0788111111'), self.create_contact("Contact2", '0788222222'), self.create_contact("Contact3", '0788333333'), self.create_contact("Contact4", '0788444444')) # create a gender field that uses strings gender = ContactField.get_or_create(self.org, 'gender', label="Gender", value_type=TEXT) c1.set_field('gender', "Male") c2.set_field('gender', "Female") c3.set_field('gender', "Female") result = Value.get_value_summary(contact_field=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals(2, result['unset']) # this is two as we have the default contact created by our unit tests self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Female", 2) self.assertResult(result, 1, "Male", 1) # create an born field that uses decimals born = ContactField.get_or_create(self.org, 'born', label="Born", value_type=DECIMAL) c1.set_field('born', 1977) c2.set_field('born', 1990) c3.set_field('born', 1977) result = Value.get_value_summary(contact_field=born)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals(2, result['unset']) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "1977", 2) self.assertResult(result, 1, "1990", 1) # ok, state field! state = ContactField.get_or_create(self.org, 'state', label="State", value_type=STATE) c1.set_field('state', "Kigali City") c2.set_field('state', "Kigali City") result = Value.get_value_summary(contact_field=state)[0] self.assertEquals(1, len(result['categories'])) self.assertEquals(2, result['set']) self.assertEquals(3, result['unset']) self.assertResult(result, 0, "1708283", 2) reg_date = ContactField.get_or_create(self.org, 'reg_date', label="Registration Date", value_type=DATETIME) now = timezone.now() c1.set_field('reg_date', now.replace(hour=9)) c2.set_field('reg_date', now.replace(hour=4)) c3.set_field('reg_date', now - timedelta(days=1)) result = Value.get_value_summary(contact_field=reg_date)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals(2, result['unset']) self.assertResult(result, 0, (now - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0), 1) self.assertResult(result, 1, now.replace(hour=0, minute=0, second=0, microsecond=0), 2)
def validate_label(self, value): if not ContactField.is_valid_label(value): raise serializers.ValidationError("Can only contain letters, numbers and hypens.") key = ContactField.make_key(value) if not ContactField.is_valid_key(key): raise serializers.ValidationError("Generated key \"%s\" is invalid or a reserved name." % key) return value
def save(self): label = self.validated_data.get('label') value_type = self.validated_data.get('value_type') if self.instance: key = self.instance.key else: key = ContactField.make_key(label) return ContactField.get_or_create(self.context['org'], self.context['user'], key, label, value_type=value_type)
def save(self): label = self.validated_data.get("label") value_type = self.validated_data.get("value_type") if self.instance: key = self.instance.key else: key = ContactField.make_key(label) return ContactField.get_or_create(self.context["org"], self.context["user"], key, label, value_type=value_type)
def setUp(self): super().setUp() self.planting_date = ContactField.get_or_create( self.org, self.admin, "planting_date", "Planting Date", value_type=Value.TYPE_DATETIME ) self.contact = self.create_contact("Ben Haggerty", number="+12065552020") self.contact.set_field(self.admin, "planting_date", "2018-06-23T13:48:12.123456Z") # create a campaign with a message event 1 day after planting date self.farmers = self.create_group("Farmers", [self.contact]) self.campaign = Campaign.create(self.org, self.admin, "Planting Reminders", self.farmers) self.event = CampaignEvent.create_message_event( self.org, self.admin, self.campaign, relative_to=self.planting_date, offset=1, unit="D", message={ "eng": "Hi @(upper(contact.name)) don't forget to plant on @(format_date(contact.planting_date))" }, base_language="eng", )
def contact_field(contact, arg): field = ContactField.get_by_key(contact.org, arg.lower()) if field is None: return MISSING_VALUE value = contact.get_field_display(field) return value or MISSING_VALUE
def restore_object(self, attrs, instance=None): """ Update our contact field """ if instance: # pragma: no cover raise ValidationError("Invalid operation") org = self.user.get_org() key = attrs.get('key', None) label = attrs.get('label') value_type = attrs.get('value_type') if not key: key = ContactField.make_key(label) return ContactField.get_or_create(org, key, label, value_type=value_type)
def pre_save(self, task): extra_fields = [] cleaned_data = self.form.cleaned_data # enumerate the columns which the user has chosen to include as fields for column in self.column_controls: if cleaned_data[column['include_field']]: label = cleaned_data[column['label_field']] label = label.strip() value_type = cleaned_data[column['type_field']] org = self.derive_org() field_key = slugify_with(label) existing_field = ContactField.get_by_label(org, label) if existing_field: field_key = existing_field.key extra_fields.append(dict(key=field_key, header=column['header'], label=label, type=value_type)) # update the extra_fields in the task's params params = json.loads(task.import_params) params['extra_fields'] = extra_fields task.import_params = json.dumps(params) return task
def validate(self, data): key = data.get("key") label = data.get("label") if not key: key = ContactField.make_key(label) if not ContactField.is_valid_key(key): raise serializers.ValidationError(_("Generated key for '%s' is invalid or a reserved name") % label) fields_count = ContactField.user_fields.filter(org=self.org).count() if not self.instance and fields_count >= ContactField.MAX_ORG_CONTACTFIELDS: raise serializers.ValidationError( "This org has %s contact fields and the limit is %s. " "You must delete existing ones before " "you can create new ones." % (fields_count, ContactField.MAX_ORG_CONTACTFIELDS) ) data["key"] = key return data
def get_contact_field(cls, path): parts = path.split(".") if len(parts) > 1: if parts[0] in ("parent", "child"): parts = parts[1:] if len(parts) < 2: return None if parts[0] == "contact": field_name = parts[1] if ContactField.is_valid_key(field_name): return parts[1] return None
def clean(self): for key in self.cleaned_data: if key.startswith('field_'): idx = key[6:] label = self.cleaned_data["label_%s" % idx] if label: if not ContactField.is_valid_label(label): raise forms.ValidationError(_("Field names can only contain letters, numbers, spaces and hypens")) elif label in RESERVED_CONTACT_FIELDS: raise forms.ValidationError(_("Field name '%s' is a reserved word") % label) return self.cleaned_data
def clean(self): # don't allow users to specify field keys or labels re_col_name_field = re.compile(r'column_\w+_label') for key, value in self.data.items(): if re_col_name_field.match(key): field_label = value field_key = slugify_with(value) if not ContactField.is_valid_label(field_label): raise ValidationError(_("Field names can only contain letters, numbers, spaces and hypens")) if field_key in RESERVED_CONTACT_FIELDS: raise ValidationError(_("%s is a reserved name for contact fields") % value) return self.cleaned_data
def create_event(event_spec, notification, campaign): org = notification.org_dest user = notification.created_by relative_to = ContactField.get_or_create( org, user, key=event_spec["relative_to"]["key"], label=event_spec["relative_to"]["label"], value_type="D", ) # create our message flow for message events if event_spec["event_type"] == CampaignEvent.TYPE_MESSAGE: message = event_spec["message"] base_language = event_spec.get("base_language") if not isinstance(message, dict): try: message = json.loads(message) except ValueError: # if it's not a language dict, turn it into one message = dict(base=message) base_language = "base" event = CampaignEvent.create_message_event( org, user, campaign, relative_to, event_spec["offset"], event_spec["unit"], message, event_spec["delivery_hour"], base_language=base_language, ) event.update_flow_name() else: flow = Flow.objects.filter(org=org, is_active=True, name=event_spec["flow"]["name"]).last() if flow: CampaignEvent.create_flow_event( org, user, campaign, relative_to, event_spec["offset"], event_spec["unit"], flow, event_spec["delivery_hour"], )
def clean(self): used_labels = [] for key in self.cleaned_data: if key.startswith('field_'): idx = key[6:] label = self.cleaned_data["label_%s" % idx] if label: if not ContactField.is_valid_label(label): raise forms.ValidationError(_("Field names can only contain letters, numbers and hypens")) if label.lower() in used_labels: raise ValidationError(_("Field names must be unique")) elif label in Contact.RESERVED_FIELDS: raise forms.ValidationError(_("Field name '%s' is a reserved word") % label) used_labels.append(label.lower()) return self.cleaned_data
def setUp(self): super(CampaignTest, self).setUp() self.farmer1 = self.create_contact("Rob Jasper", "+250788111111") self.farmer2 = self.create_contact("Mike Gordon", "+250788222222") self.nonfarmer = self.create_contact("Trey Anastasio", "+250788333333") self.farmers = self.create_group("Farmers", [self.farmer1, self.farmer2]) self.reminder_flow = self.create_flow() self.reminder2_flow = self.create_flow() # create a voice flow to make sure they work too, not a proper voice flow but # sufficient for assuring these flow types show up where they should self.voice_flow = self.create_flow() self.voice_flow.name = 'IVR flow' self.voice_flow.flow_type = 'V' self.voice_flow.save() # create a contact field for our planting date self.planting_date = ContactField.get_or_create(self.org, self.admin, 'planting_date', "Planting Date")
def form_valid(self, form): try: cleaned_data = form.cleaned_data org = self.request.user.get_org() for key in cleaned_data: if key.startswith('field_'): idx = key[6:] label = cleaned_data["label_%s" % idx] field = cleaned_data[key] show_in_table = cleaned_data["show_%s" % idx] value_type = cleaned_data['type_%s' % idx] if field == '__new_field': if label: analytics.track(self.request.user.username, 'temba.contactfield_created') key = ContactField.make_key(label) ContactField.get_or_create(org, key, label, show_in_table=show_in_table, value_type=value_type) else: if label: ContactField.get_or_create(org, field.key, label, show_in_table=show_in_table, value_type=value_type) else: ContactField.hide_field(org, field.key) if 'HTTP_X_PJAX' not in self.request.META: return HttpResponseRedirect(self.get_success_url()) else: # pragma: no cover return self.render_to_response(self.get_context_data(form=form, success_url=self.get_success_url(), success_script=getattr(self, 'success_script', None))) except IntegrityError as e: # pragma: no cover message = str(e).capitalize() errors = self.form._errors.setdefault(forms.forms.NON_FIELD_ERRORS, forms.utils.ErrorList()) errors.append(message) return self.render_to_response(self.get_context_data(form=form))
def test_fields(self): url = reverse('api.v2.fields') self.assertEndpointAccess(url) ContactField.get_or_create(self.org, self.admin, 'nick_name', "Nick Name") ContactField.get_or_create(self.org, self.admin, 'registered', "Registered On", value_type=Value.TYPE_DATETIME) ContactField.get_or_create(self.org2, self.admin2, 'not_ours', "Something Else") # no filtering with self.assertNumQueries(NUM_BASE_REQUEST_QUERIES + 1): response = self.fetchJSON(url) self.assertEqual(response.status_code, 200) self.assertEqual(response.json['next'], None) self.assertEqual(response.json['results'], [ {'key': 'registered', 'label': "Registered On", 'value_type': "datetime"}, {'key': 'nick_name', 'label': "Nick Name", 'value_type': "text"} ]) # filter by key response = self.fetchJSON(url, 'key=nick_name') self.assertEqual(response.json['results'], [{'key': 'nick_name', 'label': "Nick Name", 'value_type': "text"}])
def import_campaigns(cls, org, user, campaign_defs, same_site=False) -> List: """ Import campaigns from a list of exported campaigns """ imported = [] for campaign_def in campaign_defs: name = campaign_def[Campaign.EXPORT_NAME] campaign = None group = None # first check if we have the objects by UUID if same_site: group = ContactGroup.user_groups.filter( uuid=campaign_def[Campaign.EXPORT_GROUP]["uuid"], org=org).first() if group: # pragma: needs cover group.name = campaign_def[Campaign.EXPORT_GROUP]["name"] group.save() campaign = Campaign.objects.filter( org=org, uuid=campaign_def[Campaign.EXPORT_UUID]).first() if campaign: # pragma: needs cover campaign.name = Campaign.get_unique_name(org, name, ignore=campaign) campaign.save() # fall back to lookups by name if not group: group = ContactGroup.get_user_group_by_name( org, campaign_def[Campaign.EXPORT_GROUP]["name"]) if not campaign: campaign = Campaign.objects.filter(org=org, name=name).first() # all else fails, create the objects from scratch if not group: group = ContactGroup.create_static( org, user, campaign_def[Campaign.EXPORT_GROUP]["name"]) if not campaign: campaign_name = Campaign.get_unique_name(org, name) campaign = Campaign.create(org, user, campaign_name, group) else: campaign.group = group campaign.save() # deactivate all of our events, we'll recreate these for event in campaign.events.all(): event.release() # fill our campaign with events for event_spec in campaign_def[Campaign.EXPORT_EVENTS]: field_key = event_spec["relative_to"]["key"] if field_key == "created_on": relative_to = ContactField.system_fields.filter( org=org, key=field_key).first() else: relative_to = ContactField.get_or_create( org, user, key=field_key, label=event_spec["relative_to"]["label"], value_type="D") start_mode = event_spec.get("start_mode", CampaignEvent.MODE_INTERRUPT) # create our message flow for message events if event_spec["event_type"] == CampaignEvent.TYPE_MESSAGE: message = event_spec["message"] base_language = event_spec.get("base_language") if not isinstance(message, dict): try: message = json.loads(message) except ValueError: # if it's not a language dict, turn it into one message = dict(base=message) base_language = "base" event = CampaignEvent.create_message_event( org, user, campaign, relative_to, event_spec["offset"], event_spec["unit"], message, event_spec["delivery_hour"], base_language=base_language, start_mode=start_mode, ) event.update_flow_name() else: flow = Flow.objects.filter( org=org, is_active=True, is_system=False, uuid=event_spec["flow"]["uuid"]).first() if flow: CampaignEvent.create_flow_event( org, user, campaign, relative_to, event_spec["offset"], event_spec["unit"], flow, event_spec["delivery_hour"], start_mode=start_mode, ) imported.append(campaign) return imported
def import_campaigns(cls, exported_json, org, user, same_site=False): """ Import campaigns from our export file """ from temba.orgs.models import EARLIEST_IMPORT_VERSION if exported_json.get('version', 0) < EARLIEST_IMPORT_VERSION: raise ValueError( _("Unknown version (%s)" % exported_json.get('version', 0))) if 'campaigns' in exported_json: for campaign_spec in exported_json['campaigns']: name = campaign_spec['name'] campaign = None group = None # first check if we have the objects by id if same_site: group = ContactGroup.user_groups.filter( id=campaign_spec['group']['id'], org=org, is_active=True).first() if group: group.name = campaign_spec['group']['name'] group.save() campaign = Campaign.objects.filter( org=org, id=campaign_spec['id']).first() if campaign: campaign.name = Campaign.get_unique_name( org, name, ignore=campaign) campaign.save() # fall back to lookups by name if not group: group = ContactGroup.get_user_group( org, campaign_spec['group']['name']) if not campaign: campaign = Campaign.objects.filter(org=org, name=name).first() # all else fails, create the objects from scratch if not group: group = ContactGroup.create(org, user, campaign_spec['group']['name']) if not campaign: campaign_name = Campaign.get_unique_name(org, name) campaign = Campaign.create(org, user, campaign_name, group) else: campaign.group = group campaign.save() # we want to nuke old single message flows for event in campaign.events.all(): if event.flow.flow_type == Flow.MESSAGE: event.flow.delete() # and all of the events, we'll recreate these campaign.events.all().delete() # fill our campaign with events for event_spec in campaign_spec['events']: relative_to = ContactField.get_or_create( org, user, key=event_spec['relative_to']['key'], label=event_spec['relative_to']['label']) # create our message flow for message events if event_spec['event_type'] == MESSAGE_EVENT: event = CampaignEvent.create_message_event( org, user, campaign, relative_to, event_spec['offset'], event_spec['unit'], event_spec['message'], event_spec['delivery_hour']) event.update_flow_name() else: flow = Flow.objects.filter( org=org, is_active=True, id=event_spec['flow']['id']).first() if flow: CampaignEvent.create_flow_event( org, user, campaign, relative_to, event_spec['offset'], event_spec['unit'], flow, event_spec['delivery_hour']) # update our scheduled events for this campaign EventFire.update_campaign_events(campaign)
def test_views(self): # update the planting date for our contacts self.farmer1.set_field(self.user, 'planting_date', '1/10/2020') # get the resulting time (including minutes) planting_date = self.farmer1.get_field_value(self.planting_date) # don't log in, try to create a new campaign response = self.client.get(reverse('campaigns.campaign_create')) self.assertRedirect(response, reverse('users.user_login')) # ok log in as an org self.login(self.admin) # go to to the creation page response = self.client.get(reverse('campaigns.campaign_create')) self.assertEqual(200, response.status_code) # groups shouldn't include the group that isn't ready self.assertEqual( set(response.context['form'].fields['group'].queryset), {self.farmers}) post_data = dict(name="Planting Reminders", group=self.farmers.pk) response = self.client.post(reverse('campaigns.campaign_create'), post_data) # should redirect to read page for this campaign campaign = Campaign.objects.get() self.assertRedirect( response, reverse('campaigns.campaign_read', args=[campaign.pk])) # go to the list page, should be there as well response = self.client.get(reverse('campaigns.campaign_list')) self.assertContains(response, "Planting Reminders") # try searching for the campaign by group name response = self.client.get( reverse('campaigns.campaign_list') + "?search=farmers") self.assertContains(response, "Planting Reminders") # test no match response = self.client.get( reverse('campaigns.campaign_list') + "?search=factory") self.assertNotContains(response, "Planting Reminders") # archive a campaign post_data = dict(action='archive', objects=campaign.pk) self.client.post(reverse('campaigns.campaign_list'), post_data) response = self.client.get(reverse('campaigns.campaign_list')) self.assertNotContains(response, "Planting Reminders") # restore the campaign response = self.client.get(reverse('campaigns.campaign_archived')) self.assertContains(response, "Planting Reminders") post_data = dict(action='restore', objects=campaign.pk) self.client.post(reverse('campaigns.campaign_archived'), post_data) response = self.client.get(reverse('campaigns.campaign_archived')) self.assertNotContains(response, "Planting Reminders") response = self.client.get(reverse('campaigns.campaign_list')) self.assertContains(response, "Planting Reminders") # test viewers cannot use action archive or restore self.client.logout() # create a viewer self.viewer = self.create_user("Viewer") self.org.viewers.add(self.viewer) self.viewer.set_org(self.org) self.login(self.viewer) # go to the list page, should be there as well response = self.client.get(reverse('campaigns.campaign_list')) self.assertContains(response, "Planting Reminders") # cannot archive a campaign post_data = dict(action='archive', objects=campaign.pk) self.client.post(reverse('campaigns.campaign_list'), post_data) response = self.client.get(reverse('campaigns.campaign_list')) self.assertContains(response, "Planting Reminders") response = self.client.get(reverse('campaigns.campaign_archived')) self.assertNotContains(response, "Planting Reminders") self.client.logout() self.login(self.admin) # see if we can create a new event, should see both sms and voice flows response = self.client.get( reverse('campaigns.campaignevent_create') + "?campaign=%d" % campaign.pk) self.assertContains(response, self.reminder_flow.name) self.assertContains(response, self.voice_flow.name) self.assertEqual(200, response.status_code) post_data = dict(relative_to=self.planting_date.pk, delivery_hour=-1, base='', direction='A', offset=2, unit='D', event_type='M', flow_to_start=self.reminder_flow.pk) response = self.client.post( reverse('campaigns.campaignevent_create') + "?campaign=%d" % campaign.pk, post_data) self.assertTrue(response.context['form'].errors) self.assertIn( 'A message is required', six.text_type(response.context['form'].errors['__all__'])) post_data = dict(relative_to=self.planting_date.pk, delivery_hour=-1, base='allo!' * 500, direction='A', offset=2, unit='D', event_type='M', flow_to_start=self.reminder_flow.pk) response = self.client.post( reverse('campaigns.campaignevent_create') + "?campaign=%d" % campaign.pk, post_data) self.assertTrue(response.context['form'].errors) self.assertTrue( "Translation for 'Default' exceeds the %d character limit." % Msg.MAX_TEXT_LEN in six.text_type( response.context['form'].errors['__all__'])) post_data = dict(relative_to=self.planting_date.pk, delivery_hour=-1, base='', direction='A', offset=2, unit='D', event_type='F') response = self.client.post( reverse('campaigns.campaignevent_create') + "?campaign=%d" % campaign.pk, post_data) self.assertTrue(response.context['form'].errors) self.assertIn('Please select a flow', response.context['form'].errors['flow_to_start']) post_data = dict(relative_to=self.planting_date.pk, delivery_hour=-1, base='', direction='A', offset=2, unit='D', event_type='F', flow_to_start=self.reminder_flow.pk) response = self.client.post( reverse('campaigns.campaignevent_create') + "?campaign=%d" % campaign.pk, post_data) # should be redirected back to our campaign read page self.assertRedirect( response, reverse('campaigns.campaign_read', args=[campaign.pk])) # should now have a campaign event event = CampaignEvent.objects.get() self.assertEqual(self.reminder_flow, event.flow) self.assertEqual(self.planting_date, event.relative_to) self.assertEqual(2, event.offset) # read the campaign read page response = self.client.get( reverse('campaigns.campaign_read', args=[campaign.pk])) self.assertContains(response, "Reminder Flow") self.assertContains(response, "1") # convert our planting date to UTC and calculate with our offset utc_planting_date = planting_date.astimezone(pytz.utc) scheduled_date = utc_planting_date + timedelta(days=2) # should also have event fires scheduled for our contacts fire = EventFire.objects.get() self.assertEqual(scheduled_date.hour, fire.scheduled.hour) self.assertEqual(scheduled_date.minute, fire.scheduled.minute) self.assertEqual(scheduled_date.day, fire.scheduled.day) self.assertEqual(scheduled_date.month, fire.scheduled.month) self.assertEqual(scheduled_date.year, fire.scheduled.year) self.assertEqual(event, fire.event) post_data = dict(relative_to=self.planting_date.pk, delivery_hour=15, base='', direction='A', offset=1, unit='D', event_type='F', flow_to_start=self.reminder_flow.pk) response = self.client.post( reverse('campaigns.campaignevent_update', args=[event.pk]), post_data) # should be redirected back to our campaign event read page self.assertRedirect( response, reverse('campaigns.campaignevent_read', args=[event.pk])) # should now have update the campaign event event = CampaignEvent.objects.get() self.assertEqual(self.reminder_flow, event.flow) self.assertEqual(self.planting_date, event.relative_to) self.assertEqual(1, event.offset) # should also event fires rescheduled for our contacts fire = EventFire.objects.get() self.assertEqual(13, fire.scheduled.hour) self.assertEqual(0, fire.scheduled.minute) self.assertEqual(0, fire.scheduled.second) self.assertEqual(0, fire.scheduled.microsecond) self.assertEqual(2, fire.scheduled.day) self.assertEqual(10, fire.scheduled.month) self.assertEqual(2020, fire.scheduled.year) self.assertEqual(event, fire.event) post_data = dict(relative_to=self.planting_date.pk, delivery_hour=15, base='', direction='A', offset=2, unit='D', event_type='F', flow_to_start=self.reminder2_flow.pk) self.client.post( reverse('campaigns.campaignevent_create') + "?campaign=%d" % campaign.pk, post_data) # trying to archive our flow should fail since it belongs to a campaign post_data = dict(action='archive', objects=[self.reminder_flow.pk]) response = self.client.post(reverse('flows.flow_list'), post_data) self.reminder_flow.refresh_from_db() self.assertFalse(self.reminder_flow.is_archived) self.assertEqual( 'Reminder Flow is used inside a campaign. To archive it, first remove it from your campaigns.', response.get('Temba-Toast')) post_data = dict( action='archive', objects=[self.reminder_flow.pk, self.reminder2_flow.pk]) response = self.client.post(reverse('flows.flow_list'), post_data) self.assertEqual( 'Planting Reminder and Reminder Flow are used inside a campaign. To archive them, first remove them from your campaigns.', response.get('Temba-Toast')) CampaignEvent.objects.filter(flow=self.reminder2_flow.pk).delete() # archive the campaign post_data = dict(action='archive', objects=campaign.pk) self.client.post(reverse('campaigns.campaign_list'), post_data) response = self.client.get(reverse('campaigns.campaign_list')) self.assertNotContains(response, "Planting Reminders") # should have no event fires self.assertFalse(EventFire.objects.all()) # restore the campaign post_data = dict(action='restore', objects=campaign.pk) self.client.post(reverse('campaigns.campaign_archived'), post_data) # EventFire should be back self.assertTrue(EventFire.objects.all()) # set a planting date on our other farmer self.farmer2.set_field(self.user, 'planting_date', '1/6/2022') # should have two fire events now fires = EventFire.objects.all() self.assertEqual(2, len(fires)) fire = fires[0] self.assertEqual(2, fire.scheduled.day) self.assertEqual(10, fire.scheduled.month) self.assertEqual(2020, fire.scheduled.year) self.assertEqual(event, fire.event) fire = fires[1] self.assertEqual(2, fire.scheduled.day) self.assertEqual(6, fire.scheduled.month) self.assertEqual(2022, fire.scheduled.year) self.assertEqual(event, fire.event) # setting a planting date on our outside contact has no effect self.nonfarmer.set_field(self.user, 'planting_date', '1/7/2025') self.assertEqual(2, EventFire.objects.all().count()) # remove one of the farmers from the group response = self.client.post( reverse('contacts.contact_read', args=[self.farmer1.uuid]), dict(contact=self.farmer1.pk, group=self.farmers.pk)) self.assertEqual(200, response.status_code) # should only be one event now (on farmer 2) fire = EventFire.objects.get() self.assertEqual(2, fire.scheduled.day) self.assertEqual(6, fire.scheduled.month) self.assertEqual(2022, fire.scheduled.year) self.assertEqual(event, fire.event) # but if we add him back in, should be updated post_data = dict(name=self.farmer1.name, groups=[self.farmers.id], __urn__tel=self.farmer1.get_urn('tel').path) planting_date_field = ContactField.get_by_key(self.org, 'planting_date') self.client.post( reverse('contacts.contact_update', args=[self.farmer1.id]), post_data) response = self.client.post( reverse('contacts.contact_update_fields', args=[self.farmer1.id]), dict(contact_field=planting_date_field.id, field_value='4/8/2020')) self.assertRedirect( response, reverse('contacts.contact_read', args=[self.farmer1.uuid])) fires = EventFire.objects.all() self.assertEqual(2, len(fires)) fire = fires[0] self.assertEqual(5, fire.scheduled.day) self.assertEqual(8, fire.scheduled.month) self.assertEqual(2020, fire.scheduled.year) self.assertEqual(event, fire.event) self.assertEqual(str(fire), "%s - %s" % (fire.event, fire.contact)) event = CampaignEvent.objects.get() # get the detail page of the event response = self.client.get( reverse('campaigns.campaignevent_read', args=[event.pk])) self.assertEqual(200, response.status_code) self.assertEqual(response.context['scheduled_event_fires_count'], 0) self.assertEqual(len(response.context['scheduled_event_fires']), 2) # delete an event self.client.post( reverse('campaigns.campaignevent_delete', args=[event.pk]), dict()) self.assertFalse(CampaignEvent.objects.all()[0].is_active) response = self.client.get( reverse('campaigns.campaign_read', args=[campaign.pk])) self.assertNotContains(response, "Color Flow")
def test_dst_scheduling(self): # set our timezone to something that honors DST eastern = pytz.timezone('US/Eastern') self.org.timezone = eastern self.org.save() # create our campaign and event campaign = Campaign.create(self.org, self.admin, "Planting Reminders", self.farmers) event = CampaignEvent.create_flow_event(self.org, self.admin, campaign, relative_to=self.planting_date, offset=2, unit='D', flow=self.reminder_flow) # set the time to something pre-dst (fall back on November 4th at 2am to 1am) self.farmer1.set_field(self.user, 'planting_date', "03-11-2029 12:30:00") EventFire.update_campaign_events(campaign) # try changing our field type to something non-date, should throw with self.assertRaises(ValueError): ContactField.get_or_create(self.org, self.admin, 'planting_date', value_type=Value.TYPE_TEXT) # we should be scheduled to go off on the 5th at 12:30:10 Eastern fire = EventFire.objects.get() self.assertEqual(5, fire.scheduled.day) self.assertEqual(11, fire.scheduled.month) self.assertEqual(2029, fire.scheduled.year) self.assertEqual(12, fire.scheduled.astimezone(eastern).hour) # assert our offsets are different (we crossed DST) self.assertNotEqual( fire.scheduled.utcoffset(), self.farmer1.get_field_value(self.planting_date).utcoffset()) # the number of hours between these two events should be 49 (two days 1 hour) delta = fire.scheduled - self.farmer1.get_field_value( self.planting_date) self.assertEqual(delta.days, 2) self.assertEqual(delta.seconds, 3600) # spring forward case, this will go across a DST jump forward scenario self.farmer1.set_field(self.user, 'planting_date', "10-03-2029 02:30:00") EventFire.update_campaign_events(campaign) fire = EventFire.objects.get() self.assertEqual(12, fire.scheduled.day) self.assertEqual(3, fire.scheduled.month) self.assertEqual(2029, fire.scheduled.year) self.assertEqual(2, fire.scheduled.astimezone(eastern).hour) # assert our offsets changed (we crossed DST) self.assertNotEqual( fire.scheduled.utcoffset(), self.farmer1.get_field_value(self.planting_date).utcoffset()) # delta should be 47 hours exactly delta = fire.scheduled - self.farmer1.get_field_value( self.planting_date) self.assertEqual(delta.days, 1) self.assertEqual(delta.seconds, 82800) # release our campaign event event.release() # should be able to change our field type now ContactField.get_or_create(self.org, self.admin, 'planting_date', value_type=Value.TYPE_TEXT)
def validate_label(self, value): if value and not ContactField.is_valid_label(value): raise serializers.ValidationError("Field can only contain letters, numbers and hypens") return value
def restore_object(self, attrs, instance=None): """ Create or update our campaign """ if instance: # pragma: no cover raise ValidationError("Invalid operation") org = self.user.get_org() # parse our arguments message = attrs.get('message', None) flow = attrs.get('flow', None) if not message and not flow: raise ValidationError("Must specify either a flow or a message for the event") if message and flow: raise ValidationError("You cannot set both a flow and a message on an event, it must be only one") campaign_id = attrs.get('campaign', None) event_id = attrs.get('event', None) if not campaign_id and not event_id: raise ValidationError("You must specify either a campaign to create a new event, or an event to update") offset = attrs.get('offset') unit = attrs.get('unit') delivery_hour = attrs.get('delivery_hour') relative_to = attrs.get('relative_to') # load our contact field existing_field = ContactField.objects.filter(label=relative_to, org=org, is_active=True) if not existing_field: relative_to_field = ContactField.get_or_create(org, ContactField.make_key(relative_to), relative_to) else: relative_to_field = existing_field[0] if 'event' in attrs: event = CampaignEvent.objects.get(pk=attrs['event'], is_active=True, campaign__org=org) # we are being set to a flow if 'flow' in attrs: flow = Flow.objects.get(pk=attrs['flow'], is_active=True, org=org) event.flow = flow event.event_type = FLOW_EVENT event.message = None # we are being set to a message else: event.message = attrs['message'] # if we aren't currently a message event, we need to create our hidden message flow if event.event_type != MESSAGE_EVENT: event.flow = Flow.create_single_message(org, self.user, event.message) event.event_type = MESSAGE_EVENT # otherwise, we can just update that flow else: # set our single message on our flow event.flow.update_single_message_flow(message=attrs['message']) # update our other attributes event.offset = offset event.unit = unit event.delivery_hour = delivery_hour event.relative_to = relative_to_field event.save() event.update_flow_name() else: campaign = Campaign.objects.get(pk=attrs['campaign'], is_active=True, org=org) event_type = MESSAGE_EVENT if 'flow' in attrs: flow = Flow.objects.get(pk=attrs['flow'], is_active=True, org=org) event_type = FLOW_EVENT else: flow = Flow.create_single_message(org, self.user, message) event = CampaignEvent.objects.create(campaign=campaign, relative_to=relative_to_field, offset=offset, unit=unit, event_type=event_type, flow=flow, message=message, created_by=self.user, modified_by=self.user) event.update_flow_name() return event
def test_scheduling(self): campaign = Campaign.create(self.org, self.admin, "Planting Reminders", self.farmers) self.assertEquals("Planting Reminders", unicode(campaign)) # create a reminder for our first planting event planting_reminder = CampaignEvent.create_flow_event(self.org, self.admin, campaign, relative_to=self.planting_date, offset=0, unit='D', flow=self.reminder_flow, delivery_hour=17) self.assertEquals("Planting Date == 0 -> Color Flow", unicode(planting_reminder)) # schedule our reminders EventFire.update_campaign_events(campaign) # we should haven't any event fires created, since neither of our farmers have a planting date self.assertEquals(0, EventFire.objects.all().count()) # ok, set a planting date on one of our contacts self.farmer1.set_field(self.user, 'planting_date', "05-10-2020 12:30:10") # update our campaign events EventFire.update_campaign_events(campaign) # should have one event now fire = EventFire.objects.get() self.assertEquals(5, fire.scheduled.day) self.assertEquals(10, fire.scheduled.month) self.assertEquals(2020, fire.scheduled.year) # account for timezone difference, our org is in UTC+2 self.assertEquals(17 - 2, fire.scheduled.hour) self.assertEquals(self.farmer1, fire.contact) self.assertEquals(planting_reminder, fire.event) self.assertIsNone(fire.fired) # change the date of our date self.farmer1.set_field(self.user, 'planting_date', "06-10-2020 12:30:10") EventFire.update_campaign_events_for_contact(campaign, self.farmer1) fire = EventFire.objects.get() self.assertEquals(6, fire.scheduled.day) self.assertEquals(10, fire.scheduled.month) self.assertEquals(2020, fire.scheduled.year) self.assertEquals(self.farmer1, fire.contact) self.assertEquals(planting_reminder, fire.event) # set it to something invalid self.farmer1.set_field(self.user, 'planting_date', "what?") EventFire.update_campaign_events_for_contact(campaign, self.farmer1) self.assertFalse(EventFire.objects.all()) # now something valid again self.farmer1.set_field(self.user, 'planting_date', "07-10-2020 12:30:10") EventFire.update_campaign_events_for_contact(campaign, self.farmer1) fire = EventFire.objects.get() self.assertEquals(7, fire.scheduled.day) self.assertEquals(10, fire.scheduled.month) self.assertEquals(2020, fire.scheduled.year) self.assertEquals(self.farmer1, fire.contact) self.assertEquals(planting_reminder, fire.event) # create another reminder planting_reminder2 = CampaignEvent.create_flow_event(self.org, self.admin, campaign, relative_to=self.planting_date, offset=1, unit='D', flow=self.reminder2_flow) self.assertEquals(1, planting_reminder2.abs_offset()) # update the campaign EventFire.update_campaign_events(campaign) # should have two events now, ordered by date events = EventFire.objects.all() self.assertEquals(planting_reminder, events[0].event) self.assertEquals(7, events[0].scheduled.day) self.assertEquals(planting_reminder2, events[1].event) self.assertEquals(8, events[1].scheduled.day) # mark one of the events as inactive planting_reminder2.is_active = False planting_reminder2.save() # update the campaign EventFire.update_campaign_events(campaign) # back to only one event event = EventFire.objects.get() self.assertEquals(planting_reminder, event.event) self.assertEquals(7, event.scheduled.day) # update our date self.farmer1.set_field(self.user, 'planting_date', '09-10-2020 12:30') # should have updated event = EventFire.objects.get() self.assertEquals(planting_reminder, event.event) self.assertEquals(9, event.scheduled.day) # let's remove our contact field ContactField.hide_field(self.org, self.user, 'planting_date') # shouldn't have anything scheduled self.assertFalse(EventFire.objects.all()) # add it back in ContactField.get_or_create(self.org, self.admin, 'planting_date', "planting Date") # should be back! event = EventFire.objects.get() self.assertEquals(planting_reminder, event.event) self.assertEquals(9, event.scheduled.day) # change our fire date to sometimein the past so it gets triggered event.scheduled = timezone.now() - timedelta(hours=1) event.save() # schedule our events to fire check_campaigns_task() # should have one flow run now run = FlowRun.objects.get() self.assertEquals(event.contact, run.contact)
def create_org(self, spec, superuser, country, locations): self._log(f"\nCreating org {spec['name']}...\n") org = Org.objects.create( uuid=spec["uuid"], name=spec["name"], timezone=pytz.timezone("America/Los_Angeles"), brand="rapidpro.io", country=country, created_on=timezone.now(), created_by=superuser, modified_by=superuser, ) ContactGroup.create_system_groups(org) ContactField.create_system_fields(org) org.init_topups(100_000) # set our sequences to make ids stable across orgs with connection.cursor() as cursor: cursor.execute( "ALTER SEQUENCE contacts_contact_id_seq RESTART WITH %s", [spec["sequence_start"]]) cursor.execute( "ALTER SEQUENCE contacts_contacturn_id_seq RESTART WITH %s", [spec["sequence_start"]]) cursor.execute( "ALTER SEQUENCE contacts_contactgroup_id_seq RESTART WITH %s", [spec["sequence_start"]]) cursor.execute("ALTER SEQUENCE flows_flow_id_seq RESTART WITH %s", [spec["sequence_start"]]) cursor.execute( "ALTER SEQUENCE channels_channel_id_seq RESTART WITH %s", [spec["sequence_start"]]) cursor.execute( "ALTER SEQUENCE campaigns_campaign_id_seq RESTART WITH %s", [spec["sequence_start"]]) cursor.execute( "ALTER SEQUENCE campaigns_campaignevent_id_seq RESTART WITH %s", [spec["sequence_start"]]) cursor.execute("ALTER SEQUENCE msgs_label_id_seq RESTART WITH %s", [spec["sequence_start"]]) cursor.execute( "ALTER SEQUENCE templates_template_id_seq RESTART WITH %s", [spec["sequence_start"]]) cursor.execute( "ALTER SEQUENCE templates_templatetranslation_id_seq RESTART WITH %s", [spec["sequence_start"]]) self.create_channels(spec, org, superuser) self.create_fields(spec, org, superuser) self.create_globals(spec, org, superuser) self.create_labels(spec, org, superuser) self.create_groups(spec, org, superuser) self.create_flows(spec, org, superuser) self.create_contacts(spec, org, superuser) self.create_group_contacts(spec, org, superuser) self.create_campaigns(spec, org, superuser) self.create_templates(spec, org, superuser) self.create_classifiers(spec, org, superuser) self.create_ticketers(spec, org, superuser) return org
def import_campaigns(cls, exported_json, org, user, same_site=False): """ Import campaigns from our export file """ from temba.orgs.models import EARLIEST_IMPORT_VERSION if Flow.is_before_version(exported_json.get("version", "0"), EARLIEST_IMPORT_VERSION): # pragma: needs cover raise ValueError(_("Unknown version (%s)" % exported_json.get("version", 0))) if "campaigns" in exported_json: for campaign_spec in exported_json["campaigns"]: name = campaign_spec["name"] campaign = None group = None # first check if we have the objects by id if same_site: group = ContactGroup.user_groups.filter(uuid=campaign_spec["group"]["uuid"], org=org).first() if group: # pragma: needs cover group.name = campaign_spec["group"]["name"] group.save() campaign = Campaign.objects.filter(org=org, uuid=campaign_spec["uuid"]).first() if campaign: # pragma: needs cover campaign.name = Campaign.get_unique_name(org, name, ignore=campaign) campaign.save() # fall back to lookups by name if not group: group = ContactGroup.get_user_group(org, campaign_spec["group"]["name"]) if not campaign: campaign = Campaign.objects.filter(org=org, name=name).first() # all else fails, create the objects from scratch if not group: group = ContactGroup.create_static(org, user, campaign_spec["group"]["name"]) if not campaign: campaign_name = Campaign.get_unique_name(org, name) campaign = Campaign.create(org, user, campaign_name, group) else: campaign.group = group campaign.save() # deactivate all of our events, we'll recreate these for event in campaign.events.all(): event.release() # fill our campaign with events for event_spec in campaign_spec["events"]: field_key = event_spec["relative_to"]["key"] if field_key == "created_on": relative_to = ContactField.system_fields.filter(org=org, key=field_key).first() else: relative_to = ContactField.get_or_create( org, user, key=field_key, label=event_spec["relative_to"]["label"], value_type="D" ) start_mode = event_spec.get("start_mode", CampaignEvent.MODE_INTERRUPT) # create our message flow for message events if event_spec["event_type"] == CampaignEvent.TYPE_MESSAGE: message = event_spec["message"] base_language = event_spec.get("base_language") if not isinstance(message, dict): try: message = json.loads(message) except ValueError: # if it's not a language dict, turn it into one message = dict(base=message) base_language = "base" event = CampaignEvent.create_message_event( org, user, campaign, relative_to, event_spec["offset"], event_spec["unit"], message, event_spec["delivery_hour"], base_language=base_language, start_mode=start_mode, ) event.update_flow_name() else: flow = Flow.objects.filter( org=org, is_active=True, is_system=False, uuid=event_spec["flow"]["uuid"] ).first() if flow: CampaignEvent.create_flow_event( org, user, campaign, relative_to, event_spec["offset"], event_spec["unit"], flow, event_spec["delivery_hour"], start_mode=start_mode, ) # update our scheduled events for this campaign EventFire.update_campaign_events(campaign)
def test_field_results(self): c1 = self.create_contact("Contact1", '0788111111') c2 = self.create_contact("Contact2", '0788222222') c3 = self.create_contact("Contact3", '0788333333') self.create_contact("Contact4", '0788444444') # create a gender field that uses strings gender = ContactField.get_or_create(self.org, self.admin, 'gender', label="Gender", value_type=Value.TYPE_TEXT) c1.set_field(self.user, 'gender', "Male") c2.set_field(self.user, 'gender', "Female") c3.set_field(self.user, 'gender', "Female") result = Value.get_value_summary(contact_field=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals(2, result['unset']) # this is two as we have the default contact created by our unit tests self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Female", 2) self.assertResult(result, 1, "Male", 1) # create an born field that uses decimals born = ContactField.get_or_create(self.org, self.admin, 'born', label="Born", value_type=Value.TYPE_DECIMAL) c1.set_field(self.user, 'born', 1977) c2.set_field(self.user, 'born', 1990) c3.set_field(self.user, 'born', 1977) result = Value.get_value_summary(contact_field=born)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals(2, result['unset']) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "1977", 2) self.assertResult(result, 1, "1990", 1) # ok, state field! state = ContactField.get_or_create(self.org, self.admin, 'state', label="State", value_type=Value.TYPE_STATE) c1.set_field(self.user, 'state', "Kigali City") c2.set_field(self.user, 'state', "Kigali City") result = Value.get_value_summary(contact_field=state)[0] self.assertEquals(1, len(result['categories'])) self.assertEquals(2, result['set']) self.assertEquals(3, result['unset']) self.assertResult(result, 0, "1708283", 2) reg_date = ContactField.get_or_create(self.org, self.admin, 'reg_date', label="Registration Date", value_type=Value.TYPE_DATETIME) now = timezone.now() c1.set_field(self.user, 'reg_date', now.replace(hour=9)) c2.set_field(self.user, 'reg_date', now.replace(hour=4)) c3.set_field(self.user, 'reg_date', now - timedelta(days=1)) result = Value.get_value_summary(contact_field=reg_date)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals(2, result['unset']) self.assertResult(result, 0, now.replace(hour=0, minute=0, second=0, microsecond=0), 2) self.assertResult(result, 1, (now - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0), 1) # make sure categories returned are sorted by count, not name c2.set_field(self.user, 'gender', "Male") result = Value.get_value_summary(contact_field=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals(2, result['unset']) # this is two as we have the default contact created by our unit tests self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Male", 2) self.assertResult(result, 1, "Female", 1) # check the modified date is tracked for fields original_value = Value.objects.get(contact=c1, contact_field=gender) c1.set_field(self.user, 'gender', 'unknown') new_value = Value.objects.get(contact=c1, contact_field=gender) self.assertTrue(new_value.modified_on > original_value.modified_on) self.assertNotEqual(new_value.string_value, original_value.string_value)
def get_value_summary(cls, ruleset=None, contact_field=None, filters=None, segment=None): """ Returns the results for the passed in ruleset or contact field given the passed in filters and segments. Filters are expected in the following formats: { field: rulesetId, categories: ["Red", "Blue", "Yellow"] } Segments are expected in these formats instead: { ruleset: 1515, categories: ["Red", "Blue"] } // segmenting by another field, for those categories { groups: 124,151,151 } // segment by each each group in the passed in ids { location: "State", parent: null } // segment for each admin boundary within the parent { contact_field: "Country", values: ["US", "EN", "RW"] } // segment by a contact field for these values """ from temba.contacts.models import ContactGroup, ContactField from temba.flows.models import TrueTest, RuleSet start = time.time() results = [] if (not ruleset and not contact_field) or (ruleset and contact_field): raise ValueError("Must specify either a RuleSet or Contact field.") org = ruleset.flow.org if ruleset else contact_field.org open_ended = ruleset and ruleset.ruleset_type == RuleSet.TYPE_WAIT_MESSAGE and len(ruleset.get_rules()) == 1 # default our filters to an empty list if None are passed in if filters is None: filters = [] # build the kwargs for our subcall kwargs = dict(ruleset=ruleset, contact_field=contact_field, filters=filters) # this is our list of dependencies, that is things that will blow away our results dependencies = set() fingerprint_dict = dict(filters=filters, segment=segment) if ruleset: fingerprint_dict['ruleset'] = ruleset.id dependencies.add(RULESET_KEY % ruleset.id) if contact_field: fingerprint_dict['contact_field'] = contact_field.id dependencies.add(CONTACT_KEY % contact_field.id) for contact_filter in filters: if 'ruleset' in contact_filter: dependencies.add(RULESET_KEY % contact_filter['ruleset']) if 'groups' in contact_filter: for group_id in contact_filter['groups']: dependencies.add(GROUP_KEY % group_id) if 'location' in contact_filter: field = ContactField.get_by_label(org, contact_filter['location']) dependencies.add(CONTACT_KEY % field.id) if segment: if 'ruleset' in segment: dependencies.add(RULESET_KEY % segment['ruleset']) if 'groups' in segment: for group_id in segment['groups']: dependencies.add(GROUP_KEY % group_id) if 'location' in segment: field = ContactField.get_by_label(org, segment['location']) dependencies.add(CONTACT_KEY % field.id) # our final redis key will contain each dependency as well as a HASH representing the fingerprint of the # kwargs passed to this method, generate that hash fingerprint = hash(dict_to_json(fingerprint_dict)) # generate our key key = VALUE_SUMMARY_CACHE_KEY + ":" + str(org.id) + ":".join(sorted(list(dependencies))) + ":" + str(fingerprint) # does our value exist? r = get_redis_connection() cached = r.get(key) if cached is not None: try: return json_to_dict(cached) except: # failed decoding, oh well, go calculate it instead pass if segment: # segmenting a result is the same as calculating the result with the addition of each # category as a filter so we expand upon the passed in filters to do this if 'ruleset' in segment and 'categories' in segment: for category in segment['categories']: category_filter = list(filters) category_filter.append(dict(ruleset=segment['ruleset'], categories=[category])) # calculate our results for this segment kwargs['filters'] = category_filter (set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs) results.append(dict(label=category, open_ended=open_ended, set=set_count, unset=unset_count, categories=categories)) # segmenting by groups instead, same principle but we add group filters elif 'groups' in segment: for group_id in segment['groups']: # load our group group = ContactGroup.user_groups.get(is_active=True, org=org, pk=group_id) category_filter = list(filters) category_filter.append(dict(groups=[group_id])) # calculate our results for this segment kwargs['filters'] = category_filter (set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs) results.append(dict(label=group.name, open_ended=open_ended, set=set_count, unset_count=unset_count, categories=categories)) # segmenting by a contact field, only for passed in categories elif 'contact_field' in segment and 'values' in segment: # look up the contact field field = ContactField.get_by_label(org, segment['contact_field']) for value in segment['values']: value_filter = list(filters) value_filter.append(dict(contact_field=field.pk, values=[value])) # calculate our results for this segment kwargs['filters'] = value_filter (set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs) results.append(dict(label=value, open_ended=open_ended, set=set_count, unset=unset_count, categories=categories)) # segmenting by a location field elif 'location' in segment: # look up the contact field field = ContactField.get_by_label(org, segment['location']) # make sure they are segmenting on a location type that makes sense if field.value_type not in [STATE, DISTRICT]: raise ValueError(_("Cannot segment on location for field that is not a State or District type")) # make sure our org has a country for location based responses if not org.country: raise ValueError(_("Cannot segment by location until country has been selected for organization")) # the boundaries we will segment by parent = org.country # figure out our parent parent_osm_id = segment.get('parent', None) if parent_osm_id: parent = AdminBoundary.objects.get(osm_id=parent_osm_id) # get all the boundaries we are segmenting on boundaries = list(AdminBoundary.objects.filter(parent=parent).order_by('name')) # if the field is a district field, they need to specify the parent state if not parent_osm_id and field.value_type == DISTRICT: raise ValueError(_("You must specify a parent state to segment results by district")) # if this is a district, we can speed things up by only including those districts in our parent, build # the filter for that if parent and field.value_type == DISTRICT: location_filters = [filters, dict(location=field.pk, boundary=[b.osm_id for b in boundaries])] else: location_filters = filters # get all the contacts segment by location first (location_set_contacts, location_unset_contacts, location_results) = \ cls.get_filtered_value_summary(contact_field=field, filters=location_filters, return_contacts=True) # now get the contacts for our primary query kwargs['return_contacts'] = True kwargs['filter_contacts'] = location_set_contacts (primary_set_contacts, primary_unset_contacts, primary_results) = cls.get_filtered_value_summary(**kwargs) # build a map of osm_id to location_result osm_results = {lr['label']: lr for lr in location_results} empty_result = dict(contacts=list()) for boundary in boundaries: location_result = osm_results.get(boundary.osm_id, empty_result) # clone our primary results segmented_results = dict(label=boundary.name, boundary=boundary.osm_id, open_ended=open_ended) location_categories = list() location_contacts = set(location_result['contacts']) for category in primary_results: category_contacts = set(category['contacts']) intersection = location_contacts & category_contacts location_categories.append(dict(label=category['label'], count=len(intersection))) segmented_results['set'] = len(location_contacts & primary_set_contacts) segmented_results['unset'] = len(location_contacts & primary_unset_contacts) segmented_results['categories'] = location_categories results.append(segmented_results) results = sorted(results, key=lambda r: r['label']) else: (set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs) # Check we have and we have an OPEN ENDED ruleset if ruleset and len(ruleset.get_rules()) == 1 and isinstance(ruleset.get_rules()[0].test, TrueTest): cursor = connection.cursor() custom_sql = """SELECT w.label, count(*) AS count FROM ( SELECT regexp_split_to_table(LOWER(text), E'[^[:alnum:]_]') AS label FROM msgs_msg INNER JOIN contacts_contact ON ( msgs_msg.contact_id = contacts_contact.id ) WHERE msgs_msg.id IN ( SELECT msg_id FROM flows_flowstep_messages, flows_flowstep WHERE flowstep_id = flows_flowstep.id AND flows_flowstep.step_uuid = '%s' ) AND contacts_contact.is_test = False ) w group by w.label order by count desc;""" % ruleset.uuid cursor.execute(custom_sql) unclean_categories = get_dict_from_cursor(cursor) categories = [] ignore_words = get_stop_words('english') for category in unclean_categories: if len(category['label']) > 1 and category['label'] not in ignore_words and len(categories) < 100: categories.append(dict(label=category['label'], count=int(category['count']))) # sort by count, then alphabetically categories = sorted(categories, key=lambda c: (-c['count'], c['label'])) results.append(dict(label=unicode(_("All")), open_ended=open_ended, set=set_count, unset=unset_count, categories=categories)) # for each of our dependencies, add our key as something that depends on it pipe = r.pipeline() for dependency in dependencies: pipe.sadd(dependency, key) pipe.expire(dependency, VALUE_SUMMARY_CACHE_TIME) # and finally set our result pipe.set(key, dict_to_json(results), VALUE_SUMMARY_CACHE_TIME) pipe.execute() # leave me: nice for profiling.. #from django.db import connection as db_connection, reset_queries #print "=" * 80 #for query in db_connection.queries: # print "%s - %s" % (query['time'], query['sql'][:1000]) #print "-" * 80 #print "took: %f" % (time.time() - start) #print "=" * 80 #reset_queries() return results
def test_category_results(self): self.setup_color_gender_flow() # create a state field: # assign c1 and c2 to Kigali state = ContactField.get_or_create(self.org, 'state', label="State", value_type=STATE) district = ContactField.get_or_create(self.org, 'district', label="District", value_type=DISTRICT) self.c1.set_field('state', "Kigali City") self.c1.set_field('district', "Kigali") self.c2.set_field('state', "Kigali City") self.c2.set_field('district', "Kigali") self.run_color_gender_flow(self.c1, "red", "male", "16") self.run_color_gender_flow(self.c2, "blue", "female", "19") self.run_color_gender_flow(self.c3, "green", "male", "75") self.run_color_gender_flow(self.c4, "maroon", "female", "50") # create a group of the women ladies = self.create_group("Ladies", [self.c2, self.c4]) # get our rulesets color = RuleSet.objects.get(flow=self.flow, label="Color") gender = RuleSet.objects.get(flow=self.flow, label="Gender") age = RuleSet.objects.get(flow=self.flow, label="Age") # categories should be in the same order as our rules, should have correct counts result = Value.get_value_summary(ruleset=color)[0] self.assertEquals(3, len(result['categories'])) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Red", 2) self.assertResult(result, 1, "Blue", 1) self.assertResult(result, 2, "Green", 1) # check our age category as well result = Value.get_value_summary(ruleset=age)[0] self.assertEquals(3, len(result['categories'])) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Child", 1) self.assertResult(result, 1, "Adult", 2) self.assertResult(result, 2, "Senior", 1) # and our gender categories result = Value.get_value_summary(ruleset=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Male", 2) self.assertResult(result, 1, "Female", 2) # now filter the results and only get responses by men result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male"])])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 0) self.assertResult(result, 2, "Green", 1) # what about men that are adults? result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male"]), dict(ruleset=age.pk, categories=["Adult"])])[0] self.assertResult(result, 0, "Red", 0) self.assertResult(result, 1, "Blue", 0) self.assertResult(result, 2, "Green", 0) # union of all genders result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male", "Female"]), dict(ruleset=age.pk, categories=["Adult"])])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 1) self.assertResult(result, 2, "Green", 0) # just women adults by group result = Value.get_value_summary(ruleset=color, filters=[dict(groups=[ladies.pk]), dict(ruleset=age.pk, categories="Adult")])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 1) self.assertResult(result, 2, "Green", 0) # remove one of the women from the group ladies.update_contacts([self.c2], False) # get a new summary result = Value.get_value_summary(ruleset=color, filters=[dict(groups=[ladies.pk]), dict(ruleset=age.pk, categories="Adult")])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 0) self.assertResult(result, 2, "Green", 0) # ok, back in she goes ladies.update_contacts([self.c2], True) # do another run for contact 1 run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16") # totals should reflect the new value, not the old result = Value.get_value_summary(ruleset=color)[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 2) self.assertResult(result, 2, "Green", 1) # what if we do a partial run? self.send_message(self.flow, "red", contact=self.c1, restart_participants=True) # should change our male/female breakdown since c1 now no longer has a gender result = Value.get_value_summary(ruleset=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertResult(result, 0, "Male", 1) self.assertResult(result, 1, "Female", 2) # back to a full flow run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16") # ok, now segment by gender result = Value.get_value_summary(ruleset=color, filters=[], segment=dict(ruleset=gender.pk, categories=["Male", "Female"])) male_result = result[0] self.assertResult(male_result, 0, "Red", 0) self.assertResult(male_result, 1, "Blue", 1) self.assertResult(male_result, 2, "Green", 1) female_result = result[1] self.assertResult(female_result, 0, "Red", 1) self.assertResult(female_result, 1, "Blue", 1) self.assertResult(female_result, 2, "Green", 0) # add in a filter at the same time result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=color.pk, categories=["Blue"])], segment=dict(ruleset=gender.pk, categories=["Male", "Female"])) male_result = result[0] self.assertResult(male_result, 0, "Red", 0) self.assertResult(male_result, 1, "Blue", 1) self.assertResult(male_result, 2, "Green", 0) female_result = result[1] self.assertResult(female_result, 0, "Red", 0) self.assertResult(female_result, 1, "Blue", 1) self.assertResult(female_result, 2, "Green", 0) # ok, try segmenting by location instead result = Value.get_value_summary(ruleset=color, segment=dict(location="State")) eastern_result = result[0] self.assertEquals('171591', eastern_result['boundary']) self.assertEquals('Eastern Province', eastern_result['label']) self.assertResult(eastern_result, 0, "Red", 0) self.assertResult(eastern_result, 1, "Blue", 0) self.assertResult(eastern_result, 2, "Green", 0) kigali_result = result[1] self.assertEquals('1708283', kigali_result['boundary']) self.assertEquals('Kigali City', kigali_result['label']) self.assertResult(kigali_result, 0, "Red", 0) self.assertResult(kigali_result, 1, "Blue", 2) self.assertResult(kigali_result, 2, "Green", 0) # updating state location leads to updated data self.c2.set_field('state', "Eastern Province") result = Value.get_value_summary(ruleset=color, segment=dict(location="State")) eastern_result = result[0] self.assertEquals('171591', eastern_result['boundary']) self.assertEquals('Eastern Province', eastern_result['label']) self.assertResult(eastern_result, 0, "Red", 0) self.assertResult(eastern_result, 1, "Blue", 1) self.assertResult(eastern_result, 2, "Green", 0) kigali_result = result[1] self.assertEquals('1708283', kigali_result['boundary']) self.assertEquals('Kigali City', kigali_result['label']) self.assertResult(kigali_result, 0, "Red", 0) self.assertResult(kigali_result, 1, "Blue", 1) self.assertResult(kigali_result, 2, "Green", 0) # segment by district instead result = Value.get_value_summary(ruleset=color, segment=dict(parent="1708283", location="District")) # only on district in kigali self.assertEquals(1, len(result)) kigali_result = result[0] self.assertEquals('60485579', kigali_result['boundary']) self.assertEquals('Kigali', kigali_result['label']) self.assertResult(kigali_result, 0, "Red", 0) self.assertResult(kigali_result, 1, "Blue", 2) self.assertResult(kigali_result, 2, "Green", 0)
def test_field_results(self): (c1, c2, c3, c4) = (self.create_contact("Contact1", '0788111111'), self.create_contact("Contact2", '0788222222'), self.create_contact("Contact3", '0788333333'), self.create_contact("Contact4", '0788444444')) # create a gender field that uses strings gender = ContactField.get_or_create(self.org, 'gender', label="Gender", value_type=TEXT) c1.set_field('gender', "Male") c2.set_field('gender', "Female") c3.set_field('gender', "Female") result = Value.get_value_summary(contact_field=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals( 2, result['unset'] ) # this is two as we have the default contact created by our unit tests self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Female", 2) self.assertResult(result, 1, "Male", 1) # create an born field that uses decimals born = ContactField.get_or_create(self.org, 'born', label="Born", value_type=DECIMAL) c1.set_field('born', 1977) c2.set_field('born', 1990) c3.set_field('born', 1977) result = Value.get_value_summary(contact_field=born)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals(2, result['unset']) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "1977", 2) self.assertResult(result, 1, "1990", 1) # ok, state field! state = ContactField.get_or_create(self.org, 'state', label="State", value_type=STATE) c1.set_field('state', "Kigali City") c2.set_field('state', "Kigali City") result = Value.get_value_summary(contact_field=state)[0] self.assertEquals(1, len(result['categories'])) self.assertEquals(2, result['set']) self.assertEquals(3, result['unset']) self.assertResult(result, 0, "1708283", 2) reg_date = ContactField.get_or_create(self.org, 'reg_date', label="Registration Date", value_type=DATETIME) now = timezone.now() c1.set_field('reg_date', now.replace(hour=9)) c2.set_field('reg_date', now.replace(hour=4)) c3.set_field('reg_date', now - timedelta(days=1)) result = Value.get_value_summary(contact_field=reg_date)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals(2, result['unset']) self.assertResult( result, 0, now.replace(hour=0, minute=0, second=0, microsecond=0), 2) self.assertResult(result, 1, (now - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0), 1) # make sure categories returned are sorted by count, not name c2.set_field('gender', "Male") result = Value.get_value_summary(contact_field=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertEquals(3, result['set']) self.assertEquals( 2, result['unset'] ) # this is two as we have the default contact created by our unit tests self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Male", 2) self.assertResult(result, 1, "Female", 1)
def test_category_results(self): self.setup_color_gender_flow() # create a state field: # assign c1 and c2 to Kigali state = ContactField.get_or_create(self.org, 'state', label="State", value_type=STATE) district = ContactField.get_or_create(self.org, 'district', label="District", value_type=DISTRICT) self.c1.set_field('state', "Kigali City") self.c1.set_field('district', "Kigali") self.c2.set_field('state', "Kigali City") self.c2.set_field('district', "Kigali") self.run_color_gender_flow(self.c1, "red", "male", "16") self.run_color_gender_flow(self.c2, "blue", "female", "19") self.run_color_gender_flow(self.c3, "green", "male", "75") self.run_color_gender_flow(self.c4, "maroon", "female", "50") # create a group of the women ladies = self.create_group("Ladies", [self.c2, self.c4]) # get our rulesets color = RuleSet.objects.get(flow=self.flow, label="Color") gender = RuleSet.objects.get(flow=self.flow, label="Gender") age = RuleSet.objects.get(flow=self.flow, label="Age") # categories should be in the same order as our rules, should have correct counts result = Value.get_value_summary(ruleset=color)[0] self.assertEquals(3, len(result['categories'])) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Red", 2) self.assertResult(result, 1, "Blue", 1) self.assertResult(result, 2, "Green", 1) # check our age category as well result = Value.get_value_summary(ruleset=age)[0] self.assertEquals(3, len(result['categories'])) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Child", 1) self.assertResult(result, 1, "Adult", 2) self.assertResult(result, 2, "Senior", 1) # and our gender categories result = Value.get_value_summary(ruleset=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Male", 2) self.assertResult(result, 1, "Female", 2) # now filter the results and only get responses by men result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male"])])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 0) self.assertResult(result, 2, "Green", 1) # what about men that are adults? result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male"]), dict(ruleset=age.pk, categories=["Adult"])])[0] self.assertResult(result, 0, "Red", 0) self.assertResult(result, 1, "Blue", 0) self.assertResult(result, 2, "Green", 0) # union of all genders result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male", "Female"]), dict(ruleset=age.pk, categories=["Adult"])])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 1) self.assertResult(result, 2, "Green", 0) # just women adults by group result = Value.get_value_summary(ruleset=color, filters=[dict(groups=[ladies.pk]), dict(ruleset=age.pk, categories="Adult")])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 1) self.assertResult(result, 2, "Green", 0) # remove one of the women from the group ladies.update_contacts([self.c2], False) # get a new summary result = Value.get_value_summary(ruleset=color, filters=[dict(groups=[ladies.pk]), dict(ruleset=age.pk, categories="Adult")])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 0) self.assertResult(result, 2, "Green", 0) # ok, back in she goes ladies.update_contacts([self.c2], True) # do another run for contact 1 run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16") # totals should reflect the new value, not the old result = Value.get_value_summary(ruleset=color)[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 2) self.assertResult(result, 2, "Green", 1) # what if we do a partial run? self.send_message(self.flow, "red", contact=self.c1, restart_participants=True) # should change our male/female breakdown since c1 now no longer has a gender result = Value.get_value_summary(ruleset=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertResult(result, 0, "Male", 1) self.assertResult(result, 1, "Female", 2) # back to a full flow run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16") # ok, now segment by gender result = Value.get_value_summary(ruleset=color, filters=[], segment=dict(ruleset=gender.pk, categories=["Male", "Female"])) male_result = result[0] self.assertResult(male_result, 0, "Red", 0) self.assertResult(male_result, 1, "Blue", 1) self.assertResult(male_result, 2, "Green", 1) female_result = result[1] self.assertResult(female_result, 0, "Red", 1) self.assertResult(female_result, 1, "Blue", 1) self.assertResult(female_result, 2, "Green", 0) # segment by gender again, but use the contact field to do so result = Value.get_value_summary(ruleset=color, filters=[], segment=dict(contact_field="Gender", values=["MALE", "Female"])) male_result = result[0] self.assertResult(male_result, 0, "Red", 0) self.assertResult(male_result, 1, "Blue", 1) self.assertResult(male_result, 2, "Green", 1) female_result = result[1] self.assertResult(female_result, 0, "Red", 1) self.assertResult(female_result, 1, "Blue", 1) self.assertResult(female_result, 2, "Green", 0) # add in a filter at the same time result = Value.get_value_summary(ruleset=color, filters=[dict(ruleset=color.pk, categories=["Blue"])], segment=dict(ruleset=gender.pk, categories=["Male", "Female"])) male_result = result[0] self.assertResult(male_result, 0, "Red", 0) self.assertResult(male_result, 1, "Blue", 1) self.assertResult(male_result, 2, "Green", 0) female_result = result[1] self.assertResult(female_result, 0, "Red", 0) self.assertResult(female_result, 1, "Blue", 1) self.assertResult(female_result, 2, "Green", 0) # ok, try segmenting by location instead result = Value.get_value_summary(ruleset=color, segment=dict(location="State")) eastern_result = result[0] self.assertEquals('171591', eastern_result['boundary']) self.assertEquals('Eastern Province', eastern_result['label']) self.assertResult(eastern_result, 0, "Red", 0) self.assertResult(eastern_result, 1, "Blue", 0) self.assertResult(eastern_result, 2, "Green", 0) kigali_result = result[1] self.assertEquals('1708283', kigali_result['boundary']) self.assertEquals('Kigali City', kigali_result['label']) self.assertResult(kigali_result, 0, "Red", 0) self.assertResult(kigali_result, 1, "Blue", 2) self.assertResult(kigali_result, 2, "Green", 0) # updating state location leads to updated data self.c2.set_field('state', "Eastern Province") result = Value.get_value_summary(ruleset=color, segment=dict(location="State")) eastern_result = result[0] self.assertEquals('171591', eastern_result['boundary']) self.assertEquals('Eastern Province', eastern_result['label']) self.assertResult(eastern_result, 0, "Red", 0) self.assertResult(eastern_result, 1, "Blue", 1) self.assertResult(eastern_result, 2, "Green", 0) kigali_result = result[1] self.assertEquals('1708283', kigali_result['boundary']) self.assertEquals('Kigali City', kigali_result['label']) self.assertResult(kigali_result, 0, "Red", 0) self.assertResult(kigali_result, 1, "Blue", 1) self.assertResult(kigali_result, 2, "Green", 0) # segment by district instead result = Value.get_value_summary(ruleset=color, segment=dict(parent="1708283", location="District")) # only on district in kigali self.assertEquals(1, len(result)) kigali_result = result[0] self.assertEquals('60485579', kigali_result['boundary']) self.assertEquals('Kigali', kigali_result['label']) self.assertResult(kigali_result, 0, "Red", 0) self.assertResult(kigali_result, 1, "Blue", 2) self.assertResult(kigali_result, 2, "Green", 0) # do a sanity check on our choropleth view self.login(self.admin) response = self.client.get(reverse('flows.ruleset_choropleth', args=[color.pk]) + "?_format=json&boundary=" + self.org.country.osm_id) # response should be valid json response = json.loads(response.content) # should have breaks self.assertTrue('breaks' in response) # should have two categories, Blue and Others self.assertEquals(2, len(response['categories'])) self.assertEquals("Blue", response['categories'][0]) self.assertEquals("Others", response['categories'][1]) # assert our kigali result kigali_result = response['scores']['1708283'] self.assertEquals(1, kigali_result['score']) self.assertEquals("Kigali City", kigali_result['name']) self.assertEquals("Blue", kigali_result['results'][0]['label']) self.assertEquals("Others", kigali_result['results'][1]['label']) self.assertEquals(1, kigali_result['results'][0]['count']) self.assertEquals(0, kigali_result['results'][1]['count']) self.assertEquals(100, kigali_result['results'][0]['percentage']) self.assertEquals(0, kigali_result['results'][1]['percentage']) with patch('temba.values.models.Value.get_value_summary') as mock: mock.return_value = [] response = self.client.get(reverse('flows.ruleset_choropleth', args=[color.pk]) + "?_format=json&boundary=" + self.org.country.osm_id) # response should be valid json response = json.loads(response.content) # should have two categories, Blue and Others self.assertEquals(2, len(response['categories'])) self.assertEquals("", response['categories'][0]) self.assertEquals("", response['categories'][1]) # all counts and percentage are 0 self.assertEquals(0, response['totals']['count']) self.assertEquals(0, response['totals']['results'][0]['count']) self.assertEquals(0, response['totals']['results'][0]['percentage']) self.assertEquals(0, response['totals']['results'][1]['count']) self.assertEquals(0, response['totals']['results'][1]['percentage']) # and empty string labels self.assertEquals("", response['totals']['results'][0]['label']) self.assertEquals("", response['totals']['results'][1]['label']) # also check our analytics view response = self.client.get(reverse('flows.ruleset_analytics')) # make sure we have only one flow in it flows = json.loads(response.context['flows']) self.assertEquals(1, len(flows)) self.assertEquals(3, len(flows[0]['rules']))
def import_campaigns(cls, exported_json, org, user, same_site=False): """ Import campaigns from our export file """ from temba.orgs.models import EARLIEST_IMPORT_VERSION if Flow.is_before_version(exported_json.get('version', "0"), EARLIEST_IMPORT_VERSION): # pragma: needs cover raise ValueError(_("Unknown version (%s)" % exported_json.get('version', 0))) if 'campaigns' in exported_json: for campaign_spec in exported_json['campaigns']: name = campaign_spec['name'] campaign = None group = None # first check if we have the objects by id if same_site: group = ContactGroup.user_groups.filter(uuid=campaign_spec['group']['uuid'], org=org).first() if group: # pragma: needs cover group.name = campaign_spec['group']['name'] group.save() campaign = Campaign.objects.filter(org=org, uuid=campaign_spec['uuid']).first() if campaign: # pragma: needs cover campaign.name = Campaign.get_unique_name(org, name, ignore=campaign) campaign.save() # fall back to lookups by name if not group: group = ContactGroup.get_user_group(org, campaign_spec['group']['name']) if not campaign: campaign = Campaign.objects.filter(org=org, name=name).first() # all else fails, create the objects from scratch if not group: group = ContactGroup.create_static(org, user, campaign_spec['group']['name']) if not campaign: campaign_name = Campaign.get_unique_name(org, name) campaign = Campaign.create(org, user, campaign_name, group) else: campaign.group = group campaign.save() # we want to nuke old single message flows for event in campaign.events.all(): if event.flow.flow_type == Flow.MESSAGE: event.flow.release() # and all of the events, we'll recreate these campaign.events.all().delete() # fill our campaign with events for event_spec in campaign_spec['events']: relative_to = ContactField.get_or_create(org, user, key=event_spec['relative_to']['key'], label=event_spec['relative_to']['label']) # create our message flow for message events if event_spec['event_type'] == CampaignEvent.TYPE_MESSAGE: message = event_spec['message'] base_language = event_spec.get('base_language') if not isinstance(message, dict): try: message = json.loads(message) except ValueError: # if it's not a language dict, turn it into one message = dict(base=message) base_language = 'base' event = CampaignEvent.create_message_event(org, user, campaign, relative_to, event_spec['offset'], event_spec['unit'], message, event_spec['delivery_hour'], base_language=base_language) event.update_flow_name() else: flow = Flow.objects.filter(org=org, is_active=True, uuid=event_spec['flow']['uuid']).first() if flow: CampaignEvent.create_flow_event(org, user, campaign, relative_to, event_spec['offset'], event_spec['unit'], flow, event_spec['delivery_hour']) # update our scheduled events for this campaign EventFire.update_campaign_events(campaign)
def import_campaigns(cls, exported_json, org, user, same_site=False): """ Import campaigns from our export file """ from temba.orgs.models import EARLIEST_IMPORT_VERSION if exported_json.get('version', 0) < EARLIEST_IMPORT_VERSION: raise ValueError(_("Unknown version (%s)" % exported_json.get('version', 0))) if 'campaigns' in exported_json: for campaign_spec in exported_json['campaigns']: name = campaign_spec['name'] campaign = None group = None # first check if we have the objects by id if same_site: group = ContactGroup.user_groups.filter(uuid=campaign_spec['group']['uuid'], org=org).first() if group: group.name = campaign_spec['group']['name'] group.save() campaign = Campaign.objects.filter(org=org, uuid=campaign_spec['uuid']).first() if campaign: campaign.name = Campaign.get_unique_name(org, name, ignore=campaign) campaign.save() # fall back to lookups by name if not group: group = ContactGroup.get_user_group(org, campaign_spec['group']['name']) if not campaign: campaign = Campaign.objects.filter(org=org, name=name).first() # all else fails, create the objects from scratch if not group: group = ContactGroup.create_static(org, user, campaign_spec['group']['name']) if not campaign: campaign_name = Campaign.get_unique_name(org, name) campaign = Campaign.create(org, user, campaign_name, group) else: campaign.group = group campaign.save() # we want to nuke old single message flows for event in campaign.events.all(): if event.flow.flow_type == Flow.MESSAGE: event.flow.release() # and all of the events, we'll recreate these campaign.events.all().delete() # fill our campaign with events for event_spec in campaign_spec['events']: relative_to = ContactField.get_or_create(org, user, key=event_spec['relative_to']['key'], label=event_spec['relative_to']['label']) # create our message flow for message events if event_spec['event_type'] == CampaignEvent.TYPE_MESSAGE: event = CampaignEvent.create_message_event(org, user, campaign, relative_to, event_spec['offset'], event_spec['unit'], event_spec['message'], event_spec['delivery_hour']) event.update_flow_name() else: flow = Flow.objects.filter(org=org, is_active=True, uuid=event_spec['flow']['uuid']).first() if flow: CampaignEvent.create_flow_event(org, user, campaign, relative_to, event_spec['offset'], event_spec['unit'], flow, event_spec['delivery_hour']) # update our scheduled events for this campaign EventFire.update_campaign_events(campaign)
def test_category_results(self): self.setup_color_gender_flow() # create a state field: # assign c1 and c2 to Kigali state = ContactField.get_or_create(self.org, 'state', label="State", value_type=STATE) district = ContactField.get_or_create(self.org, 'district', label="District", value_type=DISTRICT) self.c1.set_field('state', "Kigali City") self.c1.set_field('district', "Kigali") self.c2.set_field('state', "Kigali City") self.c2.set_field('district', "Kigali") self.run_color_gender_flow(self.c1, "red", "male", "16") self.run_color_gender_flow(self.c2, "blue", "female", "19") self.run_color_gender_flow(self.c3, "green", "male", "75") self.run_color_gender_flow(self.c4, "maroon", "female", "50") # create a group of the women ladies = self.create_group("Ladies", [self.c2, self.c4]) # get our rulesets color = RuleSet.objects.get(flow=self.flow, label="Color") gender = RuleSet.objects.get(flow=self.flow, label="Gender") age = RuleSet.objects.get(flow=self.flow, label="Age") # categories should be in the same order as our rules, should have correct counts result = Value.get_value_summary(ruleset=color)[0] self.assertEquals(3, len(result['categories'])) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Red", 2) self.assertResult(result, 1, "Blue", 1) self.assertResult(result, 2, "Green", 1) # check our age category as well result = Value.get_value_summary(ruleset=age)[0] self.assertEquals(3, len(result['categories'])) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Child", 1) self.assertResult(result, 1, "Adult", 2) self.assertResult(result, 2, "Senior", 1) # and our gender categories result = Value.get_value_summary(ruleset=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertFalse(result['open_ended']) self.assertResult(result, 0, "Male", 2) self.assertResult(result, 1, "Female", 2) # now filter the results and only get responses by men result = Value.get_value_summary( ruleset=color, filters=[dict(ruleset=gender.pk, categories=["Male"])])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 0) self.assertResult(result, 2, "Green", 1) # what about men that are adults? result = Value.get_value_summary(ruleset=color, filters=[ dict(ruleset=gender.pk, categories=["Male"]), dict(ruleset=age.pk, categories=["Adult"]) ])[0] self.assertResult(result, 0, "Red", 0) self.assertResult(result, 1, "Blue", 0) self.assertResult(result, 2, "Green", 0) # union of all genders result = Value.get_value_summary( ruleset=color, filters=[ dict(ruleset=gender.pk, categories=["Male", "Female"]), dict(ruleset=age.pk, categories=["Adult"]) ])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 1) self.assertResult(result, 2, "Green", 0) # just women adults by group result = Value.get_value_summary(ruleset=color, filters=[ dict(groups=[ladies.pk]), dict(ruleset=age.pk, categories="Adult") ])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 1) self.assertResult(result, 2, "Green", 0) # remove one of the women from the group ladies.update_contacts([self.c2], False) # get a new summary result = Value.get_value_summary(ruleset=color, filters=[ dict(groups=[ladies.pk]), dict(ruleset=age.pk, categories="Adult") ])[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 0) self.assertResult(result, 2, "Green", 0) # ok, back in she goes ladies.update_contacts([self.c2], True) # do another run for contact 1 run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16") # totals should reflect the new value, not the old result = Value.get_value_summary(ruleset=color)[0] self.assertResult(result, 0, "Red", 1) self.assertResult(result, 1, "Blue", 2) self.assertResult(result, 2, "Green", 1) # what if we do a partial run? self.send_message(self.flow, "red", contact=self.c1, restart_participants=True) # should change our male/female breakdown since c1 now no longer has a gender result = Value.get_value_summary(ruleset=gender)[0] self.assertEquals(2, len(result['categories'])) self.assertResult(result, 0, "Male", 1) self.assertResult(result, 1, "Female", 2) # back to a full flow run5 = self.run_color_gender_flow(self.c1, "blue", "male", "16") # ok, now segment by gender result = Value.get_value_summary(ruleset=color, filters=[], segment=dict( ruleset=gender.pk, categories=["Male", "Female"])) male_result = result[0] self.assertResult(male_result, 0, "Red", 0) self.assertResult(male_result, 1, "Blue", 1) self.assertResult(male_result, 2, "Green", 1) female_result = result[1] self.assertResult(female_result, 0, "Red", 1) self.assertResult(female_result, 1, "Blue", 1) self.assertResult(female_result, 2, "Green", 0) # add in a filter at the same time result = Value.get_value_summary( ruleset=color, filters=[dict(ruleset=color.pk, categories=["Blue"])], segment=dict(ruleset=gender.pk, categories=["Male", "Female"])) male_result = result[0] self.assertResult(male_result, 0, "Red", 0) self.assertResult(male_result, 1, "Blue", 1) self.assertResult(male_result, 2, "Green", 0) female_result = result[1] self.assertResult(female_result, 0, "Red", 0) self.assertResult(female_result, 1, "Blue", 1) self.assertResult(female_result, 2, "Green", 0) # ok, try segmenting by location instead result = Value.get_value_summary(ruleset=color, segment=dict(location="State")) eastern_result = result[0] self.assertEquals('171591', eastern_result['boundary']) self.assertEquals('Eastern Province', eastern_result['label']) self.assertResult(eastern_result, 0, "Red", 0) self.assertResult(eastern_result, 1, "Blue", 0) self.assertResult(eastern_result, 2, "Green", 0) kigali_result = result[1] self.assertEquals('1708283', kigali_result['boundary']) self.assertEquals('Kigali City', kigali_result['label']) self.assertResult(kigali_result, 0, "Red", 0) self.assertResult(kigali_result, 1, "Blue", 2) self.assertResult(kigali_result, 2, "Green", 0) # updating state location leads to updated data self.c2.set_field('state', "Eastern Province") result = Value.get_value_summary(ruleset=color, segment=dict(location="State")) eastern_result = result[0] self.assertEquals('171591', eastern_result['boundary']) self.assertEquals('Eastern Province', eastern_result['label']) self.assertResult(eastern_result, 0, "Red", 0) self.assertResult(eastern_result, 1, "Blue", 1) self.assertResult(eastern_result, 2, "Green", 0) kigali_result = result[1] self.assertEquals('1708283', kigali_result['boundary']) self.assertEquals('Kigali City', kigali_result['label']) self.assertResult(kigali_result, 0, "Red", 0) self.assertResult(kigali_result, 1, "Blue", 1) self.assertResult(kigali_result, 2, "Green", 0) # segment by district instead result = Value.get_value_summary(ruleset=color, segment=dict(parent="1708283", location="District")) # only on district in kigali self.assertEquals(1, len(result)) kigali_result = result[0] self.assertEquals('60485579', kigali_result['boundary']) self.assertEquals('Kigali', kigali_result['label']) self.assertResult(kigali_result, 0, "Red", 0) self.assertResult(kigali_result, 1, "Blue", 2) self.assertResult(kigali_result, 2, "Green", 0) # do a sanity check on our choropleth view self.login(self.admin) response = self.client.get( reverse('flows.ruleset_choropleth', args=[color.pk]) + "?_format=json&boundary=" + self.org.country.osm_id) # response should be valid json response = json.loads(response.content) # should have breaks self.assertTrue('breaks' in response) # should have two categories, Blue and Others self.assertEquals(2, len(response['categories'])) self.assertEquals("Blue", response['categories'][0]) self.assertEquals("Others", response['categories'][1]) # assert our kigali result kigali_result = response['scores']['1708283'] self.assertEquals(1, kigali_result['score']) self.assertEquals("Kigali City", kigali_result['name']) self.assertEquals("Blue", kigali_result['results'][0]['label']) self.assertEquals("Others", kigali_result['results'][1]['label']) self.assertEquals(1, kigali_result['results'][0]['count']) self.assertEquals(0, kigali_result['results'][1]['count']) self.assertEquals(100, kigali_result['results'][0]['percentage']) self.assertEquals(0, kigali_result['results'][1]['percentage']) with patch('temba.values.models.Value.get_value_summary') as mock: mock.return_value = [] response = self.client.get( reverse('flows.ruleset_choropleth', args=[color.pk]) + "?_format=json&boundary=" + self.org.country.osm_id) # response should be valid json response = json.loads(response.content) # should have two categories, Blue and Others self.assertEquals(2, len(response['categories'])) self.assertEquals("", response['categories'][0]) self.assertEquals("", response['categories'][1]) # all counts and percentage are 0 self.assertEquals(0, response['totals']['count']) self.assertEquals(0, response['totals']['results'][0]['count']) self.assertEquals(0, response['totals']['results'][0]['percentage']) self.assertEquals(0, response['totals']['results'][1]['count']) self.assertEquals(0, response['totals']['results'][1]['percentage']) # and empty string labels self.assertEquals("", response['totals']['results'][0]['label']) self.assertEquals("", response['totals']['results'][1]['label']) # also check our analytics view response = self.client.get(reverse('flows.ruleset_analytics')) # make sure we have only one flow in it flows = json.loads(response.context['flows']) self.assertEquals(1, len(flows)) self.assertEquals(3, len(flows[0]['rules']))
def save(self): key = self.validated_data.get("key") label = self.validated_data.get("label") value_type = self.validated_data.get("value_type") return ContactField.get_or_create(self.org, self.user, key, label, value_type=value_type)
def import_campaigns(cls, exported_json, org, user, same_site=False): """ Import campaigns from our export file """ from temba.orgs.models import EARLIEST_IMPORT_VERSION if exported_json.get("version", 0) < EARLIEST_IMPORT_VERSION: raise ValueError(_("Unknown version (%s)" % exported_json.get("version", 0))) if "campaigns" in exported_json: for campaign_spec in exported_json["campaigns"]: name = campaign_spec["name"] campaign = None group = None # first check if we have the objects by id if same_site: group = ContactGroup.user_groups.filter( id=campaign_spec["group"]["id"], org=org, is_active=True ).first() if group: group.name = campaign_spec["group"]["name"] group.save() campaign = Campaign.objects.filter(org=org, id=campaign_spec["id"]).first() if campaign: campaign.name = Campaign.get_unique_name(org, name, ignore=campaign) campaign.save() # fall back to lookups by name if not group: group = ContactGroup.user_groups.filter(name=campaign_spec["group"]["name"], org=org).first() if not campaign: campaign = Campaign.objects.filter(org=org, name=name).first() # all else fails, create the objects from scratch if not group: group = ContactGroup.create(org, user, campaign_spec["group"]["name"]) if not campaign: campaign_name = Campaign.get_unique_name(org, name) campaign = Campaign.create(org, user, campaign_name, group) else: campaign.group = group campaign.save() # we want to nuke old single message flows for event in campaign.events.all(): if event.flow.flow_type == Flow.MESSAGE: event.flow.delete() # and all of the events, we'll recreate these campaign.events.all().delete() # fill our campaign with events for event_spec in campaign_spec["events"]: relative_to = ContactField.get_or_create( org, key=event_spec["relative_to"]["key"], label=event_spec["relative_to"]["label"] ) # create our message flow for message events if event_spec["event_type"] == MESSAGE_EVENT: event = CampaignEvent.create_message_event( org, user, campaign, relative_to, event_spec["offset"], event_spec["unit"], event_spec["message"], event_spec["delivery_hour"], ) event.update_flow_name() else: flow = Flow.objects.filter(org=org, id=event_spec["flow"]["id"]).first() if flow: CampaignEvent.create_flow_event( org, user, campaign, relative_to, event_spec["offset"], event_spec["unit"], flow, event_spec["delivery_hour"], ) # update our scheduled events for this campaign EventFire.update_campaign_events(campaign)
def validate_key(self, value): if value and not ContactField.is_valid_key(value): raise serializers.ValidationError("Field is invalid or a reserved name") return value
def import_campaigns(cls, exported_json, org, user, same_site=False): """ Import campaigns from our export file """ from temba.orgs.models import EARLIEST_IMPORT_VERSION if Flow.is_before_version(exported_json.get("version", "0"), EARLIEST_IMPORT_VERSION): # pragma: needs cover raise ValueError(_("Unknown version (%s)" % exported_json.get("version", 0))) if "campaigns" in exported_json: for campaign_spec in exported_json["campaigns"]: name = campaign_spec["name"] campaign = None group = None # first check if we have the objects by id if same_site: group = ContactGroup.user_groups.filter(uuid=campaign_spec["group"]["uuid"], org=org).first() if group: # pragma: needs cover group.name = campaign_spec["group"]["name"] group.save() campaign = Campaign.objects.filter(org=org, uuid=campaign_spec["uuid"]).first() if campaign: # pragma: needs cover campaign.name = Campaign.get_unique_name(org, name, ignore=campaign) campaign.save() # fall back to lookups by name if not group: group = ContactGroup.get_user_group(org, campaign_spec["group"]["name"]) if not campaign: campaign = Campaign.objects.filter(org=org, name=name).first() # all else fails, create the objects from scratch if not group: group = ContactGroup.create_static(org, user, campaign_spec["group"]["name"]) if not campaign: campaign_name = Campaign.get_unique_name(org, name) campaign = Campaign.create(org, user, campaign_name, group) else: campaign.group = group campaign.save() # deactivate all of our events, we'll recreate these for event in campaign.events.all(): event.release() # fill our campaign with events for event_spec in campaign_spec["events"]: field_key = event_spec["relative_to"]["key"] if field_key == "created_on": relative_to = ContactField.system_fields.filter(org=org, key=field_key).first() else: relative_to = ContactField.get_or_create( org, user, key=field_key, label=event_spec["relative_to"]["label"], value_type="D" ) start_mode = event_spec.get("start_mode", CampaignEvent.MODE_INTERRUPT) # create our message flow for message events if event_spec["event_type"] == CampaignEvent.TYPE_MESSAGE: message = event_spec["message"] base_language = event_spec.get("base_language") if not isinstance(message, dict): try: message = json.loads(message) except ValueError: # if it's not a language dict, turn it into one message = dict(base=message) base_language = "base" event = CampaignEvent.create_message_event( org, user, campaign, relative_to, event_spec["offset"], event_spec["unit"], message, event_spec["delivery_hour"], base_language=base_language, start_mode=start_mode, ) event.update_flow_name() else: flow = Flow.objects.filter( org=org, is_active=True, is_system=False, uuid=event_spec["flow"]["uuid"] ).first() if flow: CampaignEvent.create_flow_event( org, user, campaign, relative_to, event_spec["offset"], event_spec["unit"], flow, event_spec["delivery_hour"], start_mode=start_mode, ) # update our scheduled events for this campaign EventFire.update_campaign_events(campaign)
def get_value_summary(cls, ruleset=None, contact_field=None, filters=None, segment=None): """ Returns the results for the passed in ruleset or contact field given the passed in filters and segments. Filters are expected in the following formats: { field: rulesetId, categories: ["Red", "Blue", "Yellow"] } Segments are expected in these formats instead: { ruleset: 1515, categories: ["Red", "Blue"] } // segmenting by another field, for those categories { groups: 124,151,151 } // segment by each each group in the passed in ids { location: "State", parent: null } // segment for each admin boundary within the parent { contact_field: "Country", values: ["US", "EN", "RW"] } // segment by a contact field for these values """ from temba.contacts.models import ContactGroup, ContactField from temba.flows.models import TrueTest, RuleSet start = time.time() results = [] if (not ruleset and not contact_field) or (ruleset and contact_field): raise ValueError("Must specify either a RuleSet or Contact field.") org = ruleset.flow.org if ruleset else contact_field.org open_ended = ruleset and ruleset.ruleset_type == RuleSet.TYPE_WAIT_MESSAGE and len(ruleset.get_rules()) == 1 # default our filters to an empty list if None are passed in if filters is None: filters = [] # build the kwargs for our subcall kwargs = dict(ruleset=ruleset, contact_field=contact_field, filters=filters) # this is our list of dependencies, that is things that will blow away our results dependencies = set() fingerprint_dict = dict(filters=filters, segment=segment) if ruleset: fingerprint_dict['ruleset'] = ruleset.id dependencies.add(RULESET_KEY % ruleset.id) if contact_field: fingerprint_dict['contact_field'] = contact_field.id dependencies.add(CONTACT_KEY % contact_field.id) for contact_filter in filters: if 'ruleset' in contact_filter: dependencies.add(RULESET_KEY % contact_filter['ruleset']) if 'groups' in contact_filter: for group_id in contact_filter['groups']: dependencies.add(GROUP_KEY % group_id) if 'location' in contact_filter: field = ContactField.get_by_label(org, contact_filter['location']) dependencies.add(CONTACT_KEY % field.id) if segment: if 'ruleset' in segment: dependencies.add(RULESET_KEY % segment['ruleset']) if 'groups' in segment: for group_id in segment['groups']: dependencies.add(GROUP_KEY % group_id) if 'location' in segment: field = ContactField.get_by_label(org, segment['location']) dependencies.add(CONTACT_KEY % field.id) # our final redis key will contain each dependency as well as a HASH representing the fingerprint of the # kwargs passed to this method, generate that hash fingerprint = hash(dict_to_json(fingerprint_dict)) # generate our key key = VALUE_SUMMARY_CACHE_KEY + ":" + str(org.id) + ":".join(sorted(list(dependencies))) + ":" + str(fingerprint) # does our value exist? r = get_redis_connection() cached = r.get(key) if cached is not None: try: return json_to_dict(cached) except Exception: # failed decoding, oh well, go calculate it instead pass if segment: # segmenting a result is the same as calculating the result with the addition of each # category as a filter so we expand upon the passed in filters to do this if 'ruleset' in segment and 'categories' in segment: for category in segment['categories']: category_filter = list(filters) category_filter.append(dict(ruleset=segment['ruleset'], categories=[category])) # calculate our results for this segment kwargs['filters'] = category_filter (set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs) results.append(dict(label=category, open_ended=open_ended, set=set_count, unset=unset_count, categories=categories)) # segmenting by groups instead, same principle but we add group filters elif 'groups' in segment: for group_id in segment['groups']: # load our group group = ContactGroup.user_groups.get(org=org, pk=group_id) category_filter = list(filters) category_filter.append(dict(groups=[group_id])) # calculate our results for this segment kwargs['filters'] = category_filter (set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs) results.append(dict(label=group.name, open_ended=open_ended, set=set_count, unset_count=unset_count, categories=categories)) # segmenting by a contact field, only for passed in categories elif 'contact_field' in segment and 'values' in segment: # look up the contact field field = ContactField.get_by_label(org, segment['contact_field']) for value in segment['values']: value_filter = list(filters) value_filter.append(dict(contact_field=field.pk, values=[value])) # calculate our results for this segment kwargs['filters'] = value_filter (set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs) results.append(dict(label=value, open_ended=open_ended, set=set_count, unset=unset_count, categories=categories)) # segmenting by a location field elif 'location' in segment: # look up the contact field field = ContactField.get_by_label(org, segment['location']) # make sure they are segmenting on a location type that makes sense if field.value_type not in [Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD]: raise ValueError(_("Cannot segment on location for field that is not a State or District type")) # make sure our org has a country for location based responses if not org.country: raise ValueError(_("Cannot segment by location until country has been selected for organization")) # the boundaries we will segment by parent = org.country # figure out our parent parent_osm_id = segment.get('parent', None) if parent_osm_id: parent = AdminBoundary.objects.get(osm_id=parent_osm_id) # get all the boundaries we are segmenting on boundaries = list(AdminBoundary.objects.filter(parent=parent).order_by('name')) # if the field is a district field, they need to specify the parent state if not parent_osm_id and field.value_type == Value.TYPE_DISTRICT: raise ValueError(_("You must specify a parent state to segment results by district")) if not parent_osm_id and field.value_type == Value.TYPE_WARD: raise ValueError(_("You must specify a parent state to segment results by ward")) # if this is a district, we can speed things up by only including those districts in our parent, build # the filter for that if parent and field.value_type in [Value.TYPE_DISTRICT, Value.TYPE_WARD]: location_filters = [filters, dict(location=field.pk, boundary=[b.osm_id for b in boundaries])] else: location_filters = filters # get all the contacts segment by location first (location_set_contacts, location_unset_contacts, location_results) = \ cls.get_filtered_value_summary(contact_field=field, filters=location_filters, return_contacts=True) # now get the contacts for our primary query kwargs['return_contacts'] = True kwargs['filter_contacts'] = location_set_contacts (primary_set_contacts, primary_unset_contacts, primary_results) = cls.get_filtered_value_summary(**kwargs) # build a map of osm_id to location_result osm_results = {lr['label']: lr for lr in location_results} empty_result = dict(contacts=list()) for boundary in boundaries: location_result = osm_results.get(boundary.osm_id, empty_result) # clone our primary results segmented_results = dict(label=boundary.name, boundary=boundary.osm_id, open_ended=open_ended) location_categories = list() location_contacts = set(location_result['contacts']) for category in primary_results: category_contacts = set(category['contacts']) intersection = location_contacts & category_contacts location_categories.append(dict(label=category['label'], count=len(intersection))) segmented_results['set'] = len(location_contacts & primary_set_contacts) segmented_results['unset'] = len(location_contacts & primary_unset_contacts) segmented_results['categories'] = location_categories results.append(segmented_results) results = sorted(results, key=lambda r: r['label']) else: (set_count, unset_count, categories) = cls.get_filtered_value_summary(**kwargs) # Check we have and we have an OPEN ENDED ruleset if ruleset and len(ruleset.get_rules()) == 1 and isinstance(ruleset.get_rules()[0].test, TrueTest): cursor = connection.cursor() custom_sql = """SELECT w.label, count(*) AS count FROM ( SELECT regexp_split_to_table(LOWER(text), E'[^[:alnum:]_]') AS label FROM msgs_msg INNER JOIN contacts_contact ON ( msgs_msg.contact_id = contacts_contact.id ) WHERE msgs_msg.id IN ( SELECT msg_id FROM flows_flowstep_messages, flows_flowstep WHERE flowstep_id = flows_flowstep.id AND flows_flowstep.step_uuid = '%s' ) AND contacts_contact.is_test = False ) w group by w.label order by count desc;""" % ruleset.uuid cursor.execute(custom_sql) unclean_categories = get_dict_from_cursor(cursor) categories = [] org_languages = [lang.name.lower() for lang in org.languages.filter(orgs=None).distinct()] if 'english' not in org_languages: org_languages.append('english') ignore_words = [] for lang in org_languages: ignore_words += safe_get_stop_words(lang) for category in unclean_categories: if len(category['label']) > 1 and category['label'] not in ignore_words and len(categories) < 100: categories.append(dict(label=category['label'], count=int(category['count']))) # sort by count, then alphabetically categories = sorted(categories, key=lambda c: (-c['count'], c['label'])) results.append(dict(label=unicode(_("All")), open_ended=open_ended, set=set_count, unset=unset_count, categories=categories)) # for each of our dependencies, add our key as something that depends on it pipe = r.pipeline() for dependency in dependencies: pipe.sadd(dependency, key) pipe.expire(dependency, VALUE_SUMMARY_CACHE_TIME) # and finally set our result pipe.set(key, dict_to_json(results), VALUE_SUMMARY_CACHE_TIME) pipe.execute() # leave me: nice for profiling.. #from django.db import connection as db_connection, reset_queries #print "=" * 80 #for query in db_connection.queries: # print "%s - %s" % (query['time'], query['sql'][:1000]) #print "-" * 80 #print "took: %f" % (time.time() - start) #print "=" * 80 #reset_queries() return results
def test_scheduling(self): campaign = Campaign.objects.create(name="Planting Reminders", group=self.farmers, org=self.org, created_by=self.admin, modified_by=self.admin) self.assertEquals("Planting Reminders", unicode(campaign)) # create a reminder for our first planting event planting_reminder = CampaignEvent.objects.create(campaign=campaign, relative_to=self.planting_date, offset=0, flow=self.reminder_flow, delivery_hour=17, created_by=self.admin, modified_by=self.admin) self.assertEquals("Planting Date == 0 -> Color Flow", unicode(planting_reminder)) # schedule our reminders EventFire.update_campaign_events(campaign) # we should haven't any event fires created, since neither of our farmers have a planting date self.assertEquals(0, EventFire.objects.all().count()) # ok, set a planting date on one of our contacts self.farmer1.set_field('planting_date', "05-10-2020 12:30:10") # update our campaign events EventFire.update_campaign_events(campaign) # should have one event now fire = EventFire.objects.get() self.assertEquals(5, fire.scheduled.day) self.assertEquals(10, fire.scheduled.month) self.assertEquals(2020, fire.scheduled.year) # account for timezone difference, our org is in UTC+2 self.assertEquals(17 - 2, fire.scheduled.hour) self.assertEquals(self.farmer1, fire.contact) self.assertEquals(planting_reminder, fire.event) self.assertIsNone(fire.fired) # change the date of our date self.farmer1.set_field('planting_date', "06-10-2020 12:30:10") EventFire.update_campaign_events_for_contact(campaign, self.farmer1) fire = EventFire.objects.get() self.assertEquals(6, fire.scheduled.day) self.assertEquals(10, fire.scheduled.month) self.assertEquals(2020, fire.scheduled.year) self.assertEquals(self.farmer1, fire.contact) self.assertEquals(planting_reminder, fire.event) # set it to something invalid self.farmer1.set_field('planting_date', "what?") EventFire.update_campaign_events_for_contact(campaign, self.farmer1) self.assertFalse(EventFire.objects.all()) # now something valid again self.farmer1.set_field('planting_date', "07-10-2020 12:30:10") EventFire.update_campaign_events_for_contact(campaign, self.farmer1) fire = EventFire.objects.get() self.assertEquals(7, fire.scheduled.day) self.assertEquals(10, fire.scheduled.month) self.assertEquals(2020, fire.scheduled.year) self.assertEquals(self.farmer1, fire.contact) self.assertEquals(planting_reminder, fire.event) # create another reminder planting_reminder2 = CampaignEvent.objects.create(campaign=campaign, relative_to=self.planting_date, offset=1, flow=self.reminder2_flow, created_by=self.admin, modified_by=self.admin) self.assertEquals(1, planting_reminder2.abs_offset()) # update the campaign EventFire.update_campaign_events(campaign) # should have two events now, ordered by date events = EventFire.objects.all() self.assertEquals(planting_reminder, events[0].event) self.assertEquals(7, events[0].scheduled.day) self.assertEquals(planting_reminder2, events[1].event) self.assertEquals(8, events[1].scheduled.day) # mark one of the events as inactive planting_reminder2.is_active = False planting_reminder2.save() # update the campaign EventFire.update_campaign_events(campaign) # back to only one event event = EventFire.objects.get() self.assertEquals(planting_reminder, event.event) self.assertEquals(7, event.scheduled.day) # update our date self.farmer1.set_field('planting_date', '09-10-2020 12:30') # should have updated event = EventFire.objects.get() self.assertEquals(planting_reminder, event.event) self.assertEquals(9, event.scheduled.day) # let's remove our contact field ContactField.hide_field(self.org, 'planting_date') # shouldn't have anything scheduled self.assertFalse(EventFire.objects.all()) # add it back in ContactField.get_or_create(self.org, 'planting_date', "planting Date") # should be back! event = EventFire.objects.get() self.assertEquals(planting_reminder, event.event) self.assertEquals(9, event.scheduled.day) # try firing the event event.fire() # should have one flow run now run = FlowRun.objects.get() self.assertEquals(event.contact, run.contact)
def save(self): """ Update our contact """ name = self.validated_data.get("name") fields = self.validated_data.get("fields") language = self.validated_data.get("language") # treat empty names as None if not name: name = None changed = [] if self.instance: if self.parsed_urns is not None: self.instance.update_urns(self.user, self.parsed_urns) # update our name and language if name != self.instance.name: self.instance.name = name changed.append("name") else: self.instance = Contact.get_or_create_by_urns( self.org, self.user, name, urns=self.parsed_urns, language=language, force_urn_update=True) # Contact.get_or_create doesn't nullify language so do that here if "language" in self.validated_data and language is None: self.instance.language = language.lower() if language else None changed.append("language") # save our contact if it changed if changed: self.instance.save(update_fields=changed, handle_update=True) # update our fields if fields is not None: for key, value in fields.items(): existing_by_key = ContactField.user_fields.filter( org=self.org, key__iexact=key, is_active=True).first() if existing_by_key: self.instance.set_field(self.user, existing_by_key.key, value) continue elif self.new_fields and key in self.new_fields: new_field = ContactField.get_or_create( org=self.org, user=self.user, key=regex.sub("[^A-Za-z0-9]+", "_", key).lower(), label=key) self.instance.set_field(self.user, new_field.key, value) # TODO as above, need to get users to stop updating via label existing_by_label = ContactField.get_by_label(self.org, key) if existing_by_label: self.instance.set_field(self.user, existing_by_label.key, value) # update our contact's groups if self.group_objs is not None: self.instance.update_static_groups(self.user, self.group_objs) return self.instance