def serialize_field_value(contact, field, value):
org = contact.org
# parse as all value data types
str_value = str(value)[:640]
dt_value = org.parse_datetime(value)
num_value = org.parse_number(value)
loc_value = None
# for locations, if it has a '>' then it is explicit, look it up that way
if AdminBoundary.PATH_SEPARATOR in str_value:
loc_value = contact.org.parse_location_path(str_value)
# otherwise, try to parse it as a name at the appropriate level
else:
if field.value_type == ContactField.TYPE_WARD:
district_field = ContactField.get_location_field(org, ContactField.TYPE_DISTRICT)
district_value = contact.get_field_value(district_field)
if district_value:
loc_value = org.parse_location(str_value, AdminBoundary.LEVEL_WARD, district_value)
elif field.value_type == ContactField.TYPE_DISTRICT:
state_field = ContactField.get_location_field(org, ContactField.TYPE_STATE)
if state_field:
state_value = contact.get_field_value(state_field)
if state_value:
loc_value = org.parse_location(str_value, AdminBoundary.LEVEL_DISTRICT, state_value)
elif field.value_type == ContactField.TYPE_STATE:
loc_value = org.parse_location(str_value, AdminBoundary.LEVEL_STATE)
if loc_value is not None and len(loc_value) > 0:
loc_value = loc_value[0]
else:
loc_value = None
# all fields have a text value
field_dict = {"text": str_value}
# set all the other fields that have a non-zero value
if dt_value is not None:
field_dict["datetime"] = timezone.localtime(dt_value, org.timezone).isoformat()
if num_value is not None:
field_dict["number"] = format_number(num_value)
if loc_value:
if loc_value.level == AdminBoundary.LEVEL_STATE:
field_dict["state"] = loc_value.path
elif loc_value.level == AdminBoundary.LEVEL_DISTRICT:
field_dict["district"] = loc_value.path
field_dict["state"] = AdminBoundary.strip_last_path(loc_value.path)
elif loc_value.level == AdminBoundary.LEVEL_WARD:
field_dict["ward"] = loc_value.path
field_dict["district"] = AdminBoundary.strip_last_path(loc_value.path)
field_dict["state"] = AdminBoundary.strip_last_path(field_dict["district"])
return field_dict
def setUpLocations(self):
"""
Installs some basic test location data for Rwanda
"""
self.country = AdminBoundary.create(osm_id="171496",
name="Rwanda",
level=0)
self.state1 = AdminBoundary.create(osm_id="1708283",
name="Kigali City",
level=1,
parent=self.country)
self.state2 = AdminBoundary.create(osm_id="171591",
name="Eastern Province",
level=1,
parent=self.country)
self.district1 = AdminBoundary.create(osm_id="R1711131",
name="Gatsibo",
level=2,
parent=self.state2)
self.district2 = AdminBoundary.create(osm_id="1711163",
name="Kayônza",
level=2,
parent=self.state2)
self.district3 = AdminBoundary.create(osm_id="3963734",
name="Nyarugenge",
level=2,
parent=self.state1)
self.district4 = AdminBoundary.create(osm_id="1711142",
name="Rwamagana",
level=2,
parent=self.state2)
self.ward1 = AdminBoundary.create(osm_id="171113181",
name="Kageyo",
level=3,
parent=self.district1)
self.ward2 = AdminBoundary.create(osm_id="171116381",
name="Kabare",
level=3,
parent=self.district2)
self.ward3 = AdminBoundary.create(osm_id="VMN.49.1_1",
name="Bukure",
level=3,
parent=self.district4)
BoundaryAlias.create(self.org, self.admin, self.state1, "Kigari")
self.country.update_path()
self.org.country = self.country
self.org.save(update_fields=("country", ))
def setUp(self):
self.maxDiff = 4096
self.mock_server = mock_server
# if we are super verbose, turn on debug for sql queries
if self.get_verbosity() > 2:
settings.DEBUG = True
# make sure we start off without any service users
Group.objects.get(name="Service Users").user_set.clear()
self.clear_cache()
self.create_anonymous_user()
self.superuser = User.objects.create_superuser(username="******",
email="*****@*****.**",
password="******")
# create different user types
self.non_org_user = self.create_user("NonOrg")
self.user = self.create_user("User", ("Viewers", ))
self.editor = self.create_user("Editor")
self.admin = self.create_user("Administrator")
self.surveyor = self.create_user("Surveyor")
self.customer_support = self.create_user("support",
("Customer Support", ))
# setup admin boundaries for Rwanda
self.country = AdminBoundary.create(osm_id="171496",
name="Rwanda",
level=0)
self.state1 = AdminBoundary.create(osm_id="1708283",
name="Kigali City",
level=1,
parent=self.country)
self.state2 = AdminBoundary.create(osm_id="171591",
name="Eastern Province",
level=1,
parent=self.country)
self.district1 = AdminBoundary.create(osm_id="R1711131",
name="Gatsibo",
level=2,
parent=self.state2)
self.district2 = AdminBoundary.create(osm_id="1711163",
name="Kayônza",
level=2,
parent=self.state2)
self.district3 = AdminBoundary.create(osm_id="3963734",
name="Nyarugenge",
level=2,
parent=self.state1)
self.district4 = AdminBoundary.create(osm_id="1711142",
name="Rwamagana",
level=2,
parent=self.state2)
self.ward1 = AdminBoundary.create(osm_id="171113181",
name="Kageyo",
level=3,
parent=self.district1)
self.ward2 = AdminBoundary.create(osm_id="171116381",
name="Kabare",
level=3,
parent=self.district2)
self.ward3 = AdminBoundary.create(osm_id="VMN.49.1_1",
name="Bukure",
level=3,
parent=self.district4)
self.country.update_path()
self.org = Org.objects.create(
name="Temba",
timezone=pytz.timezone("Africa/Kigali"),
country=self.country,
brand=settings.DEFAULT_BRAND,
created_by=self.user,
modified_by=self.user,
)
self.org.initialize(topup_size=1000)
# add users to the org
self.user.set_org(self.org)
self.org.viewers.add(self.user)
self.editor.set_org(self.org)
self.org.editors.add(self.editor)
self.admin.set_org(self.org)
self.org.administrators.add(self.admin)
self.surveyor.set_org(self.org)
self.org.surveyors.add(self.surveyor)
self.superuser.set_org(self.org)
# welcome topup with 1000 credits
self.welcome_topup = self.org.topups.all()[0]
# a single Android channel
self.channel = Channel.create(
self.org,
self.user,
"RW",
"A",
name="Test Channel",
address="+250785551212",
device="Nexus 5X",
secret="12345",
config={Channel.CONFIG_FCM_ID: "123"},
)
# don't cache anon user between tests
from temba import utils
utils._anon_user = None
clear_flow_users()
def import_file(self, filename, file):
admin_json = geojson.loads(file.read())
# we keep track of all the osm ids we've seen because we remove all admin levels at this level
# which weren't seen. (they have been removed)
seen_osm_ids = set()
osm_id = None
# parse our filename.. they are in the format:
# 192787admin2_simplified.json
match = regex.match(r"(\w+\d+)admin(\d)(_simplified)?\.json$",
filename, regex.V0)
level = None
is_simplified = None
if match:
level = int(match.group(2))
is_simplified = True if match.group(3) else False
else:
# else parse other filenames that are in
# admin_level_0_simplified.json format.
match = regex.match(r"admin_level_(\d)(_simplified)?\.json$",
filename, regex.V0)
if match:
level = int(match.group(1))
is_simplified = True if match.group(2) else False
else:
self.stdout.write(
self.style.WARNING(
f"Skipping '{filename}', doesn't match file pattern."))
return None, set()
# for each of our features
for feature in admin_json["features"]:
# what level are we?
props = feature.properties
# get parent id which is set in new file format
parent_osm_id = str(props.get("parent_id"))
# if parent_osm_id is not set and not LEVEL_COUNTRY check for old file format
if parent_osm_id == "None" and level != AdminBoundary.LEVEL_COUNTRY:
if level == AdminBoundary.LEVEL_STATE:
parent_osm_id = str(props["is_in_country"])
elif level == AdminBoundary.LEVEL_DISTRICT:
parent_osm_id = str(props["is_in_state"])
osm_id = str(props["osm_id"])
name = props.get("name", "")
if not name or name == "None" or level == AdminBoundary.LEVEL_COUNTRY:
name = props.get("name_en", "")
# try to find parent, bail if we can't
parent = None
if parent_osm_id != "None":
parent = AdminBoundary.objects.filter(
osm_id=parent_osm_id).first()
if not parent:
self.stdout.write(
self.style.SUCCESS(
f"Skipping {name} ({osm_id}) as parent {parent_osm_id} not found."
))
continue
# try to find existing admin level by osm_id
boundary = AdminBoundary.objects.filter(osm_id=osm_id)
# didn't find it? what about by name?
if not boundary:
boundary = AdminBoundary.objects.filter(parent=parent,
name__iexact=name)
# skip over items with no geometry
if not feature["geometry"] or not feature["geometry"][
"coordinates"]:
continue # pragma: can't cover
polygons = []
if feature["geometry"]["type"] == "Polygon":
polygons.append(Polygon(*feature["geometry"]["coordinates"]))
elif feature["geometry"]["type"] == "MultiPolygon":
for polygon in feature["geometry"]["coordinates"]:
polygons.append(Polygon(*polygon))
else:
raise Exception(
"Error importing %s, unknown geometry type '%s'" %
(name, feature["geometry"]["type"]))
geometry = MultiPolygon(polygons)
kwargs = dict(osm_id=osm_id, name=name, level=level, parent=parent)
if is_simplified:
kwargs["simplified_geometry"] = geometry
# if this is an update, just update with those fields
if boundary:
if not parent:
kwargs["path"] = name
else:
kwargs[
"path"] = parent.path + AdminBoundary.PADDED_PATH_SEPARATOR + name
self.stdout.write(
self.style.SUCCESS(f" ** updating {name} ({osm_id})"))
boundary = boundary.first()
boundary.update(**kwargs)
# update any children
boundary.update_path()
# otherwise, this is new, so create it
else:
self.stdout.write(
self.style.SUCCESS(f" ** adding {name} ({osm_id})"))
AdminBoundary.create(**kwargs)
# keep track of this osm_id
seen_osm_ids.add(osm_id)
# now remove any unseen boundaries
if osm_id:
last_boundary = AdminBoundary.objects.filter(osm_id=osm_id).first()
if last_boundary:
self.stdout.write(
self.style.SUCCESS(
f" ** removing unseen boundaries ({osm_id})"))
country = last_boundary.get_root()
unseen_boundaries = country.get_descendants().filter(
level=level).exclude(osm_id__in=seen_osm_ids)
deleted_count = 0
for unseen_boundary in unseen_boundaries:
unseen_boundary.release()
deleted_count += 1
if deleted_count > 0:
self.stdout.write(
f" ** Unseen boundaries removed: {deleted_count}")
return country, seen_osm_ids
else:
return None, set()
else:
return None, set()
def setUp(self):
self.maxDiff = 4096
self.mock_server = mock_server
# if we are super verbose, turn on debug for sql queries
if self.get_verbosity() > 2:
settings.DEBUG = True
# make sure we start off without any service users
Group.objects.get(name="Service Users").user_set.clear()
self.clear_cache()
self.create_anonymous_user()
self.superuser = User.objects.create_superuser(username="******", email="*****@*****.**", password="******")
# create different user types
self.non_org_user = self.create_user("NonOrg")
self.user = self.create_user("User")
self.editor = self.create_user("Editor")
self.admin = self.create_user("Administrator")
self.surveyor = self.create_user("Surveyor")
self.customer_support = self.create_user("support", ("Customer Support",))
# setup admin boundaries for Rwanda
self.country = AdminBoundary.create(osm_id="171496", name="Rwanda", level=0)
self.state1 = AdminBoundary.create(osm_id="1708283", name="Kigali City", level=1, parent=self.country)
self.state2 = AdminBoundary.create(osm_id="171591", name="Eastern Province", level=1, parent=self.country)
self.district1 = AdminBoundary.create(osm_id="R1711131", name="Gatsibo", level=2, parent=self.state2)
self.district2 = AdminBoundary.create(osm_id="1711163", name="Kayônza", level=2, parent=self.state2)
self.district3 = AdminBoundary.create(osm_id="3963734", name="Nyarugenge", level=2, parent=self.state1)
self.district4 = AdminBoundary.create(osm_id="1711142", name="Rwamagana", level=2, parent=self.state2)
self.ward1 = AdminBoundary.create(osm_id="171113181", name="Kageyo", level=3, parent=self.district1)
self.ward2 = AdminBoundary.create(osm_id="171116381", name="Kabare", level=3, parent=self.district2)
self.ward3 = AdminBoundary.create(osm_id="VMN.49.1_1", name="Bukure", level=3, parent=self.district4)
self.country.update_path()
self.org = Org.objects.create(
name="Temba",
timezone=pytz.timezone("Africa/Kigali"),
country=self.country,
brand=settings.DEFAULT_BRAND,
created_by=self.user,
modified_by=self.user,
)
self.org.initialize(topup_size=1000)
# add users to the org
self.user.set_org(self.org)
self.org.viewers.add(self.user)
self.editor.set_org(self.org)
self.org.editors.add(self.editor)
self.admin.set_org(self.org)
self.org.administrators.add(self.admin)
self.surveyor.set_org(self.org)
self.org.surveyors.add(self.surveyor)
self.superuser.set_org(self.org)
# welcome topup with 1000 credits
self.welcome_topup = self.org.topups.all()[0]
# a single Android channel
self.channel = Channel.create(
self.org,
self.user,
"RW",
"A",
name="Test Channel",
address="+250785551212",
device="Nexus 5X",
secret="12345",
gcm_id="123",
)
# don't cache anon user between tests
from temba import utils
utils._anon_user = None
clear_flow_users()
# reset our simulation to False
Contact.set_simulation(False)
def import_file(self, filename, file):
admin_json = geojson.loads(file.read())
# we keep track of all the osm ids we've seen because we remove all admin levels at this level
# which weren't seen. (they have been removed)
seen_osm_ids = []
osm_id = None
# parse our filename.. they are in the format:
# 192787admin2_simplified.json
match = regex.match(
r'(\w\d+)admin(\d)(_simplified)?\.json$', filename, regex.V0)
level = None
is_simplified = None
if match:
level = int(match.group(2))
is_simplified = True if match.group(3) else False
else:
# else parse other filenames that are in
# admin_level_0_simplified.json format.
match = regex.match(
r'admin_level_(\d)(_simplified)?\.json$', filename, regex.V0)
if match:
level = int(match.group(1))
is_simplified = True if match.group(2) else False
elif not match:
print("Skipping '%s', doesn't match file pattern." % filename)
return
# for each of our features
for feature in admin_json['features']:
# what level are we?
props = feature.properties
# get parent id which is set in new file format
parent_osm_id = props.get('parent_id')
# if parent_osm_id is not set and not LEVEL_COUNTRY check for old file format
if not parent_osm_id and level != AdminBoundary.LEVEL_COUNTRY:
if level == AdminBoundary.LEVEL_STATE:
parent_osm_id = props['is_in_country']
elif level == AdminBoundary.LEVEL_DISTRICT:
parent_osm_id = props['is_in_state']
osm_id = props['osm_id']
name = props.get('name', '')
if not name or name == 'None' or level == AdminBoundary.LEVEL_COUNTRY:
name = props.get('name_en', '')
# try to find parent, bail if we can't
parent = None
if parent_osm_id and parent_osm_id != 'None':
parent = AdminBoundary.objects.filter(osm_id=parent_osm_id).first()
if not parent:
print("Skipping %s (%s) as parent %s not found." %
(name, osm_id, parent_osm_id))
continue
# try to find existing admin level by osm_id
boundary = AdminBoundary.objects.filter(osm_id=osm_id)
# didn't find it? what about by name?
if not boundary:
boundary = AdminBoundary.objects.filter(parent=parent, name__iexact=name)
# skip over items with no geometry
if not feature['geometry'] or not feature['geometry']['coordinates']:
continue
polygons = []
if feature['geometry']['type'] == 'Polygon':
polygons.append(Polygon(*feature['geometry']['coordinates']))
elif feature['geometry']['type'] == 'MultiPolygon':
for polygon in feature['geometry']['coordinates']:
polygons.append(Polygon(*polygon))
else:
raise Exception("Error importing %s, unknown geometry type '%s'" % (
name, feature['geometry']['type']))
geometry = MultiPolygon(polygons)
kwargs = dict(osm_id=osm_id, name=name, level=level, parent=parent)
if is_simplified:
kwargs['simplified_geometry'] = geometry
else:
kwargs['geometry'] = geometry
# if this is an update, just update with those fields
if boundary:
if not parent:
kwargs['path'] = name
else:
kwargs['path'] = parent.path + AdminBoundary.PADDED_PATH_SEPARATOR + name
print(" ** updating %s (%s)" % (name, osm_id))
boundary = boundary.first()
boundary.update(**kwargs)
# update any children
boundary.update_path()
# otherwise, this is new, so create it
else:
print(" ** adding %s (%s)" % (name, osm_id))
AdminBoundary.create(**kwargs)
# keep track of this osm_id
seen_osm_ids.append(osm_id)
# now remove any unseen boundaries
if osm_id:
last_boundary = AdminBoundary.objects.filter(osm_id=osm_id).first()
if last_boundary:
print(" ** removing unseen boundaries (%s)" % (osm_id))
country = last_boundary.get_root()
country.get_descendants().filter(level=level).exclude(osm_id__in=seen_osm_ids).delete()
return country
def import_file(self, filename, file):
admin_json = geojson.loads(file.read())
# we keep track of all the osm ids we've seen because we remove all admin levels at this level
# which weren't seen. (they have been removed)
seen_osm_ids = set()
osm_id = None
# parse our filename.. they are in the format:
# 192787admin2_simplified.json
match = regex.match(r"(\w+\d+)admin(\d)(_simplified)?\.json$", filename, regex.V0)
level = None
is_simplified = None
if match:
level = int(match.group(2))
is_simplified = True if match.group(3) else False
else:
# else parse other filenames that are in
# admin_level_0_simplified.json format.
match = regex.match(r"admin_level_(\d)(_simplified)?\.json$", filename, regex.V0)
if match:
level = int(match.group(1))
is_simplified = True if match.group(2) else False
else:
self.stdout.write(self.style.WARNING(f"Skipping '{filename}', doesn't match file pattern."))
return None, set()
# for each of our features
for feature in admin_json["features"]:
# what level are we?
props = feature.properties
# get parent id which is set in new file format
parent_osm_id = props.get("parent_id")
# if parent_osm_id is not set and not LEVEL_COUNTRY check for old file format
if not parent_osm_id and level != AdminBoundary.LEVEL_COUNTRY:
if level == AdminBoundary.LEVEL_STATE:
parent_osm_id = props["is_in_country"]
elif level == AdminBoundary.LEVEL_DISTRICT:
parent_osm_id = props["is_in_state"]
osm_id = props["osm_id"]
name = props.get("name", "")
if not name or name == "None" or level == AdminBoundary.LEVEL_COUNTRY:
name = props.get("name_en", "")
# try to find parent, bail if we can't
parent = None
if parent_osm_id and parent_osm_id != "None":
parent = AdminBoundary.objects.filter(osm_id=parent_osm_id).first()
if not parent:
self.stdout.write(
self.style.SUCCESS(f"Skipping {name} ({osm_id}) as parent {parent_osm_id} not found.")
)
continue
# try to find existing admin level by osm_id
boundary = AdminBoundary.objects.filter(osm_id=osm_id)
# didn't find it? what about by name?
if not boundary:
boundary = AdminBoundary.objects.filter(parent=parent, name__iexact=name)
# skip over items with no geometry
if not feature["geometry"] or not feature["geometry"]["coordinates"]:
continue # pragma: can't cover
polygons = []
if feature["geometry"]["type"] == "Polygon":
polygons.append(Polygon(*feature["geometry"]["coordinates"]))
elif feature["geometry"]["type"] == "MultiPolygon":
for polygon in feature["geometry"]["coordinates"]:
polygons.append(Polygon(*polygon))
else:
raise Exception("Error importing %s, unknown geometry type '%s'" % (name, feature["geometry"]["type"]))
geometry = MultiPolygon(polygons)
kwargs = dict(osm_id=osm_id, name=name, level=level, parent=parent)
if is_simplified:
kwargs["simplified_geometry"] = geometry
else:
kwargs["geometry"] = geometry
# if this is an update, just update with those fields
if boundary:
if not parent:
kwargs["path"] = name
else:
kwargs["path"] = parent.path + AdminBoundary.PADDED_PATH_SEPARATOR + name
self.stdout.write(self.style.SUCCESS(f" ** updating {name} ({osm_id})"))
boundary = boundary.first()
boundary.update(**kwargs)
# update any children
boundary.update_path()
# otherwise, this is new, so create it
else:
self.stdout.write(self.style.SUCCESS(f" ** adding {name} ({osm_id})"))
AdminBoundary.create(**kwargs)
# keep track of this osm_id
seen_osm_ids.add(osm_id)
# now remove any unseen boundaries
if osm_id:
last_boundary = AdminBoundary.objects.filter(osm_id=osm_id).first()
if last_boundary:
self.stdout.write(self.style.SUCCESS(f" ** removing unseen boundaries ({osm_id})"))
country = last_boundary.get_root()
unseen_boundaries = country.get_descendants().filter(level=level).exclude(osm_id__in=seen_osm_ids)
deleted_count = 0
for unseen_boundary in unseen_boundaries:
unseen_boundary.release()
deleted_count += 1
if deleted_count > 0:
self.stdout.write(f" ** Unseen boundaries removed: {deleted_count}")
return country, seen_osm_ids
else:
return None, set()
else:
return None, set()