def _prepare_countries(self, prepared_data):
countries = {
country_name(division)
for division in prepared_data['division_ids']
}
return list(countries)
def get_context_data(self, **kwargs):
query = self.request.GET.get('q')
results = Location.objects.filter(Q(name__icontains=query) | Q(id__startswith=query))
if self.request.GET.get('feature_type'):
feature_type = self.request.GET['feature_type']
results = results.filter(feature_type=feature_type)
context = {
'results': []
}
for result in results[:10]:
location = {
'id': result.id,
'text': '{}, {} ({} - {})'.format(result.name,
country_name(result.division_id),
result.feature_type,
result.id),
'geometry': json.loads(result.geometry.simplify(tolerance=0.01).geojson),
}
context['results'].append(location)
return context
def get_context_data(self, **kwargs):
query = self.request.GET.get('q')
results = Location.objects.filter(
Q(name__icontains=query) | Q(id__startswith=query))
if self.request.GET.get('feature_type'):
feature_type = self.request.GET['feature_type']
results = results.filter(feature_type=feature_type)
context = {'results': []}
for result in results[:10]:
location = {
'id':
result.id,
'text':
'{}, {} ({} - {})'.format(result.name,
country_name(result.division_id),
result.feature_type, result.id),
'geometry':
json.loads(result.geometry.simplify(tolerance=0.01).geojson),
}
context['results'].append(location)
return context
def organization_autocomplete(request):
term = request.GET.get('q')
response = {
'results': []
}
if term:
organizations = Organization.objects.filter(organizationname__value__icontains=term)[:10]
for organization in organizations:
result = {
'id': organization.id,
'text': organization.name.get_value().value,
'aliases': organization.alias_list,
'country': None,
}
if organization.division_id.get_value():
result['country'] = country_name(organization.division_id.get_value().value)
response['results'].append(result)
return HttpResponse(json.dumps(response), content_type='application/json')
def organization_autocomplete(request):
term = request.GET.get('q')
response = {'results': []}
if term:
organizations = Organization.objects.filter(
organizationname__value__icontains=term)[:10]
for organization in organizations:
result = {
'id': organization.id,
'text': organization.name.get_value().value,
'aliases': organization.alias_list,
'country': None,
}
if organization.division_id.get_value():
result['country'] = country_name(
organization.division_id.get_value().value)
response['results'].append(result)
return HttpResponse(json.dumps(response), content_type='application/json')
def index_organizations(self, doc_id=None, update=False):
self.stdout.write(self.style.HTTP_NOT_MODIFIED('\n Indexing organizations ... '))
if doc_id:
orgs = Organization.objects.filter(uuid=doc_id)
else:
orgs = Organization.objects.all()
documents = []
for organization in orgs:
org_id = str(organization.uuid)
# Skip this record if we're updating and it already exists
if update and self.check_index(org_id):
continue
name = organization.name.get_value()
try:
content = [name.value]
except AttributeError:
continue
aliases = organization.aliases.get_list()
if aliases:
content.extend(al.get_value().value for al in aliases)
classes = organization.classification.get_list()
class_count = len(classes)
if classes:
content.extend(cl.get_value().value for cl in classes)
hq = organization.headquarters.get_value()
if hq:
content.extend([hq.value])
first_cited = self.format_date(organization.firstciteddate.get_value())
last_cited = self.format_date(organization.lastciteddate.get_value())
open_ended = organization.open_ended.get_value()
division_ids, countries = set(), set()
# Start by getting the division ID recorded for this org
org_division_id = organization.division_id.get_value()
if org_division_id:
division_ids.update([org_division_id.value])
org_country = country_name(org_division_id)
countries.update([org_country])
country_count = len(countries)
# Grab foreign key sets
emplacements = organization.emplacementorganization_set.all()
exactloc_names, admin_names, admin_l1_names = set(), set(), set()
for emp in emplacements:
emp = emp.object_ref
site = emp.site.get_value()
if site:
exactloc_name = site.value.name
emp_division_id = site.value.division_id
exactloc_names.add(exactloc_name)
if site.value.adminlevel1:
admin_l1_names.add(site.value.adminlevel1.name)
if emp_division_id:
division_ids.update([emp_division_id])
emp_country = country_name(emp_division_id)
countries.update([emp_country])
last_site_exists = len(exactloc_names)
areas = set()
assocs = organization.associationorganization_set.all()
for assoc in assocs:
area = assoc.object_ref.area.get_value()
if area:
area_name = area.value.name
areas.update([area_name])
parent_names, parent_ids = [], []
parents = organization.parent_organization.all()
parent_count = len(parents)
published = all([p.value.published for p in parents])
if parent_count == 0:
documents.append({
'id': 'composition-{}'.format(org_id),
'composition_parent_id_s': org_id,
'composition_parent_name_s': name,
'published_b': published,
'entity_type': 'Composition',
'content': 'Composition',
})
for parent in parents:
# `parent_organization` returns a list of CompositionChilds,
# so we have to jump through some hoops to get the foreign
# key value
composition = parent.object_ref
parent = composition.parent.get_value().value
parent_name = parent.name.get_value().value
parent_names.append(parent_name)
start_date = self.format_date(composition.startdate.get_value())
end_date = self.format_date(composition.enddate.get_value())
classification = composition.classification.get_value()
open_ended = composition.open_ended.get_value()
composition_daterange = None
if start_date and end_date:
start_date = start_date.split('T')[0]
end_date = end_date.split('T')[0]
args = [start_date, end_date]
if dateparser.parse(start_date) > dateparser.parse(end_date):
args = [end_date, start_date]
composition_daterange = '[{0} TO {1}]'.format(*args)
elif start_date and not end_date and open_ended and open_ended.value.strip() == 'Y':
composition_daterange = '[{} TO *]'.format(start_date.split('T')[0])
elif start_date and not end_date and open_ended and open_ended.value.strip() == 'N':
date_string = start_date.split('T')[0]
composition_daterange = '[{} TO {}]'.format(date_string, date_string)
elif not start_date and end_date:
composition_daterange = '[* TO {}]'.format(end_date.split('T')[0])
if composition_daterange:
documents.append({
'id': 'composition-{}'.format(composition.id),
'composition_parent_id_s': str(parent.uuid),
'composition_parent_pk_i': parent.id,
'composition_parent_name_s': parent_name,
'composition_child_id_s': org_id,
'composition_child_pk_i': organization.id,
'composition_child_name_s': name,
'composition_daterange_dr': composition_daterange,
'published_b': published,
'entity_type': 'Composition',
'content': 'Composition',
})
memberships = []
mems = organization.membershiporganizationmember_set.all()
for membership in mems:
# Similar to parents, we have to traverse the directed graph
# in order to get the entities we want
org = membership.object_ref.organization.get_value().value
if org.name.get_value():
memberships.append(org.name.get_value())
# Convert sets to lists, for indexing
division_ids, countries = list(division_ids), list(countries)
exactloc_names = list(exactloc_names)
areas = list(areas)
# Add some attributes to the global index
for attr in (parent_names, countries, exactloc_names, areas):
content.extend(attr)
content = '; '.join(content)
document = {
'id': org_id,
'entity_type': 'Organization',
'content': content,
'location': '', # disabled until we implement map search
'published_b': organization.published,
'country_ss': countries,
'division_id_ss': division_ids,
'start_date_dt': first_cited,
'end_date_dt': last_cited,
'open_ended_s': open_ended,
'organization_name_s': name,
'organization_parent_name_ss': parent_names,
'organization_parent_count_i': parent_count,
'organization_membership_ss': memberships,
'organization_classification_ss': classes,
'organization_classification_count_i': class_count,
'organization_alias_ss': aliases,
'organization_headquarters_s': hq,
'organization_exact_location_ss': exactloc_names,
'organization_site_count_i': last_site_exists,
'organization_country_count_i': country_count,
'organization_area_ss': areas,
'organization_start_date_dt': first_cited,
'organization_end_date_dt': first_cited,
'organization_adminlevel1_ss': list(admin_l1_names),
'text': content
}
documents.append(document)
if update:
self.updated_count += 1
else:
self.added_count += 1
self.add_to_index(documents)
def index_violations(self, doc_id=None, update=False):
self.stdout.write(self.style.HTTP_NOT_MODIFIED('\n Indexing violations ... '))
if doc_id:
violations = Violation.objects.filter(uuid=doc_id)
else:
violations = Violation.objects.all()
documents = []
for violation in violations:
viol_id = str(violation.uuid)
if update and self.check_index(viol_id):
continue
content = []
description = violation.description.get_value()
if description:
description = description.value
vtypes = violation.types.get_list()
vtype_count = 0
if vtypes:
vtypes = list(str(vt.get_value().value) for vt in vtypes)
vtype_count = len(vtypes)
perps = violation.perpetrator.get_list()
perp_names, perp_aliases = [], []
perp_count = 0
if perps:
# Move from PerpetratorPerson -> Person
perps = list(perp.get_value().value for perp in perps)
perp_count = len(perps)
for perp in perps:
aliases = perp.aliases.get_list()
if aliases:
perp_aliases.extend(al.get_value().value
for al in aliases)
perp_names.append(perp.name.get_value().value)
perp_orgs = violation.perpetratororganization.get_list()
perp_org_names, perp_org_aliases = [], []
perp_org_count = 0
if perp_orgs:
# Move from PerpetratorOrganization -> Organization
perp_orgs = list(perp.get_value().value for perp in perp_orgs)
perp_org_count = len(perp_orgs)
for perp in perp_orgs:
org_aliases = perp.aliases.get_list()
if org_aliases:
perp_org_aliases.extend(al.get_value().value
for al in org_aliases)
perp_org_names.append(perp.name.get_value().value)
perp_org_classes = list(cls.value for cls in
violation.violationperpetratorclassification_set.all())
perp_org_class_count = len(perp_org_classes)
division_id = violation.division_id.get_value()
country = []
if division_id:
country = [country_name(division_id)]
else:
division_id = []
location_description = violation.locationdescription.get_value()
if location_description:
location_description = location_description.value
location_name = violation.location_name.get_value()
if location_name:
location_name = location_name.value.name
osmname = violation.osmname.get_value()
if osmname:
osmname = osmname.value
admin_l1_name = violation.adminlevel1.get_value()
if admin_l1_name:
admin_l1_name = admin_l1_name.value.name
admin_l2_name = violation.adminlevel2.get_value()
if admin_l2_name:
admin_l2_name = admin_l2_name.value.name
start_date = self.format_date(violation.startdate.get_value())
end_date = self.format_date(violation.enddate.get_value())
first_allegation = self.format_date(violation.first_allegation.get_value())
last_update = self.format_date(violation.last_update.get_value())
status = violation.status.get_value()
if status:
status = status.value
global_index = [perp_names, perp_aliases, perp_org_names, perp_org_aliases,
perp_org_classes, vtypes, country]
# We need to make solo attributes into lists to extend the `content`
# list; before doing that, check to see that each attribute actually
# exists
for single_attr in (description, location_description,
location_name, osmname, admin_l1_name,
admin_l2_name):
if single_attr:
global_index.append([single_attr])
for attr in global_index:
content.extend(attr)
content = '; '.join([c for c in content if c])
document = {
'id': viol_id,
'entity_type': 'Violation',
'content': content,
'location': '',
'published_b': violation.published,
'country_ss': country,
'division_id_ss': division_id,
'start_date_dt': start_date,
'end_date_dt': end_date,
'violation_type_ss': vtypes,
'violation_type_count_i': vtype_count,
'violation_description_t': description,
'violation_start_date_dt': start_date,
'violation_end_date_dt': end_date,
'violation_first_allegation_dt': first_allegation,
'violation_last_update_dt': last_update,
'violation_status_s': status,
'violation_location_description_s': location_description,
'violation_location_name_s': location_name,
'violation_osmname_s': osmname,
'violation_adminlevel1_s': admin_l1_name,
'violation_adminlevel2_s': admin_l2_name,
'perpetrator_ss': perps,
'perpetrator_count_i': perp_count,
'perpetrator_alias_ss': perp_aliases,
'perpetrator_organization_ss': perp_orgs,
'perpetrator_organization_count_i': perp_org_count,
'perpetrator_organization_alias_ss': perp_org_aliases,
'perpetrator_classification_ss': perp_org_classes,
'perpetrator_classification_count_i': perp_org_class_count,
'text': content
}
documents.append(document)
if update:
self.updated_count += 1
else:
self.added_count += 1
self.add_to_index(documents)
def index_people(self, doc_id=None, update=False):
self.stdout.write(self.style.HTTP_NOT_MODIFIED('\n Indexing people ... '))
if doc_id:
people = Person.objects.filter(uuid=doc_id)
else:
people = Person.objects.all()
documents = []
for person in people:
person_id = str(person.uuid)
if update and self.check_index(person_id):
continue
name = person.name.get_value()
content = [name.value]
aliases = person.aliases.get_list()
alias_count = 0
if aliases:
content.extend(al.get_value().value for al in aliases)
alias_count = len(aliases)
affiliations = person.memberships
memberships = [mem.object_ref for mem in affiliations]
division_ids, countries = set(), set()
# Start by getting the division ID recorded for the person
person_division_id = person.division_id.get_value()
if person_division_id:
division_ids.update([person_division_id.value])
person_country = country_name(person_division_id)
countries.update([person_country])
ranks, roles, titles = set(), set(), set()
latest_rank, latest_role, latest_title = None, None, None
first_cited = None
last_cited = None
# Get most recent information
if memberships:
latest_title = memberships[0].title.get_value()
latest_role = memberships[0].role.get_value()
most_recent_unit = memberships[0].organization.get_value()
most_recent_rank = memberships[0].rank.get_value()
latest_rank = most_recent_rank
else:
latest_title, latest_role, most_recent_unit, most_recent_rank, latest_rank = None, None, None, None, None
for membership in memberships:
org = membership.organization.get_value()
if org:
org = org.value
# Check to see if we can extend first/last cited dates
fcd = membership.firstciteddate.get_value()
if fcd:
if first_cited:
if fcd.value < first_cited.value:
first_cited = fcd
else:
first_cited = fcd
lcd = membership.lastciteddate.get_value()
if lcd:
if last_cited:
if lcd.value > last_cited.value:
last_cited = lcd
else:
last_cited = lcd
assignment_range = None
if fcd and lcd:
fcd = self.format_date(fcd).split('T')[0]
lcd = self.format_date(lcd).split('T')[0]
args = [fcd, lcd]
if dateparser.parse(fcd) > dateparser.parse(lcd):
args = [lcd, fcd]
assignment_range = '[{0} TO {1}]'.format(*args)
elif fcd:
fcd = self.format_date(fcd).split('T')[0]
assignment_range = '[{} TO *]'.format(fcd)
elif lcd:
lcd = self.format_date(lcd).split('T')[0]
assignment_range = '[* TO {}]'.format(lcd)
role = membership.role.get_value()
if role and role.value.value == 'Commander' and assignment_range:
published = all([person.published, org.published])
commander = {
'id': 'commander-{}'.format(membership.id),
'commander_person_id_s': person_id,
'commander_person_name_s': name,
'commander_org_id_s': org.uuid,
'commander_org_name_s': org.name.get_value().value,
'commander_assignment_range_dr': assignment_range,
'published_b': published,
'entity_type': 'Commander',
'content': 'Commander',
}
documents.append(commander)
elif role:
roles.add(role.value.value)
rank = membership.rank.get_value()
if rank:
ranks.add(rank.value.value)
title = membership.title.get_value()
if title:
titles.add(title.value)
# We also want to index the person based on the countries
# their member units have operated in
org_division_id = org.division_id.get_value()
if org_division_id:
division_ids.update([org_division_id.value])
org_country = country_name(org_division_id)
countries.update([org_country])
# Convert sets to lists, for indexing
division_ids, countries = list(division_ids), list(countries)
ranks, roles, titles = list(ranks), list(roles), list(titles)
# Add some attributes to the global index
for attr in (roles, ranks, titles, countries):
content.extend(attr)
content = '; '.join(content)
first_cited = self.format_date(first_cited)
last_cited = self.format_date(last_cited)
document = {
'id': person_id,
'entity_type': 'Person',
'content': content,
'location': '', # disabled until we implement map search
'published_b': person.published,
'country_ss': countries,
'division_id_ss': division_ids,
'person_title_ss': titles,
'person_name_s': name,
'person_alias_ss': aliases,
'person_alias_count_i': alias_count,
'person_role_ss': roles,
'person_rank_ss': ranks,
'person_most_recent_rank_s': most_recent_rank,
'person_most_recent_unit_s': most_recent_unit,
'person_title_ss': titles,
'person_current_rank_s': latest_rank,
'person_current_role_s': latest_role,
'person_current_title_s': latest_title,
'person_first_cited_dt': first_cited,
'person_last_cited_dt': last_cited,
'start_date_dt': first_cited,
'end_date_dt': last_cited,
'text': content
}
documents.append(document)
if update:
self.updated_count += 1
else:
self.added_count += 1
self.add_to_index(documents)
def division_choices():
division_ids = OrganizationDivisionId.objects.distinct('value').order_by('value')
return [(r.value, country_name(r.value)) for r in division_ids]
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
# Determine if the user is logged in
authenticated = self.request.user.is_authenticated
# Generate link to download a CSV of this record
params = '?download_etype=Organization&entity_id={0}'.format(
str(context['organization'].uuid))
context['download_url'] = reverse('download') + params
# Commanders of this unit
context['person_members'] = []
if authenticated:
person_members = context[
'organization'].membershippersonorganization_set.all()
else:
person_members = context[
'organization'].membershippersonorganization_set.filter(
object_ref__membershippersonmember__value__published=True)
for membership in person_members:
context['person_members'].append(membership.object_ref)
# Organizational members of this unit
context['org_members'] = []
if authenticated:
org_members = context[
'organization'].membershiporganizationorganization_set.all()
else:
org_members = context[
'organization'].membershiporganizationorganization_set.filter(
value__published=True)
if org_members:
org_members = (mem.object_ref for mem in org_members)
context['org_members'] = sorted(
org_members,
key=lambda x:
(country_name(x.member.get_value().value.division_id.get_value(
).value), x.member.get_value().value.name.get_value().value))
# Other units that this unit is a member of
context['memberships'] = []
if authenticated:
memberships = context[
'organization'].membershiporganizationmember_set.all()
else:
memberships = context[
'organization'].membershiporganizationmember_set.filter(
object_ref__membershiporganizationorganization__value__published
=True)
if memberships:
memberships = (mem.object_ref for mem in memberships)
context['memberships'] = sorted(
memberships,
key=lambda x:
(country_name(x.organization.get_value().value.
division_id.get_value().value),
x.organization.get_value().value.name.get_value().value))
# Child units
context['subsidiaries'] = []
if authenticated:
children = context['organization'].child_organization.all()
else:
children = context['organization'].child_organization.filter(
object_ref__compositionchild__value__published=True)
for child in children:
context['subsidiaries'].append(child.object_ref)
# Incidents that this unit perpetrated
context['events'] = []
if authenticated:
events = context[
'organization'].violationperpetratororganization_set.all()
else:
events = context[
'organization'].violationperpetratororganization_set.filter(
object_ref__published=True)
for event in events:
context['events'].append(event.object_ref)
context['emplacements'] = context['organization'].emplacements
site_ids = context['emplacements'].values_list(
'emplacement__emplacementsite__value')
context['sites'] = serialize('geojson',
Location.objects.filter(id__in=site_ids),
geometry_field='geometry')
context['associations'] = context['organization'].associations
area_ids = context['associations'].values_list(
'association__associationarea__value')
context['areas'] = serialize('geojson',
Location.objects.filter(id__in=area_ids),
geometry_field='geometry')
context['parents'] = []
context['parents_list'] = []
if authenticated:
parents = context['organization'].parent_organization.all()
else:
parents = context['organization'].parent_organization.filter(
object_ref__compositionparent__value__published=True)
# "parent" is a CompositionChild
for parent in parents:
context['parents'].append(
parent.object_ref.parent.get_value().value)
org_data = {'when': '', 'url': ''}
when = None
if parent.object_ref.enddate.get_value():
# Make the query using the raw date string, to accomodate
# fuzzy dates
when = repr(parent.object_ref.enddate.get_value().value)
org_data['when'] = when
# Display a formatted date
org_data['display_date'] = str(
parent.object_ref.enddate.get_value())
kwargs = {'org_id': str(context['organization'].uuid)}
ajax_route = 'command-chain'
if when:
kwargs['when'] = when
ajax_route = 'command-chain-bounded'
command_chain_url = reverse(ajax_route, kwargs=kwargs)
org_data['url'] = command_chain_url
context['parents_list'].append(org_data)
context['sources'] = self.get_sources(context)
return context
def prepare_countries(self, object):
division_id = object.division_id.get_value()
if division_id:
return [country_name(division_id)]