def test_get_current_records_and_filter_by_source_and_type_empty_data(self):
""" Verify that we raise the expected Exception if this is called before data is populated"""
sdn_source = "Specially Designated Nationals (SDN) - Treasury Department"
sdn_type = "Individual"
with self.assertRaises(SDNFallbackDataEmptyError):
SDNFallbackData.get_current_records_and_filter_by_source_and_type(sdn_source, sdn_type)
def test_get_current_records_and_filter_by_source_and_type(self):
""" Verify the query is done for current records by source and by optional sdn_type. """
sdn_metadata_current = factories.SDNFallbackMetadataFactory.create(import_state="Current")
sdn_metadata_discard = factories.SDNFallbackMetadataFactory.create(import_state="Discard")
sdn_source = "Specially Designated Nationals (SDN) - Treasury Department"
isn_source = "Nonproliferation Sanctions (ISN) - State Department"
sdn_type = "Individual"
rows = [
[sdn_source, sdn_type, sdn_metadata_current],
[sdn_source, "Entity", sdn_metadata_current],
[isn_source, "", sdn_metadata_current],
[sdn_source, sdn_type, sdn_metadata_discard],
]
for row in rows:
source, sdn_type, sdn_fallback_metadata = row
factories.SDNFallbackDataFactory.create(
sdn_fallback_metadata=sdn_fallback_metadata,
source=source,
sdn_type=sdn_type,
)
filtered_records_sdn_individual = SDNFallbackData.get_current_records_and_filter_by_source_and_type(
sdn_source, sdn_type)
self.assertEqual(len(filtered_records_sdn_individual), 1)
self.assertEqual(filtered_records_sdn_individual[0].sdn_fallback_metadata, sdn_metadata_current)
filtered_records_sdn_entity = SDNFallbackData.get_current_records_and_filter_by_source_and_type(
sdn_source, "Entity")
self.assertEqual(len(filtered_records_sdn_entity), 1)
filtered_records_isn = SDNFallbackData.get_current_records_and_filter_by_source_and_type(
isn_source, "")
self.assertEqual(len(filtered_records_isn), 1)
def checkSDNFallback(name, city, country):
"""
Performs an SDN check against the SDNFallbackData
First, filter the SDNFallbackData records by source, type and country.
Then, compare the provided name/city against each record and return whether we find a match.
The check uses the following properties:
1. Order of words doesn’t matter
2. Number of times that a given word appears doesn’t matter
3. Punctuation between words or at the beginning/end of a given word doesn’t matter
4. If a subset of words match, it still counts as a match
5. Capitalization doesn’t matter
"""
hit_count = 0
records = SDNFallbackData.get_current_records_and_filter_by_source_and_type(
'Specially Designated Nationals (SDN) - Treasury Department',
'Individual')
records = records.filter(countries__contains=country)
processed_name, processed_city = process_text(name), process_text(city)
for record in records:
record_names, record_addresses = set(record.names.split()), set(
record.addresses.split())
if (processed_name.issubset(record_names)
and processed_city.issubset(record_addresses)):
hit_count = hit_count + 1
return hit_count
def populate_sdn_fallback_data(sdn_csv_string, metadata_entry):
"""
Process CSV data and create SDNFallbackData records
Args:
sdn_csv_string (str): String of the sdn csv
metadata_entry (SDNFallbackMetadata): Instance of the current SDNFallbackMetadata class
"""
sdn_csv_reader = csv.DictReader(io.StringIO(sdn_csv_string))
processed_records = []
for row in sdn_csv_reader:
sdn_source, sdn_type, names, addresses, alt_names, ids = (
row['source'] or '', row['type'] or '', row['name']
or '', row['addresses'] or '', row['alt_names'] or '', row['ids']
or '')
processed_names = ' '.join(
process_text(' '.join(filter(None, [names, alt_names]))))
processed_addresses = ' '.join(process_text(addresses))
countries = extract_country_information(addresses, ids)
processed_records.append(
SDNFallbackData(sdn_fallback_metadata=metadata_entry,
source=sdn_source,
sdn_type=sdn_type,
names=processed_names,
addresses=processed_addresses,
countries=countries))
# Bulk create should be more efficient for a few thousand records without needing to use SQL directly.
SDNFallbackData.objects.bulk_create(processed_records)
def test_file_format(self, csv_string):
populate_sdn_fallback_data_and_metadata(csv_string)
records = SDNFallbackData.get_current_records_and_filter_by_source_and_type('Specially Designated Nationals (SDN) - Treasury Department', 'Individual')
self.assertEqual(len(records), 1)
self.assertEqual(process_text(records.first().names), process_text('Victor Conrad Wendy Brock'))
self.assertEqual(process_text(records.first().addresses), process_text('17472 Christie Stream Apt. 976 North Kristinaport, HI 91033, SN'))
self.assertEqual(records.first().countries, 'SN')
def test_fields(self):
"Verify all fields are correctly populated"
new_data = SDNFallbackData(
sdn_fallback_metadata=self.sdn_metadata,
sdn_id="123abc",
source="Specially Designated Nationals (SDN) - Treasury Department",
sdn_type="Individual",
names="maria giuseppe",
addresses="123 main street",
countries="US",
)
new_data.full_clean()
new_data.save()
self.assertEqual(len(SDNFallbackData.objects.all()), 1)
actual_data = SDNFallbackData.objects.all()[0]
self.assertEqual(actual_data.sdn_fallback_metadata, self.sdn_metadata)
self.assertEqual(actual_data.sdn_id, "123abc")
self.assertEqual(
actual_data.source,
"Specially Designated Nationals (SDN) - Treasury Department")
self.assertEqual(actual_data.sdn_type, "Individual")
self.assertEqual(actual_data.names, "maria giuseppe")
self.assertEqual(actual_data.addresses, "123 main street")
self.assertEqual(actual_data.countries, "US")
