def test_save_csv_files_datawarehouse(self, mocked_get_storage_backend):
# Mocking the storage backend to local file system with tmp directory.
# In this test case we don't want to make usage of the remote Object
# Store.
mocked_get_storage_backend.return_value = FileSystemStorage(
location=self.file_backend_tmp_dir)
# Creating a few objects in the database.
for i in range(3):
SignalFactory.create()
datawarehouse.save_csv_files_datawarehouse()
# Checking if we have files on the correct locations and that they
# do have some content.
signals_csv = path.join(self.file_backend_tmp_dir, 'signals.csv')
locations_csv = path.join(self.file_backend_tmp_dir, 'locations.csv')
reporters_csv = path.join(self.file_backend_tmp_dir, 'reporters.csv')
categories_csv = path.join(self.file_backend_tmp_dir, 'categories.csv')
statuses_csv = path.join(self.file_backend_tmp_dir, 'statuses.csv')
sla_csv = path.join(self.file_backend_tmp_dir, 'sla.csv')
self.assertTrue(path.exists(signals_csv))
self.assertTrue(path.getsize(signals_csv))
self.assertTrue(path.exists(locations_csv))
self.assertTrue(path.getsize(locations_csv))
self.assertTrue(path.exists(reporters_csv))
self.assertTrue(path.getsize(reporters_csv))
self.assertTrue(path.exists(categories_csv))
self.assertTrue(path.getsize(categories_csv))
self.assertTrue(path.exists(statuses_csv))
self.assertTrue(path.getsize(statuses_csv))
self.assertTrue(path.getsize(sla_csv))
def setUp(self):
self.signal = SignalFactory.create(reporter__email='*****@*****.**')
self.signal_no_email = SignalFactory.create(reporter__email='')
self.parent_signal = SignalFactory.create(
reporter__email='*****@*****.**')
self.child_signal = SignalFactory.create(reporter__email='*****@*****.**',
parent=self.parent_signal)
def setUp(self):
self.parent_signal = SignalFactory.create(
status__state=workflow.GESPLITST, parent=None, reporter__email='*****@*****.**')
self.child_signal_1 = SignalFactory.create(
status__state=workflow.GEMELD, parent=self.parent_signal)
self.child_signal_2 = SignalFactory.create(
status__state=workflow.GEMELD, parent=self.parent_signal)
def test_create_initial_child_signals_parent_is_child_not_allowed(
self, validate_address):
parent_signal = SignalFactory.create()
child_signal = SignalFactory.create(parent=parent_signal)
signal_count = Signal.objects.count()
parent_signal_count = Signal.objects.filter(
parent_id__isnull=True).count()
child_signal_count = Signal.objects.filter(
parent_id__isnull=False).count()
self.assertEqual(signal_count, 2)
self.assertEqual(parent_signal_count, 1)
self.assertEqual(child_signal_count, 1)
initial_data = [
copy.deepcopy(self.initial_data_base),
copy.deepcopy(self.initial_data_base),
]
initial_data[0]['parent'] = child_signal.pk
initial_data[1]['parent'] = child_signal.pk
response = self.client.post(self.list_endpoint,
initial_data,
format='json')
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(Signal.objects.count(), 2)
self.assertEqual(
Signal.objects.filter(parent_id__isnull=True).count(), 1)
self.assertEqual(
Signal.objects.filter(parent_id__isnull=False).count(), 1)
def test_create_initial_child_signals_max_exceeded(self, validate_address):
parent_signal = SignalFactory.create()
SignalFactory.create(parent=parent_signal)
SignalFactory.create(parent=parent_signal)
signal_count = Signal.objects.count()
parent_signal_count = Signal.objects.filter(
parent_id__isnull=True).count()
child_signal_count = Signal.objects.filter(
parent_id__isnull=False).count()
self.assertEqual(signal_count, 3)
self.assertEqual(parent_signal_count, 1)
self.assertEqual(child_signal_count, 2)
initial_data = []
for _ in range(2):
data = copy.deepcopy(self.initial_data_base)
data['parent'] = parent_signal.pk
initial_data.append(data)
response = self.client.post(self.list_endpoint,
initial_data,
format='json')
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(Signal.objects.count(), 3)
self.assertEqual(
Signal.objects.filter(parent_id__isnull=True).count(), 1)
self.assertEqual(
Signal.objects.filter(parent_id__isnull=False).count(), 2)
def setUp(self):
self.priority_low = SignalFactory.create(
priority__priority=Priority.PRIORITY_LOW)
self.priority_high = SignalFactory.create(
priority__priority=Priority.PRIORITY_HIGH)
self.priority_normal = SignalFactory.create(
priority__priority=Priority.PRIORITY_NORMAL)
def test_anonymize_reporters_less_than_x_days_ago(self):
allowed_states = [
AFGEHANDELD, GEANNULEERD, GESPLITST, VERZOEK_TOT_AFHANDELING
]
with freeze_time(timezone.now() - timezone.timedelta(days=1)):
for allowed_state in allowed_states:
SignalFactory.create(status__state=allowed_state)
self.assertEqual(Reporter.objects.count(), 4)
anonymize_reporters(days=2)
self.assertEqual(Reporter.objects.count(), 4)
self.assertEqual(
Reporter.objects.filter(
Q(email_anonymized=True) | Q(phone_anonymized=True)).count(),
0)
self.assertEqual(
Reporter.objects.filter(email_anonymized=False,
phone_anonymized=False).count(), 4)
for reporter in Reporter.objects.all():
self.assertIsNotNone(reporter.email)
self.assertIsNotNone(reporter.phone)
self.assertFalse(reporter.email_anonymized)
self.assertFalse(reporter.phone_anonymized)
def setUp(self):
self.signal_anonymous = SignalFactory.create(reporter__phone='', reporter__email='')
self.signal_has_email = SignalFactory.create(
reporter__phone='', reporter__email='*****@*****.**')
self.signal_has_phone = SignalFactory.create(
reporter__phone='0123456789', reporter__email='')
self.signal_has_both = SignalFactory.create(
reporter__phone='0123456789', reporter__email='*****@*****.**')
def test_status_data_success(self):
# We need two Signal objects.
SignalFactory.create()
SignalFactory.create()
response = self.client.get('/status/data')
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content, b'Data OK 2 Signal')
def setUp(self):
# Forcing authentication
self.client.force_authenticate(user=self.superuser)
SignalFactory.create(id=1)
SignalFactory.create(id=2)
SignalFactory.create(id=3, priority__priority=Priority.PRIORITY_HIGH)
SignalFactory.create(id=4)
SignalFactory.create(id=5, priority__priority=Priority.PRIORITY_HIGH)
def setUp(self):
self.main_cat_1 = ParentCategoryFactory.create()
self.main_cat_2 = ParentCategoryFactory.create()
self.s1 = SignalFactory.create(category_assignment__category__parent=self.main_cat_1)
self.s2 = SignalFactory.create(category_assignment__category__parent=self.main_cat_2)
# We are testing the authenticated part of the API, hence:
self.client.force_authenticate(user=self.superuser)
def setUp(self):
def get_email():
return f'{uuid.uuid4()}@example.com'
self.signal = SignalFactory.create(reporter__email=get_email())
self.signal_no_email = SignalFactory.create(reporter__email='')
self.parent_signal = SignalFactory.create(reporter__email=get_email())
self.child_signal = SignalFactory.create(reporter__email=get_email(),
parent=self.parent_signal)
def setUp(self):
self.signal = SignalFactory.create(
category_assignment__category__parent__name=
'Overlast Bedrijven en Horeca',
category_assignment__category__name='Geluidsoverlast muziek')
self.signal_other_category = SignalFactory.create(
category_assignment__category__parent__name=
'Some other main category',
category_assignment__category__name='Some other category')
def test_filter_kind_child_signal(self):
"""
Filter Signals that are a child Signal (3 child Signals are created so we expect 3)
"""
parent_one = SignalFactory.create()
SignalFactory.create_batch(1, parent=parent_one)
parent_two = SignalFactory.create()
SignalFactory.create_batch(2, parent=parent_two)
params = {'kind': 'child_signal'}
result_ids = self._request_filter_signals(params)
self.assertEqual(3, len(result_ids))
def test_filter_maincategory_and_category_slugs_not_a_unique_slug(self):
parent_category = ParentCategoryFactory.create(name='not_unique')
category = CategoryFactory.create(name='not_unique', parent=parent_category)
SignalFactory.create(category_assignment__category=category)
params = {
'maincategory_slug': [parent_category.slug, ],
'category_slug': [category.slug, ]
}
result_ids = self._request_filter_signals(params)
self.assertEqual(1, len(result_ids))
def test_filter_kind_exclude_parent_signal(self):
"""
We expect only "normal" and "child" Signals
"""
parent_one = SignalFactory.create()
SignalFactory.create_batch(1, parent=parent_one)
parent_two = SignalFactory.create()
SignalFactory.create_batch(2, parent=parent_two)
params = {'kind': 'exclude_parent_signal'}
result_ids = self._request_filter_signals(params)
self.assertEqual(23, len(result_ids))
def setUp(self):
self.signal = SignalFactory.create()
self.user = UserFactory.create()
self.location = self.signal.location
self.location.created_by = self.user.username
self.location.save()
def setUp(self):
self.signal = SignalFactory.create()
self.user = UserFactory.create()
self.category_assignment = self.signal.category_assignment
self.category_assignment.created_by = self.user.username
self.category_assignment.save()
def test_create_initial_child_signals(self, validate_address):
parent_signal = SignalFactory.create()
signal_count = Signal.objects.count()
parent_signal_count = Signal.objects.filter(
parent_id__isnull=True).count()
child_signal_count = Signal.objects.filter(
parent_id__isnull=False).count()
self.assertEqual(signal_count, 1)
self.assertEqual(parent_signal_count, 1)
self.assertEqual(child_signal_count, 0)
initial_data = []
for _ in range(2):
data = copy.deepcopy(self.initial_data_base)
data['parent'] = parent_signal.pk
initial_data.append(data)
response = self.client.post(self.list_endpoint,
initial_data,
format='json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
self.assertEqual(Signal.objects.count(),
signal_count + len(initial_data))
self.assertEqual(
Signal.objects.filter(parent_id__isnull=True).count(),
parent_signal_count)
self.assertEqual(
Signal.objects.filter(parent_id__isnull=False).count(),
len(initial_data))
def test_create_child_signal_transform_source(self, validate_address):
parent_signal = SignalFactory.create()
signal_count = Signal.objects.count()
source, *_ = SourceFactory.create_batch(4)
SourceFactory.create(
name=settings.API_TRANSFORM_SOURCE_OF_CHILD_SIGNAL_TO)
initial_data = copy.deepcopy(self.initial_data_base)
initial_data['source'] = source.name
initial_data['parent'] = parent_signal.pk
with self.settings(FEATURE_FLAGS={
'API_TRANSFORM_SOURCE_IF_A_SIGNAL_IS_A_CHILD': True
}):
response = self.client.post(self.list_endpoint,
initial_data,
format='json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
self.assertEqual(Signal.objects.count(), signal_count + 1)
response_data = response.json()
self.assertNotEqual(response_data['source'], source.name)
self.assertEqual(response_data['source'],
settings.API_TRANSFORM_SOURCE_OF_CHILD_SIGNAL_TO)
signal = Signal.objects.get(pk=response_data['id'])
self.assertNotEqual(signal.source, source.name)
self.assertEqual(signal.source,
settings.API_TRANSFORM_SOURCE_OF_CHILD_SIGNAL_TO)
def test_create_locations_csv(self):
signal = SignalFactory.create()
location = signal.location
csv_file = datawarehouse.create_locations_csv(self.csv_tmp_dir)
self.assertEqual(path.join(self.csv_tmp_dir, 'locations.csv'),
csv_file)
with open(csv_file) as opened_csv_file:
reader = csv.DictReader(opened_csv_file)
for row in reader:
self.assertEqual(row['id'], str(location.id))
self.assertEqual(row['_signal_id'], str(location._signal_id))
self.assertEqual(row['lat'], str(location.geometrie.x))
self.assertEqual(row['lng'], str(location.geometrie.y))
self.assertEqual(row['stadsdeel'],
str(location.get_stadsdeel_display()))
self.assertEqual(row['buurt_code'], str(location.buurt_code))
self.assertDictEqual(json.loads(row['address']),
location.address)
self.assertEqual(row['address_text'],
str(location.address_text))
# noqa Disabled because the Postgres format is slightly different than the Python format, so we decided to comment these checks for now
# self.assertEqual(row['created_at'], str(location.created_at))
# self.assertEqual(row['updated_at'], str(location.updated_at))
self.assertEqual(row['extra_properties'], 'null')
def test_create_category_assignments_csv(self):
signal = SignalFactory.create()
category_assignment = signal.category_assignment
csv_file = datawarehouse._create_category_assignments_csv(
self.csv_tmp_dir)
self.assertEqual(path.join(self.csv_tmp_dir, 'categories.csv'),
csv_file)
with open(csv_file) as opened_csv_file:
reader = csv.DictReader(opened_csv_file)
for row in reader:
self.assertEqual(row['id'], str(category_assignment.id))
self.assertEqual(row['_signal_id'],
str(category_assignment._signal_id))
self.assertEqual(row['main'],
str(category_assignment.category.parent.name))
self.assertEqual(row['sub'],
str(category_assignment.category.name))
self.assertEqual(
row['departments'], ', '.join(
category_assignment.category.departments.values_list(
'name', flat=True)))
self.assertEqual(row['created_at'],
str(category_assignment.created_at))
self.assertEqual(row['updated_at'],
str(category_assignment.updated_at))
self.assertEqual(json.loads(row['extra_properties']), None)
def test_create_locations_csv(self):
signal = SignalFactory.create()
location = signal.location
csv_file = datawarehouse._create_locations_csv(self.csv_tmp_dir)
self.assertEqual(path.join(self.csv_tmp_dir, 'locations.csv'),
csv_file)
with open(csv_file) as opened_csv_file:
reader = csv.DictReader(opened_csv_file)
for row in reader:
self.assertEqual(row['id'], str(location.id))
self.assertEqual(row['_signal_id'], str(location._signal_id))
self.assertEqual(row['lat'], str(location.geometrie.x))
self.assertEqual(row['lng'], str(location.geometrie.y))
self.assertEqual(row['stadsdeel'],
str(location.get_stadsdeel_display()))
self.assertEqual(row['buurt_code'], str(location.buurt_code))
self.assertDictEqual(json.loads(row['address']),
location.address)
self.assertEqual(row['address_text'],
str(location.address_text))
self.assertEqual(row['created_at'], str(location.created_at))
self.assertEqual(row['updated_at'], str(location.updated_at))
self.assertEqual(json.loads(row['extra_properties']), None)
def test_create_category_assignments_csv(self):
signal = SignalFactory.create()
category_assignment = signal.category_assignment
csv_file = datawarehouse.create_category_assignments_csv(
self.csv_tmp_dir)
self.assertEqual(path.join(self.csv_tmp_dir, 'categories.csv'),
csv_file)
with open(csv_file) as opened_csv_file:
reader = csv.DictReader(opened_csv_file)
for row in reader:
self.assertEqual(row['id'], str(category_assignment.id))
self.assertEqual(row['_signal_id'],
str(category_assignment._signal_id))
self.assertEqual(row['main'],
str(category_assignment.category.parent.name))
self.assertEqual(row['sub'],
str(category_assignment.category.name))
self.assertEqual(
row['departments'], ', '.join(
category_assignment.category.departments.values_list(
'name', flat=True)))
# noqa Disabled because the Postgres format is slightly different than the Python format, so we decided to comment these checks for now
# self.assertEqual(row['created_at'], str(category_assignment.created_at))
# self.assertEqual(row['updated_at'], str(category_assignment.updated_at))
self.assertEqual(row['extra_properties'], 'null')
def test_cannot_access_attachments(self):
# SIA must not publicly expose attachments
signal = SignalFactory.create()
uuid = signal.signal_id
response = self.client.get(self.attachment_endpoint.format(uuid=uuid))
self.assertEqual(response.status_code, 405)
def test_create_signals_csv(self):
signal = SignalFactory.create()
csv_file = datawarehouse._create_signals_csv(self.csv_tmp_dir)
self.assertEqual(path.join(self.csv_tmp_dir, 'signals.csv'), csv_file)
with open(csv_file) as opened_csv_file:
reader = csv.DictReader(opened_csv_file)
for row in reader:
self.assertEqual(row['id'], str(signal.id))
self.assertEqual(row['signal_uuid'], str(signal.signal_id))
self.assertEqual(row['source'], str(signal.source))
self.assertEqual(row['text'], str(signal.text))
self.assertEqual(row['text_extra'], str(signal.text_extra))
self.assertEqual(row['location_id'], str(signal.location_id))
self.assertEqual(row['status_id'], str(signal.status_id))
self.assertEqual(row['category_assignment_id'],
str(signal.category_assignment_id))
self.assertEqual(row['reporter_id'], str(signal.reporter_id))
self.assertEqual(row['incident_date_start'],
str(signal.incident_date_start))
self.assertEqual(row['incident_date_end'],
str(signal.incident_date_end))
self.assertEqual(row['created_at'], str(signal.created_at))
self.assertEqual(row['updated_at'], str(signal.updated_at))
self.assertEqual(row['operational_date'], '')
self.assertEqual(row['expire_date'], '')
self.assertEqual(row['image'], str(signal.image))
self.assertEqual(row['upload'], '')
self.assertDictEqual(json.loads(row['extra_properties']),
signal.extra_properties)
def test_create_default_notification_message_integration_test(self):
signal = SignalFactory.create()
message = create_default_notification_message(signal)
self.assertIn(str(signal.id), message)
self.assertIn(signal.text, message)
def test_create_statuses_csv(self):
signal = SignalFactory.create()
status = signal.status
csv_file = datawarehouse.create_statuses_csv(self.csv_tmp_dir)
self.assertEqual(path.join(self.csv_tmp_dir, 'statuses.csv'), csv_file)
with open(csv_file) as opened_csv_file:
reader = csv.DictReader(opened_csv_file)
for row in reader:
self.assertEqual(row['id'], str(status.id))
self.assertEqual(row['_signal_id'], str(status._signal_id))
self.assertEqual(row['text'], str(status.text))
self.assertEqual(row['user'], str(status.user))
self.assertEqual(row['target_api'], '')
self.assertEqual(row['state_display'],
status.get_state_display())
self.assertEqual(row['extern'], str(status.extern))
# noqa Disabled because the Postgres format is slightly different than the Python format, so we decided to comment these checks for now
# self.assertEqual(row['created_at'], str(status.created_at))
# self.assertEqual(row['updated_at'], str(status.updated_at))
self.assertEqual(row['extra_properties'], 'null')
self.assertEqual(row['state'], status.state)
def _create_signals(cls):
"""
Creates SIGNALS_DISTRIBUTE_DAYS_CNT signals. One per day backwards starting from yesterday
current time.
Creates SIGNALS_DISTRIBUTE_HOURS_CNT signals every hour backwards from now
If today is Friday 16:00, and SIGNALS_DISTRIBUTE_DAYS_CNT and SIGNALS_DISTRIBUTE_HOURS_CNT
are both set to 3, we will generate the following signals:
- Per day: Thursday (yesterday) 16:00, Wednesday (day before yesterday) 16:00, Tuesday 16:00
- Per hour: Friday 16:00 (now), Friday 15:00 (now minus 1 hour), Friday 14:00 (now minus 2h)
"""
now = datetime.utcnow()
# Generate created/updated times for new signals.
times = [now - timedelta(days=idx + 1)
for idx in range(cls.SIGNALS_DISTRIBUTE_DAYS_CNT)] + \
[now - timedelta(hours=idx)
for idx in range(cls.SIGNALS_DISTRIBUTE_HOURS_CNT)]
cls.states = 3 * [BEHANDELING] + 2 * [ON_HOLD] + (len(times) - 3 - 2) * [GEMELD]
shuffle(cls.states)
cls.sub_categories = [CategoryFactory.create() for _ in range(cls.SUBCATEGORY_CNT)]
for idx, time in enumerate(times):
with freeze_time(time):
signal = SignalFactory.create()
StatusFactory(_signal=signal, state=cls.states[idx])
category_assignment = CategoryAssignmentFactory(_signal=signal,
category=cls.sub_categories[
idx % len(cls.sub_categories)])
signal.category_assignment = category_assignment
signal.save()
cls.signals.append(signal)
def test_generate_creeerZaak_Lk01(self):
current_tz = timezone.get_current_timezone()
location = Point(4.1234, 52.1234)
signal = SignalFactory.create(incident_date_end=None,
location__geometrie=location)
seq_no = _generate_sequence_number(signal)
xml_message = _generate_creeerZaak_Lk01(signal, seq_no)
self.assertXmlDocument(xml_message)
self.assertIn(
f'<StUF:referentienummer>{signal.sia_id}.{seq_no}</StUF:referentienummer>',
xml_message)
self.assertIn(
'<StUF:tijdstipBericht>{}</StUF:tijdstipBericht>'.format(
signal.created_at.astimezone(current_tz).strftime(
'%Y%m%d%H%M%S')), xml_message)
self.assertIn(
'<ZKN:startdatum>{}</ZKN:startdatum>'.format(
signal.incident_date_start.astimezone(current_tz).strftime(
'%Y%m%d')), xml_message)
incident_date_end = signal.created_at + timedelta(days=3)
self.assertIn(
'<ZKN:einddatumGepland>{}</ZKN:einddatumGepland>'.format(
incident_date_end.astimezone(current_tz).strftime('%Y%m%d')),
xml_message)
self.assertIn(
'<StUF:extraElement naam="Ycoordinaat">{}</StUF:extraElement>'.
format(location.y), xml_message)
self.assertIn(
'<StUF:extraElement naam="Xcoordinaat">{}</StUF:extraElement>'.
format(location.x), xml_message)
