def test_scramble_v3_counts_command(self):
from django.db.models import Q, F
from model_bakery.recipe import Recipe
from random import randint
group_aggregate_recipe = Recipe(GroupAggregate,
count=randint(0, 1000),
count_scrambled=None)
group_aggregate_recipe.make(_quantity=100)
for ga in GroupAggregate.objects.all():
self.assertIsNotNone(ga.count)
self.assertIsNone(ga.count_scrambled)
# Do the scrambling
call_man_command('scramble_v3_counts')
differ_count = 0
for ga in GroupAggregate.objects.all():
self.assertIsNotNone(ga.count_scrambled)
self.assertIn(ga.count_scrambled,
(ga.count - 1, ga.count, ga.count + 1))
if ga.count_scrambled != ga.count:
differ_count += 1
# check all records have their scrambled counts set
assert not GroupAggregate.objects.filter(count_scrambled=None).exists()
# check that all scrambled counts are within valid range
assert not GroupAggregate.objects.filter(
Q(count_scrambled__gt=F('count') + 1)
| Q(count_scrambled__lt=F('count') - 1))
# Make sure that a significant amount of counts_scrambled were actually changed from the original
self.assertGreater(differ_count, 50)
def test_get_15min_aggregation_timezone_with_both_v1_and_v2_records(self):
# Insert some v2 records for each hour
for i in range(0, 24):
timestamp_str = datetime.now().replace(
hour=i, minute=0,
second=0).astimezone().replace(microsecond=0).isoformat()
response = self.client.post(
self.POST_URL_V2,
create_new_v2_json(timestamp_str=timestamp_str),
**POST_AUTHORIZATION_HEADER,
content_type='application/json')
self.assertEqual(response.status_code, 200)
# Then run the parse_ingress script
parser = TelcameraParser()
parser.consume(end_at_empty_queue=True)
# Complete aggregate because the query in the endpoint depends on it
call_man_command('complete_aggregate', 'continuousaggregate_cmsa15min')
# test whether the endpoint responds correctly
response = self.client.get(self.URL, **GET_AUTHORIZATION_HEADER)
self.assertEqual(response.status_code, 200)
self.assertGreater(len(json.loads(response.content)), 0)
def test_scramble_v2_counts_command(self):
count_aggregate_recipe = Recipe(
CountAggregate,
count_in=randint(0, 1000),
count_out=randint(0, 1000),
count=randint(0, 1000),
count_in_scrambled=None,
count_out_scrambled=None,
count_scrambled=None,
)
count_aggregate_recipe.make(_quantity=100)
# Do the scrambling
call_man_command('scramble_v2_counts')
differ_count_in = 0
differ_count_out = 0
differ_count = 0
for ca in CountAggregate.objects.all():
assert ca.count_in_scrambled is not None
assert ca.count_in_scrambled in (ca.count_in - 1, ca.count_in,
ca.count_in + 1)
if ca.count_in_scrambled != ca.count_in:
differ_count_in += 1
assert ca.count_out_scrambled is not None
assert ca.count_out_scrambled in (ca.count_out - 1, ca.count_out,
ca.count_out + 1)
if ca.count_out_scrambled != ca.count_out:
differ_count_out += 1
assert ca.count_scrambled is not None
assert ca.count_scrambled in (ca.count - 1, ca.count, ca.count + 1)
if ca.count_scrambled != ca.count:
differ_count += 1
# check all records have their scrambled counts set
assert not CountAggregate.objects.filter(
Q(count_in_scrambled=None) | Q(count_out_scrambled=None)
| Q(count_scrambled=None)).exists()
# check that all scrambled counts are within valid range
assert not CountAggregate.objects.filter(
Q(count_in_scrambled__gt=F('count_in') + 1)
| Q(count_in_scrambled__lt=F('count_in') - 1),
Q(count_out_scrambled__gt=F('count_out') + 1)
| Q(count_out_scrambled__lt=F('count_out') - 1),
Q(count_scrambled__gt=F('count') + 1)
| Q(count_scrambled__lt=F('count') - 1),
)
# Make sure that a significant amount of counts_scrambled were actually changed from the original
assert differ_count_in > 50
assert differ_count_out > 50
assert differ_count > 50
def test_changing_is_active_to_existing_status(self):
# The sensor is already active. Now try to set it to active again.
out = call_man_command('set_bool_field', self.objectnummer,
'is_active', 'true')
self.assertEqual(
out.strip(),
f"The sensor '{self.objectnummer}'.is_active is already True. Nothing has changed."
)
def test_changing_is_active_to_false_and_back_to_true(self):
out = call_man_command('set_bool_field', self.objectnummer,
'is_active', 'false')
self.assertEqual(
out.strip(),
f"The sensor '{self.objectnummer}'.is_active was successfully changed to False."
)
sensor = Sensors.objects.get(objectnummer=self.objectnummer)
self.assertEqual(sensor.is_active, False)
out = call_man_command('set_bool_field', self.objectnummer,
'is_active', 'true')
self.assertEqual(
out.strip(),
f"The sensor '{self.objectnummer}'.is_active was successfully changed to True."
)
sensor = Sensors.objects.get(objectnummer=self.objectnummer)
self.assertEqual(sensor.is_active, True)
def test_vanilla(self, client):
# Add records every 5min for multiple days
Message.objects.all().delete()
test_days = 2
today = date.today()
start_date = today - timedelta(days=test_days)
the_dt = datetime(start_date.year, start_date.month, start_date.day)
while the_dt < datetime(today.year, today.month, today.day):
for sensor_name in self.sensor_names:
test_post = json.loads(TEST_POST)
test_post['data'][0]['sensor'] = sensor_name
test_post['data'][0]['timestamp_start'] = the_dt.isoformat()
client.post(self.URL,
json.dumps(test_post),
**AUTHORIZATION_HEADER,
content_type='application/json')
the_dt += timedelta(minutes=5)
# Then run the parse_ingress script
parser = TelcameraParser()
parser.consume(end_at_empty_queue=True)
# Make sure we've got source data
assert Observation.objects.all().count() > 100
# Run the aggregator
call_man_command('complete_aggregate', 'continuousaggregate_cmsa15min')
# Do we have any records in the continuous aggregate table?
assert Cmsa15Min.objects.all().count() > 500
# Take a record in the middle of the data in the continuous aggregate table
# and check whether the record is made up of exactly 3 messages (one every 5 min)
last_record = Cmsa15Min.objects\
.filter(sensor=self.sensor_names[0])\
.filter(timestamp_rounded__gte=(today - timedelta(days=1)).isoformat())\
.order_by('timestamp_rounded')\
.first()
assert last_record.basedonxmessages == 3
def test_monitor_incoming_messages(self):
# There is no message yet, so we expect an error
with pytest.raises(AssertionError):
call_man_command('monitor_v2_incoming_messages')
# Insert one observation
baker.make(Observation)
# There is one recent observation so we don't expect an error now
call_man_command('monitor_v2_incoming_messages')
# We travel 20 minutes in the future, and there we do expect an error
with time_machine.travel(now() + timedelta(minutes=20)):
with pytest.raises(AssertionError):
call_man_command('monitor_v2_incoming_messages')
def test_changing_non_existing_sensor_fails(self):
out = call_man_command('set_bool_field', 'does not exist', 'is_active',
'false')
self.assertEqual(
out.strip(),
f"No sensor exists for the objectnummer 'does not exist'")