def test_update_marker(self):
executions_queue = self.get_queue()
dumper = Dumper(queue=executions_queue,
export_dir='/tmp',
batch_size=5,
max_files_per_sleep=1,
file_prefix='st2-stuff-',
file_format='json')
# Batch 1
batch = self.execution_apis[0:5]
new_marker = dumper._update_marker(batch)
self.assertTrue(new_marker is not None)
timestamps = [
isotime.parse(execution.end_timestamp) for execution in batch
]
max_timestamp = max(timestamps)
self.assertEqual(new_marker, max_timestamp)
# Batch 2
batch = self.execution_apis[0:5]
new_marker = dumper._update_marker(batch)
timestamps = [
isotime.parse(execution.end_timestamp) for execution in batch
]
max_timestamp = max(timestamps)
self.assertEqual(new_marker, max_timestamp)
dumper._write_marker_to_db.assert_called_with(new_marker)
def test_update_marker_out_of_order_batch(self):
executions_queue = self.get_queue()
dumper = Dumper(
queue=executions_queue,
export_dir="/tmp",
batch_size=5,
max_files_per_sleep=1,
file_prefix="st2-stuff-",
file_format="json",
)
timestamps = [
isotime.parse(execution.end_timestamp)
for execution in self.execution_apis
]
max_timestamp = max(timestamps)
# set dumper persisted timestamp to something less than min timestamp in the batch
test_timestamp = max_timestamp + datetime.timedelta(hours=1)
dumper._persisted_marker = test_timestamp
new_marker = dumper._update_marker(self.execution_apis)
self.assertTrue(new_marker < test_timestamp)
# Assert we rolled back the marker.
self.assertEqual(dumper._persisted_marker, max_timestamp)
self.assertEqual(new_marker, max_timestamp)
dumper._write_marker_to_db.assert_called_with(new_marker)
def test_update_marker(self):
executions_queue = self.get_queue()
dumper = Dumper(queue=executions_queue,
export_dir='/tmp', batch_size=5,
max_files_per_sleep=1,
file_prefix='st2-stuff-', file_format='json')
# Batch 1
batch = self.execution_apis[0:5]
new_marker = dumper._update_marker(batch)
self.assertTrue(new_marker is not None)
timestamps = [isotime.parse(execution.end_timestamp) for execution in batch]
max_timestamp = max(timestamps)
self.assertEqual(new_marker, max_timestamp)
# Batch 2
batch = self.execution_apis[0:5]
new_marker = dumper._update_marker(batch)
timestamps = [isotime.parse(execution.end_timestamp) for execution in batch]
max_timestamp = max(timestamps)
self.assertEqual(new_marker, max_timestamp)
dumper._write_marker_to_db.assert_called_with(new_marker)
def test_update_marker_out_of_order_batch(self):
executions_queue = self.get_queue()
dumper = Dumper(queue=executions_queue,
export_dir='/tmp', batch_size=5,
max_files_per_sleep=1,
file_prefix='st2-stuff-', file_format='json')
timestamps = [isotime.parse(execution.end_timestamp) for execution in self.execution_apis]
max_timestamp = max(timestamps)
# set dumper persisted timestamp to something less than min timestamp in the batch
test_timestamp = max_timestamp + datetime.timedelta(hours=1)
dumper._persisted_marker = test_timestamp
new_marker = dumper._update_marker(self.execution_apis)
self.assertTrue(new_marker < test_timestamp)
# Assert we rolled back the marker.
self.assertEqual(dumper._persisted_marker, max_timestamp)
self.assertEqual(new_marker, max_timestamp)
dumper._write_marker_to_db.assert_called_with(new_marker)