def test_get_marker_from_db(self):
marker_dt = date_utils.get_datetime_utc_now() - datetime.timedelta(minutes=5)
marker_db = DumperMarkerDB(marker=isotime.format(marker_dt, offset=False),
updated_at=date_utils.get_datetime_utc_now())
DumperMarker.add_or_update(marker_db)
exec_exporter = ExecutionsExporter(None, None)
export_marker = exec_exporter._get_export_marker_from_db()
self.assertEqual(export_marker, date_utils.add_utc_tz(marker_dt))
def test_get_marker_from_db(self):
marker_dt = date_utils.get_datetime_utc_now() - datetime.timedelta(minutes=5)
marker_db = DumperMarkerDB(marker=isotime.format(marker_dt, offset=False),
updated_at=date_utils.get_datetime_utc_now())
DumperMarker.add_or_update(marker_db)
exec_exporter = ExecutionsExporter(None, None)
export_marker = exec_exporter._get_export_marker_from_db()
self.assertEqual(export_marker, date_utils.add_utc_tz(marker_dt))
def test_write_marker_to_db_marker_exists(self):
executions_queue = self.get_queue()
dumper = Dumper(queue=executions_queue,
export_dir='/tmp',
batch_size=5,
max_files_per_sleep=1,
file_prefix='st2-stuff-',
file_format='json')
timestamps = [
isotime.parse(execution.end_timestamp)
for execution in self.execution_apis
]
max_timestamp = max(timestamps)
first_marker_db = dumper._write_marker_to_db(max_timestamp)
second_marker_db = dumper._write_marker_to_db(max_timestamp +
datetime.timedelta(
hours=1))
markers = DumperMarker.get_all()
self.assertEqual(len(markers), 1)
final_marker_id = markers[0].id
self.assertEqual(first_marker_db.id, final_marker_id)
self.assertEqual(second_marker_db.id, final_marker_id)
self.assertEqual(markers[0].marker, second_marker_db.marker)
self.assertTrue(
second_marker_db.updated_at > first_marker_db.updated_at)
def _write_marker_to_db(self, new_marker):
LOG.info('Updating marker in db to: %s', new_marker)
markers = DumperMarker.get_all()
if len(markers) > 1:
LOG.exception('More than one dumper marker found. Using first found one.')
marker = isotime.format(new_marker, offset=False)
updated_at = date_utils.get_datetime_utc_now()
if markers:
marker_id = markers[0]['id']
else:
marker_id = None
marker_db = DumperMarkerDB(id=marker_id, marker=marker, updated_at=updated_at)
return DumperMarker.add_or_update(marker_db)
def test_dumper_marker_crud(self):
saved = DumperMarkerModelTest._create_save_dumper_marker()
retrieved = DumperMarker.get_by_id(saved.id)
self.assertEqual(saved.marker, retrieved.marker, "Same marker was not returned.")
# test update
time_now = date_utils.get_datetime_utc_now()
retrieved.updated_at = time_now
saved = DumperMarker.add_or_update(retrieved)
retrieved = DumperMarker.get_by_id(saved.id)
self.assertEqual(retrieved.updated_at, time_now, "Update to marker failed.")
# cleanup
DumperMarkerModelTest._delete([retrieved])
try:
retrieved = DumperMarker.get_by_id(saved.id)
except ValueError:
retrieved = None
self.assertIsNone(retrieved, "managed to retrieve after failure.")
def _write_marker_to_db(self, new_marker):
LOG.info('Updating marker in db to: %s', new_marker)
marker = DumperMarker.get_all()
if len(marker) > 1:
LOG.exception(
'More than one dumper marker found. Using first found one.')
marker_db = None
if marker:
marker = marker[0]
marker_db = DumperMarkerDB(id=marker['id'])
else:
marker_db = DumperMarkerDB()
marker_db.marker = isotime.format(new_marker, offset=False)
marker_db.updated_at = date_utils.get_datetime_utc_now()
return DumperMarker.add_or_update(marker_db)
def test_dumper_marker_crud(self):
saved = DumperMarkerModelTest._create_save_dumper_marker()
retrieved = DumperMarker.get_by_id(saved.id)
self.assertEqual(saved.marker, retrieved.marker,
'Same marker was not returned.')
# test update
time_now = date_utils.get_datetime_utc_now()
retrieved.updated_at = time_now
saved = DumperMarker.add_or_update(retrieved)
retrieved = DumperMarker.get_by_id(saved.id)
self.assertEqual(retrieved.updated_at, time_now, 'Update to marker failed.')
# cleanup
DumperMarkerModelTest._delete([retrieved])
try:
retrieved = DumperMarker.get_by_id(saved.id)
except StackStormDBObjectNotFoundError:
retrieved = None
self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def _get_export_marker_from_db(self):
try:
markers = DumperMarker.get_all()
except:
return None
else:
if len(markers) >= 1:
marker = markers[0]
return isotime.parse(marker.marker)
else:
return None
def _write_marker_to_db(self, new_marker):
LOG.info("Updating marker in db to: %s", new_marker)
markers = DumperMarker.get_all()
if len(markers) > 1:
LOG.exception(
"More than one dumper marker found. Using first found one.")
marker = isotime.format(new_marker, offset=False)
updated_at = date_utils.get_datetime_utc_now()
if markers:
marker_id = markers[0]["id"]
else:
marker_id = None
marker_db = DumperMarkerDB(id=marker_id,
marker=marker,
updated_at=updated_at)
return DumperMarker.add_or_update(marker_db)
def test_write_marker_to_db_marker_exists(self):
executions_queue = self.get_queue()
dumper = Dumper(queue=executions_queue,
export_dir='/tmp', batch_size=5,
max_files_per_sleep=1,
file_prefix='st2-stuff-', file_format='json')
timestamps = [isotime.parse(execution.end_timestamp) for execution in self.execution_apis]
max_timestamp = max(timestamps)
first_marker_db = dumper._write_marker_to_db(max_timestamp)
second_marker_db = dumper._write_marker_to_db(max_timestamp + datetime.timedelta(hours=1))
markers = DumperMarker.get_all()
self.assertEqual(len(markers), 1)
final_marker_id = markers[0].id
self.assertEqual(first_marker_db.id, final_marker_id)
self.assertEqual(second_marker_db.id, final_marker_id)
self.assertEqual(markers[0].marker, second_marker_db.marker)
self.assertTrue(second_marker_db.updated_at > first_marker_db.updated_at)
def _create_save_dumper_marker():
created = DumperMarkerDB()
created.marker = '2015-06-11T00:35:15.260439Z'
created.updated_at = date_utils.get_datetime_utc_now()
return DumperMarker.add_or_update(created)
def _create_save_dumper_marker():
created = DumperMarkerDB()
created.marker = '2015-06-11T00:35:15.260439Z'
created.updated_at = date_utils.get_datetime_utc_now()
return DumperMarker.add_or_update(created)
