def get_all(self, driver_name, action):
try:
LOCK_BY_DRIVER.acquire(driver_name)
driver = utils.get_drivers_by_name([driver_name])[0]
LOG.info("run driver get_all: %s", driver_name)
now = format_utcnow()
events = driver.get_all(action)
count = self.process_output_func(events)
LOG.info("run driver get_all: %s done (%s events)", driver_name,
count)
if driver.should_delete_outdated_entities():
vertices_to_delete = \
self._find_vertices_to_delete(driver_name, now)
if vertices_to_delete:
delete_events = self._to_events(vertices_to_delete)
count_deleted = self.process_output_func(delete_events)
LOG.info(
"run driver delete outdated vertices: %s done "
"(%s events)", driver_name, count_deleted)
count += count_deleted
return count
except Exception:
LOG.exception("run driver get_all: %s Failed", driver_name)
finally:
LOCK_BY_DRIVER.release(driver_name)
return 0
def test_snapshot_event_transform(self):
sample_timestamp = format_utcnow()
for event in events:
event[DSProps.DATASOURCE_ACTION] = DatasourceAction.SNAPSHOT
event[DSProps.SAMPLE_DATE] = sample_timestamp
wrapper = self.transformers[CETUS_POD_DATASOURCE].transform(event)
vertex = wrapper.vertex
self._validate_vertex_props(vertex, event)
neighbors = wrapper.neighbors
self.assertThat(neighbors, matchers.HasLength(1))
self._validate_neighbors(neighbors, vertex.vertex_id)
def _to_events(vertices):
return ({
DSProps.ENTITY_TYPE: CONSISTENCY_DATASOURCE,
DSProps.DATASOURCE_ACTION: DatasourceAction.UPDATE,
DSProps.SAMPLE_DATE: format_utcnow(),
DSProps.EVENT_TYPE: GraphAction.DELETE_ENTITY,
VProps.VITRAGE_ID: vertex[VProps.VITRAGE_ID],
VProps.ID: vertex.get(VProps.ID, None),
VProps.VITRAGE_TYPE: vertex[VProps.VITRAGE_TYPE],
VProps.VITRAGE_CATEGORY: vertex[VProps.VITRAGE_CATEGORY],
VProps.IS_REAL_VITRAGE_ID: True
} for vertex in vertices)
def _to_events(self, vertices, action):
for vertex in vertices:
event = {
DSProps.ENTITY_TYPE: CONSISTENCY_DATASOURCE,
DSProps.DATASOURCE_ACTION: DatasourceAction.UPDATE,
DSProps.SAMPLE_DATE: datetime.format_utcnow(),
DSProps.EVENT_TYPE: action,
VProps.VITRAGE_ID: vertex[VProps.VITRAGE_ID],
VProps.ID: vertex.get(VProps.ID, None),
VProps.VITRAGE_TYPE: vertex[VProps.VITRAGE_TYPE],
VProps.VITRAGE_CATEGORY: vertex[VProps.VITRAGE_CATEGORY],
VProps.IS_REAL_VITRAGE_ID: True
}
yield event
def _add_sampling_time(entity):
entity[DSProps.SAMPLE_DATE] = datetime_utils.format_utcnow()
def _add_default_properties(event):
event[DSProps.DATASOURCE_ACTION] = AType.UPDATE
event[DSProps.ENTITY_TYPE] = VITRAGE_DATASOURCE
event[VProps.UPDATE_TIMESTAMP] = str(datetime_utils.utcnow(False))
event[VProps.VITRAGE_SAMPLE_TIMESTAMP] = datetime_utils.format_utcnow()