def test_remove_deleted_buildings(self):
building_collection = Building.collection_name()
Building.remove_deleted_buildings()
query = {
"$or" : [
{
"edilizia" : {"$exists": False},
"deleted_easyroom" : { "$exists": True }
},
{
"easyroom" : {"$exists": False},
"deleted_edilizia" : { "$exists": True }
}
]
}
options = {"multi" : True}
self.assertEqual(self.pm.remove.call_count, 2)
valid_ids = Building._pm.get_collection_ids(building_collection)
query_bv = {
"_id" : {
"$nin" : list(valid_ids)
}
}
call1 = call(building_collection, query, options)
call2 = call("buildingview", query_bv, options)
self.pm.remove.assert_has_calls([call1, call2], any_order = True)
def test_remove_deleted_buildings(self):
building_collection = Building.collection_name()
Building.remove_deleted_buildings()
query = {
"$or": [{
"edilizia": {
"$exists": False
},
"deleted_easyroom": {
"$exists": True
}
}, {
"easyroom": {
"$exists": False
},
"deleted_edilizia": {
"$exists": True
}
}]
}
options = {"multi": True}
self.assertEqual(self.pm.remove.call_count, 2)
valid_ids = Building._pm.get_collection_ids(building_collection)
query_bv = {"_id": {"$nin": list(valid_ids)}}
call1 = call(building_collection, query, options)
call2 = call("buildingview", query_bv, options)
self.pm.remove.assert_has_calls([call1, call2], any_order=True)
def _clean_unmarked_buildings(self):
"""
After an update batch is completed, buildings not updated are to be
considered as "removed" by the supplied data source, and, hence, a logic
"delete" operation is performed, by adding a delete_<namespace> key
to the building object.
A building is completely removed from database if every source that once
stated it existed now has a deleted_<namespace> key set.
What this algorithm does is to look for Buildings with updated_at field
older than this last batch_date (i.e., untouched), and add the logic
delete key to those buildings. Finally, it looks for buildings that
had all data logically deleted and removes them physically from DB.
Return value: None
"""
# Make sure the current update is performed as a perfect snapshot,
# removing also "untouched" buildings
n_removed, b_removed = Building.remove_untouched_keys(
self.get_namespace(), self.batch_date
)
b_removed = [ b["b_id"] for b in b_removed ]
if b_removed:
Logger.info(
n_removed,
"previously existing buildings are not present",
"in this snapshot:",
", ".join(b_removed)
)
n_destroyed, b_destroyed = Building.remove_deleted_buildings()
b_destroyed = [ b["b_id"] for b in b_destroyed ]
if n_destroyed:
Logger.info(
n_destroyed,
"buildings were effectively removed from database",
"since no data source affirms its existence:",
", ".join(b_destroyed)
)
def _clean_unmarked_buildings(self):
"""
After an update batch is completed, buildings not updated are to be
considered as "removed" by the supplied data source, and, hence, a logic
"delete" operation is performed, by adding a delete_<namespace> key
to the building object.
A building is completely removed from database if every source that once
stated it existed now has a deleted_<namespace> key set.
What this algorithm does is to look for Buildings with updated_at field
older than this last batch_date (i.e., untouched), and add the logic
delete key to those buildings. Finally, it looks for buildings that
had all data logically deleted and removes them physically from DB.
Return value: None
"""
# Make sure the current update is performed as a perfect snapshot,
# removing also "untouched" buildings
n_removed, b_removed = Building.remove_untouched_keys(
self.get_namespace(), self.batch_date)
b_removed = [b["b_id"] for b in b_removed]
if b_removed:
Logger.info(n_removed,
"previously existing buildings are not present",
"in this snapshot:", ", ".join(b_removed))
n_destroyed, b_destroyed = Building.remove_deleted_buildings()
b_destroyed = [b["b_id"] for b in b_destroyed]
if n_destroyed:
Logger.info(
n_destroyed,
"buildings were effectively removed from database",
"since no data source affirms its existence:",
", ".join(b_destroyed))
