def test_remove_deleted_buildings(self): building_collection = Building.collection_name() Building.remove_deleted_buildings() query = { "$or" : [ { "edilizia" : {"$exists": False}, "deleted_easyroom" : { "$exists": True } }, { "easyroom" : {"$exists": False}, "deleted_edilizia" : { "$exists": True } } ] } options = {"multi" : True} self.assertEqual(self.pm.remove.call_count, 2) valid_ids = Building._pm.get_collection_ids(building_collection) query_bv = { "_id" : { "$nin" : list(valid_ids) } } call1 = call(building_collection, query, options) call2 = call("buildingview", query_bv, options) self.pm.remove.assert_has_calls([call1, call2], any_order = True)
def test_remove_deleted_buildings(self): building_collection = Building.collection_name() Building.remove_deleted_buildings() query = { "$or": [{ "edilizia": { "$exists": False }, "deleted_easyroom": { "$exists": True } }, { "easyroom": { "$exists": False }, "deleted_edilizia": { "$exists": True } }] } options = {"multi": True} self.assertEqual(self.pm.remove.call_count, 2) valid_ids = Building._pm.get_collection_ids(building_collection) query_bv = {"_id": {"$nin": list(valid_ids)}} call1 = call(building_collection, query, options) call2 = call("buildingview", query_bv, options) self.pm.remove.assert_has_calls([call1, call2], any_order=True)
def _clean_unmarked_buildings(self): """ After an update batch is completed, buildings not updated are to be considered as "removed" by the supplied data source, and, hence, a logic "delete" operation is performed, by adding a delete_<namespace> key to the building object. A building is completely removed from database if every source that once stated it existed now has a deleted_<namespace> key set. What this algorithm does is to look for Buildings with updated_at field older than this last batch_date (i.e., untouched), and add the logic delete key to those buildings. Finally, it looks for buildings that had all data logically deleted and removes them physically from DB. Return value: None """ # Make sure the current update is performed as a perfect snapshot, # removing also "untouched" buildings n_removed, b_removed = Building.remove_untouched_keys( self.get_namespace(), self.batch_date ) b_removed = [ b["b_id"] for b in b_removed ] if b_removed: Logger.info( n_removed, "previously existing buildings are not present", "in this snapshot:", ", ".join(b_removed) ) n_destroyed, b_destroyed = Building.remove_deleted_buildings() b_destroyed = [ b["b_id"] for b in b_destroyed ] if n_destroyed: Logger.info( n_destroyed, "buildings were effectively removed from database", "since no data source affirms its existence:", ", ".join(b_destroyed) )
def _clean_unmarked_buildings(self): """ After an update batch is completed, buildings not updated are to be considered as "removed" by the supplied data source, and, hence, a logic "delete" operation is performed, by adding a delete_<namespace> key to the building object. A building is completely removed from database if every source that once stated it existed now has a deleted_<namespace> key set. What this algorithm does is to look for Buildings with updated_at field older than this last batch_date (i.e., untouched), and add the logic delete key to those buildings. Finally, it looks for buildings that had all data logically deleted and removes them physically from DB. Return value: None """ # Make sure the current update is performed as a perfect snapshot, # removing also "untouched" buildings n_removed, b_removed = Building.remove_untouched_keys( self.get_namespace(), self.batch_date) b_removed = [b["b_id"] for b in b_removed] if b_removed: Logger.info(n_removed, "previously existing buildings are not present", "in this snapshot:", ", ".join(b_removed)) n_destroyed, b_destroyed = Building.remove_deleted_buildings() b_destroyed = [b["b_id"] for b in b_destroyed] if n_destroyed: Logger.info( n_destroyed, "buildings were effectively removed from database", "since no data source affirms its existence:", ", ".join(b_destroyed))