def vacuum_all(self, limit=None, time_limit=None, unupdated=False):
logger.debug('Begin vacuum_all(limit=%s, time_limit=%s, unupdated=%s)', limit, time_limit, unupdated)
##TODO delete SCIFields with SCFilterId not found in SCFilter
self.plugins = self.load_plugins()
self.ts = self.term_stat('SupplierCatalogItemFields Vacuum', len(self.plugins))
now = start_time = datetime.now()
try:
transaction.begin()
for plug in self.plugins.itervalues():
supplier_catalog_filter_id = plug.supplier_catalog_filter_id()
### Generate a bloom filter set of SCIF id's in VersionModel
model_name = plug.version_model() + 'Model'
VersionModel = getattr(model, model_name)
query = DBSession.query(VersionModel.supplier_catalog_item_field_id)
s = BloomFilter(capacity=query.count() + 1)
self.ts['sub_total'] = query.count()
for (supplier_catalog_item_field_id, ) in query.yield_per(100):
s.add(supplier_catalog_item_field_id)
self.ts['sub_done'] += 1
del query
### Iterate through SCIFields, deleting any that don't appear in the bloom filter.
query = DBSession.query(SupplierCatalogItemFieldModel)
query = query.filter(SupplierCatalogItemFieldModel.supplier_catalog_filter_id == supplier_catalog_filter_id)
if unupdated is not True:
query = query.filter(SupplierCatalogItemFieldModel.updated != None)
if limit is not None:
query = query.order_by(SupplierCatalogItemFieldModel.vacuumed.nullsfirst())
query = query.limit(limit)
logger.debug("LIMIT %i, supplier_catalog_filter_id %s", limit, supplier_catalog_filter_id)
self.ts['sub_done'] = 0
self.ts['sub_total'] = query.count()
for supplier_catalog_item_field in query.yield_per(100):
if supplier_catalog_item_field.id not in s:
logger.debug("Deleting SupplierCatalogItemField %s", supplier_catalog_item_field.id)
DBSession.delete(supplier_catalog_item_field)
else:
supplier_catalog_item_field.vacuumed = now
if self.ts['sub_done'] % 1000 == 0:
DBSession.flush()
self.ts['sub_done'] += 1
del query
DBSession.flush()
if time_limit is not None:
if datetime.now() > start_time + time_limit:
logger.info("Reached Time Limit at %i of %i", self.ts['done'], self.ts['total'])
transaction.commit()
break;
self.ts['done'] += 1
transaction.commit()
except Exception:
logger.exception("Caught Exception: ")
transaction.abort()
finally:
self.ts.finish()
logger.debug('End vacuum()')
def vacuum_all(self, limit=None):
logger.debug('Begin vacuum_all(limit=%s)', limit)
self.plugins = self.load_plugins()
ts = self.term_stat('SupplierSpecialItemVersion Vacuum', len(self.plugins))
tx = transaction.get()
try:
#s = set()
s = ScalableBloomFilter()
query = DBSession.query(SupplierSpecialModel.id)
for (supplier_special_id, ) in query.yield_per(100):
s.add(supplier_special_id)
for plug in self.plugins.itervalues():
supplier_special_filter_id = plug.supplier_special_filter_id()
model_name = plug.version_model() + 'Model'
VersionModel = getattr(model, model_name)
query = DBSession.query(VersionModel)
if limit:
query = query.order_by(VersionModel.vacuumed.nullsfirst())
query = query.limit(limit)
ts['sub_done'] = 0
ts['sub_total'] = query.count()
for supplier_special_item_version in query.yield_per(10):
if supplier_special_item_version.supplier_special_id not in s:
logger.debug("Deleting %s %s", model_name, supplier_special_item_version.id)
DBSession.delete(supplier_special_item_version)
ts['sub_done'] += 1
if ts['sub_done'] % 1000 == 0:
DBSession.flush()
DBSession.flush()
ts['done'] += 1
except Exception:
logger.exception('Caught Exception: ')
tx.abort()
finally:
ts.finish()
transaction.commit()
logger.debug('End vacuum_all()')
