def update_all(self, limit=None, time_limit=None): """Update All""" logger.debug("Begin update_all()") ts = self.term_stat('Products Update') start_time = datetime.now() tx = transaction.get() try: query = DBSession.query(ProductModel) if limit is not None: query = query.order_by(ProductModel.updated.nullsfirst()) query = query.limit(limit) ts['total'] = query.count() for product in query.yield_per(100): self.update_one(product) if ts['done'] % 100 == 0: DBSession.flush() if time_limit is not None: if datetime.now() > start_time + time_limit: logger.info("Reached Time Limit at %i of %i", ts['done'], ts['total']) break; ts['done'] += 1 except Exception: logger.exception("Caught Exception: ") tx.abort() finally: ts.finish() logger.debug("End update_all()")
def update_all(self, limit=None, time_limit=None): """Update All""" logger.debug("Begin update_all(limit=%s)", limit) transaction.begin() self.ts = self.term_stat('SupplierCatalogItemField Update') start_time = datetime.now() try: self.plugins = self.load_plugins() query = DBSession.query(SupplierCatalogItemFieldModel) if limit is not None: query = query.order_by(SupplierCatalogItemFieldModel.updated.nullsfirst()) query = query.limit(limit) self.ts['total'] = query.count() for supplier_catalog_item_field in query.yield_per(10000): self.update_one(supplier_catalog_item_field) if self.ts['done'] % 10000 == 0: DBSession.flush() if time_limit is not None: if datetime.now() > start_time + time_limit: logger.info("Reached Time Limit at %i of %i", ts['done'], ts['total']) break; self.ts['done'] += 1 del query transaction.commit() except Exception: logger.exception("Caught Exception: ") transaction.abort() finally: self.ts.finish() logger.debug("End update_all()")
def vacuum_all(self, limit=None, time_limit=None, unupdated=False): logger.debug('Begin vacuum_all(limit=%s, time_limit=%s, unupdated=%s)', limit, time_limit, unupdated) ##TODO delete SCIFields with SCFilterId not found in SCFilter self.plugins = self.load_plugins() self.ts = self.term_stat('SupplierCatalogItemFields Vacuum', len(self.plugins)) now = start_time = datetime.now() try: transaction.begin() for plug in self.plugins.itervalues(): supplier_catalog_filter_id = plug.supplier_catalog_filter_id() ### Generate a bloom filter set of SCIF id's in VersionModel model_name = plug.version_model() + 'Model' VersionModel = getattr(model, model_name) query = DBSession.query(VersionModel.supplier_catalog_item_field_id) s = BloomFilter(capacity=query.count() + 1) self.ts['sub_total'] = query.count() for (supplier_catalog_item_field_id, ) in query.yield_per(100): s.add(supplier_catalog_item_field_id) self.ts['sub_done'] += 1 del query ### Iterate through SCIFields, deleting any that don't appear in the bloom filter. query = DBSession.query(SupplierCatalogItemFieldModel) query = query.filter(SupplierCatalogItemFieldModel.supplier_catalog_filter_id == supplier_catalog_filter_id) if unupdated is not True: query = query.filter(SupplierCatalogItemFieldModel.updated != None) if limit is not None: query = query.order_by(SupplierCatalogItemFieldModel.vacuumed.nullsfirst()) query = query.limit(limit) logger.debug("LIMIT %i, supplier_catalog_filter_id %s", limit, supplier_catalog_filter_id) self.ts['sub_done'] = 0 self.ts['sub_total'] = query.count() for supplier_catalog_item_field in query.yield_per(100): if supplier_catalog_item_field.id not in s: logger.debug("Deleting SupplierCatalogItemField %s", supplier_catalog_item_field.id) DBSession.delete(supplier_catalog_item_field) else: supplier_catalog_item_field.vacuumed = now if self.ts['sub_done'] % 1000 == 0: DBSession.flush() self.ts['sub_done'] += 1 del query DBSession.flush() if time_limit is not None: if datetime.now() > start_time + time_limit: logger.info("Reached Time Limit at %i of %i", self.ts['done'], self.ts['total']) transaction.commit() break; self.ts['done'] += 1 transaction.commit() except Exception: logger.exception("Caught Exception: ") transaction.abort() finally: self.ts.finish() logger.debug('End vacuum()')
def setup(self): try: new_attrs = {} new_attrs.update(self.attrs) new_attrs.update(self.do_get_dependencies()) self.obj = self.klass(**new_attrs) DBSession.add(self.obj) DBSession.flush() return self.obj except: DBSession.rollback() raise
def vacuum_all(self, limit=None): logger.debug('Begin vacuum_all(limit=%s)', limit) self.plugins = self.load_plugins() ts = self.term_stat('SupplierSpecialItemVersion Vacuum', len(self.plugins)) tx = transaction.get() try: #s = set() s = ScalableBloomFilter() query = DBSession.query(SupplierSpecialModel.id) for (supplier_special_id, ) in query.yield_per(100): s.add(supplier_special_id) for plug in self.plugins.itervalues(): supplier_special_filter_id = plug.supplier_special_filter_id() model_name = plug.version_model() + 'Model' VersionModel = getattr(model, model_name) query = DBSession.query(VersionModel) if limit: query = query.order_by(VersionModel.vacuumed.nullsfirst()) query = query.limit(limit) ts['sub_done'] = 0 ts['sub_total'] = query.count() for supplier_special_item_version in query.yield_per(10): if supplier_special_item_version.supplier_special_id not in s: logger.debug("Deleting %s %s", model_name, supplier_special_item_version.id) DBSession.delete(supplier_special_item_version) ts['sub_done'] += 1 if ts['sub_done'] % 1000 == 0: DBSession.flush() DBSession.flush() ts['done'] += 1 except Exception: logger.exception('Caught Exception: ') tx.abort() finally: ts.finish() transaction.commit() logger.debug('End vacuum_all()')
def load_all(self, limit=None, item_versions_loaded=None, supplier_id=None): """Load All""" logger.debug("Begin load_all(limit=%s, item_versions_loaded=%s)", limit, item_versions_loaded) self.ts = self.term_stat('SupplierSpecialItemVersion Load') tx = transaction.get() try: self.plugins = self.load_plugins() query = DBSession.query(SupplierSpecialModel) alt_query = query.filter(SupplierSpecialModel.supplier_special_item_versions_loaded == None) if alt_query.count() > 0: query = alt_query.order_by(desc(SupplierSpecialModel.begin_date)) else: query = query.order_by(SupplierSpecialModel.supplier_special_item_versions_loaded.nullsfirst()) del alt_query if supplier_id is not None: query = query.filter(SupplierSpecialModel.supplier_id == supplier_id) else: query = query.filter(SupplierSpecialModel.supplier_id != None) if limit is not None: query = query.limit(limit) self.ts['total'] = query.count() for supplier_special in query.yield_per(10): self.load_one(supplier_special) supplier_special.supplier_special_item_versions_loaded = datetime.now() if self.ts['done'] % 1000 == 0 : DBSession.flush() self.ts['done'] += 1 except Exception: logger.exception('Caught Exception: ') tx.abort() finally: self.ts.finish() transaction.commit() logger.debug("End load_all()")