def test_delete_existing_object(self): """ Test the method _delete_object(self, obj) when the object exists Test scenario: When the data is in the database, the delete operation should complete successfully """ self.aMox.StubOutWithMock(session, 'query') session.query(model.PowerVCMapping).AndReturn(query) self.aMox.StubOutWithMock(query, 'filter_by') query.filter_by(id=self.powerVCMapping['id']).AndReturn(query) self.aMox.StubOutWithMock(query, 'one') query.one().AndReturn(self.powerVCMapping) self.aMox.StubOutWithMock(session, 'begin') session.begin(subtransactions=True).AndReturn(transaction(None, None)) self.aMox.StubOutWithMock(session, 'delete') returnValue = session.delete(self.powerVCMapping).AndReturn(True) self.aMox.ReplayAll() self.powervcagentdb._delete_object(self.powerVCMapping) self.aMox.VerifyAll() self.assertEqual(returnValue, True) self.aMox.UnsetStubs()
def memoize_bed(string, sql): """Function to memoize a BED12 file for fast access""" # records = dict() beds = [] engine = create_engine("sqlite:///{}".format(sql)) DBBASE.metadata.create_all(engine) session = Session(bind=engine, autocommit=True, autoflush=True) counter = 0 with open(string, "rb") as parser: pos = parser.tell() # This will be 0 as we are at the beginning for line in parser: fields = line.split(b"\t") if len(fields) != 12 or (fields and fields[0][0] == b"#"): header = True else: header = False if not header: name = fields[3].decode() if "ID=" in name: groups = dict(re.findall("([^(;|=)]*)=([^;]*)", name)) name = groups["ID"] beds.append(BedIndex(name, pos)) counter += 1 pos += len(line) if len(beds) > 10**6: session.begin(subtransactions=True) session.add_all(beds) session.commit() beds = [] session.begin(subtransactions=True) session.add_all(beds) session.commit() session.close() res = engine.execute("select count(*) from bed").fetchone() assert res[0] == counter, (res[0], counter)
def test_create_object(self): """ Test the method def _create_object(self, obj_type, sync_key, local_id=None, pvc_id=None) """ obj_type = "Network" sync_key = utils.gen_network_sync_key( self.fakePowerVCNetwork.powerNetInstance) local_id = self.fakeOSNetwork.fakeOSNetworkInstance['id'] pvc_id = self.fakePowerVCNetwork.powerNetInstance['id'] inputPowerVCMObj = model.PowerVCMapping(obj_type, sync_key) self.aMox.StubOutWithMock(session, 'begin') session.begin(subtransactions=True).AndReturn(transaction(None, None)) self.aMox.StubOutWithMock(model, 'PowerVCMapping') model.PowerVCMapping(obj_type, sync_key).AndReturn(inputPowerVCMObj) self.aMox.StubOutWithMock(session, 'add') session.add(inputPowerVCMObj).AndReturn("") self.aMox.ReplayAll() self.powervcagentdb._create_object(obj_type, sync_key, update_data=None, local_id=local_id, pvc_id=pvc_id) self.aMox.VerifyAll() self.assertEqual(self.powerVCMapping.local_id, inputPowerVCMObj.local_id) self.assertEqual(self.powerVCMapping.pvc_id, inputPowerVCMObj.pvc_id) self.assertEqual(self.powerVCMapping.status, inputPowerVCMObj.status) self.aMox.UnsetStubs()
def test_create_object(self): """ Test the method def _create_object(self, obj_type, sync_key, local_id=None, pvc_id=None) """ obj_type = "Network" sync_key = utils.gen_network_sync_key( self.fakePowerVCNetwork.powerNetInstance) local_id = self.fakeOSNetwork.fakeOSNetworkInstance['id'] pvc_id = self.fakePowerVCNetwork.powerNetInstance['id'] inputPowerVCMObj = model.PowerVCMapping(obj_type, sync_key) self.aMox.StubOutWithMock(session, 'begin') session.begin(subtransactions=True).AndReturn(transaction(None, None)) self.aMox.StubOutWithMock(model, 'PowerVCMapping') model.PowerVCMapping(obj_type, sync_key).AndReturn(inputPowerVCMObj) self.aMox.StubOutWithMock(session, 'add') session.add(inputPowerVCMObj).AndReturn("") self.aMox.ReplayAll() self.powervcagentdb._create_object( obj_type, sync_key, update_data=None, local_id=local_id, pvc_id=pvc_id) self.aMox.VerifyAll() self.assertEqual( self.powerVCMapping.local_id, inputPowerVCMObj.local_id) self.assertEqual(self.powerVCMapping.pvc_id, inputPowerVCMObj.pvc_id) self.assertEqual(self.powerVCMapping.status, inputPowerVCMObj.status) self.aMox.UnsetStubs()
class Reindexer: def __init__(self, clear, progressive, batch_size): # type: (bool, bool, int) -> None self.clear = clear self.progressive = progressive self.batch_size = int(batch_size or 0) self.index_service = get_service("indexing") self.index = self.index_service.app_state.indexes["default"] self.adapted = self.index_service.adapted self.session = Session(bind=db.session.get_bind(None, None), autocommit=True) self.indexed = set() # type: Set[str] self.cleared = set() # type: Set[str] strategy = progressive_mode if self.progressive else single_transaction self.strategy = strategy(self.index, clear=self.clear) def reindex_all(self): next(self.strategy) # starts generator indexed_classes = self.index_service.app_state.indexed_classes for cls in sorted(indexed_classes, key=lambda c: c.__name__): self.reindex_class(cls) try: self.strategy.send(STOP) except StopIteration: pass try: self.strategy.close() except StopIteration: pass def reindex_class(self, cls): # type: (Entity) -> None current_object_type = cls._object_type() if not self.clear and current_object_type not in self.cleared: self.strategy.send(current_object_type) self.cleared.add(current_object_type) adapter = self.adapted.get(current_object_type) if not adapter or not adapter.indexable: return name = cls.__name__ with self.session.begin(): query = self.session.query(cls).options(sa.orm.lazyload("*")) try: count = query.count() except Exception as e: current_app.logger.error( "Indexing error on class {}: {}".format(name, repr(e)) ) return print("*" * 79) print(f"{name}") if count == 0: print("*" * 79) print(f"{name}") return print("*" * 79) print(f"{name}") with tqdm(total=count) as bar: self.reindex_batch(query, current_object_type, adapter, bar) if not self.batch_size: self.strategy.send(COMMIT) self.strategy.send(COMMIT) def reindex_batch(self, query, current_object_type, adapter, bar): count = 0 for obj in query.yield_per(1000): count += 1 if obj.object_type != current_object_type: # may happen if obj is a subclass and its parent class # is also indexable bar.update() continue object_key = obj.object_key if object_key in self.indexed: bar.update() continue document = self.index_service.get_document(obj, adapter) self.strategy.send(document) self.indexed.add(object_key) if self.batch_size and (count % self.batch_size) == 0: bar.update() self.strategy.send(COMMIT) bar.update()
class Reindexer: def __init__(self, clear, progressive, batch_size): # type: (bool, bool, int) -> None self.clear = clear self.progressive = progressive self.batch_size = int(batch_size or 0) self.index_service = get_service("indexing") self.index = self.index_service.app_state.indexes["default"] # pyre-fixme[16]: `Service` has no attribute `adapted`. self.adapted = self.index_service.adapted self.session = Session(bind=db.session.get_bind(None, None), autocommit=True) self.indexed = set() # type: Set[str] self.cleared = set() # type: Set[str] strategy = progressive_mode if self.progressive else single_transaction self.strategy = strategy(self.index, clear=self.clear) def reindex_all(self): next(self.strategy) # starts generator indexed_classes = self.index_service.app_state.indexed_classes for cls in sorted(indexed_classes, key=lambda c: c.__name__): self.reindex_class(cls) try: self.strategy.send(STOP) except StopIteration: pass try: self.strategy.close() except StopIteration: pass def reindex_class(self, cls): # type: (Entity) -> None current_object_type = cls._object_type() if not self.clear and current_object_type not in self.cleared: self.strategy.send(current_object_type) self.cleared.add(current_object_type) adapter = self.adapted.get(current_object_type) if not adapter or not adapter.indexable: return # pyre-fixme[16]: `Entity` has no attribute `__name__`. name = cls.__name__ with self.session.begin(): query = self.session.query(cls).options(sa.orm.lazyload("*")) try: count = query.count() except Exception as e: current_app.logger.error( f"Indexing error on class {name}: {repr(e)}") return print("*" * 79) print(f"{name}") if count == 0: print("*" * 79) print(f"{name}") return print("*" * 79) print(f"{name}") with tqdm(total=count) as bar: self.reindex_batch(query, current_object_type, adapter, bar) if not self.batch_size: self.strategy.send(COMMIT) self.strategy.send(COMMIT) def reindex_batch(self, query, current_object_type, adapter, bar): count = 0 for obj in query.yield_per(1000): count += 1 if obj.object_type != current_object_type: # may happen if obj is a subclass and its parent class # is also indexable bar.update() continue object_key = obj.object_key if object_key in self.indexed: bar.update() continue document = self.index_service.get_document(obj, adapter) self.strategy.send(document) self.indexed.add(object_key) if self.batch_size and (count % self.batch_size) == 0: bar.update() self.strategy.send(COMMIT) bar.update()
class InboxSession(object): """ Inbox custom ORM (with SQLAlchemy compatible API). Parameters ---------- engine : <sqlalchemy.engine.Engine> A configured database engine to use for this session versioned : bool Do you want to enable the transaction log? ignore_soft_deletes : bool Whether or not to ignore soft-deleted objects in query results. namespace_id : int Namespace to limit query results with. """ def __init__(self, engine, versioned=True, ignore_soft_deletes=True, namespace_id=None): # TODO: support limiting on namespaces assert engine, "Must set the database engine" args = dict(bind=engine, autoflush=True, autocommit=False) self.ignore_soft_deletes = ignore_soft_deletes if ignore_soft_deletes: args['query_cls'] = InboxQuery self._session = Session(**args) if versioned: from inbox.models.transaction import create_revisions @event.listens_for(self._session, 'after_flush') def after_flush(session, flush_context): """ Hook to log revision snapshots. Must be post-flush in order to grab object IDs on new objects. """ create_revisions(session) def query(self, *args, **kwargs): q = self._session.query(*args, **kwargs) if self.ignore_soft_deletes: return q.options(IgnoreSoftDeletesOption()) else: return q def add(self, instance): if not self.ignore_soft_deletes or not instance.is_deleted: self._session.add(instance) else: raise Exception("Why are you adding a deleted object?") def add_all(self, instances): if True not in [i.is_deleted for i in instances] or \ not self.ignore_soft_deletes: self._session.add_all(instances) else: raise Exception("Why are you adding a deleted object?") def delete(self, instance): if self.ignore_soft_deletes: instance.mark_deleted() # just to make sure self._session.add(instance) else: self._session.delete(instance) def begin(self): self._session.begin() def commit(self): self._session.commit() def rollback(self): self._session.rollback() def flush(self): self._session.flush() def close(self): self._session.close() def expunge(self, obj): self._session.expunge(obj) def merge(self, obj): return self._session.merge(obj) @property def no_autoflush(self): return self._session.no_autoflush
def reindex(clear=False, progressive=False, batch_size=None): """ Reindex all content; optionally clear index before. All is done in a single transaction by default. :param clear: clear index content. :param progressive: don't run in a single transaction. :param batch_size: number of documents to process before writing to the index. Unused in single transaction mode. If `None` then all documents of same content type are written at once. """ svc = current_app.services["indexing"] adapted = svc.adapted index = svc.app_state.indexes["default"] session = Session(bind=current_app.db.session.get_bind(None, None), autocommit=True) setattr(session, "_model_changes", {}) # please flask-sqlalchemy <= 1.0 indexed = set() cleared = set() if batch_size is not None: batch_size = int(batch_size) strategy_kw = dict(clear=clear, progressive=progressive, batch_size=batch_size) strategy = progressive_mode if progressive else single_transaction strategy = strategy(index, **strategy_kw) next(strategy) # starts generator count_indexed = 0 for cls in sorted(svc.app_state.indexed_classes, key=lambda c: c.__name__): current_object_type = cls._object_type() if not clear and current_object_type not in cleared: strategy.send(current_object_type) cleared.add(current_object_type) adapter = adapted.get(current_object_type) if not adapter or not adapter.indexable: continue name = cls.__name__ with session.begin(): q = session.query(cls).options(sa.orm.lazyload("*")) count = q.count() if count == 0: print("{}: 0".format(name)) continue widgets = [ name, ": ", pb.Counter(), "/{}".format(count), " ", pb.Timer(), " ", pb.Percentage(), " ", pb.Bar(), " ", pb.ETA(), ] progress = pb.ProgressBar(widgets=widgets, maxval=count) progress.start() count_current = 0 for obj in q.yield_per(1000): if obj.object_type != current_object_type: # may happen if obj is a subclass and its parent class is also # indexable continue object_key = obj.object_key if object_key in indexed: continue document = svc.get_document(obj, adapter) strategy.send(document) indexed.add(object_key) count_indexed += 1 count_current += 1 try: progress.update(count_current) except ValueError: pass if batch_size is not None and (count_current % batch_size) == 0: strategy.send(COMMIT) if batch_size is None: strategy.send(COMMIT) progress.finish() strategy.send(COMMIT) try: strategy.send(STOP) except StopIteration: pass try: strategy.close() except StopIteration: pass
def reindex(clear=False, progressive=False, batch_size=None): """ Reindex all content; optionally clear index before. All is done in a single transaction by default. :param clear: clear index content. :param progressive: don't run in a single transaction. :param batch_size: number of documents to process before writing to the index. Unused in single transaction mode. If `None` then all documents of same content type are written at once. """ svc = current_app.services['indexing'] adapted = svc.adapted index = svc.app_state.indexes['default'] session = Session(bind=current_app.db.session.get_bind(None, None), autocommit=True) setattr(session, '_model_changes', {}) # please flask-sqlalchemy <= 1.0 indexed = set() cleared = set() if batch_size is not None: batch_size = int(batch_size) strategy_kw = dict(clear=clear, progressive=progressive, batch_size=batch_size) strategy = progressive_mode if progressive else single_transaction strategy = strategy(index, **strategy_kw) next(strategy) # starts generator count_indexed = 0 for cls in sorted(svc.app_state.indexed_classes, key=lambda c: c.__name__): current_object_type = cls._object_type() if not clear and current_object_type not in cleared: strategy.send(current_object_type) cleared.add(current_object_type) adapter = adapted.get(current_object_type) if not adapter or not adapter.indexable: continue name = cls.__name__ with session.begin(): q = session.query(cls).options(sa.orm.lazyload('*')) count = q.count() if count == 0: print("{}: 0".format(name)) continue widgets = [name, ': ', pb.Counter(), '/{}'.format(count), ' ', pb.Timer(), ' ', pb.Percentage(), ' ', pb.Bar(), ' ', pb.ETA(), ] progress = pb.ProgressBar(widgets=widgets, maxval=count) progress.start() count_current = 0 for obj in q.yield_per(1000): if obj.object_type != current_object_type: # may happen if obj is a subclass and its parent class is also # indexable continue object_key = obj.object_key if object_key in indexed: continue document = svc.get_document(obj, adapter) strategy.send(document) indexed.add(object_key) count_indexed += 1 count_current += 1 try: progress.update(count_current) except ValueError: pass if batch_size is not None and (count_current % batch_size) == 0: strategy.send(COMMIT) if batch_size is None: strategy.send(COMMIT) progress.finish() strategy.send(COMMIT) try: strategy.send(STOP) except StopIteration: pass try: strategy.close() except StopIteration: pass
class Catalog(object): def __init__(self, engine): self.engine = engine self.session = Session(bind=self.engine) @subtransaction def add_item(self, code, description, long_description=None): citem = models.Item(code=code, description=description, long_description=long_description) self.session.add(citem) return citem @subtransaction def add_items(self, items): for item in items: # KeyErrors not caught if missing required field try: long_desc = item['long description'] except KeyError: long_desc = None citem = models.Item(code=item['code'], description=item['description'], description_lower=item['description'].lower(), long_description=long_desc) primary=True for cat in item['categories']: q = self.session.query(models.Category).\ filter(models.Category.name == cat) cater = q.first() icater = models.ItemCategory(item=citem, category=cater, primary=primary) citem.categories.append(icater) primary=False self.session.add(citem) def add_category(self, name): self.add_categories([name]) @subtransaction def add_categories(self, names): for name in names: cater = models.Category(name=name) self.session.add(cater) # def remove_category: should make sure that zero use # def rename_category @subtransaction def add_item_category(self, item_id, category_id, primary=False): catitem = models.ItemCategory(item_id=item_id, category_id=category_id, primary=primary) self.session.add(catitem) return catitem # def remove_item_category def get_item(self, code, as_object=False): q = self.session.query(models.Item).filter(models.Item.code == code) item = q.first() if item is None: return None if as_object is True: return item return (item.id, item.code, item.description, item.long_description) def remove_item(self, code): self.session.begin(subtransactions=True) q = self.session.query(models.Item).filter(models.Item.code == code).\ delete() self.session.commit() @subtransaction def update_item(self, code, description=None, long_description=None, new_code=None): q = self.session.query(models.Item).filter(models.Item.code == code) item = q.first() if new_code is not None: item.code = new_code if description is not None: item.description = description item.description_lower = description.lower() if long_description is not None: item.long_description = long_description def get_stock(self, code, as_object=False): q = self.session.query(models.Item, models.StockItem).\ with_entities(models.StockItem).\ filter(models.Item.code == code).\ filter(models.Item.id == models.StockItem.item_id) res = q.first() if res is None: return None if as_object is True: return res return (res.id, res.price, res.count) @subtransaction def add_stock(self, items): for item in items: q = self.session.query(models.Item).\ filter(models.Item.code == item['code']) citem = q.first() stock = models.StockItem(item=citem, count=item['count'], price=item['price']) self.session.add(stock) @subtransaction def update_stock(self, code, count, price): q = self.session.query(models.Item, models.StockItem).\ with_entities(models.StockItem).\ join(models.StockItem.item).\ filter(models.Item.code == code) stock = q.first() if stock is None: q = self.session.query(models.Item).\ filter(models.Item.code == code) item = q.first() if item is not None: stock = models.StockItem(item=item, count=count, price=price) self.session.add(stock) else: raise KeyError('Unknown item code') else: stock.count += count stock.price = price @subtransaction def add_items_with_stock(self, items): categories = set() for item in items: for x in item['categories']: categories.add(x) self.add_categories(categories) self.add_items(items) self.add_stock(items) def list_items(self, sort_key='description', ascending=True, page=1, page_size=10): sq = self.session.query(models.Reservation.stock_item_id, func.sum(models.Reservation.count).\ label('reserved')).\ group_by(models.Reservation.stock_item_id).\ subquery() q = self.session.query(models.Item, models.Category, models.ItemCategory, models.StockItem, sq.c.reserved).\ with_entities(models.Item.code, models.Item.description, models.Category.name, models.StockItem.price, models.StockItem.count, sq.c.reserved).\ join(models.StockItem).\ join(models.ItemCategory, models.Item.id == models.ItemCategory.item_id).\ join(models.Category, models.Category.id == models.ItemCategory.category_id).\ filter(models.ItemCategory.primary == True).\ outerjoin(sq, models.StockItem.id == sq.c.stock_item_id) q = ordering(q, ascending, sort_key) q = pagination(q, page, page_size) res = [item_to_json(*x) for x in q] return res def search_items(self, prefix, price_range, sort_key='description', ascending=True, page=1, page_size=10): sq = self.session.query(models.Reservation.stock_item_id, func.sum(models.Reservation.count).\ label('reserved')).\ group_by(models.Reservation.stock_item_id).\ subquery() q = self.session.query(models.Item, models.Category, models.ItemCategory, models.StockItem, sq.c.reserved).\ with_entities(models.Item.code, models.Item.description, models.Category.name, models.StockItem.price, models.StockItem.count, sq.c.reserved).\ join(models.StockItem).\ join(models.ItemCategory, models.Item.id == models.ItemCategory.item_id).\ join(models.Category, models.Category.id == models.ItemCategory.category_id).\ filter(models.StockItem.price.between(*price_range), models.Item.description_lower.like('{:s}%'.format(prefix.lower())), models.ItemCategory.primary == True).\ outerjoin(sq, models.StockItem.id == sq.c.stock_item_id) q = ordering(q, ascending, sort_key) q = pagination(q, page, page_size) res = [item_to_json(*x) for x in q] return res def list_items_by_prices(self, prices, sort_key='price', prefix=None, ascending=True, page=1, page_size=10): pgs = pg_cases(prices) pg_case = sqla.case(pgs, else_ = -1).label('price_group') sq = self.session.query(models.Reservation.stock_item_id, func.sum(models.Reservation.count).\ label('reserved')).\ group_by(models.Reservation.stock_item_id).\ subquery() q = self.session.query(models.Item, models.Category, models.ItemCategory, models.StockItem, sq.c.reserved).\ with_entities(pg_case, models.Item.code, models.Item.description, models.Category.name, models.StockItem.price, models.StockItem.count, sq.c.reserved) if prefix is not None: q = q.filter(models.Item.description_lower.like('{:s}%'.format(prefix.lower()))) q = q.join(models.StockItem.item).\ outerjoin(sq, models.StockItem.id == sq.c.stock_item_id).\ filter(models.ItemCategory.item_id == models.Item.id, models.ItemCategory.category_id == models.Category.id, models.ItemCategory.primary == True, pg_case >= 0) q = pg_ordering(q, ascending) q = pagination(q, page, page_size) def to_dict(x): tmp = item_to_json(*x[1:]) tmp.update({'price_group': x[0]}) return tmp res = [to_dict(x) for x in q] return res def _get_reservations(self, stock_id): q = self.session.query(func.sum(models.Reservation.count)).\ filter(models.StockItem.id == stock_id).\ filter(models.StockItem.id == models.Reservation.stock_item_id) res = q.first() if res is None or res[0] is None: return 0 return res[0] def _get_reservation(self, basket_item_id): q = self.session.query(models.Basket, models.BasketItem, models.Reservation).\ with_entities(models.Reservation).\ join(models.Reservation.basket_item).\ filter(models.BasketItem.id == basket_item_id) res = q.first() return res @subtransaction def _update_reservation(self, stock, basket_item): reservations = self._get_reservations(basket_item.stock_item_id) # can reserve (scount - reservations) reservation = self._get_reservation(basket_item.id) if reservation is not None: rcount = min(basket_item.count, stock.count - reservations + reservation.count) reservation.count = rcount else: rcount = min(basket_item.count, stock.count - reservations) reservation = models.Reservation(stock_item=stock, basket_item=basket_item, count=rcount) self.session.add(reservation)
def reindex(clear=False): """ Reindex all content; optionally clear index before. All is done in a single transaction. """ svc = current_app.services['indexing'] adapted = svc.adapted index = svc.app_state.indexes['default'] session = Session(bind=current_app.db.session.get_bind(None, None), autocommit=True) setattr(session, '_model_changes', {}) # please flask-sqlalchemy <= 1.0 indexed = set() cleared = set() with AsyncWriter(index) as writer: if clear: print "*" * 80 print "WILL CLEAR INDEX BEFORE REINDEXING" print "*" * 80 writer.writer.mergetype = CLEAR for cls in sorted(svc.app_state.indexed_classes, key=lambda c: c.__name__): current_object_type = cls._object_type() if not clear and current_object_type not in cleared: writer.delete_by_term('object_type', current_object_type) cleared.add(current_object_type) adapter = adapted.get(current_object_type) if not adapter or not adapter.indexable: continue name = cls.__name__ with session.begin(): q = session.query(cls).options(sa.orm.lazyload('*')) count = q.count() if count == 0: print "{}: 0".format(name) continue widgets = [name, ': ', pb.Counter(), '/{}'.format(count), ' ', pb.Timer(), ' ',pb.Percentage(), ' ', pb.Bar(), ' ', pb.ETA(), ] progress = pb.ProgressBar(widgets=widgets, maxval=count) progress.start() count_current = 0 for obj in q.yield_per(1000): if obj.object_type != current_object_type: # may happen if obj is a subclass and its parent class is also # indexable continue object_key = obj.object_key if object_key in indexed: continue document = svc.get_document(obj, adapter) writer.add_document(**document) indexed.add(object_key) count_current += 1 try: progress.update(count_current) except ValueError: pass progress.finish()
def reindex(clear=False, progressive=False, batch_size=None): """Reindex all content; optionally clear index before. All is done in asingle transaction by default. :param clear: clear index content. :param progressive: don't run in a single transaction. :param batch_size: number of documents to process before writing to the index. Unused in single transaction mode. If `None` then all documents of same content type are written at once. """ index_service = get_service("indexing") adapted = index_service.adapted index = index_service.app_state.indexes["default"] session = Session(bind=db.session.get_bind(None, None), autocommit=True) session._model_changes = {} # please flask-sqlalchemy <= 1.0 indexed = set() cleared = set() if batch_size is not None: batch_size = int(batch_size) strategy = progressive_mode if progressive else single_transaction strategy = strategy(index, clear=clear, progressive=progressive, batch_size=batch_size) next(strategy) # starts generator for cls in sorted(index_service.app_state.indexed_classes, key=lambda c: c.__name__): current_object_type = cls._object_type() if not clear and current_object_type not in cleared: strategy.send(current_object_type) cleared.add(current_object_type) adapter = adapted.get(current_object_type) if not adapter or not adapter.indexable: continue name = cls.__name__ with session.begin(): query = session.query(cls).options(sa.orm.lazyload("*")) try: count = query.count() except Exception as e: current_app.logger.error( "Indexing error on class {}: {}".format(name, repr(e))) continue print("*" * 79) print("{}".format(name)) if count == 0: print("*" * 79) print("{}".format(name)) continue print("*" * 79) print("{}".format(name)) count_current = 0 with tqdm(total=count) as bar: for obj in query.yield_per(1000): if obj.object_type != current_object_type: # may happen if obj is a subclass and its parent class # is also indexable bar.update() continue object_key = obj.object_key if object_key in indexed: bar.update() continue document = index_service.get_document(obj, adapter) strategy.send(document) indexed.add(object_key) if batch_size is not None and (count_current % batch_size) == 0: bar.update() strategy.send(COMMIT) bar.update() if batch_size is None: strategy.send(COMMIT) strategy.send(COMMIT) try: strategy.send(STOP) except StopIteration: pass try: strategy.close() except StopIteration: pass
# statement being rolled back. session.rollback() # commit ##################################################### # Commit the transaction and expires the state of all # instances present after the commit is complete.. session.commit() # close ###################################################### # Issues a expunge_all(), and releases any # transactional/connection resources. session.close() # transaction demarcation in auto commit mode session = Session(autocommit=True) session.begin() try: item1 = session.query(User).get(1) item2 = session.query(User).get(2) item1.foo = 'bar' item2.bar = 'foo' session.commit() except: session.rollback() raise with session.begin(): item1 = session.query(User).get(1) item2 = session.query(User).get(2) item1.foo = 'bar' item2.bar = 'foo'