def all_fields(self): session = self.create_session() try: for window in windowed_query( session.query( AuthorItem.Model).filter(AuthorItem.Model.org != None), AuthorItem.Model.org, 1000): yield window finally: session.close()
def missing_authors(self): session = self.create_session() try: q = session.query(AuthorItem.Model).filter(or_(AuthorItem.Model.measures == None, AuthorItem.Model.org == None, AuthorItem.Model.hasCo == None)) for window in windowed_query(q, AuthorItem.Model.id, 1000): yield window finally: session.close()
def all_fields(self): session = self.create_session() try: # to do: only get unprocessed author ids for window in windowed_query( session.query(AuthorItem.Model).filter( AuthorItem.Model.hasCo == True), AuthorItem.Model.id, 1000): yield window finally: session.close()
def missing_authors(self): session = self.create_session() try: q = session.query(AuthorItem.Model).filter( or_(AuthorItem.Model.measures == None, AuthorItem.Model.org == None, AuthorItem.Model.hasCo == None)) for window in windowed_query(q, AuthorItem.Model.id, 1000): yield window finally: session.close()
def next_window(self): session = self.create_session() window_size = 1000 start_row = self.state.get('start_row', 0) ffwd = True self.logger.debug('Starting row %d' % start_row) try: rows = windowed_query(session.query(FOSItem.Model), FOSItem.Model.field_name, window_size, asc=False) for row in rows: if ffwd: for i in range(0, start_row): # fast-forward seen rows... self.logger.debug('Fast-forwarding...') rows.next() ffwd = False yield row self.state['start_row'] = self.state.get('start_row', 0) + 1 finally: session.close()