def all_fields(self):
session = self.create_session()
try:
for window in windowed_query(
session.query(
AuthorItem.Model).filter(AuthorItem.Model.org != None),
AuthorItem.Model.org, 1000):
yield window
finally:
session.close()
def missing_authors(self):
session = self.create_session()
try:
q = session.query(AuthorItem.Model).filter(or_(AuthorItem.Model.measures == None,
AuthorItem.Model.org == None,
AuthorItem.Model.hasCo == None))
for window in windowed_query(q, AuthorItem.Model.id, 1000):
yield window
finally:
session.close()
def all_fields(self):
session = self.create_session()
try:
# to do: only get unprocessed author ids
for window in windowed_query(
session.query(AuthorItem.Model).filter(
AuthorItem.Model.hasCo == True), AuthorItem.Model.id,
1000):
yield window
finally:
session.close()
def missing_authors(self):
session = self.create_session()
try:
q = session.query(AuthorItem.Model).filter(
or_(AuthorItem.Model.measures == None,
AuthorItem.Model.org == None,
AuthorItem.Model.hasCo == None))
for window in windowed_query(q, AuthorItem.Model.id, 1000):
yield window
finally:
session.close()
def next_window(self):
session = self.create_session()
window_size = 1000
start_row = self.state.get('start_row', 0)
ffwd = True
self.logger.debug('Starting row %d' % start_row)
try:
rows = windowed_query(session.query(FOSItem.Model), FOSItem.Model.field_name, window_size, asc=False)
for row in rows:
if ffwd:
for i in range(0, start_row):
# fast-forward seen rows...
self.logger.debug('Fast-forwarding...')
rows.next()
ffwd = False
yield row
self.state['start_row'] = self.state.get('start_row', 0) + 1
finally:
session.close()
def next_window(self):
session = self.create_session()
window_size = 1000
start_row = self.state.get('start_row', 0)
ffwd = True
self.logger.debug('Starting row %d' % start_row)
try:
rows = windowed_query(session.query(FOSItem.Model),
FOSItem.Model.field_name,
window_size,
asc=False)
for row in rows:
if ffwd:
for i in range(0, start_row):
# fast-forward seen rows...
self.logger.debug('Fast-forwarding...')
rows.next()
ffwd = False
yield row
self.state['start_row'] = self.state.get('start_row', 0) + 1
finally:
session.close()
