def __call__(self, key=None, reverse=False):
if self.task.source:
state = self.task.get_state()
error = self.task.target.models.errors.alias('error')
table = self.task.source.table.alias('table')
# Filter by key if provided
if key is not None:
row = self.task.source.last(key)
if row is None:
return
where = sa.and_(
error.c.state_id == state.id,
error.c.row_id == row.id,
)
else:
where = error.c.state_id == state.id
# Ordering
if reverse:
order_by = error.c.id.desc()
else:
order_by = error.c.id
# Query if all tables stored in same database
if self.task.target.samedb and self.task.source.samedb:
query = (
sa.select([error, table], use_labels=True).
select_from(
error.
join(table, error.c.row_id == table.c.id)
).
where(where).
order_by(order_by)
)
for row in windowed_query(self.task.target.engine, query, table.c.id):
item = strip_prefix(row, 'error_')
item['row'] = create_row(strip_prefix(row, 'table_'))
yield item
# Query if some tables are stored in external database
else:
query = error.select(where).order_by(order_by)
for err in windowed_query(self.task.target.engine, query, error.c.id):
query = table.select(table.c.id == err['row_id'])
row = self.task.source.engine.execute(query).first()
if row:
yield Row(err, row=create_row(row))
def rows(self):
if self.source:
table = self.source.table
query = table.select(
table.c.id > self.get_state().offset).order_by(table.c.id)
for row in windowed_query(self.source.engine, query, table.c.id):
yield create_row(row)
def __call__(self, key=None, reverse=False):
if self.task.source:
state = self.task.get_state()
error = self.task.target.models.errors.alias('error')
table = self.task.source.table.alias('table')
# Filter by key if provided
if key is not None:
row = self.task.source.last(key)
if row is None:
return
where = sa.and_(
error.c.state_id == state.id,
error.c.row_id == row.id,
)
else:
where = error.c.state_id == state.id
# Ordering
if reverse:
order_by = error.c.id.desc()
else:
order_by = error.c.id
# Query if all tables stored in same database
if self.task.target.samedb and self.task.source.samedb:
query = (sa.select(
[error, table], use_labels=True).select_from(
error.join(table, error.c.row_id == table.c.id)).where(
where).order_by(order_by))
for row in windowed_query(self.task.target.engine, query,
table.c.id):
item = strip_prefix(row, 'error_')
item['row'] = create_row(strip_prefix(row, 'table_'))
yield item
# Query if some tables are stored in external database
else:
query = error.select(where).order_by(order_by)
for err in windowed_query(self.task.target.engine, query,
error.c.id):
query = table.select(table.c.id == err['row_id'])
row = self.task.source.engine.execute(query).first()
if row:
yield Row(err, row=create_row(row))
def last(self, key=None):
if key:
query = self.table.select().where(self.table.c.key == serkey(key)).order_by(self.table.c.id.desc())
else:
query = self.table.select().order_by(self.table.c.id.desc())
row = self.engine.execute(query).first()
return create_row(row) if row else None
def last(self, key=None):
if key:
query = self.table.select().where(
self.table.c.key == serkey(key)).order_by(
self.table.c.id.desc())
else:
query = self.table.select().order_by(self.table.c.id.desc())
row = self.engine.execute(query).first()
return create_row(row) if row else None
def merge(self):
"""Merge all duplicate value, newer values overwrites older values.
Dicts will be merged recursively.
After merge, old values will be left as is, use compact to remove them.
"""
query = self.table.select().order_by(self.table.c.key, self.table.c.created)
rows = (create_row(row) for row in windowed_query(self.engine, query, self.table.c.id))
self.append(merge_rows((row.key, row.value) for row in rows))
return self
def merge(self):
"""Merge all duplicate value, newer values overwrites older values.
Dicts will be merged recursively.
After merge, old values will be left as is, use compact to remove them.
"""
query = self.table.select().order_by(self.table.c.key,
self.table.c.created)
rows = (create_row(row)
for row in windowed_query(self.engine, query, self.table.c.id))
self.append(merge_rows((row.key, row.value) for row in rows))
return self
def call(self, pipe, limit=10, table=False, include=None, exclude=None):
from databot.db.utils import create_row
rows = [create_row(row) for row in self.rows(pipe, limit)]
if rows:
exclude = exclude.split(',') if exclude else None
include = include.split(',') if include else None
if table:
self.bot.output.table(rows, exclude=exclude, include=include)
else:
for row in rows:
self.bot.output.key_value(row.key, row.value, exclude=exclude)
else:
self.info('No items found.')
def call(self, pipe, limit=10, table=False, include=None, exclude=None):
from databot.db.utils import create_row
rows = [create_row(row) for row in self.rows(pipe, limit)]
if rows:
exclude = exclude.split(',') if exclude else None
include = include.split(',') if include else None
if table:
self.bot.output.table(rows, exclude=exclude, include=include)
else:
for row in rows:
self.bot.output.key_value(row.key,
row.value,
exclude=exclude)
else:
self.info('No items found.')
def getall(self, key, reverse=False):
order_by = self.table.c.id.desc() if reverse else self.table.c.id
query = self.table.select().where(
self.table.c.key == serkey(key)).order_by(order_by)
for row in windowed_query(self.engine, query, self.table.c.id):
yield create_row(row)
def rows(self, desc=False):
order_by = self.table.c.id.desc() if desc else self.table.c.id
query = self.table.select().order_by(order_by)
for row in windowed_query(self.engine, query, self.table.c.id):
yield create_row(row)
def getall(self, key, reverse=False):
order_by = self.table.c.id.desc() if reverse else self.table.c.id
query = self.table.select().where(self.table.c.key == serkey(key)).order_by(order_by)
for row in windowed_query(self.engine, query, self.table.c.id):
yield create_row(row)
def rows(self, desc=False):
order_by = self.table.c.id.desc() if desc else self.table.c.id
query = self.table.select().order_by(order_by)
for row in windowed_query(self.engine, query, self.table.c.id):
yield create_row(row)
def rows(self):
if self.source:
table = self.source.table
query = table.select(table.c.id > self.get_state().offset).order_by(table.c.id)
for row in windowed_query(self.source.engine, query, table.c.id):
yield create_row(row)