def from_queue(self, max_date, batch_limit=50, kind=None):
from r2.models import is_banned_IP, Account, Thing
keep_trying = True
min_id = None
s = self.queue_table
while keep_trying:
where = [s.c.date < max_date]
if min_id:
where.append(s.c.uid > min_id)
if kind:
where.append(s.c.kind == kind)
res = sa.select([
s.c.to_addr, s.c.account_id, s.c.from_name, s.c.fullname,
s.c.body, s.c.kind, s.c.ip, s.c.date, s.c.uid, s.c.msg_hash,
s.c.fr_addr, s.c.reply_to
],
sa.and_(*where),
order_by=s.c.uid,
limit=batch_limit).execute()
res = res.fetchall()
if not res: break
# batch load user accounts
aids = [x[1] for x in res if x[1] > 0]
accts = Account._byID(aids, data=True,
return_dict=True) if aids else {}
# batch load things
tids = [x[3] for x in res if x[3]]
things = Thing._by_fullname(tids, data=True,
return_dict=True) if tids else {}
# make sure no IPs have been banned in the mean time
ips = set(x[6] for x in res)
ips = dict((ip, is_banned_IP(ip)) for ip in ips)
# get the lower bound date for next iteration
min_id = max(x[8] for x in res)
# did we not fetch them all?
keep_trying = (len(res) == batch_limit)
for (addr, acct, fname, fulln, body, kind, ip, date, uid, msg_hash,
fr_addr, reply_to) in res:
yield (accts.get(acct), things.get(fulln), addr, fname, date,
ip, ips[ip], kind, msg_hash, body, fr_addr, reply_to)
def from_queue(self, max_date, batch_limit = 50, kind = None):
from r2.models import is_banned_IP, Account, Thing
keep_trying = True
min_id = None
s = self.queue_table
while keep_trying:
where = [s.c.date < max_date]
if min_id:
where.append(s.c.uid > min_id)
if kind:
where.append(s.c.kind == kind)
res = sa.select([s.c.to_addr, s.c.account_id,
s.c.from_name, s.c.fullname, s.c.body,
s.c.kind, s.c.ip, s.c.date, s.c.uid,
s.c.msg_hash, s.c.fr_addr, s.c.reply_to],
sa.and_(*where),
order_by = s.c.uid, limit = batch_limit).execute()
res = res.fetchall()
if not res: break
# batch load user accounts
aids = [x[1] for x in res if x[1] > 0]
accts = Account._byID(aids, data = True,
return_dict = True) if aids else {}
# batch load things
tids = [x[3] for x in res if x[3]]
things = Thing._by_fullname(tids, data = True,
return_dict = True) if tids else {}
# make sure no IPs have been banned in the mean time
ips = set(x[6] for x in res)
ips = dict((ip, is_banned_IP(ip)) for ip in ips)
# get the lower bound date for next iteration
min_id = max(x[8] for x in res)
# did we not fetch them all?
keep_trying = (len(res) == batch_limit)
for (addr, acct, fname, fulln, body, kind, ip, date, uid,
msg_hash, fr_addr, reply_to) in res:
yield (accts.get(acct), things.get(fulln), addr,
fname, date, ip, ips[ip], kind, msg_hash, body,
fr_addr, reply_to)