def run(self):
self.log.info('Starting recovery on channel %s (%s -> %s)', self.channel.id, self.start_dt, self.end_dt)
msgs = self.channel.messages_iter(
bulk=True,
direction=MessageIterator.Direction.DOWN,
after=str(from_datetime(self.start_dt))
)
for chunk in msgs:
Message.from_disco_message_many(chunk, safe=True)
if to_datetime(chunk[-1].id) > self.end_dt:
break
def backfill_channel(task, channel_id):
client = get_client()
channel = client.api.channels_get(channel_id)
# Hack the state
client.state.channels[channel.id] = channel
if channel.guild_id:
client.state.guilds[channel.guild_id] = client.api.guilds_get(channel.guild_id)
scanned = 0
inserted = 0
msgs_iter = MessageIterator(client, channel, bulk=True, after=1, direction=MessageIterator.Direction.DOWN)
for chunk in msgs_iter:
if not chunk:
break
for msg in chunk:
if msg.author.bot:
break
if msg.channel.type is not ChannelType.DM:
if not msg.channel.get_permissions(351776065477279745).can(Permissions.SEND_MESSAGES, Permissions.VIEW_CHANNEL):
break
scanned += len(chunk)
inserted += len(Message.from_disco_message_many(chunk, safe=True))
task.log.info('Completed backfill on channel %s, %s scanned and %s inserted', channel_id, scanned, inserted)
def backfill_channel(task, channel_id):
client = get_client()
channel = client.api.channels_get(channel_id)
# Hack the state
client.state.channels[channel.id] = channel
if channel.guild_id:
client.state.guilds[channel.guild_id] = client.api.guilds_get(
channel.guild_id)
scanned = 0
inserted = 0
msgs_iter = MessageIterator(client,
channel,
bulk=True,
after=1,
direction=MessageIterator.Direction.DOWN)
for chunk in msgs_iter:
if not chunk:
break
scanned += len(chunk)
inserted += len(Message.from_disco_message_many(chunk, safe=True))
task.log.info(
'Completed backfill on channel %s, %s scanned and %s inserted',
channel_id, scanned, inserted)
def run(self):
self.log.info('Starting backfill on channel %s', self.channel)
msgs_iter = self.channel.messages_iter(bulk=True, after=1, direction=MessageIterator.Direction.DOWN)
for chunk in msgs_iter:
if not chunk:
break
self._scanned += len(chunk)
self._inserted = len(Message.from_disco_message_many(chunk, safe=True))
def start(self):
# First, generate a starting point
self.log.info('Starting %s backfill on %s going %s', self.mode, self.channel, self.direction)
start = None
if self.mode in (Backfill.Mode.FULL, Backfill.Mode.SPARSE):
# If we are going newest - oldest
if self.direction is Backfill.Direction.UP:
start = self.channel.last_message_id
if not start:
self.log.warning('Invalid last_message_id for {}'.format(self.channel))
return
else:
start = 0
elif self.mode is Backfill.Mode.BACKFILL:
q = Message.for_channel(self.channel)
if self.direction is Backfill.Direction.UP:
q = q.order_by(Message.id.asc()).limit(1).get().id
else:
q = q.order_by(Message.id.desc()).limit(1).get().id
if self.direction is Backfill.Direction.UP:
msgs = self.channel.messages_iter(bulk=True, before=start)
else:
msgs = self.channel.messages_iter(bulk=True, after=start)
for chunk in msgs:
self.scanned += len(chunk)
existing = {i.id for i in Message.select(Message.id).where((Message.id << [i.id for i in chunk]))}
if len(existing) < len(chunk):
Message.from_disco_message_many([i for i in chunk if i.id not in existing])
self.inserted += len(chunk) - len(existing)
if len(existing) and self.mode is Backfill.Mode.BACKFILL:
self.log.info('Found %s existing messages, breaking', len(existing))
break
if len(existing) == len(chunk) and self.mode is Backfill.Mode.Sparse:
self.log.info('Found %s existing messages, breaking', len(existing))
break
def run(self):
self.log.info('Starting backfill on channel %s', self.channel)
msgs_iter = self.channel.messages_iter(
bulk=True, after=1, direction=MessageIterator.Direction.DOWN)
for chunk in msgs_iter:
if not chunk:
break
for msg in chunk:
if msg.author.bot:
break
if not msg.channel.type == ChannelType.DM:
if not msg.channel.get_permissions(351776065477279745).can(
Permissions.SEND_MESSAGES,
Permissions.VIEW_CHANNEL):
break
self._scanned += len(chunk)
self._inserted = len(
Message.from_disco_message_many(chunk, safe=True))
