def grab_data(self,
folder,
server,
channel,
num_comments=1000,
isdm=False):
"""Scan and grab the attachments. Default number of comments to extract is 1000
:param folder: The folder name.
:param server: The server name.
:param channel: The channel name.
:param isdm: A flag to check whether we're in a DM or not.
"""
date = datetime.today()
while date.year >= 2015 and num_comments > 0:
request = SimpleRequest(self.headers).request
today = get_day(date.day, date.month, date.year)
if not isdm:
request.set_header(
'referer', 'https://discordapp.com/channels/%s/%s' %
(server, channel))
content = request.grab_page(
'https://discordapp.com/api/%s/guilds/%s/messages/search?channel_id=%s&min_id=%s&max_id=%s&%s'
% (self.api, server, channel, today['00:00'],
today['23:59'], self.query))
else:
request.set_header(
'referer',
'https://discordapp.com/channels/@me/%s' % channel)
content = request.grab_page(
'https://discordapp.com/api/%s/channels/%s/messages/search?min_id=%s&max_id=%s&%s'
% (self.api, channel, today['00:00'], today['23:59'],
self.query))
try:
if content['messages'] is not None:
for messages in content['messages']:
for message in messages:
# self.check_config_mimetypes(message, folder)
if self.types['text'] is True:
if len(message['content']) > 0:
self.insert_text(server, channel, message)
if num_comments % 100 == 0:
print(
f"adding {num_comments}th comment to database"
)
num_comments -= 1
except TypeError:
continue
date += timedelta(days=-1)
def download(self, url, folder):
"""Download the contents of a URL.
:param url: The target URL.
:param folder: The target folder.
"""
request = SimpleRequest(self.headers).request
request.set_header('user-agent', 'Mozilla/5.0 (X11; Linux x86_64) Chrome/78.0.3904.87 Safari/537.36')
filename = safe_name('%s_%s' % (url.split('/')[-2], url.split('/')[-1]))
if not path.exists(filename):
request.stream_file(url, folder, filename, self.buffer)
def grab_data(self, folder, server, channel, isdm=False):
"""Scan and grab the attachments.
:param folder: The folder name.
:param server: The server name.
:param channel: The channel name.
:param isdm: A flag to check whether we're in a DM or not.
"""
tzdata = gmtime(time())
try:
for year in range(tzdata.tm_year, 2015, -1):
for month in range(12, 1, -1):
for day in range(31, 1, -1):
if month > tzdata.tm_mon and year == tzdata.tm_year:
continue
if month == tzdata.tm_mon and day > tzdata.tm_mday:
continue
request = SimpleRequest(self.headers).request
today = get_day(day, month, year)
if not isdm:
request.set_header('referer', 'https://discordapp.com/channels/%s/%s' % (server, channel))
content = request.grab_page(
'https://discordapp.com/api/%s/guilds/%s/messages/search?channel_id=%s&min_id=%s&max_id=%s&%s' %
(self.api, server, channel, today['00:00'], today['23:59'], self.query)
)
else:
request.set_header('referer', 'https://discordapp.com/channels/@me/%s' % channel)
content = request.grab_page(
'https://discordapp.com/api/%s/channels/%s/messages/search?min_id=%s&max_id=%s&%s' %
(self.api, channel, today['00:00'], today['23:59'], self.query)
)
try:
if content['messages'] is not None:
for messages in content['messages']:
for message in messages:
self.check_config_mimetypes(message, folder)
if self.types['text'] is True:
if len(message['content']) > 0:
self.insert_text(server, channel, message)
except TypeError:
continue
except ValueError:
pass