def __init__(self, jsonfile='discord.json'):
with open(jsonfile, 'r') as config_file:
config = json_to_array(config_file.read())
self.headers = {
'user-agent': config['agent'],
'authorization': config['token']
}
self.types = config['types']
self.query = create_query_body(images=config['query']['images'],
files=config['query']['files'],
embeds=config['query']['embeds'],
links=config['query']['links'],
videos=config['query']['videos'],
nsfw=config['query']['nsfw'])
self.directs = config['directs']
self.servers = config['servers']
self.channels = []
for server in self.servers:
#assume channels is a list per server, collect all in one flat list
self.channels.extend(self.servers[server])
if S3_ENDPOINT_URL is not None:
#for local testing using min.io
s3 = boto3.client("s3", endpoint_url=S3_ENDPOINT_URL)
else:
s3 = boto3.client("s3")
self.s3 = s3
def __init__(self):
with open('discord.json', 'r') as config_file:
config = json_to_array(config_file.read())
self.pages = config['pages']
self.query = config['query']
self.types = config['types'].replace(' ', '').split(',')
self.header = {
'user-agent': config['agent'],
'authorization': config['token']
}
self.servers = config['servers']
def grab_page(self, url, binary = False):
try:
domain, path = py3_url_split(url)
conn = HTTPSConnection(domain, 443)
conn.request('GET', path, headers=self.headers)
resp = conn.getresponse()
if str(resp.status)[0] == '2':
return resp.read() if binary else json_to_array(resp.read())
else:
stderr.write('\nReceived HTTP %s error: %s' % (resp.status, resp.reason))
except Exception:
stderr.write('\nUnknown exception occurred when grabing page contents.')
def grab_page(
self,
url,
binary=False,
error='Received HTTP {0} error when gathering JSON data.\n'):
domain, path = py3_url_split(url)
conn = HTTPSConnection(domain, 443)
conn.request('GET', path, headers=self.headers)
resp = conn.getresponse()
if resp.status == 200:
return resp.read() if binary else json_to_array(resp.read())
else:
stderr.write(error.format(resp.status))
stdout.write(resp.read().decode('iso-8859-1'))
return ''
def grab_page(self, url, binary = False):
try:
opener_headers = []
for key, val in self.headers.items():
opener_headers.append((key, val.encode('iso-8859-1')))
opener = build_opener()
opener.addheaders = opener_headers
install_opener(opener)
return urlopen(url).read() if binary else json_to_array(urlopen(url).read())
except HTTPError as err:
stderr.write('\nReceived HTTP %s error: %s' % (err.code, err.reason))
except Exception:
stderr.write('\nUnknown exception occurred when grabing page contents.')
def grab_page(
self,
url,
binary=False,
error='Received HTTP {0} error when gathering JSON data.\n'):
opener_headers = []
for k, v in zip(self.headers.keys(), self.headers.values()):
opener_headers.append((k, v.encode('iso-8859-1')))
opener = build_opener()
opener.addheaders = opener_headers
install_opener(opener)
try:
return urlopen(url).read() if binary else json_to_array(
urlopen(url).read())
except HTTPError as err:
stderr.write(error.format(err.code))
return ''
def __init__(self, jsonfile='discord.json'):
with open(jsonfile, 'r') as config_file:
config = json_to_array(config_file.read())
self.headers = {
'user-agent': config['agent'],
'authorization': config['token']
}
self.types = config['types']
self.query = create_query_body(images=config['query']['images'],
files=config['query']['files'],
embeds=config['query']['embeds'],
links=config['query']['links'],
videos=config['query']['videos'],
nsfw=config['query']['nsfw'])
self.directs = config['directs']
self.servers = config['servers']