def __init__(self, jsonfile='discord.json'): with open(jsonfile, 'r') as config_file: config = json_to_array(config_file.read()) self.headers = { 'user-agent': config['agent'], 'authorization': config['token'] } self.types = config['types'] self.query = create_query_body(images=config['query']['images'], files=config['query']['files'], embeds=config['query']['embeds'], links=config['query']['links'], videos=config['query']['videos'], nsfw=config['query']['nsfw']) self.directs = config['directs'] self.servers = config['servers'] self.channels = [] for server in self.servers: #assume channels is a list per server, collect all in one flat list self.channels.extend(self.servers[server]) if S3_ENDPOINT_URL is not None: #for local testing using min.io s3 = boto3.client("s3", endpoint_url=S3_ENDPOINT_URL) else: s3 = boto3.client("s3") self.s3 = s3
def __init__(self): with open('discord.json', 'r') as config_file: config = json_to_array(config_file.read()) self.pages = config['pages'] self.query = config['query'] self.types = config['types'].replace(' ', '').split(',') self.header = { 'user-agent': config['agent'], 'authorization': config['token'] } self.servers = config['servers']
def grab_page(self, url, binary = False): try: domain, path = py3_url_split(url) conn = HTTPSConnection(domain, 443) conn.request('GET', path, headers=self.headers) resp = conn.getresponse() if str(resp.status)[0] == '2': return resp.read() if binary else json_to_array(resp.read()) else: stderr.write('\nReceived HTTP %s error: %s' % (resp.status, resp.reason)) except Exception: stderr.write('\nUnknown exception occurred when grabing page contents.')
def grab_page( self, url, binary=False, error='Received HTTP {0} error when gathering JSON data.\n'): domain, path = py3_url_split(url) conn = HTTPSConnection(domain, 443) conn.request('GET', path, headers=self.headers) resp = conn.getresponse() if resp.status == 200: return resp.read() if binary else json_to_array(resp.read()) else: stderr.write(error.format(resp.status)) stdout.write(resp.read().decode('iso-8859-1')) return ''
def grab_page(self, url, binary = False): try: opener_headers = [] for key, val in self.headers.items(): opener_headers.append((key, val.encode('iso-8859-1'))) opener = build_opener() opener.addheaders = opener_headers install_opener(opener) return urlopen(url).read() if binary else json_to_array(urlopen(url).read()) except HTTPError as err: stderr.write('\nReceived HTTP %s error: %s' % (err.code, err.reason)) except Exception: stderr.write('\nUnknown exception occurred when grabing page contents.')
def grab_page( self, url, binary=False, error='Received HTTP {0} error when gathering JSON data.\n'): opener_headers = [] for k, v in zip(self.headers.keys(), self.headers.values()): opener_headers.append((k, v.encode('iso-8859-1'))) opener = build_opener() opener.addheaders = opener_headers install_opener(opener) try: return urlopen(url).read() if binary else json_to_array( urlopen(url).read()) except HTTPError as err: stderr.write(error.format(err.code)) return ''
def __init__(self, jsonfile='discord.json'): with open(jsonfile, 'r') as config_file: config = json_to_array(config_file.read()) self.headers = { 'user-agent': config['agent'], 'authorization': config['token'] } self.types = config['types'] self.query = create_query_body(images=config['query']['images'], files=config['query']['files'], embeds=config['query']['embeds'], links=config['query']['links'], videos=config['query']['videos'], nsfw=config['query']['nsfw']) self.directs = config['directs'] self.servers = config['servers']