def __init__(self, host, timeout = CONFIG.TIME_OUT, proxy = CONFIG.PROXY, user_agents_type = ''):
self.timeout = timeout
self.proxy = proxy
self.store_cookie = False
self.redirect = True #flag redirect when check found Location in header
self.rand_useragent = True #random useragents flag
if user_agents_type == 'bot':
self.user_agents = Bot_User_Agent()
else:
self.user_agents = User_Agent()
self.user_agent = self.user_agents.user_agent
HTTPConnection.debuglevel = 10
######################################
self.url = ''
self.method = 'GET'
self.headers = {
'User-Agent': self.user_agent,
'Accept-Encoding': 'gzip, deflate',
'Connection': 'close'
}
######################################
https = False
if self.proxy != None:
host = proxy
if host.startswith('http'):
parser = parse.urlparse(host)
self.host = parser.hostname
self.port = parser.port
if self.port == None:
if parser.scheme == 'https':
self.port = 443
else:
self.port = 80
if parser.scheme == 'https':
https = True
_user = parser.username if parser.username else ''
_pass = parser.password if parser.password else ''
_login = str(_user) + ':' + str(_pass)
else:
spliter = host.split(':')
self.host = spliter[0]
if len(spliter) < 2:
self.port = 80
if len(_login) > 1:
self.headers.update({'Authorization': "Basic " + b64encode(_login.encode('ascii')).decode('utf-8')})
if https:
self.request = HTTPSConnection(self.host,self.port, timeout = self.timeout)
else:
self.request = HTTPConnection(self.host,self.port, timeout = self.timeout)
