def requestdirect(url, close=True, redirect=True, followredirect=False, error=False, proxy=None, post=None, headers=None, mobile=False, limit=None, referer=None, cookie=None, output='', timeout='30', httpsskip=False, use_web_proxy=False, XHR=False, IPv4=False):
try:
urlhost = re.findall('([\w]+[.][\w]+)$', urlparse.urlparse(url.strip().lower()).netloc)[0]
if headers == None:
headers = {'Connection' : 'keep-alive'}
headers['User-Agent'] = client.randomagent()
res = client.request(url = PROXY_URL + url, close=close, redirect=redirect, followredirect=followredirect, error=error, proxy=proxy, post=post, headers=headers, mobile=mobile, limit=limit, referer=referer, cookie=cookie, output=output, timeout=timeout, httpsskip=httpsskip, use_web_proxy=use_web_proxy, XHR=XHR, IPv4=IPv4)
page_data_string = client.getPageDataBasedOnOutput(res, output)
#print page_data_string
pattern = re.compile('<script[\s\S]+?/script>')
page_data_string = re.sub(pattern, '', page_data_string)
try:
page_data_string = page_data_string.replace('\n','')
#page_data_string = page_data_string.replace('\r','r').replace('\n','<br/>').replace('\w','').replace('\.','').replace('\t','').replace('\ ','')
except Exception as e:
log('FAIL','requestdirect-1', '%s' % e, dolog=False)
#print page_data_string
try:
page_data_stringx = json.dumps(page_data_string)
page_data_stringx = page_data_stringx.replace('\\','')
page_data_stringx = page_data_stringx[1:-1]
page_data_string = page_data_stringx
except Exception as e:
log('FAIL','requestdirect-2', '%s' % e, dolog=False)
#print page_data_string
#page_data_string = str(page_data_string)
try:
r = unicode(page_data_string, "utf-8")
page_data_string = r
except Exception as e:
log('FAIL','requestdirect-3', '%s' % e, dolog=False)
try:
r = str(page_data_string)
page_data_string = r
except Exception as e:
log('FAIL','requestdirect-4', '%s' % e, dolog=False)
page_data_string = page_data_string.replace('https://unblock.co/browse.php?', '')
page_data_string = page_data_string.replace('/browse.php?u=', '')
page_data_string = page_data_string.replace('b=4', '')
page_data_string = page_data_string.replace('u=', '')
page_data_string = page_data_string.replace('&http', 'http')
page_data_string = page_data_string.replace('/http', 'http')
try:
page_data_string = page_data_string.decode('utf-8')
except:
pass
try:
page_data_string = urllib.unquote_plus(page_data_string)
except:
pass
try:
page_data_string = page_data_string.encode('utf-8')
except:
pass
return client.getResponseDataBasedOnOutput(page_data_string, res, output)
except Exception as e:
log('ERROR','requestdirect', '%s' % e)
return None
def requestdirect(url,
close=True,
redirect=True,
followredirect=False,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
limit=None,
referer=None,
cookie=None,
output='',
timeout='30',
httpsskip=False,
use_web_proxy=False,
XHR=False,
IPv4=False):
#try:
print "Requesting: %s Using via: %s" % (url, PROXY_URL)
urlhost = re.findall('([\w]+[.][\w]+)$',
urlparse.urlparse(url.strip().lower()).netloc)[0]
if headers == None:
headers = {'Connection': 'keep-alive'}
headers['User-Agent'] = client.randomagent()
res = client.request(url=PROXY_URL + url,
close=close,
redirect=redirect,
followredirect=followredirect,
error=error,
proxy=proxy,
post=post,
headers=headers,
mobile=mobile,
limit=limit,
referer=referer,
cookie=cookie,
output=output,
timeout=timeout,
httpsskip=httpsskip,
use_web_proxy=use_web_proxy,
XHR=XHR,
IPv4=IPv4)
page_data_string = client.getPageDataBasedOnOutput(res, output)
page_data_string = page_data_string.decode('utf-8')
page_data_string = urllib.unquote_plus(page_data_string)
page_data_string = page_data_string.encode('utf-8')
page_data_string = page_data_string.replace(
'/o.php?b=4&f=frame&mobile=&u=', '')
page_data_string = page_data_string.replace(
'/o.php?b=4&mobile=&u=', '')
page_data_string = page_data_string.replace('/o.php?b=4&mobile=&u=', '')
# page_data_string_t = None
# regex = r'{.*[token:]}]}'
# matches = re.finditer(regex, page_data_string)
# for matchNum, match in enumerate(matches):
# page_data_string_t = match.group()
# break
# if page_data_string_t != None and 'token' in page_data_string_t:
# page_data_string = page_data_string_t
return client.getResponseDataBasedOnOutput(page_data_string, res, output)
#except Exception as e:
# print "Error: %s - %s" % (name, e)
# return None
def requestdirect(url,
close=True,
redirect=True,
followredirect=False,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
limit=None,
referer=None,
cookie=None,
output='',
timeout='30',
httpsskip=False,
use_web_proxy=False,
XHR=False,
IPv4=False):
#try:
control.log("Requesting: %s Using via: %s" % (url, PROXY_URL))
urlhost = re.findall('([\w]+[.][\w]+)$',
urlparse.urlparse(url.strip().lower()).netloc)[0]
if headers == None:
headers = {'Connection': 'keep-alive'}
else:
headers['Connection'] = 'keep-alive'
res = client.request(url=PROXY_URL + url,
close=close,
redirect=redirect,
followredirect=followredirect,
error=error,
proxy=proxy,
post=post,
headers=headers,
mobile=mobile,
limit=limit,
referer=referer,
cookie=cookie,
output=output,
timeout=timeout,
httpsskip=httpsskip,
use_web_proxy=use_web_proxy,
XHR=XHR,
IPv4=IPv4)
page_data_string = client.getPageDataBasedOnOutput(res, output)
pattern = re.compile('<script[\s\S]+?/script>')
page_data_string = re.sub(pattern, '', page_data_string)
try:
page_data_string = page_data_string.replace('\n', '')
#page_data_string = page_data_string.replace('\r','r').replace('\n','<br/>').replace('\w','').replace('\.','').replace('\t','').replace('\ ','')
except Exception as e:
control.log("Error1: %s - %s" % (name, e))
#print page_data_string
page_data_string = json.dumps(page_data_string)
page_data_string = page_data_string.replace('\\', '')
page_data_string = page_data_string[1:-1]
#print page_data_string
#page_data_string = str(page_data_string)
try:
r = unicode(page_data_string, "utf-8")
page_data_string = r
except Exception as e:
control.log("Error2: %s - %s" % (name, e))
try:
r = str(page_data_string)
page_data_string = r
except Exception as e:
control.log("Error3: %s - %s" % (name, e))
PROXY_PART0A = PROXY_PART0 % urlhost
PROXY_PART1A = PROXY_PART1 % urlhost
PROXY_PART3A = PROXY_PART3 % (urlhost, urlhost)
PROXY_PART3A_REPLACE = PROXY_PART3_REPLACE % urlhost
page_data_string = page_data_string.replace(PROXY_PART0A,
PROXY_PART1_REPLACE)
page_data_string = page_data_string.replace(PROXY_PART1A,
PROXY_PART1_REPLACE)
page_data_string = page_data_string.replace(PROXY_PART2A,
PROXY_PART2_REPLACE)
page_data_string = page_data_string.replace(PROXY_PART2B,
PROXY_PART2_REPLACE)
page_data_string = page_data_string.replace(PROXY_PART3A,
PROXY_PART3A_REPLACE)
return client.getResponseDataBasedOnOutput(page_data_string, res, output)
# except Exception as e:
# print "Error: %s - %s" % (name, e)
# return None
def requestdirect(url,
close=True,
redirect=True,
followredirect=False,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
limit=None,
referer=None,
cookie=None,
output='',
timeout='30',
httpsskip=False,
use_web_proxy=False,
XHR=False,
IPv4=False):
print "Requesting: %s Using via: %s" % (url, PROXY_URL)
urlhost = re.findall('([\w]+[.][\w]+)$',
urlparse.urlparse(url.strip().lower()).netloc)[0]
if headers == None:
headers = {'Connection': 'keep-alive'}
headers['User-Agent'] = client.randomagent()
res = client.request(url=PROXY_URL + url,
close=close,
redirect=redirect,
followredirect=followredirect,
error=error,
proxy=proxy,
post=post,
headers=headers,
mobile=mobile,
limit=limit,
referer=referer,
cookie=cookie,
output=output,
timeout=timeout,
httpsskip=httpsskip,
use_web_proxy=use_web_proxy,
XHR=XHR,
IPv4=IPv4)
page_data_string = client.getPageDataBasedOnOutput(res, output)
pattern = re.compile('<script[\s\S]+?/script>')
page_data_string = re.sub(pattern, '', page_data_string)
try:
page_data_string = page_data_string.replace('\n', '')
#page_data_string = page_data_string.replace('\r','r').replace('\n','<br/>').replace('\w','').replace('\.','').replace('\t','').replace('\ ','')
except Exception as e:
control.log("Error1: %s - %s" % (name, e))
#print page_data_string
page_data_string = json.dumps(page_data_string)
page_data_string = page_data_string.replace('\\', '')
page_data_string = page_data_string[1:-1]
#print page_data_string
#page_data_string = str(page_data_string)
try:
r = unicode(page_data_string, "utf-8")
page_data_string = r
except Exception as e:
control.log("Error2: %s - %s" % (name, e))
try:
r = str(page_data_string)
page_data_string = r
except Exception as e:
control.log("Error3: %s - %s" % (name, e))
page_data_string = page_data_string.replace(
'https://www.xperienc.com/browsexp.php?', '')
page_data_string = page_data_string.replace('b=40', '')
page_data_string = page_data_string.replace('u=', '')
page_data_string = page_data_string.replace('browsexp.php?', '')
page_data_string = page_data_string.replace('&http', 'http')
page_data_string = page_data_string.replace('/http', 'http')
page_data_string = page_data_string.decode('utf-8')
page_data_string = urllib.unquote_plus(page_data_string)
page_data_string = page_data_string.encode('utf-8')
return client.getResponseDataBasedOnOutput(page_data_string, res, output)