def requestdirect(url, close=True, redirect=True, followredirect=False, error=False, proxy=None, post=None, headers=None, mobile=False, limit=None, referer=None, cookie=None, output='', timeout='30', httpsskip=False, use_web_proxy=False, XHR=False, IPv4=False): try: urlhost = re.findall('([\w]+[.][\w]+)$', urlparse.urlparse(url.strip().lower()).netloc)[0] if headers == None: headers = {'Connection' : 'keep-alive'} headers['User-Agent'] = client.randomagent() res = client.request(url = PROXY_URL + url, close=close, redirect=redirect, followredirect=followredirect, error=error, proxy=proxy, post=post, headers=headers, mobile=mobile, limit=limit, referer=referer, cookie=cookie, output=output, timeout=timeout, httpsskip=httpsskip, use_web_proxy=use_web_proxy, XHR=XHR, IPv4=IPv4) page_data_string = client.getPageDataBasedOnOutput(res, output) #print page_data_string pattern = re.compile('<script[\s\S]+?/script>') page_data_string = re.sub(pattern, '', page_data_string) try: page_data_string = page_data_string.replace('\n','') #page_data_string = page_data_string.replace('\r','r').replace('\n','<br/>').replace('\w','').replace('\.','').replace('\t','').replace('\ ','') except Exception as e: log('FAIL','requestdirect-1', '%s' % e, dolog=False) #print page_data_string try: page_data_stringx = json.dumps(page_data_string) page_data_stringx = page_data_stringx.replace('\\','') page_data_stringx = page_data_stringx[1:-1] page_data_string = page_data_stringx except Exception as e: log('FAIL','requestdirect-2', '%s' % e, dolog=False) #print page_data_string #page_data_string = str(page_data_string) try: r = unicode(page_data_string, "utf-8") page_data_string = r except Exception as e: log('FAIL','requestdirect-3', '%s' % e, dolog=False) try: r = str(page_data_string) page_data_string = r except Exception as e: log('FAIL','requestdirect-4', '%s' % e, dolog=False) page_data_string = page_data_string.replace('https://unblock.co/browse.php?', '') page_data_string = page_data_string.replace('/browse.php?u=', '') page_data_string = page_data_string.replace('b=4', '') page_data_string = page_data_string.replace('u=', '') page_data_string = page_data_string.replace('&http', 'http') page_data_string = page_data_string.replace('/http', 'http') try: page_data_string = page_data_string.decode('utf-8') except: pass try: page_data_string = urllib.unquote_plus(page_data_string) except: pass try: page_data_string = page_data_string.encode('utf-8') except: pass return client.getResponseDataBasedOnOutput(page_data_string, res, output) except Exception as e: log('ERROR','requestdirect', '%s' % e) return None
def requestdirect(url, close=True, redirect=True, followredirect=False, error=False, proxy=None, post=None, headers=None, mobile=False, limit=None, referer=None, cookie=None, output='', timeout='30', httpsskip=False, use_web_proxy=False, XHR=False, IPv4=False): #try: print "Requesting: %s Using via: %s" % (url, PROXY_URL) urlhost = re.findall('([\w]+[.][\w]+)$', urlparse.urlparse(url.strip().lower()).netloc)[0] if headers == None: headers = {'Connection': 'keep-alive'} headers['User-Agent'] = client.randomagent() res = client.request(url=PROXY_URL + url, close=close, redirect=redirect, followredirect=followredirect, error=error, proxy=proxy, post=post, headers=headers, mobile=mobile, limit=limit, referer=referer, cookie=cookie, output=output, timeout=timeout, httpsskip=httpsskip, use_web_proxy=use_web_proxy, XHR=XHR, IPv4=IPv4) page_data_string = client.getPageDataBasedOnOutput(res, output) page_data_string = page_data_string.decode('utf-8') page_data_string = urllib.unquote_plus(page_data_string) page_data_string = page_data_string.encode('utf-8') page_data_string = page_data_string.replace( '/o.php?b=4&f=frame&mobile=&u=', '') page_data_string = page_data_string.replace( '/o.php?b=4&mobile=&u=', '') page_data_string = page_data_string.replace('/o.php?b=4&mobile=&u=', '') # page_data_string_t = None # regex = r'{.*[token:]}]}' # matches = re.finditer(regex, page_data_string) # for matchNum, match in enumerate(matches): # page_data_string_t = match.group() # break # if page_data_string_t != None and 'token' in page_data_string_t: # page_data_string = page_data_string_t return client.getResponseDataBasedOnOutput(page_data_string, res, output) #except Exception as e: # print "Error: %s - %s" % (name, e) # return None
def requestdirect(url, close=True, redirect=True, followredirect=False, error=False, proxy=None, post=None, headers=None, mobile=False, limit=None, referer=None, cookie=None, output='', timeout='30', httpsskip=False, use_web_proxy=False, XHR=False, IPv4=False): #try: control.log("Requesting: %s Using via: %s" % (url, PROXY_URL)) urlhost = re.findall('([\w]+[.][\w]+)$', urlparse.urlparse(url.strip().lower()).netloc)[0] if headers == None: headers = {'Connection': 'keep-alive'} else: headers['Connection'] = 'keep-alive' res = client.request(url=PROXY_URL + url, close=close, redirect=redirect, followredirect=followredirect, error=error, proxy=proxy, post=post, headers=headers, mobile=mobile, limit=limit, referer=referer, cookie=cookie, output=output, timeout=timeout, httpsskip=httpsskip, use_web_proxy=use_web_proxy, XHR=XHR, IPv4=IPv4) page_data_string = client.getPageDataBasedOnOutput(res, output) pattern = re.compile('<script[\s\S]+?/script>') page_data_string = re.sub(pattern, '', page_data_string) try: page_data_string = page_data_string.replace('\n', '') #page_data_string = page_data_string.replace('\r','r').replace('\n','<br/>').replace('\w','').replace('\.','').replace('\t','').replace('\ ','') except Exception as e: control.log("Error1: %s - %s" % (name, e)) #print page_data_string page_data_string = json.dumps(page_data_string) page_data_string = page_data_string.replace('\\', '') page_data_string = page_data_string[1:-1] #print page_data_string #page_data_string = str(page_data_string) try: r = unicode(page_data_string, "utf-8") page_data_string = r except Exception as e: control.log("Error2: %s - %s" % (name, e)) try: r = str(page_data_string) page_data_string = r except Exception as e: control.log("Error3: %s - %s" % (name, e)) PROXY_PART0A = PROXY_PART0 % urlhost PROXY_PART1A = PROXY_PART1 % urlhost PROXY_PART3A = PROXY_PART3 % (urlhost, urlhost) PROXY_PART3A_REPLACE = PROXY_PART3_REPLACE % urlhost page_data_string = page_data_string.replace(PROXY_PART0A, PROXY_PART1_REPLACE) page_data_string = page_data_string.replace(PROXY_PART1A, PROXY_PART1_REPLACE) page_data_string = page_data_string.replace(PROXY_PART2A, PROXY_PART2_REPLACE) page_data_string = page_data_string.replace(PROXY_PART2B, PROXY_PART2_REPLACE) page_data_string = page_data_string.replace(PROXY_PART3A, PROXY_PART3A_REPLACE) return client.getResponseDataBasedOnOutput(page_data_string, res, output) # except Exception as e: # print "Error: %s - %s" % (name, e) # return None
def requestdirect(url, close=True, redirect=True, followredirect=False, error=False, proxy=None, post=None, headers=None, mobile=False, limit=None, referer=None, cookie=None, output='', timeout='30', httpsskip=False, use_web_proxy=False, XHR=False, IPv4=False): print "Requesting: %s Using via: %s" % (url, PROXY_URL) urlhost = re.findall('([\w]+[.][\w]+)$', urlparse.urlparse(url.strip().lower()).netloc)[0] if headers == None: headers = {'Connection': 'keep-alive'} headers['User-Agent'] = client.randomagent() res = client.request(url=PROXY_URL + url, close=close, redirect=redirect, followredirect=followredirect, error=error, proxy=proxy, post=post, headers=headers, mobile=mobile, limit=limit, referer=referer, cookie=cookie, output=output, timeout=timeout, httpsskip=httpsskip, use_web_proxy=use_web_proxy, XHR=XHR, IPv4=IPv4) page_data_string = client.getPageDataBasedOnOutput(res, output) pattern = re.compile('<script[\s\S]+?/script>') page_data_string = re.sub(pattern, '', page_data_string) try: page_data_string = page_data_string.replace('\n', '') #page_data_string = page_data_string.replace('\r','r').replace('\n','<br/>').replace('\w','').replace('\.','').replace('\t','').replace('\ ','') except Exception as e: control.log("Error1: %s - %s" % (name, e)) #print page_data_string page_data_string = json.dumps(page_data_string) page_data_string = page_data_string.replace('\\', '') page_data_string = page_data_string[1:-1] #print page_data_string #page_data_string = str(page_data_string) try: r = unicode(page_data_string, "utf-8") page_data_string = r except Exception as e: control.log("Error2: %s - %s" % (name, e)) try: r = str(page_data_string) page_data_string = r except Exception as e: control.log("Error3: %s - %s" % (name, e)) page_data_string = page_data_string.replace( 'https://www.xperienc.com/browsexp.php?', '') page_data_string = page_data_string.replace('b=40', '') page_data_string = page_data_string.replace('u=', '') page_data_string = page_data_string.replace('browsexp.php?', '') page_data_string = page_data_string.replace('&http', 'http') page_data_string = page_data_string.replace('/http', 'http') page_data_string = page_data_string.decode('utf-8') page_data_string = urllib.unquote_plus(page_data_string) page_data_string = page_data_string.encode('utf-8') return client.getResponseDataBasedOnOutput(page_data_string, res, output)