def __init__(self,
db_username,
db_password,
db_name,
db_proxy_table,
db_host="127.0.0.1"):
self.userAgent = UserAgent()
self.proxyGetter = Proxy(db_username, db_password, db_host, db_name,
db_proxy_table)
self.proxy_list = []
self.current_proxy = None
self.proxy_time_start_used = 0
self.logger = logging.getLogger()
self.logger.addHandler(handler)
self.logger.setLevel(0)
def __init__(self, user_agent=False, proxy=False):
self.proxy = {}
self.session = requests.Session()
if user_agent:
self.session.headers.update(
{'User-agent', '{}'.format(UserAgent().get())})
if proxy:
ip = Proxy().get()
self.proxy['http'] = 'http://{}'.format(ip)
self.proxy['https'] = 'https://{}'.format(ip)
class Google(Base):
baseurl = 'http://www.google.com/'
search = urljoin(baseurl, '/search')
luckyopts = {'hl': 'en', 'btnI': 'I', 'aq': 'f', 'safe': 'off'}
calcopts = {'hl': 'en', 'safe': 'off', 'c2coff': 1, 'btnG': 'Search'}
spellcheck_opts = {'hl': 'en', 'aq': 'f', 'safe': 'off'}
correct = re.compile(r'Did you mean.*?:.*?</font>.*?<a.*?>\s*(.*?)\s*</a>',
re.I + re.DOTALL)
reConversionDetected = re.compile('More about (calculator|currency)')
reConversionResult = re.compile('<h2 class=r>.*?<b>(.*?)<\/b><\/h2>')
def __init__(self):
self.ua = UserAgent(handlers=[NoRedirects, NoErrors])
def lucky(self, query):
opts = dict(self.luckyopts.items())
opts['q'] = query
result = self.ua.openurl(self.search, opts=opts, referer=self.baseurl,
size=1024)
if not result.startswith('http'):
raise NonRedirectResponse
return '%s = %s' % (query, result)
def spellcheck(self, query):
opts = dict(self.spellcheck_opts)
opts['q'] = query
result = self.ua.openurl(self.search, opts=opts, referer=self.baseurl)
try:
result = self.correct.search(result).group(1)
result = stripHTML(result)
except:
result = query
return result
def calculator(self, query):
opts = dict(self.calcopts)
opts['q'] = query
doc = self.ua.openurl(self.search, opts=opts)
if not self.reConversionDetected.search(doc):
raise Exception, 'no conversion detected'
response = self.reConversionResult.search(doc).group(1)
response = stripHTML(response)
return response
def _func(*args, **kwargs):
log.warning('HERE MNA --- MM1')
if request.method != method:
log.warning('view_method_error|url=%s,method=%s',
get_request_url().encode('utf-8'), request.method)
return error_handler()
if isinstance(form, FormValidator):
if method == 'GET':
formdata = request.args
else:
formdata = request.form
try:
data = form.normalize(formdata)
except Exception as ex:
log.warning(
'view_params_error|format=form,url=%s,error=%s,body=%s',
get_request_url().encode('utf-8'), ex,
request.get_data())
return error_handler()
else:
if data_format == 'JSON':
log.warning('HERE MNA --- MM')
request_body = request.get_data()
try:
data = jsonutils.from_json(request_body)
except Exception as ex:
log.warning(
'view_params_error|format=json,url=%s,error=%s,body=%s',
get_request_url().encode('utf-8'), ex,
request_body)
#return error_handler()
data = {}
if form is not None:
params_errors = [
e.message for e in form.iter_errors(data)
]
if params_errors:
log.warning(
'view_params_error|format=json,url=%s,error=json_validotor:%s,body=%s',
get_request_url().encode('utf-8'),
';'.join(params_errors), request_body)
return error_handler()
else:
data = request.values
data['request_ip'] = get_request_ip(request)
if parse_ua:
data['request_ua'] = UserAgent(
(request.headers.get('User-Agent', '')))
return func(data, *args, **kwargs)
def map(key, logline):
try:
parsed = ApacheLogParser().parse(logline)
except:
return
# timestamp of format yyyymmdd.
timestamp = ymd(parsed['epoch'])
# dimension attributes are strings.
dimensions = {}
dimensions['host'] = parsed['host']
dimensions['method'] = parsed['method']
dimensions['path'] = parsed['path']
dimensions['status'] = parsed['status']
dimensions['referrer'] = parsed['referrer']
dimensions['agent'] = UserAgent(parsed['agent']).classify()
# measurements are integers.
measurements = {}
measurements['bytes'] = int(parsed['size'])
measurements['requests'] = 1
yield timestamp, dimensions, measurements
from splinter import Browser
from selenium import webdriver
from getproxies import *
from useragent import UserAgent
proxies = get_proxies()
url = "https://www.expressvpn.com/what-is-my-ip"
for proxy in proxies:
print("Proxy: " + proxy)
ua = UserAgent()
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % proxy)
chrome_options.add_argument('user-agent=' + ua.get_user_agent("chrome"))
try:
browser = Browser('chrome', options=chrome_options)
browser.visit(url)
ip_address = browser.find_by_css('p[class="ip-address"]').text
if ip_address == proxy['ip']:
print("IP address match with Proxy: " + ip_address)
browser.quit()
break
except Exception as e:
browser.quit()
print("Trying another proxy....")
continue
def test_classify(self):
user_agent_string = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-GB; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5'
ua = UserAgent(user_agent_string)
self.assertEqual(ua.classify(), "firefox")
self.assertTrue(ua.is_browser())
self.assertFalse(ua.is_robot())
def __init__(self):
self.ua = UserAgent(handlers=[NoRedirects, NoErrors])
def test_classify(self):
user_agent_string = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-GB; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5'
ua = UserAgent(user_agent_string)
self.assertEqual(ua.classify(), "firefox")
self.assertTrue(ua.is_browser())
self.assertFalse(ua.is_robot())
class RequestProxy:
def __init__(self,
db_username,
db_password,
db_name,
db_proxy_table,
db_host="127.0.0.1"):
self.userAgent = UserAgent()
self.proxyGetter = Proxy(db_username, db_password, db_host, db_name,
db_proxy_table)
self.proxy_list = []
self.current_proxy = None
self.proxy_time_start_used = 0
self.logger = logging.getLogger()
self.logger.addHandler(handler)
self.logger.setLevel(0)
def generate_random_request_headers(self):
headers = {
"Connection": "close", # another way to cover tracks
"User-Agent": self.userAgent.generate_random_user_agent()
} # select a random user agent
return headers
def get_random_proxy_list(self):
return self.proxyGetter.query_random_proxy()
def generate_proxied_requests(self, url, method="GET", req_timeout=20):
headers = self.generate_random_request_headers()
if not self.proxy_list:
self.proxy_list = self.get_random_proxy_list()
if not self.proxy_list:
raise Exception("No proxy list")
if not self.current_proxy:
self.current_proxy = self.randomize_proxy()
elif (time.time() - self.proxy_time_start_used) > 600:
# Change proxy for every 10 minutes
self.current_proxy = self.randomize_proxy()
proxies = {
"http": "http://{}".format(self.current_proxy),
"https": "https://{}".format(self.current_proxy)
}
try:
request = requests.request(url=url,
method=method,
headers=headers,
proxies=proxies,
timeout=req_timeout)
# Avoid HTTP request errors
if request.status_code == 409:
raise ConnectionError(
"HTTP Response [409] - Possible Cloudflare DNS resolution error"
)
elif request.status_code == 403:
raise ConnectionError(
"HTTP Response [403] - Permission denied error")
elif request.status_code == 503:
raise ConnectionError(
"HTTP Response [503] - Service unavailable error")
print('RR Status {}'.format(request.status_code))
return request
except ConnectionError:
try:
self.proxy_list.remove(self.current_proxy)
except ValueError:
pass
self.logger.debug(
"Proxy unreachable - Removed Straggling proxy: {0} PL Size = {1}"
.format(self.current_proxy, len(self.proxy_list)))
self.randomize_proxy()
except ReadTimeout:
try:
self.proxy_list.remove(self.current_proxy)
except ValueError:
pass
self.logger.debug(
"Read timed out - Removed Straggling proxy: {0} PL Size = {1}".
format(self.current_proxy, len(self.proxy_list)))
self.randomize_proxy()
except ChunkedEncodingError:
try:
self.proxy_list.remove(self.current_proxy)
except ValueError:
pass
self.logger.debug(
"Wrong server chunked encoding - Removed Straggling proxy: {0} PL Size = {1}"
.format(self.current_proxy, len(self.proxy_list)))
self.randomize_proxy()
except TooManyRedirects:
try:
self.proxy_list.remove(self.current_proxy)
except ValueError:
pass
self.logger.debug(
"Too many redirects - Removed Straggling proxy: {0} PL Size = {1}"
.format(self.current_proxy, len(self.proxy_list)))
self.randomize_proxy()
def randomize_proxy(self):
if len(self.proxy_list) == 0:
raise Exception("list is empty")
rand_proxy = random.choice(self.proxy_list)
while not rand_proxy:
rand_proxy = random.choice(self.proxy_list)
self.current_proxy = rand_proxy
self.proxy_time_start_used = time.time()
return rand_proxy
def get_token(ip):
ua = UserAgent('POST', '{}/getSecure'.format(ip))
r = ua.SendRequest()
return read_response(r)
def get_token(ip):
ua = UserAgent('POST', '{}/getSecure'.format(ip))
r = ua.SendRequest()
return read_response(r)
if __name__ == '__main__':
ip = '54.209.150.110'
headers = DEFAULT_HEADERS
parameters = {
'username': '******',
'token': get_token(ip),
}
# Activity #1
ua = UserAgent('POST', '{}/'.format(ip))
r = ua.SendRequest()
f = read_response(r)
print('Activity 1 Flag: ' + f)
# Activity #2
ua = UserAgent('POST',
'{}/getFlag2'.format(ip),
parameters=parameters,
ie_headers=True)
r = ua.SendRequest()
f = read_response(r)
print('Activity 2 Flag: ' + f)
# Activity #3
ua = UserAgent('POST',