def __init__(self, target):
self.target = target
self.repart = urlparse(target['url'])
self.redirect = ''
self.type = 'Open Redirection'
self.logger = logger()
self.db = Database()
class cloudproxies:
def __init__(self, ):
self.db = Database()
maxIndex = self.getIndex()
for index in range(1, maxIndex + 1):
self.getProxy(index)
def connect(self, index):
url = 'http://cloudproxies.com/proxylist/?page=%d' % index
r = requests.get(url, timeout=10)
return r.content
def getIndex(self, ):
content = self.connect(1)
tree = html.fromstring(content)
maxIndex = tree.xpath(
'//*[@class="pagination"]/li/a/@data-ci-pagination-page')[-1]
return int(maxIndex)
def getProxy(self, index):
content = self.connect(index)
tree = html.fromstring(content)
ip = tree.xpath('//*[@id="ContentTable"]/tbody/tr/td[3]/text()')
port = tree.xpath('//*[@id="ContentTable"]/tbody/tr/td[4]/a/text()')
if ip:
self.db.insert(ip)
class hidemy:
def __init__(self,):
self.db = Database()
try:
maxIndex,totalPages = self.getIndex()
for x in range(0,maxIndex,maxIndex/totalPages):
content = self.connect(x)
tree = html.fromstring(content)
ips = tree.xpath('//*[@class="tdl"]/text()')
ports = tree.xpath('//*[@id="content-section"]/section[1]/div/table/tbody/tr/td[2]/text()')
self.db.insert(ips)
except Exception as e:
pass
def connect(self,index):
url = 'https://hidemy.name/en/proxy-list/?start=%d' % index
r = requests.get(url,timeout=10)
return r.content
def getIndex(self,):
content = self.connect(0)
tree = html.fromstring(content)
maxIndex = tree.xpath('//*[@id="content-section"]/section[1]/div/div[4]/ul/li/a/@href')[-1].split('=')[1].split('#')[0]
totalPages = tree.xpath('//*[@id="content-section"]/section[1]/div/div[4]/ul/li/a/text()')[-1]
return int(maxIndex),int(totalPages)
class tor:
def __init__(self, ):
self.db = Database()
r = requests.get('http://torstatus.blutmagie.de/', timeout=10)
tree = html.fromstring(r.content)
torNodes = tree.xpath('//*[@class="who"]/text()')
self.db.insert(torNodes)
class proxylist:
def __init__(self, ):
self.db = Database()
maxIndex = self.getMax()
for index in range(0, maxIndex + 1):
self.getproxy(index)
def connect(self, index):
url = 'https://proxy-list.org/english/index.php?p=%d' % index
r = requests.get(url, timeout=10)
tree = html.fromstring(r.text)
return tree
def getMax(self, ):
tree = self.connect(1)
indexes = tree.xpath(
'//*[@id="content"]/div[4]/div[5]/div[2]/a/text()')
indexes.pop()
return int(indexes[-1])
def getproxy(self, index):
tree = self.connect(index)
raw = [
base64.b64decode(ip.split("('")[1].split("')")[0])
for ip in tree.xpath('//*[@class="proxy"]/script/text()')
]
ips = list()
for info in raw:
ips.append(info.decode().split(':')[0])
self.db.insert(ips)
class samair:
def __init__(self,):
self.db = Database()
maxIndex = self.maxIndex()
for index in range(0,maxIndex+1):
self.get(index)
def maxIndex(self,):
url = "http://samair.ru/proxy/"
r = requests.get(url,timeout=10)
tree = html.fromstring(r.content)
indexes = tree.xpath('//*[@id="navbar"]/ul/li/a/text()')
indexes.pop()
return int(indexes[-1])
def get(self,index):
url = 'http://samair.ru/proxy/list-IP-port/proxy-%d.htm' % index
r = requests.get(url,timeout=10)
tree = html.fromstring(r.content)
raw = tree.xpath('//div[@class="singleprice order-msg"]/pre/text()')
if raw:
raw.pop(0)
raw = raw[0].split('\n')
ips = [ip.split(':')[0] for ip in raw if ip]
self.db.insert(ips)
def __init__(self, ):
self.db = Database()
url = 'http://rebro.weebly.com/uploads/2/7/3/7/27378307/rebroproxy-all-113326062014.txt'
r = requests.get(url, timeout=10)
ips = list()
for raw in r.text.split():
ips.append(raw.split(':')[0])
self.db.insert(ips)
def __init__(self, ):
self.db = Database()
url = 'http://www.httptunnel.ge/ProxyListForFree.aspx'
r = requests.get(url, timeout=10)
tree = html.fromstring(r.content)
raw = tree.xpath('//*[@target="_new"]/text()')
ips = [ip.split(':')[0] for ip in raw]
self.db.insert(ips)
def __init__(self, ):
self.db = Database()
url = 'https://nordvpn.com/wp-admin/admin-ajax.php?searchParameters[0][name]=proxy-country&searchParameters[0][value]=&searchParameters[1][name]=proxy-ports&searchParameters[1][value]=&offset=0&limit=100000&action=getProxies'
r = requests.get(url, timeout=10)
ips = list()
for proxy in r.json():
ips.append(proxy['ip'])
self.db.insert(ips)
def __init__(self, ):
self.db = Database()
maxIndex = int(self.getmax())
for index in range(0, maxIndex + 1):
tree = self.connect(index)
raw = tree.xpath('//*[@title="View this Proxy details"]/text()')
ips = [ip.strip() for ip in raw if ip.strip()]
self.db.insert(ips)
def __init__(self,):
self.db = Database()
url = 'http://multiproxy.org/txt_all/proxy.txt'
r = requests.get(url,timeout=10)
ips = list()
for raw in r.text.split():
ips.append(raw.split(':')[0])
self.db.insert(ips)
class gatherproxy:
def __init__(self, ):
self.db = Database()
countries = self.getCountries()
for country in countries:
t = Thread(target=self.proxybyCountry, args=(country, ))
t.start()
def proxybyCountry(self, country):
country = country.strip()
maxIndex = self.totalPages(country)
for index in range(1, maxIndex + 1):
self.getProxies(country, index)
def getCountries(self, ):
url = 'http://www.gatherproxy.com/proxylistbycountry'
r = requests.get(url, timeout=10)
tree = html.fromstring(r.content)
countries = [
country.split('(')[0]
for country in tree.xpath('//*[@class="pc-list"]/li/a/text()')
]
return countries
def connect(self, country, index):
url = 'http://www.gatherproxy.com/proxylist/country/?c=%s' % country
data = {
'Country': country,
'Filter': '',
'PageIdx': index,
'Uptime': 0
}
r = requests.post(url, data=data, timeout=10)
return r.content
def totalPages(self, country):
try:
content = self.connect(country, 1)
tree = html.fromstring(content)
maxIndex = tree.xpath('//*[@id="psbform"]/div/a/text()')
if maxIndex:
return int(maxIndex[-1])
else:
return 0
except Exception as e:
pass
def getProxies(self, country, index):
content = self.connect(country, index)
tree = html.fromstring(content)
try:
ips = [
ip.split("document.write('")[1].split("')")[0] for ip in
tree.xpath('//*[@id="tblproxy"]/tr/td[2]/script/text()')
]
self.db.insert(ips)
except Exception as e:
pass
def check_database(self):
try:
db = Database()
data = {}
db.insert_record("result", data)
return True
except Exception as e:
print("[-]%s Database connection refuse%s" % (logger.R, logger.W))
return False
class HEXAA:
def __init__(self):
self._config = read_config()
self._app = Flask(__name__,
template_folder='templates',
static_folder='static')
# set session secret
self._app.secret_key = os.environ.get("SESSION_SECRET")
# connect to database
self._connect_to_database(self._config['database'])
# load api blueprints
self._load_api_blueprints()
# load portal bluprint
self._load_portal_blueprints()
# create required buckets
self._make_storage_buckets()
def _connect_to_database(self, config):
self._db = Database()
self._db.connect(**config)
def _make_storage_buckets(self):
minio_key = os.environ.get(
"MINIO_ACCESS_KEY") or self._config["minio"]["key"]
minio_secret = os.environ.get(
"MINIO_SECRET_KEY") or self._config["minio"]["secret"]
miniocl = Minio(self._config["minio"]["url"],
minio_key,
minio_secret,
secure=False)
for bucket in ["pictures", "submissions", "testsuites"]:
if not miniocl.bucket_exists(bucket):
miniocl.make_bucket(bucket)
def _load_api_blueprints(self):
self._app.register_blueprint(auth_service)
self._app.register_blueprint(users_api, url_prefix='/api')
self._app.register_blueprint(groups_api, url_prefix='/api')
self._app.register_blueprint(submission_api, url_prefix='/api')
self._app.register_blueprint(assignments_api,
url_prefix='/api/groups/<groupId>')
self._app.register_blueprint(announcements_api,
url_prefix='/api/groups/<groupId>')
self._app.register_blueprint(testsuites_api,
url_prefix='/api/groups/<groupId>')
def _load_portal_blueprints(self):
self._app.register_blueprint(main_pages)
self._app.register_blueprint(user_pages)
self._app.register_blueprint(group_pages)
self._app.register_blueprint(assignments_pages)
self._app.register_blueprint(announcements_pages)
self._app.register_blueprint(testsuites_pages)
self._app.register_blueprint(submissions_pages)
self._app.register_blueprint(members_page)
self._app.register_blueprint(settings_page)
def __init__(self, *args, **kwargs):
if os.environ.get('FLASK_DEBUG') == '1':
g.db = Database(debugging=True)
else:
g.db = Database()
if session.get('user_id'):
g.user = g.db.query(Users).get(session['user_id'])
g.categories = g.db.query(Categories).all()
def total():
"""Get statistics about the application"""
db = Database()
totalIps = len(db.total())
totalRequests, totalAllowed, totalBlocked = db.statistics()
return {
'ips': totalIps,
'requests': totalRequests,
'allowed': totalAllowed,
'blocked': totalBlocked
}
def __init__(self,):
self.db = Database()
try:
maxIndex,totalPages = self.getIndex()
for x in range(0,maxIndex,maxIndex/totalPages):
content = self.connect(x)
tree = html.fromstring(content)
ips = tree.xpath('//*[@class="tdl"]/text()')
ports = tree.xpath('//*[@id="content-section"]/section[1]/div/table/tbody/tr/td[2]/text()')
self.db.insert(ips)
except Exception as e:
pass
def __init__(self, user, db):
if user:
self._user = user
if not db:
raise ValueError('db should be present')
database = Database(db)
self.db = database
def setUpClass(cls):
client = MongoClient(MONGODB_URI)
cls.dbName = urlparse(MONGODB_URI).path[1:]
cls.db = client[cls.dbName]
cls.playlists_collection = cls.db['playlists']
cls.playlists_collection.remove({})
cls.dbHandler = Database()
async def create_db_pool() -> None:
"""Create and bind db_pool before start serving requests"""
create_redis_pool = functools.partial(aioredis.create_redis_pool,
encoding="utf-8")
redis_uri = f"redis://{REDIS_HOST}:{REDIS_PORT}"
redis = await trio_asyncio.run_asyncio(create_redis_pool, redis_uri)
app.db_pool = Database(redis)
def Launch_Locator(password):
db = Database(password)
cur = db.get_cur()
halligan_two = os.path.join('static_files/', 'Halligan_2.png')
cur.execute("""SELECT x, y from marauder_demhoes order by id desc limit 1 """)
row = cur.fetchone()
if row is not None:
x = int(float(row[0]))
y = int(float(row[1]))
image = Image.open(halligan_two)
radius = 5 # point radius
draw_image = ImageDraw.Draw(image)
draw_image.ellipse((x-radius, y-radius, x+radius, y+radius),
fill='blue', outline='red')
image.show()
image.close()
def find(ip):
"""Validate if the IP is malicious"""
db = Database()
result = db.find(ip)
if result:
db.statsInsert(ip, 'blocked')
return {'isMalicious': True}
else:
db.statsInsert(ip, 'allowed')
return {'isMalicious': False}
def dump(password, output_file):
db = Database(password)
cur = db.get_cur()
fp = open('output/{}'.format(output_file), 'w+')
cur.execute(q2)
access_points = cur.fetchall()
res = []
for f in access_points:
print f
msg = {
'floor_id': f[0],
'location_id': f[1],
'x': f[2],
'y': f[3],
'direction': f[4],
'macs': f[5].split(','),
'rss': map(float, f[6].split(','))
}
print json.dumps(msg)
res.append(msg)
json.dump(res, fp)
fp.close()
class xroxy:
def __init__(self, ):
self.db = Database()
maxIndex = int(self.getmax())
for index in range(0, maxIndex + 1):
tree = self.connect(index)
raw = tree.xpath('//*[@title="View this Proxy details"]/text()')
ips = [ip.strip() for ip in raw if ip.strip()]
self.db.insert(ips)
def connect(self, index):
url = 'http://www.xroxy.com/proxylist.php?pnum=%d' % index
r = requests.get(url, timeout=10)
tree = html.fromstring(r.text)
return tree
def getmax(self, ):
tree = self.connect(0)
maxIndex = int(
tree.xpath(
'//*[@id="content"]/table[2]/tr/td[1]/table/tr[2]/td/small/b/text()'
)[0]) / 10
return maxIndex
class MyApplication(object):
def __init__(self):
self.database = Database()
self.initiateApp()
def initiateApp(self):
app = self.make_app()
app.listen(8888)
def make_app(self):
db = self.database.get_motor_connection()
return Application(api_routes(),
db=db,
cookie_secret=settings.CookieSecret,
debug=settings.Debug,
key_version=settings.KeyVersion,
version=settings.Version,
login_url='api/login')
from urllib.parse import urlparse,quote,unquote,parse_qs
from config.config import get_file
from library.sendrequests import request
from library.DiffResponse import diff
from db.db import Database
from db.logger import logger
db = Database()
# Fuzzing the scope to detect if scope privilege escalation exists.
class ScopeScan:
def __init__(self,target):
self.scope = get_file('scope.txt')
self.target = target
self.redirect = ''
self.type = 'Scope Escalation'
self.logger = logger()
def fetch_redirect(self):
url = self.target['url']
reparts = urlparse(url)
query = parse_qs(reparts.query)
redirect = query['redirect_uri']
self.redirect = redirect[0]
return redirect
def mutate(self,scope):
repart = urlparse(self.target['url'])
query = parse_qs(repart.query)
query['scope'] = scope.split(' ')
def _connect_to_database(self, config):
self._db = Database()
self._db.connect(**config)
def Analyse(password):
db = Database(password)
cur = db.get_cur()
count = 0
sum_data = [[0 for i in range(5)] for j in range(5)]
for ts_id, id_list in enumerate(ids):
for t_id, ID in enumerate(id_list):
names[str(ID)] = titles[ts_id][t_id]
data = {}
if len(id_list) != 5:
continue
cur.execute("""SET SESSION group_concat_max_len = 1000000""")
cur.execute("""SELECT accesspoint.location_id, GROUP_CONCAT(MAC) as MAC_list,
GROUP_CONCAT(strength) as strength_list,
GROUP_CONCAT(std_dev) as std_list from marauder_accesspoint
join marauder_location on location.id=accesspoint.location_id
where location.id in (%s,%s,%s,%s,%s)
group by accesspoint.location_id,x,y,direction""", id_list)
all_macs = []
for row in cur.fetchall():
data[str(row[0])] = {"mac": row[1].split(","),
"strengths": [float(r) for r in row[2].split(",")]}
# "stds": [float(r) for r in row[3].split(",")]}
all_macs.append( row[1].split(",") )
if not all_macs:
continue
count += 1
unique_macs = set(flatten(all_macs))
unique = len(unique_macs)
if PRINT_TABLES:
print "All {} accesspoints".format(unique)
ks = [" " * 20]
for k in sorted(data.keys(),key=lambda x: int(names[x].split()[0])):
ks.append(names[str(k)].rjust(15))
print ' '.join(ks)
mac_dict = {}
for mac_addr in unique_macs:
mac_dict[mac_addr] = []
MAC_Counts = []
for k,vals in sorted(data.iteritems(),key=lambda x: int(names[x[0]].split()[0])):
MAC_counter = 0
for mac_addr in unique_macs:
index = get_index(vals["mac"], mac_addr)
if index != -1:
mac_dict[mac_addr].append(str(round(vals["strengths"][index],2)).rjust(15))
MAC_counter += 1
else:
mac_dict[mac_addr].append("-".rjust(15))
MAC_Counts.append(MAC_counter)
for k,v in mac_dict.iteritems():
print k.rjust(20), ' '.join(v)
print "Total Macs".rjust(20), ' '.join([str(x).rjust(15) for x in MAC_Counts])
intermacs = intersection(all_macs)
stds = []
print "Comparing {} common mac addresses of {} unique".format(len(intermacs),unique)
for imac in intermacs:
L = {}
for k in data.keys():
L[k] = []
for k,vals in sorted(data.iteritems()):
L[k].append( vals["strengths"][vals["mac"].index(imac)] )
ks = [" " * 10]
for k in sorted(L.keys(),key=lambda x: int(names[x].split()[0])):
ks.append(names[str(k)].rjust(10))
print ' '.join(ks)
sorted_items = sorted(L.iteritems(),key=lambda x: int(names[x[0]].split()[0]))
for i,(k,v) in enumerate(sorted_items):
d = [names[str(k)].ljust(10)]
for j,(key,val) in enumerate(sorted_items):
diff = difference(val,v)
sum_data[i][j] += diff
d.append(str( round(diff ,2) ).rjust(10) )
print ' '.join(d)
print
print
if count == 0:
return
print "Average Table:"
for i in sum_data:
print ' '.join([str( round(x / float(count), 2) ).rjust(10) for x in i])
from db.db import Database
import numpy
D = Database('boxoffice.db')
movies = D.GetAllMovies()
genre_table = D.GetAllGenreStats()
genre_dict = dict([(x[0], x) for x in genre_table])
msize = len(movies)
m_num = 0
for movie in [x for x in movies]:
m_num += 1
print "Getting features for movie: " + str(movie[0])
print "Movie " + str(m_num) + " out of " + str(msize)
movie_id = movie[0]
score = D.GetRottenScoreByMovieId(movie_id)
if not score:
try:
print "Skipping movie b/c score is null id: " + str(
movie[0]) + " name: " + movie[1]
except:
pass
continue
score = score[0][0]
if score == 0:
try:
print "Skipping movie b/c score==0 id: " + str(
movie[0]) + " name: " + movie[1]
from console_interface.main_menu import Menu
from db.db import Database
db = Database()
db.connect('lab1')
m = Menu(db)
m.main_loop()
from flask import Flask, redirect, url_for, request, render_template
import os
from db.db import Database
app = Flask(__name__)
PORT = os.environ.get('PORT') or 5000
dbHandler = Database()
@app.route('/')
def showSummary():
playlists = dbHandler.get_top_playlists()
user_count = dbHandler.user_count()
playlist_count = dbHandler.playlist_count()
print('user count')
print(user_count)
print('playlist_count:')
print(playlist_count)
for playlist in playlists:
print(playlist['name'])
playlist['image_link'] = ''
if (playlist['images'] and len(playlist['images']) > 0):
playlist['image_link'] = playlist['images'][0]['url']
print('length playlists: ')
print(playlists.count())
return render_template('summary.html', playlists=playlists, user_count=user_count, playlist_count=playlist_count)
@app.route('/new', methods=['POST'])
def new():
from db.db import Database
import os
from PIL import Image, ImageDraw
SHOW = False # Determines if images are shown or merely saved
password = os.environ.get('SIRIUS_PASSWORD')
radius = 5 # point radius
if password is None:
raise Exception('No password available')
db = Database(password)
cur = db.get_cur()
path_offset = "../Static_Files/" # Path offset from cwd to image directory
def overlay(FID1, FID2):
"""Given 2 arrays of (x,y) and an image will plot points on the same
image. This is useful when plotting test points and gathered points
on the same map. The saved image will be saved as floor_FID1_AND_FID2.png.
The image paths are assumed the same and the image from FID1 is used.
Points from FID1 are in blue while points from FID2 are in red."""
cur.execute("""SELECT imagePath from floor where id=%s""",[FID1])
floor_path = cur.fetchone()
if not floor_path:
return
imagePath = floor_path[0] #extracting string from tuple
ax1.plot([0, 100], [std, std], 'k')
ax1.plot([0, 100], [-std, -std], 'k')
ax1.axis([0, 100, -100, 100])
ax1.set_title(str(name) + " Sample Run Error")
ax1.set_xlabel("Ground Truth Audience Film Rating")
ax1.set_ylabel("Estimated Audience Film Rating")
ax2.hist(guess)
ax2.axis([0, 100, 0, 100])
ax2.set_title(str(name))
# ax2.set_xlabel("Estimated Audience Film Rating")
return Mean, Median, R2, sortedError
D = Database('boxoffice.db')
F1 = np.array(D.GetAllFeatures())
F1 = np.array([x for x in F1 if all(x[5:] != -1)])
algorithms = [
SVR(kernel='rbf', C=1, gamma=.002, degree=3),
AdaBoostRegressor(DecisionTreeRegressor(max_depth=15), n_estimators=200),
GradientBoostingRegressor(loss='huber', n_estimators=200),
RandomForestRegressor(n_estimators=200)
]
names = [
"SVR", "Boosted Decision Tree", "Gradient Boosting Regression",
"Random Forest Regression"
]
def __init__(self, **kwargs):
self._user = kwargs.get('user')
database = Database(kwargs.get('db'))
self.db = database
