def alternatives():
"""
Get alternatives from GitHub
"""
if request.args.get('url'):
url = request.args.get('url')
if '.onion' in url:
status, final_url = mirror_tests.test_onion(url, 'web')
if status != 200:
result = 'down'
else:
result = 'up'
else:
status, final_url = mirror_tests.test_domain(url, '', 'web', '')
if status != 200:
result = 'down'
else:
result = 'up'
else:
url = False
result = 'none'
domain_choice = request.args.get('domain_choice')
alternatives_list = repo_utilities.check(domain_choice)
if not alternatives_list['exists']:
alternatives = False
else:
alternatives = alternatives_list['available_alternatives']
return render_template('alternatives.html',
domain_choice=domain_choice,
alternatives=alternatives,
url=url,
result=result,
admin=current_user.admin)
def public_alternatives():
"""
Listing Alternatives
"""
if request.args.get('url'):
alternatives = request.args.get('alternatives')
url = request.args.get('url')
if '.onion' in url:
status, final_url = mirror_tests.test_onion(url, 'web')
if status != 200:
result = 'down'
else:
result = 'up'
else:
status, final_url = mirror_tests.test_domain(url, '', 'web', '')
if status != 200:
result = 'down'
else:
result = 'up'
else:
url = False
result = 'none'
domain_choice = request.args.get('domain_choice')
alternatives_list = repo_utilities.check(domain_choice)
if not alternatives_list['exists']:
alternatives = False
else:
alternatives = alternatives_list['available_alternatives']
return render_template('public_alternatives.html',
title='Home',
domain_choice=domain_choice,
alternatives=alternatives,
url=url,
result=result)
def mirror_detail(**kwargs):
"""
List and test mirrors for a domain
:arg kwargs:
:kwarg domain
:kwarg proxy
:kwarg mode
:kwarg test
:returns: json with data
"""
output = {}
domain = kwargs['domain']
logger.debug(f"Listing {domain}...")
output['domain'] = domain
domain_data = check(domain)
exists = domain_data['exists']
if exists:
current_alternatives = domain_data['available_alternatives']
else:
logger.debug(f"{domain} doesn't exist in the mirror list.")
return False
if kwargs['mode'] == 'console':
print(f"Alternatives: {current_alternatives}")
if ('test' in kwargs) and (kwargs['test']):
logger.debug(f"Testing {domain}...")
mresp, murl = test_domain(domain, kwargs['proxy'], kwargs['mode'], '')
if kwargs['mode'] == 'console':
print(f"Response code on domain: {mresp}, url: {murl}")
output[domain] = mresp
output['current_alternatives'] = current_alternatives
for alternative in current_alternatives:
if alternative['proto'] == 'http' or alternative[
'proto'] == 'https':
mresp, murl = test_domain(alternative['url'], kwargs['proxy'],
kwargs['mode'], alternative['proto'])
if kwargs['mode'] == 'console':
print(f"Response code on mirror: {mresp}, url: {murl}")
alternative['result'] = mresp
elif alternative['proto'] == 'tor':
mresp, murl = test_onion(alternative['url'], kwargs['mode'])
if kwargs['mode'] == 'console':
print(
f"Onion {alternative['url']}... Response code: {mresp} ... URL: {murl}"
)
alternative['result'] = mresp
elif alternative['proto'] == 'ipfs':
pass
else:
pass
delete_deprecated(domain)
else:
output = "Not Tested."
return output
def new_domain():
"""
Add New Domain
"""
if not current_user.admin:
flash('Have to be an admin!')
return redirect(url_for('profile'))
else:
form = DomainForm()
if request.method == 'POST':
form_domain = form.domain.data
paths_ignore = form.paths_ignore.data
ext_ignore = form.ext_ignore.data
s3_storage_bucket = form.s3_storage_bucket.data
azure_profile_name = form.azure_profile_name.data
inactive = form.inactive.data
# Does this domain exist in the database or github?
db_check = Domain.query.filter_by(domain=form_domain).first()
gh_check = repo_utilities.check(form_domain)
if not gh_check['exists']: # not in GH
# Add to Github
added = automation.new_add(
domain=form_domain,
mode='web'
)
if not db_check: # not in db either
domain = Domain(
domain=form_domain,
paths_ignore=paths_ignore,
ext_ignore=ext_ignore,
s3_storage_bucket=s3_storage_bucket,
azure_profile_name=azure_profile_name,
inactive=inactive
)
db.session.add(domain)
db.session.commit()
flash("Domain Added!")
else: # in db already
if db_check.inactive: #set to inactive
domain.inactive = False
db.session.commit()
flash("Domain Added, set to active again!")
else: # exists in GH
if db_check: #domain exists in db
if db_check.inactive: #set to inactive
domain.inactive = False
db.session.commit()
else:
flash("Domain already exists!")
return redirect(url_for('admin_domains', status='active'))
else:
return render_template('edit_domain.html', form=form, new=True)
def delete_domain(domain):
"""
Delete domain
:arg domain
:returns nothing
"""
print(f"Deleting {domain}...")
domain_data = check(domain)
exists = domain_data['exists']
print(
f"Preexisting: {exists}, current Alternatives: {domain_data['available_alternatives']}"
)
if not exists:
print("Domain doesn't exist!")
return False
else:
removed = remove_domain(domain)
return removed
def cross_check(domain):
"""
Making sure all domain alternatives have database entry
"""
repo_list = repo_utilities.check(domain)
load_dotenv()
engine = db.create_engine(os.environ['DATABASE_URL'])
connection = engine.connect()
metadata = db.MetaData()
domains = db.Table('domains',
metadata,
autoload=True,
autoload_with=engine)
mirrors = db.Table('mirrors',
metadata,
autoload=True,
autoload_with=engine)
for alt in repo_list['available_alternatives']:
query = db.select([mirrors]).where(mirrors.c.mirror_url == alt['url'])
result = connection.execute(query)
domain_row = result.fetchone()
if domain_row == None:
print("DIDN'T LIKE IT")
query = db.select([domains]).where(domains.c.domain == domain)
result = connection.execute(query)
domain_row = result.fetchone()
alternative = {
'mirror_type': alt['type'],
'mirror_url': alt['url'],
'domain_id': domain_row.id,
'protocol': alt['proto']
}
insert = mirrors.insert().values(**alternative)
result = connection.execute(insert)
return repo_list
def domains_v2():
"""
Returns in JSON format all alternatives for a domain/url
"""
# Is this public?
configs = get_configs()
if configs['api_requests'] == 'auth':
# Auth token in headers
try:
auth_token = Token.query.filter_by(auth_token=request.headers.get('Authorization')).first()
except:
return {"alternatives" : "Database Error with token!"}
if not auth_token:
return {"alternatives": "Unauthorized!"}
req_data = request.get_json()
url = req_data['url']
if not url:
return {"alternatives" : 'None'}
domain_data = check(url)
alternatives = {"alternatives": domain_data['available_alternatives']}
return alternatives
def add_alternative():
"""
Add new alternative
"""
if not current_user.admin:
flash('Have to be an admin!')
return redirect(url_for('profile'))
else:
form = AltForm()
if request.method == 'POST':
domain_id = request.form.get('domain_id')
domain = Domain.query.filter(Domain.id==domain_id).first_or_404()
mirror_type = form.mirror_type.data
mirror_url = form.mirror_url.data
service = request.form.get('service')
if mirror_type == 'mirror':
protocol = 'http'
elif mirror_type == 'eotk':
protocol = 'tor'
else:
protocol = 'https'
# is it already there?
db_exists = False
test_mirror = Mirror.query.filter(Mirror.mirror_url==mirror_url).first()
if test_mirror and test_mirror.mirror_url == mirror_url:
db_exists = True
# github
domain_data = repo_utilities.check(domain.domain)
gh_exists = False
for alt in domain_data['available_alternatives']:
if mirror_url == alt['url']:
gh_exists = True
if (gh_exists and db_exists) and not test_mirror.inactive:
flash("Mirror Already Exists!!")
return redirect(url_for('edit_domain', id=domain_id))
# Add to github
if not gh_exists:
if mirror_type == 'proxy':
gh_mt = service
elif mirror_type == 'eotk':
gh_mt = 'onion'
else:
gh_mt = mirror_type
added = automation.new_add(
domain=domain.domain,
mirror_type=gh_mt,
existing=mirror_url,
mode='web',
)
if 'failed' in added:
flash("No Alternative added!")
return redirect(url_for('edit_domain', id=domain_id))
elif db_exists and test_mirror.inactive:
test_mirror.inactive = False
db.session.commit()
flash("Mirror added back, made active!")
else:
alternative = Mirror(mirror_type=mirror_type, mirror_url=added, domain_id=domain_id, protocol=protocol, inactive=False)
db.session.add(alternative)
db.session.commit()
flash('Alternative Added')
return redirect(url_for('edit_domain', id=domain_id))
else:
form.domain_id.data = request.args.get('id')
return render_template('edit_alternative.html', form=form)
def new_add(**kwargs):
"""
Add new domain, mirror, onion or ipfs
:kwarg <domain>
:kwarg [mirror_type]
:kwarg [existing]
:kwarg [mode]
:returns True or False
"""
mirror = ""
domain_data = check(kwargs['domain'])
if 'mirror_type' not in kwargs:
kwargs['mirror_type'] = None
proto = None
mtype = None
if 'exists' not in domain_data:
exists = False
else:
exists = domain_data['exists']
if 'available_alternatives' in domain_data:
current_alternatives = domain_data['available_alternatives']
else:
current_alternatives = []
if 'mode' in kwargs and kwargs['mode'] == 'console':
print(
f"Preexisting: {exists}, current alternatives {current_alternatives}"
)
print(f"Adding entry to {kwargs['mirror_type']} ...")
else:
kwargs['mode'] = 'web'
if kwargs['mirror_type'] == 'cloudfront':
proto = 'https'
mtype = 'proxy'
if kwargs['existing']:
mirror = kwargs['existing']
else:
mirror = cloudfront_add(domain=kwargs['domain'],
mode=kwargs['mode'])
elif kwargs['mirror_type'] == 'azure':
proto = 'https'
mtype = 'proxy'
if kwargs['existing']:
mirror = kwargs['existing']
else:
mirror = azure_add(domain=kwargs['domain'])
elif kwargs['mirror_type'] == 'fastly':
proto = 'https'
mtype = 'proxy'
if kwargs['existing']:
mirror = kwargs['existing']
else:
mirror = fastly_add(domain=kwargs['domain'])
elif kwargs['mirror_type'] == 'onion':
proto = 'tor'
mtype = 'eotk'
if 'existing' in kwargs:
mirror = kwargs['existing']
else:
print("Need to include existing url!")
return "failed: Need to include existing url!"
elif kwargs['mirror_type'] == 'ipfs':
proto = 'https'
mtype = 'ipfs'
if 'existing' not in kwargs:
mirror = ipfs_add(domain=kwargs['domain'])
else:
mirror = kwargs['existing']
elif kwargs['mirror_type'] == 'mirror':
proto = 'http'
mtype = 'mirror'
if 'existing' not in kwargs:
print("You didn't include URL - mirror type needs that!")
return "failed: You didn't include URL - mirror type needs that!"
mirror = kwargs['existing']
else:
# Just adding domain, not adding any alternative
mirror = 'no_alternatives'
if not mirror:
print(f"Sorry, mirror not created for {kwargs['domain']}!")
return "failed: Sorry, mirror not created!"
domain_listing = add(domain=kwargs['domain'],
mirror=mirror,
pre=exists,
proto=proto,
mtype=mtype,
mode=kwargs['mode'])
return mirror
def replace_mirror(**kwargs):
"""
Replace Mirror or Onion
:kwarg <domain>
:kwarg <replace>
:kwarg [existing]
:kwarg [mirror_type]
:kwarg [mode]
:returns True or False
"""
mode = kwargs['mode']
if mode == 'console':
print(f"Replacing mirror for: {kwargs['domain']}...")
domain_data = check(kwargs['domain'])
exists = domain_data['exists']
current_alternatives = domain_data['available_alternatives']
if not exists:
if mode == 'console':
print("Domain doesn't exist!")
return False
if 'mirror_type' not in kwargs:
if mode == 'console':
print("Need mirror type here!!")
return False
if kwargs['mirror_type'] == 'onion':
proto = 'tor'
mtype = 'eotk'
elif kwargs['mirror_type'] == 'mirror':
proto = 'http'
mtype = 'mirror'
elif kwargs['mirror_type'] == 'cloudfront':
proto = 'https'
mtype = 'proxy'
else:
proto = 'https'
mtype = 'proxy'
if 'existing' in kwargs and kwargs[
'existing']: # replacing with existing...
domain_listing = add(domain=kwargs['domain'],
mirror=kwargs['existing'],
pre=exists,
replace=kwargs['replace'],
proto=proto,
mtype=mtype,
mode=mode)
else: # need to create a new mirror from the old automatically
if kwargs['mirror_type'] == 'cloudfront':
mirror = cloudfront_replace(kwargs['domain'], kwargs['replace'])
domain_listing = add(domain=kwargs['domain'],
mirror=mirror,
pre=exists,
replace=kwargs['replace'],
proto=proto,
mtype=mtype,
mode=mode)
else:
if mode == 'console':
print("Sorry, only cloudfront is automated!!")
return False
return True