def forgot_password():
configs = get_configs()
if request.method == 'POST': # Got an email address
email = request.form.get('email')
user = User.query.filter_by(email=email).first()
if user:
# create token
token = secrets.token_hex(20)
message = f"You recently requested a password reset. Please visit this url to reset your password: {configs['web_url']}/forgot_password?token={token}"
sent = send_email(email, "Password Reset", message)
if sent:
user.last_token = token
db.session.commit()
flash('Sent reset information by email')
return redirect(url_for('home'))
else:
flash('No user by that email address!')
return redirect(url_for('home'))
elif request.method == 'GET':
if request.args.get('token'): #Got a token
user = User.query.filter_by(
last_token=request.args.get('token')).all()[0]
if user:
return render_template('new_password.html', user=user)
else:
flash('Unauthorized!')
return redirect(url_for('home'))
else:
return render_template('reset_password.html')
def domain_log_reports(domain, report_type):
"""
Reports of log reports
"""
configs = get_configs()
# get filtered list
file_list = get_file_list(region=configs['region'],
profile=configs['profile'],
bucket=configs['log_storage_bucket'],
domain=domain,
filter='Output')
if not file_list:
return False
# Sort by date
sorted_list = sorted(file_list, key=lambda i: i['date'], reverse=True)
if report_type == 'latest':
output_contents = get_output_contents(
bucket=configs['log_storage_bucket'],
profile=configs['profile'],
region=configs['region'],
output_file=sorted_list[0]['file_name'],
local_tmp=configs['local_tmp'])
return output_contents
def add_s3_storage(domain, s3):
"""
Add Storage Bucket
"""
configs = get_configs()
load_dotenv()
engine = db.create_engine(os.environ['DATABASE_URL'])
connection = engine.connect()
metadata = db.MetaData()
domains = db.Table('domains',
metadata,
autoload=True,
autoload_with=engine)
query = db.select([domains]).where(domains.c.domain == domain)
result = connection.execute(query)
row = result.fetchone()
logger.debug(f"Domain: {row} S3 storage {type(row.s3_storage_bucket)}")
if row.s3_storage_bucket:
return 'bucket_exists'
else:
s3simple = S3Simple(region_name=configs['region'],
profile=configs['profile'],
bucket_name=s3)
new_bucket = s3simple.s3_new_bucket()
row.s3_storage_bucket = s3
session.commit()
return 'bucket_created'
def ooni_reports(date):
"""
Get ooni report details
:arg date
"""
configs = get_configs()
load_dotenv()
engine = db.create_engine(os.environ['DATABASE_URL'])
connection = engine.connect()
metadata = db.MetaData()
ooni_reports = db.Table('ooni_reports',
metadata,
autoload=True,
autoload_with=engine)
query = f"select * from ooni_reports where date_reported > '{date}'"
ooni_report_list = connection.execute(query).fetchall()
number_of_ooni_reports = len(ooni_report_list)
number_of_ooni_problems = 0
ooni_problems = []
for orep in ooni_report_list:
if (((orep['blocked'] != 'false') and (orep['failure'] != None))
or (orep['dns_consistency'] == 'inconsistent')):
oprob = {}
number_of_ooni_problems += 1
oprob['url_accessed'] = orep['url_accessed']
oprob['country'] = orep['country']
oprob['failure'] = orep['failure']
oprob['dns_consistency'] = orep['dns_consistency']
ooni_problems.append(oprob)
return number_of_ooni_reports, number_of_ooni_problems, ooni_problems
def cloudfront_replace(domain, replace):
"""
Replaces cloudfront distribution
:param <domain>
:param <replace>
"""
# Find distribution based on replace domain
configs = get_configs()
session = boto3.Session(profile_name=configs['profile'])
client = session.client('cloudfront', region_name=configs['region'])
distributions = []
truncated = True
marker = ''
while truncated:
distribution_list = client.list_distributions(Marker=marker)
distributions.extend(distribution_list['DistributionList']['Items'])
if not distribution_list['DistributionList']['IsTruncated']:
truncated = False
else:
marker = distribution_list['DistributionList']['NextMarker']
for distribution in distributions:
if distribution['DomainName'] == replace: #it's the one to be replaced
delete_id = distribution['Id']
if not delete_id:
print("Can't find right distribution - check domain name!")
return
# Get config
distro_config = client.get_distribution_config(Id=delete_id)
logger.debug(f"Configuration: {distro_config}")
etag = distro_config['ETag']
disable_config = dict(distro_config['DistributionConfig'])
disable_config['Enabled'] = False
# Update config to disable
response = client.update_distribution(DistributionConfig=disable_config,
Id=delete_id,
IfMatch=etag)
d_etag = response['ETag']
# Wait for it...
logger.debug("Waiting for distribution to be disabled...")
waiter = client.get_waiter('distribution_deployed')
waiter.wait(Id=delete_id)
# Delete it.
response = client.delete_distribution(Id=delete_id, IfMatch=d_etag)
# Create a new distribution
new_mirror = cloudfront_add(domain=domain)
return new_mirror
def domain_listing():
"""
Lists all domains in mirror
:returns list of domains from mirror.
"""
configs = get_configs()
g = Github(configs['API_key'])
repo = g.get_repo(configs['repo'])
mirrors_object = repo.get_contents(configs['file'])
mirrors_decoded = mirrors_object.decoded_content
mirrors = json.loads(str(mirrors_decoded, "utf-8"))
return mirrors
def check(url):
"""
Function to check to see what mirrors, nodes, and onions exist on a domain
:param url
:return list with current available mirrors
"""
configs = get_configs()
g = Github(configs['API_key'])
repo = g.get_repo(configs['repo'])
mirrors_object = repo.get_contents(configs['file'])
mirrors_decoded = mirrors_object.decoded_content
mirrors = json.loads(str(mirrors_decoded, "utf-8"))
for site in mirrors['sites']:
if site_match(site['main_domain'], url):
exists = True
if 'available_mirrors' in site:
available_mirrors = site['available_mirrors']
else:
available_mirrors = []
if 'available_onions' in site:
available_onions = site['available_onions']
else:
available_onions = []
if 'available_ipfs_nodes' in site:
available_ipfs_nodes = site['available_ipfs_nodes']
else:
available_ipfs_nodes = []
if 'available_alternatives' in site:
available_alternatives = site['available_alternatives']
else:
available_alternatives = []
return {
'main_domain': site['main_domain'],
'requested_url': url,
'exists': exists,
'available_mirrors': available_mirrors,
'available_onions': available_onions,
'available_ipfs_nodes': available_ipfs_nodes,
'available_alternatives': available_alternatives
}
# No match
return {"exists": False, "alternatives": 'None'}
def domain_log_list(domain, num):
"""
List of domain logs
"""
configs = get_configs()
# get filtered list
file_list = get_file_list(region=configs['region'],
profile=configs['profile'],
bucket=configs['log_storage_bucket'],
domain=domain,
filter='Raw')
if not file_list:
return False
sorted_list = sorted(file_list, key=lambda i: i['date'], reverse=True)
return sorted_list[0:num]
def save_mirrors(new_mirrors, commit_msg):
configs = get_configs()
g = Github(configs['API_key'])
repo = g.get_repo(configs['repo'])
mirrors_object = repo.get_contents(configs['file'])
old_mirrors_decoded = mirrors_object.decoded_content
old_mirrors = json.loads(str(old_mirrors_decoded, "utf-8"))
new_mirrors_encoded = json.loads(new_mirrors)
if old_mirrors != new_mirrors_encoded:
result = repo.update_file(configs['file'], commit_msg, new_mirrors,
mirrors_object.sha)
print(f"Repo Result: {result}")
if 'commit' in result:
return True
else:
return False
else:
print("Nothing saved! Mirrors unchanged!")
return False
def domains_v2():
"""
Returns in JSON format all alternatives for a domain/url
"""
# Is this public?
configs = get_configs()
if configs['api_requests'] == 'auth':
# Auth token in headers
try:
auth_token = Token.query.filter_by(auth_token=request.headers.get('Authorization')).first()
except:
return {"alternatives" : "Database Error with token!"}
if not auth_token:
return {"alternatives": "Unauthorized!"}
req_data = request.get_json()
url = req_data['url']
if not url:
return {"alternatives" : 'None'}
domain_data = check(url)
alternatives = {"alternatives": domain_data['available_alternatives']}
return alternatives
def generate_admin_report(**kwargs):
"""
Generate a report with important data - email if mode is daemon
:arg kwargs
:kwarg mode
:kwarg user_id (not used at this time)
:returns nothing
"""
logger.debug("Creating Admin Report...")
configs = get_configs()
load_dotenv()
engine = db.create_engine(os.environ['DATABASE_URL'])
connection = engine.connect()
metadata = db.MetaData()
users = db.Table('users', metadata, autoload=True, autoload_with=engine)
domains = db.Table('domains',
metadata,
autoload=True,
autoload_with=engine)
dgdomains = db.Table('dg_domains',
metadata,
autoload=True,
autoload_with=engine)
system_settings = db.Table('system_settings',
metadata,
autoload=True,
autoload_with=engine)
# System settings
system_raw_dates = get_sys_info(all=True)
last_logfile_analysis = system_raw_dates['last_logfile_analysis'].strftime(
'%A %B %d, %Y at %I:%M %p %Z')
last_ooni_report_generated = system_raw_dates[
'last_ooni_report_generated'].strftime('%A %B %d, %Y at %I:%M %p %Z')
last_domain_test = system_raw_dates['last_domain_test'].strftime(
'%A %B %d, %Y at %I:%M %p %Z')
# List admins
user_query = db.select([users]).where(users.c.admin == True)
admin_list = connection.execute(user_query).fetchall()
# List Domain Group Owners
# TODO: Generate reports for domain group owners
dg_query = "select * from users, domain_groups where CAST(users.domain_group_id as integer)=domain_groups.id and domain_groups.name != 'None'"
dg_list = connection.execute(dg_query).fetchall()
# Get last date
last_email_report_sent = get_sys_info(request='last_email_report_sent',
update=True)
reports = db.Table('reports',
metadata,
autoload=True,
autoload_with=engine)
report_query = db.select(
[reports]).where(reports.c.date_reported > last_email_report_sent)
report_list = connection.execute(report_query).fetchall()
important_reports = ""
number_of_reports = len(report_list)
number_of_problems = 0
for report in report_list:
if ((report['domain_status'] != 200)
or (report['mirror_status'] != 200)):
number_of_problems += 1
translated_report = translate_reports(report)
important_reports += translated_report
number_of_ooni_reports, number_of_ooni_problems, ooni_problems = ooni_reports(
last_email_report_sent)
for problem in ooni_problems:
orept = f"OONI: URL Accessed: {problem['url_accessed']} Kind of Failure: {problem['failure']} DNS Consistency: {problem['dns_consistency']}\n"
important_reports += orept
if kwargs['mode'] == 'daemon':
if important_reports:
message_to_send = f""" Reporting problematic Domains and/or Alternatives since {last_email_report_sent}:
There were {number_of_reports} domain testing reports, and {number_of_problems} problems.
There were {number_of_ooni_reports} reports from OONI, with {number_of_ooni_problems} of problems.
The last domain test was {last_domain_test}.
The last logfile analysis was done on {last_logfile_analysis}.
and the last OONI report was generated on {last_ooni_report_generated}.
All detailed problem reports are below:
{important_reports}
"""
else:
message_to_send = f"""No Problematic Domains or Alternatives since {last_email_report_sent}.
The last domain test was {last_domain_test}.
The last logfile analysis was done on {last_logfile_analysis}.
and the last OONI report was generated on {last_ooni_report_generated}.
You might want to check the system."""
for user in admin_list:
if user['notifications'] and user['active']:
email = send_email(user['email'], "Report From BC APP",
message_to_send)
logger.debug(f"Message Sent to {user['email']}: {email}")
else:
if important_reports:
print(
f""" Reporting problematic Domains and/or Alternatives for Today:
There were {number_of_reports} domain testing reports, and {number_of_problems} problems.
There were {number_of_ooni_reports} reports from OONI, with {number_of_ooni_problems} of problems.
The last domain test was {last_domain_test}.
The last logfile analysis was done on {last_logfile_analysis}.
The last OONI report was generated on {last_ooni_report_generated}.
All detailed problem reports are below:
{important_reports}
""")
else:
print(f"""No problems reported since {last_email_report_sent}.
The last domain test was {last_domain_test}.
The last logfile analysis was done on {last_logfile_analysis}.
and the last OONI report was generated on {last_ooni_report_generated}.
You might want to check the system.""")
return
def cloudfront_add_logging(domain):
"""
Add logging for a cloudfront distribution
"""
load_dotenv()
engine = db.create_engine(os.environ['DATABASE_URL'])
connection = engine.connect()
metadata = db.MetaData()
domains = db.Table('domains',
metadata,
autoload=True,
autoload_with=engine)
query = db.select([domains]).where(domains.c.domain == domain)
result = connection.execute(query)
row = result.fetchone()
logger.debug(f"Domain: {row} S3 storage {row.s3_storage_bucket}")
if not row.s3_storage_bucket:
logger.debug("No S3 Storage set up!")
return False
# Find distribution based on domain
configs = get_configs()
session = boto3.Session(profile_name=configs['profile'])
client = session.client('cloudfront', region_name=configs['region'])
distributions = []
truncated = True
marker = ''
while truncated:
distribution_list = client.list_distributions(Marker=marker)
distributions.extend(distribution_list['DistributionList']['Items'])
if not distribution_list['DistributionList']['IsTruncated']:
truncated = False
else:
marker = distribution_list['DistributionList']['NextMarker']
edit_id = False
for distribution in distributions:
if domain in distribution['Origins']['Items'][0][
'DomainName']: # it's what we want to edit
edit_id = distribution['Id']
if not edit_id:
logger.debug("Can't find right distribution - check domain name!")
return False
# Get config
distro_config = client.get_distribution_config(Id=edit_id)
logger.debug(f"Configuration: {distro_config}")
etag = distro_config['ETag']
new_config = dict(distro_config['DistributionConfig'])
new_config['Logging'] = {
'Enabled': True,
'IncludeCookies': False,
'Bucket': row.s3_storage_bucket,
'Prefix': ''
}
# Update config to add logging
response = client.update_distribution(DistributionConfig=new_config,
Id=edit_id,
IfMatch=etag)
d_etag = response['ETag']
logger.debug(f"Response: {response}")
# Wait for it...
logger.debug("Waiting for distribution to be reconfigured...")
waiter = client.get_waiter('distribution_deployed')
waiter.wait(Id=edit_id)
logger.debug("Distribution updated!")
return True
def cloudfront_add(**kwargs):
"""
creates new cloudfront distribution
:params kwargs
:kwarg <domain>
:kwarg [mode]
:returns url
"""
configs = get_configs()
now = str(datetime.datetime.now())
session = boto3.Session(profile_name=configs['profile'])
client = session.client('cloudfront', region_name=configs['region'])
domain = get_final_domain(f"http://{kwargs['domain']}")
logger.debug(f"For domain: {domain}")
cdn_id = "Custom-" + domain
response = client.create_distribution(
DistributionConfig={
'CallerReference': now,
'Origins': {
'Quantity':
1,
'Items': [{
'Id': cdn_id,
'DomainName': domain,
'CustomOriginConfig': {
'HTTPPort': 80,
'HTTPSPort': 443,
'OriginProtocolPolicy': 'match-viewer',
'OriginSslProtocols': {
'Quantity': 3,
'Items': ['TLSv1', 'TLSv1.1', 'TLSv1.2']
},
'OriginReadTimeout': 30,
'OriginKeepaliveTimeout': 5
}
}]
},
'DefaultCacheBehavior': {
'TargetOriginId': cdn_id,
'ForwardedValues': {
'QueryString': True,
'Cookies': {
'Forward': 'none'
}
},
'TrustedSigners': {
'Enabled': False,
'Quantity': 0
},
'ViewerProtocolPolicy': 'redirect-to-https',
'MinTTL': 0
},
'Comment': 'CDN for ' + domain,
'PriceClass': 'PriceClass_All',
'Enabled': True,
'ViewerCertificate': {
'CloudFrontDefaultCertificate': True
}
})
logger.debug(f"Response: {response}")
distro_id = response['Distribution']['Id']
if 'mode' in kwargs and kwargs['mode'] == 'console':
wait = input("Wait for distribution (y/N)?")
if wait.lower() == 'y':
logger.debug("And now we wait...")
waiter = client.get_waiter('distribution_deployed')
waiter.wait(Id=distro_id,
WaiterConfig={
'Delay': 60,
'MaxAttempts': 30
})
return response['Distribution']['DomainName']
def get_ooni_data(range):
"""
Get data from OONI S3 bucket
"""
last_ooni_report_generated = get_sys_info(
request='last_ooni_report_generated', update=True)
configs = get_configs()
bucket = 'ooni-data-eu-fra'
session = boto3.Session(profile_name=configs['profile'])
client = session.client('s3')
#get date range
now = datetime.datetime.now()
then = now - datetime.timedelta(days=range)
delta = datetime.timedelta(days=1)
logger.debug(f"Now: {now} Then: {then}")
engine = db.create_engine(configs['database_url'])
connection = engine.connect()
metadata = db.MetaData()
ooni_reports = db.Table('ooni_reports',
metadata,
autoload=True,
autoload_with=engine)
file_list = []
logger.debug("Getting OONI file list from S3...")
while then <= now:
date_str = then.strftime('%Y%m%d')
file_date = 'raw/' + date_str
then += delta
date_report_list = client.list_objects_v2(Bucket=bucket,
Prefix=file_date)
for s3_file in date_report_list['Contents']:
if ('webconnectivity' in s3_file['Key']) and ('jsonl'
in s3_file['Key']):
file_list.append(s3_file['Key'])
# Process Files
domain_list, mirror_list = lists()
matching_domain_data = {}
for domain in domain_list:
matching_domain_data[domain['name']] = []
for file in file_list:
file_parts = file.split('/')
local_name = ('-').join(file_parts)
local_file_path = configs['local_tmp'] + '/' + local_name
logger.debug(f"Downloading to: {local_file_path}")
with open(local_file_path, 'wb') as file_data:
client.download_fileobj(bucket, file, file_data)
data = []
with gzip.open(local_file_path) as raw_file:
line = raw_file.readline()
json_data = json.loads(line)
data.append(json_data)
os.remove(local_file_path)
for jdata in data:
logger.debug(f"input: {jdata['input']}")
domain_name = False
for domain in domain_list:
match = site_match(domain['name'], jdata['input'])
if match:
domain_name = domain['name']
domain_id = domain['id']
if not domain_name:
logger.debug("No match.")
continue
date_reported = datetime.datetime.strptime(
jdata['measurement_start_time'], '%Y-%m-%d %H:%M:%S')
matching_domain_data[domain_name] = {
'domain_id': domain_id,
'url_accessed': jdata['input'],
'country': jdata['probe_cc'],
'blocked': jdata['test_keys']['blocking'],
'dns_consistency': jdata['test_keys']['dns_consistency'],
'date_reported': date_reported
}
for key in jdata['test_keys']['requests']:
for s_key in key:
if s_key == 'failure':
matching_domain_data[domain_name]['failure'] = key[
'failure']
print(
f"Matching Domain Data for {domain_name}:{matching_domain_data[domain_name]}"
)
# Make report
ooni_report_data = matching_domain_data[domain_name]
insert = ooni_reports.insert().values(**ooni_report_data)
result = connection.execute(insert)
return
def send_report(domain_data, mode):
"""
Make a report to the database with
console-generated tests
:arg domain_data
:returns nothing
"""
data_pretty = json.dumps(domain_data)
logger.debug(f"Domain Data: {data_pretty}")
now = datetime.datetime.now()
host_name = socket.gethostname()
host_ip = socket.gethostbyname(host_name)
configs = get_configs()
engine = db.create_engine(configs['database_url'])
connection = engine.connect()
metadata = db.MetaData()
domains = db.Table('domains',
metadata,
autoload=True,
autoload_with=engine)
mirrors = db.Table('mirrors',
metadata,
autoload=True,
autoload_with=engine)
reports = db.Table('reports',
metadata,
autoload=True,
autoload_with=engine)
query = db.select([domains])
result = connection.execute(query).fetchall()
domain_id = False
inactive = False
for entry in result:
d_id, domain, ext, paths, s3_storage, azure_profile, inactive = entry
logger.debug(f"Entry: {entry}")
if domain in domain_data['domain']:
domain_id = d_id
if inactive:
logger.debug("Inactive Domain!")
return False
logger.debug(f"Domain ID: {domain_id}")
if not domain_id: # we've not seen it before, add it
logger.debug("New Domain!")
insert = domains.insert().values(domain=domain_data['domain'])
result = connection.execute(insert)
domain_id = result.inserted_primary_key[0]
logger.debug(f"Domain ID: {domain_id}")
# Add mirrors
if (('current_alternatives' not in domain_data)
or (not domain_data['current_alternatives'])):
logger.debug("Not reporting on v1 data!")
return False
for current_alternative in domain_data['current_alternatives']:
query = db.select([mirrors])
result = connection.execute(query).fetchall()
mirror_id = False
for entry in result:
m_id, m_url, d_id, proto, m_type, inactive = entry
if current_alternative['url'] == m_url:
mirror_id = m_id
logger.debug(f"Mirror ID: {mirror_id}")
if not mirror_id: # add it
logger.debug("New Alternative!")
insert = mirrors.insert().values(
mirror_url=current_alternative['url'],
domain_id=domain_id,
mirror_type=current_alternative['type'],
protocol=current_alternative['proto'])
result = connection.execute(insert)
mirror_id = result.inserted_primary_key[0]
logger.debug(f"Mirror ID: {mirror_id}")
# Make report
report_data = {
'date_reported': now,
'domain_id': domain_id,
'mirror_id': mirror_id,
'user_agent': f'BC APP {mode}',
'domain_status': domain_data[domain_data['domain']],
'mirror_status': current_alternative['result'],
'ip': host_ip
}
insert = reports.insert().values(**report_data)
result = connection.execute(insert)
return True
def analyze(unzip, percent, num, daemon, range, domain):
import faulthandler
faulthandler.enable()
# update system info
last_logfile_analysis = get_sys_info(request='last_logfile_analysis',
update=True)
configs = get_configs()
now = datetime.datetime.now()
now_string = now.strftime('%d-%b-%Y:%H:%M:%S')
load_dotenv()
engine = db.create_engine(os.environ['DATABASE_URL'])
connection = engine.connect()
metadata = db.MetaData()
domains = db.Table('domains',
metadata,
autoload=True,
autoload_with=engine)
domains_list = []
query = db.select([domains])
result = connection.execute(query).fetchall()
for line in result:
domains_list.append({
'id': line[0],
'name': line[1],
's3_bucket': line[4],
'azure_profile': line[5]
})
for dm in domains_list:
if ((domain == 'all') or (dm['name'] == domain)):
# First, is there an azure profile set?
if ('azure_profile' in dm) and (dm['azure_profile']):
logger.debug(
f"Domain: {dm['name']}: Azure Profile: {dm['azure_profile']}"
)
retrieve_logs(profile_name=dm['azure_profile'],
range=range,
s3_bucket=dm['s3_bucket'])
try:
s3simple = S3Simple(region_name=configs['region'],
profile=configs['profile'],
bucket_name=dm['s3_bucket'])
except:
logger.warning(f"No bucket set for domain {dm['name']}")
continue
# get the file list to analyze
# read from S3
#logger.debug(f"Getting files from S3 bucket {dm['s3_bucket']}...")
file_list = s3simple.s3_bucket_contents()
if not file_list:
continue
logger.debug(f"File List: {file_list}")
compiled_data = {
'nginx': [],
'cloudfront': [],
'fastly': [],
'azure': []
}
logger.debug(f"Analyzing {dm['name']}...")
for ifile in file_list:
if 'LogAnalysis' in ifile:
continue
if (('.gz' in ifile or '.bz2' in ifile) and not unzip):
continue
logger.debug(f"Processing file: {ifile}")
if ifile[-1] == '/':
directory = configs['local_tmp'] + '/' + ifile
if not os.path.isdir(directory):
os.mkdir(directory)
continue
file_date = filter_and_get_date(ifile)
if not file_date:
logger.warning("Couldn't find date in logs!")
continue
numdays = (now - file_date).days
if numdays > range:
continue
#download
local_path = configs['local_tmp'] + '/' + ifile
#logger.debug(f"Downloading ... domain: {dm['name']} to {local_path}")
try:
s3simple.download_file(file_name=ifile,
output_file=local_path)
except:
continue
# Add to aggregate
file_parts = ifile.split('.')
ext = file_parts[-1]
if ext == 'bz2' or ext == 'gz':
if unzip:
if ext == 'bz2':
raw_data = str(sh.bunzip2("-k", "-c", local_path))
else:
raw_data = str(sh.gunzip("-k", "-c", local_path))
else:
continue
else:
with open(local_path) as f:
raw_data = f.read()
#logger.debug(f"Files data: {raw_data}")
compiled_log_data, log_type = analyze_file(
raw_data, dm['name'])
if not compiled_log_data:
logger.warning("No Data!")
continue
compiled_data[log_type] += compiled_log_data
#logger.debug(f"Deleting local temporary file {local_path}...")
os.remove(local_path)
for log_type in compiled_data:
logger.debug(f"Log type: {log_type}")
#logger.debug(f"Analyzed data: {compiled_data[log_type]}")
if not compiled_data[log_type]:
continue
analyzed_log_data = analyze_data(compiled_data[log_type],
log_type)
(output_text, first_date, last_date, hits,
home_page_hits) = output(domain=dm['name'],
data=analyzed_log_data,
percent=percent,
num=num)
logger.debug(output_text)
logger.debug("Saving log analysis file...")
key = 'LogAnalysis_' + dm[
'name'] + '_' + log_type + '_' + now_string + '.json'
body = str(analyzed_log_data)
s3simple.put_to_s3(key=key, body=body)
logger.debug("Saving output file....")
key = 'LogAnalysisOutput_' + dm[
'name'] + '_' + log_type + '_' + now_string + '.txt'
s3simple.put_to_s3(key=key, body=output_text)
logger.debug("Sending Report to Database...")
report_save(domain=dm['name'],
datetime=now,
report_text=output_text,
hits=hits,
home_page_hits=home_page_hits,
first_date_of_log=first_date,
last_date_of_log=last_date,
log_type=log_type)
return
def move_logs(daemon, zip, recursive, range):
"""
Move logs from local to s3
"""
configs = get_configs()
now = datetime.datetime.now()
now_string = now.strftime('%d-%b-%Y:%H:%M:%S')
logger.debug("Reading Local Files...")
if configs['paths']:
# open paths file
with open(configs['paths']) as pathfile:
raw_path_list = pathfile.read()
paths = raw_path_list.split('\n')
for fpath in paths:
if not fpath:
continue
domain, path = fpath.split('|')
load_dotenv()
engine = db.create_engine(os.environ['DATABASE_URL'])
connection = engine.connect()
metadata = db.MetaData()
domains = db.Table('domains',
metadata,
autoload=True,
autoload_with=engine)
domains_list = []
query = db.select([domains])
result = connection.execute(query).fetchall()
for line in result:
domains_list.append({
'id': line[0],
'name': line[1],
's3_bucket': line[4]
})
domain_match = False
for db_domain in domains_list:
if ((db_domain['name'] == domain) and (db_domain['s3_bucket'])):
domain_match = True
s3simple = S3Simple(region_name=configs['region'],
profile=configs['profile'],
bucket_name=db_domain['s3_bucket'])
if not domain_match:
logger_debug("No s3 bucket match!")
continue
if not os.path.exists(path):
logger.critical("Path doesn't exist!")
return
if not os.path.isdir(path):
files = [path]
else:
files = get_list(path, recursive, range)
logger.debug(f"Path: {path}")
# send to S3
for file_name in files:
if 'access' not in file_name:
continue
file_path = file_name.split('/')
file_parts = file_path[-1].split('.')
just_file_name = '.'.join(file_path)
logger.debug(f"File Name {just_file_name}")
ext = file_parts[-1]
if ((ext == 'bz2' or ext == 'gz')) and not zip:
continue
logger.debug("sending to s3...")
s3_file = 'RawLogFile_' + domain + '_' + now_string + '_' + just_file_name
s3simple.send_file_to_s3(local_file=file_name, s3_file=s3_file)
def azure_add(**kwargs):
configs = get_configs()
domain = get_final_domain(f"http://{kwargs['domain']}")
# Tenant ID for your Azure subscription
TENANT_ID = configs['azure_tenant_id']
# Your service principal App ID
CLIENT = configs['azure_app']
# Your service principal password
KEY = configs['azure_key']
logger.debug("Authenticating...")
credentials = ServicePrincipalCredentials(client_id=CLIENT,
secret=KEY,
tenant=TENANT_ID)
endpoint_name = kwargs['domain'][0:6] + '1'
endpoint_name_confirm = input(
f"Azure CDN name/subdomain ({endpoint_name}.azureedge.net)?")
if endpoint_name_confirm:
endpoint_name = endpoint_name_confirm
endpoint_full_name = endpoint_name + '.azureedge.net'
my_resource_group = 'bypasscensorship'
region = 'West India'
tier = 'Standard_Akamai'
cdn_client = CdnManagementClient(credentials, configs['azure_sub_id'])
cdn_list = cdn_client.profiles.list()
print("List of Azure CDNs:")
count = 0
names = []
for profile in cdn_list:
pf_vars = vars(profile)
print(f"{count}: Name: {pf_vars['name']}")
names.append(pf_vars['name'])
endpoints_list = cdn_client.endpoints.list_by_profile(
my_resource_group, pf_vars['name'])
ep_count = 0
for endpoint in endpoints_list:
ep_count += 1
print(f"Number of endpoints: {ep_count}")
count += 1
cdn_choice = input(f"Which CDN to add {endpoint_full_name} to?")
if not cdn_choice:
return False
else:
cdn_name = names[int(cdn_choice)]
cdn_confirm = input(f"Add to {cdn_name} (Y/n)?")
if cdn_confirm.lower() == 'n':
return False
logger.debug("Adding...")
endpoint_poller = cdn_client.endpoints.create(
my_resource_group, cdn_name, endpoint_name, {
"location": region,
"origin_host_header": domain,
"origins": [{
"name": cdn_name,
"host_name": domain
}]
})
endpoint = endpoint_poller.result()
logger.debug("Done!")
return endpoint_full_name
def retrieve_logs(**kwargs):
"""
Retrieve logs from storage
"""
configs = get_configs()
now = datetime.datetime.now()
logger.debug("Grabbing files from Azure...")
# Create a client
container_name = "insights-logs-azurecdnaccesslog" # it's always called this
container = ContainerClient.from_connection_string(
conn_str=configs['azure_storage_conn_string'],
container_name=container_name)
date_match = re.compile(
"y=[0-9]{4}\/m=[0-9]{2}\/d=[0-9]{2}\/h=[0-9]{2}\/m=[0-9]{2}")
# List blobs
blob_list = container.list_blobs()
for blob in blob_list:
try:
log_date_string = date_match.search(blob.name).group(0)
log_date = datetime.datetime.strptime(log_date_string,
"y=%Y/m=%m/d=%d/h=%H/m=%M")
#print(f"Date: {log_date} & {log_date_string}")
except:
continue
# right profile?
#logger.debug(f"Profile: {kwargs['profile_name']} blob.name: {blob.name}")
if kwargs['profile_name'].upper() not in blob.name:
#logger.debug('Not right profile')
continue
# TODO: Find right range of files, copy to S3
numdays = (now - log_date).days
if numdays > kwargs['range']:
logger.debug(f"File {blob.name} too old!")
continue
file_date = datetime.datetime.strftime(log_date, "%Y-%m-%d-%H-%M")
s3_filename = "Azure_CDN_log_" + kwargs[
'profile_name'] + "_" + file_date + ".json"
local_path = configs['local_tmp'] + "/" + s3_filename
#logger.debug(f"Local path: {local_path}")
get_blob = BlobClient.from_connection_string(
conn_str=configs['azure_storage_conn_string'],
container_name=container_name,
blob_name=blob.name)
with open(local_path, "wb") as tmp_blob:
blob_data = get_blob.download_blob()
blob_data.readinto(tmp_blob)
#upload to S3
s3simple = S3Simple(region_name=configs['region'],
profile=configs['profile'],
bucket_name=kwargs['s3_bucket'])
s3simple.send_file_to_s3(local_file=local_path, s3_file=s3_filename)
os.remove(local_path)
return
"""
Bypass Censorship API
Allows for testing and requests for BP mirrors from an API
"""
import sys
sys.path.insert(0, '.')
from system_utilities import get_configs
from app import app
if __name__ == '__main__':
configs = get_configs()
log = configs['log_level']
logger = logging.getLogger('logger') # instantiate clogger
logger.setLevel(logging.DEBUG) # pass DEBUG and higher values to handler
ch = logging.StreamHandler() # use StreamHandler, which prints to stdout
ch.setLevel(configs['log_level']) # ch handler uses the configura
# create formatter
# display the function name and logging level in columnar format if
# logging mode is 'DEBUG'
formatter = logging.Formatter('[%(funcName)24s] [%(levelname)8s] %(message)s')
# add formatter to ch
ch.setFormatter(formatter)
logger.addHandler(ch)
def automation(testing, domain, test, proxy, existing, delete, domain_list,
log, mirror_type, replace, remove, report, mode,
generate_report, s3, ooni, missing):
configs = get_configs()
logger.debug(f"Repo: {configs['repo']}")
if domain:
if delete:
delete = delete_domain(domain)
if mode == 'console':
if not delete:
print(f"Domain {domain} not deleted from github.")
else:
print(f"Domain {domain} deleted from github, and {delete}")
elif replace:
delete_deprecated(domain)
replace_mirror(domain=domain,
existing=existing,
replace=replace,
mirror_type=mirror_type,
mode=mode)
elif remove:
delete_deprecated(domain)
removed = remove_mirror(domain=domain, remove=remove)
if mode == 'console':
print(removed)
elif log:
add_logging(domain=domain, mirror_type=mirror_type, mode=mode)
elif s3:
s3_storage_add = add_s3_storage(domain=domain, s3=s3)
if mode == 'console':
print(f"Result: {s3_storage_add}")
elif mirror_type or existing:
domain = strip_www(domain)
delete_deprecated(domain)
new_add(domain=domain,
mirror_type=mirror_type,
existing=existing,
mode=mode)
domain_testing(proxy, mode, domain)
elif report:
domain_reporting(domain=domain, mode=mode)
elif missing:
missing_mirrors(domain=domain)
else:
domain_data = mirror_detail(domain=domain,
proxy=proxy,
mode=mode,
test=test)
if mode == 'console':
if not domain_data:
print("No data returned!")
return
elif missing:
missing_mirrors(missing=missing)
elif ooni:
get_ooni_data(ooni)
elif testing:
if mode == 'console':
test = input("Test all (Y/n)?")
else:
test = 'y'
if test.lower() != 'n':
domain_testing(proxy, mode, '')
elif generate_report:
generate_admin_report(mode=mode, user_id=None)
elif domain_list: #assuming console mode
dlist = domain_listing()
print(f""" List of all domains, mirrors and onions
___________________________________________________
{dlist}
___________________________________________________
""")
else:
click.echo("Invalid parameters! try --help")
return
