def forgot_password(): configs = get_configs() if request.method == 'POST': # Got an email address email = request.form.get('email') user = User.query.filter_by(email=email).first() if user: # create token token = secrets.token_hex(20) message = f"You recently requested a password reset. Please visit this url to reset your password: {configs['web_url']}/forgot_password?token={token}" sent = send_email(email, "Password Reset", message) if sent: user.last_token = token db.session.commit() flash('Sent reset information by email') return redirect(url_for('home')) else: flash('No user by that email address!') return redirect(url_for('home')) elif request.method == 'GET': if request.args.get('token'): #Got a token user = User.query.filter_by( last_token=request.args.get('token')).all()[0] if user: return render_template('new_password.html', user=user) else: flash('Unauthorized!') return redirect(url_for('home')) else: return render_template('reset_password.html')
def domain_log_reports(domain, report_type): """ Reports of log reports """ configs = get_configs() # get filtered list file_list = get_file_list(region=configs['region'], profile=configs['profile'], bucket=configs['log_storage_bucket'], domain=domain, filter='Output') if not file_list: return False # Sort by date sorted_list = sorted(file_list, key=lambda i: i['date'], reverse=True) if report_type == 'latest': output_contents = get_output_contents( bucket=configs['log_storage_bucket'], profile=configs['profile'], region=configs['region'], output_file=sorted_list[0]['file_name'], local_tmp=configs['local_tmp']) return output_contents
def add_s3_storage(domain, s3): """ Add Storage Bucket """ configs = get_configs() load_dotenv() engine = db.create_engine(os.environ['DATABASE_URL']) connection = engine.connect() metadata = db.MetaData() domains = db.Table('domains', metadata, autoload=True, autoload_with=engine) query = db.select([domains]).where(domains.c.domain == domain) result = connection.execute(query) row = result.fetchone() logger.debug(f"Domain: {row} S3 storage {type(row.s3_storage_bucket)}") if row.s3_storage_bucket: return 'bucket_exists' else: s3simple = S3Simple(region_name=configs['region'], profile=configs['profile'], bucket_name=s3) new_bucket = s3simple.s3_new_bucket() row.s3_storage_bucket = s3 session.commit() return 'bucket_created'
def ooni_reports(date): """ Get ooni report details :arg date """ configs = get_configs() load_dotenv() engine = db.create_engine(os.environ['DATABASE_URL']) connection = engine.connect() metadata = db.MetaData() ooni_reports = db.Table('ooni_reports', metadata, autoload=True, autoload_with=engine) query = f"select * from ooni_reports where date_reported > '{date}'" ooni_report_list = connection.execute(query).fetchall() number_of_ooni_reports = len(ooni_report_list) number_of_ooni_problems = 0 ooni_problems = [] for orep in ooni_report_list: if (((orep['blocked'] != 'false') and (orep['failure'] != None)) or (orep['dns_consistency'] == 'inconsistent')): oprob = {} number_of_ooni_problems += 1 oprob['url_accessed'] = orep['url_accessed'] oprob['country'] = orep['country'] oprob['failure'] = orep['failure'] oprob['dns_consistency'] = orep['dns_consistency'] ooni_problems.append(oprob) return number_of_ooni_reports, number_of_ooni_problems, ooni_problems
def cloudfront_replace(domain, replace): """ Replaces cloudfront distribution :param <domain> :param <replace> """ # Find distribution based on replace domain configs = get_configs() session = boto3.Session(profile_name=configs['profile']) client = session.client('cloudfront', region_name=configs['region']) distributions = [] truncated = True marker = '' while truncated: distribution_list = client.list_distributions(Marker=marker) distributions.extend(distribution_list['DistributionList']['Items']) if not distribution_list['DistributionList']['IsTruncated']: truncated = False else: marker = distribution_list['DistributionList']['NextMarker'] for distribution in distributions: if distribution['DomainName'] == replace: #it's the one to be replaced delete_id = distribution['Id'] if not delete_id: print("Can't find right distribution - check domain name!") return # Get config distro_config = client.get_distribution_config(Id=delete_id) logger.debug(f"Configuration: {distro_config}") etag = distro_config['ETag'] disable_config = dict(distro_config['DistributionConfig']) disable_config['Enabled'] = False # Update config to disable response = client.update_distribution(DistributionConfig=disable_config, Id=delete_id, IfMatch=etag) d_etag = response['ETag'] # Wait for it... logger.debug("Waiting for distribution to be disabled...") waiter = client.get_waiter('distribution_deployed') waiter.wait(Id=delete_id) # Delete it. response = client.delete_distribution(Id=delete_id, IfMatch=d_etag) # Create a new distribution new_mirror = cloudfront_add(domain=domain) return new_mirror
def domain_listing(): """ Lists all domains in mirror :returns list of domains from mirror. """ configs = get_configs() g = Github(configs['API_key']) repo = g.get_repo(configs['repo']) mirrors_object = repo.get_contents(configs['file']) mirrors_decoded = mirrors_object.decoded_content mirrors = json.loads(str(mirrors_decoded, "utf-8")) return mirrors
def check(url): """ Function to check to see what mirrors, nodes, and onions exist on a domain :param url :return list with current available mirrors """ configs = get_configs() g = Github(configs['API_key']) repo = g.get_repo(configs['repo']) mirrors_object = repo.get_contents(configs['file']) mirrors_decoded = mirrors_object.decoded_content mirrors = json.loads(str(mirrors_decoded, "utf-8")) for site in mirrors['sites']: if site_match(site['main_domain'], url): exists = True if 'available_mirrors' in site: available_mirrors = site['available_mirrors'] else: available_mirrors = [] if 'available_onions' in site: available_onions = site['available_onions'] else: available_onions = [] if 'available_ipfs_nodes' in site: available_ipfs_nodes = site['available_ipfs_nodes'] else: available_ipfs_nodes = [] if 'available_alternatives' in site: available_alternatives = site['available_alternatives'] else: available_alternatives = [] return { 'main_domain': site['main_domain'], 'requested_url': url, 'exists': exists, 'available_mirrors': available_mirrors, 'available_onions': available_onions, 'available_ipfs_nodes': available_ipfs_nodes, 'available_alternatives': available_alternatives } # No match return {"exists": False, "alternatives": 'None'}
def domain_log_list(domain, num): """ List of domain logs """ configs = get_configs() # get filtered list file_list = get_file_list(region=configs['region'], profile=configs['profile'], bucket=configs['log_storage_bucket'], domain=domain, filter='Raw') if not file_list: return False sorted_list = sorted(file_list, key=lambda i: i['date'], reverse=True) return sorted_list[0:num]
def save_mirrors(new_mirrors, commit_msg): configs = get_configs() g = Github(configs['API_key']) repo = g.get_repo(configs['repo']) mirrors_object = repo.get_contents(configs['file']) old_mirrors_decoded = mirrors_object.decoded_content old_mirrors = json.loads(str(old_mirrors_decoded, "utf-8")) new_mirrors_encoded = json.loads(new_mirrors) if old_mirrors != new_mirrors_encoded: result = repo.update_file(configs['file'], commit_msg, new_mirrors, mirrors_object.sha) print(f"Repo Result: {result}") if 'commit' in result: return True else: return False else: print("Nothing saved! Mirrors unchanged!") return False
def domains_v2(): """ Returns in JSON format all alternatives for a domain/url """ # Is this public? configs = get_configs() if configs['api_requests'] == 'auth': # Auth token in headers try: auth_token = Token.query.filter_by(auth_token=request.headers.get('Authorization')).first() except: return {"alternatives" : "Database Error with token!"} if not auth_token: return {"alternatives": "Unauthorized!"} req_data = request.get_json() url = req_data['url'] if not url: return {"alternatives" : 'None'} domain_data = check(url) alternatives = {"alternatives": domain_data['available_alternatives']} return alternatives
def generate_admin_report(**kwargs): """ Generate a report with important data - email if mode is daemon :arg kwargs :kwarg mode :kwarg user_id (not used at this time) :returns nothing """ logger.debug("Creating Admin Report...") configs = get_configs() load_dotenv() engine = db.create_engine(os.environ['DATABASE_URL']) connection = engine.connect() metadata = db.MetaData() users = db.Table('users', metadata, autoload=True, autoload_with=engine) domains = db.Table('domains', metadata, autoload=True, autoload_with=engine) dgdomains = db.Table('dg_domains', metadata, autoload=True, autoload_with=engine) system_settings = db.Table('system_settings', metadata, autoload=True, autoload_with=engine) # System settings system_raw_dates = get_sys_info(all=True) last_logfile_analysis = system_raw_dates['last_logfile_analysis'].strftime( '%A %B %d, %Y at %I:%M %p %Z') last_ooni_report_generated = system_raw_dates[ 'last_ooni_report_generated'].strftime('%A %B %d, %Y at %I:%M %p %Z') last_domain_test = system_raw_dates['last_domain_test'].strftime( '%A %B %d, %Y at %I:%M %p %Z') # List admins user_query = db.select([users]).where(users.c.admin == True) admin_list = connection.execute(user_query).fetchall() # List Domain Group Owners # TODO: Generate reports for domain group owners dg_query = "select * from users, domain_groups where CAST(users.domain_group_id as integer)=domain_groups.id and domain_groups.name != 'None'" dg_list = connection.execute(dg_query).fetchall() # Get last date last_email_report_sent = get_sys_info(request='last_email_report_sent', update=True) reports = db.Table('reports', metadata, autoload=True, autoload_with=engine) report_query = db.select( [reports]).where(reports.c.date_reported > last_email_report_sent) report_list = connection.execute(report_query).fetchall() important_reports = "" number_of_reports = len(report_list) number_of_problems = 0 for report in report_list: if ((report['domain_status'] != 200) or (report['mirror_status'] != 200)): number_of_problems += 1 translated_report = translate_reports(report) important_reports += translated_report number_of_ooni_reports, number_of_ooni_problems, ooni_problems = ooni_reports( last_email_report_sent) for problem in ooni_problems: orept = f"OONI: URL Accessed: {problem['url_accessed']} Kind of Failure: {problem['failure']} DNS Consistency: {problem['dns_consistency']}\n" important_reports += orept if kwargs['mode'] == 'daemon': if important_reports: message_to_send = f""" Reporting problematic Domains and/or Alternatives since {last_email_report_sent}: There were {number_of_reports} domain testing reports, and {number_of_problems} problems. There were {number_of_ooni_reports} reports from OONI, with {number_of_ooni_problems} of problems. The last domain test was {last_domain_test}. The last logfile analysis was done on {last_logfile_analysis}. and the last OONI report was generated on {last_ooni_report_generated}. All detailed problem reports are below: {important_reports} """ else: message_to_send = f"""No Problematic Domains or Alternatives since {last_email_report_sent}. The last domain test was {last_domain_test}. The last logfile analysis was done on {last_logfile_analysis}. and the last OONI report was generated on {last_ooni_report_generated}. You might want to check the system.""" for user in admin_list: if user['notifications'] and user['active']: email = send_email(user['email'], "Report From BC APP", message_to_send) logger.debug(f"Message Sent to {user['email']}: {email}") else: if important_reports: print( f""" Reporting problematic Domains and/or Alternatives for Today: There were {number_of_reports} domain testing reports, and {number_of_problems} problems. There were {number_of_ooni_reports} reports from OONI, with {number_of_ooni_problems} of problems. The last domain test was {last_domain_test}. The last logfile analysis was done on {last_logfile_analysis}. The last OONI report was generated on {last_ooni_report_generated}. All detailed problem reports are below: {important_reports} """) else: print(f"""No problems reported since {last_email_report_sent}. The last domain test was {last_domain_test}. The last logfile analysis was done on {last_logfile_analysis}. and the last OONI report was generated on {last_ooni_report_generated}. You might want to check the system.""") return
def cloudfront_add_logging(domain): """ Add logging for a cloudfront distribution """ load_dotenv() engine = db.create_engine(os.environ['DATABASE_URL']) connection = engine.connect() metadata = db.MetaData() domains = db.Table('domains', metadata, autoload=True, autoload_with=engine) query = db.select([domains]).where(domains.c.domain == domain) result = connection.execute(query) row = result.fetchone() logger.debug(f"Domain: {row} S3 storage {row.s3_storage_bucket}") if not row.s3_storage_bucket: logger.debug("No S3 Storage set up!") return False # Find distribution based on domain configs = get_configs() session = boto3.Session(profile_name=configs['profile']) client = session.client('cloudfront', region_name=configs['region']) distributions = [] truncated = True marker = '' while truncated: distribution_list = client.list_distributions(Marker=marker) distributions.extend(distribution_list['DistributionList']['Items']) if not distribution_list['DistributionList']['IsTruncated']: truncated = False else: marker = distribution_list['DistributionList']['NextMarker'] edit_id = False for distribution in distributions: if domain in distribution['Origins']['Items'][0][ 'DomainName']: # it's what we want to edit edit_id = distribution['Id'] if not edit_id: logger.debug("Can't find right distribution - check domain name!") return False # Get config distro_config = client.get_distribution_config(Id=edit_id) logger.debug(f"Configuration: {distro_config}") etag = distro_config['ETag'] new_config = dict(distro_config['DistributionConfig']) new_config['Logging'] = { 'Enabled': True, 'IncludeCookies': False, 'Bucket': row.s3_storage_bucket, 'Prefix': '' } # Update config to add logging response = client.update_distribution(DistributionConfig=new_config, Id=edit_id, IfMatch=etag) d_etag = response['ETag'] logger.debug(f"Response: {response}") # Wait for it... logger.debug("Waiting for distribution to be reconfigured...") waiter = client.get_waiter('distribution_deployed') waiter.wait(Id=edit_id) logger.debug("Distribution updated!") return True
def cloudfront_add(**kwargs): """ creates new cloudfront distribution :params kwargs :kwarg <domain> :kwarg [mode] :returns url """ configs = get_configs() now = str(datetime.datetime.now()) session = boto3.Session(profile_name=configs['profile']) client = session.client('cloudfront', region_name=configs['region']) domain = get_final_domain(f"http://{kwargs['domain']}") logger.debug(f"For domain: {domain}") cdn_id = "Custom-" + domain response = client.create_distribution( DistributionConfig={ 'CallerReference': now, 'Origins': { 'Quantity': 1, 'Items': [{ 'Id': cdn_id, 'DomainName': domain, 'CustomOriginConfig': { 'HTTPPort': 80, 'HTTPSPort': 443, 'OriginProtocolPolicy': 'match-viewer', 'OriginSslProtocols': { 'Quantity': 3, 'Items': ['TLSv1', 'TLSv1.1', 'TLSv1.2'] }, 'OriginReadTimeout': 30, 'OriginKeepaliveTimeout': 5 } }] }, 'DefaultCacheBehavior': { 'TargetOriginId': cdn_id, 'ForwardedValues': { 'QueryString': True, 'Cookies': { 'Forward': 'none' } }, 'TrustedSigners': { 'Enabled': False, 'Quantity': 0 }, 'ViewerProtocolPolicy': 'redirect-to-https', 'MinTTL': 0 }, 'Comment': 'CDN for ' + domain, 'PriceClass': 'PriceClass_All', 'Enabled': True, 'ViewerCertificate': { 'CloudFrontDefaultCertificate': True } }) logger.debug(f"Response: {response}") distro_id = response['Distribution']['Id'] if 'mode' in kwargs and kwargs['mode'] == 'console': wait = input("Wait for distribution (y/N)?") if wait.lower() == 'y': logger.debug("And now we wait...") waiter = client.get_waiter('distribution_deployed') waiter.wait(Id=distro_id, WaiterConfig={ 'Delay': 60, 'MaxAttempts': 30 }) return response['Distribution']['DomainName']
def get_ooni_data(range): """ Get data from OONI S3 bucket """ last_ooni_report_generated = get_sys_info( request='last_ooni_report_generated', update=True) configs = get_configs() bucket = 'ooni-data-eu-fra' session = boto3.Session(profile_name=configs['profile']) client = session.client('s3') #get date range now = datetime.datetime.now() then = now - datetime.timedelta(days=range) delta = datetime.timedelta(days=1) logger.debug(f"Now: {now} Then: {then}") engine = db.create_engine(configs['database_url']) connection = engine.connect() metadata = db.MetaData() ooni_reports = db.Table('ooni_reports', metadata, autoload=True, autoload_with=engine) file_list = [] logger.debug("Getting OONI file list from S3...") while then <= now: date_str = then.strftime('%Y%m%d') file_date = 'raw/' + date_str then += delta date_report_list = client.list_objects_v2(Bucket=bucket, Prefix=file_date) for s3_file in date_report_list['Contents']: if ('webconnectivity' in s3_file['Key']) and ('jsonl' in s3_file['Key']): file_list.append(s3_file['Key']) # Process Files domain_list, mirror_list = lists() matching_domain_data = {} for domain in domain_list: matching_domain_data[domain['name']] = [] for file in file_list: file_parts = file.split('/') local_name = ('-').join(file_parts) local_file_path = configs['local_tmp'] + '/' + local_name logger.debug(f"Downloading to: {local_file_path}") with open(local_file_path, 'wb') as file_data: client.download_fileobj(bucket, file, file_data) data = [] with gzip.open(local_file_path) as raw_file: line = raw_file.readline() json_data = json.loads(line) data.append(json_data) os.remove(local_file_path) for jdata in data: logger.debug(f"input: {jdata['input']}") domain_name = False for domain in domain_list: match = site_match(domain['name'], jdata['input']) if match: domain_name = domain['name'] domain_id = domain['id'] if not domain_name: logger.debug("No match.") continue date_reported = datetime.datetime.strptime( jdata['measurement_start_time'], '%Y-%m-%d %H:%M:%S') matching_domain_data[domain_name] = { 'domain_id': domain_id, 'url_accessed': jdata['input'], 'country': jdata['probe_cc'], 'blocked': jdata['test_keys']['blocking'], 'dns_consistency': jdata['test_keys']['dns_consistency'], 'date_reported': date_reported } for key in jdata['test_keys']['requests']: for s_key in key: if s_key == 'failure': matching_domain_data[domain_name]['failure'] = key[ 'failure'] print( f"Matching Domain Data for {domain_name}:{matching_domain_data[domain_name]}" ) # Make report ooni_report_data = matching_domain_data[domain_name] insert = ooni_reports.insert().values(**ooni_report_data) result = connection.execute(insert) return
def send_report(domain_data, mode): """ Make a report to the database with console-generated tests :arg domain_data :returns nothing """ data_pretty = json.dumps(domain_data) logger.debug(f"Domain Data: {data_pretty}") now = datetime.datetime.now() host_name = socket.gethostname() host_ip = socket.gethostbyname(host_name) configs = get_configs() engine = db.create_engine(configs['database_url']) connection = engine.connect() metadata = db.MetaData() domains = db.Table('domains', metadata, autoload=True, autoload_with=engine) mirrors = db.Table('mirrors', metadata, autoload=True, autoload_with=engine) reports = db.Table('reports', metadata, autoload=True, autoload_with=engine) query = db.select([domains]) result = connection.execute(query).fetchall() domain_id = False inactive = False for entry in result: d_id, domain, ext, paths, s3_storage, azure_profile, inactive = entry logger.debug(f"Entry: {entry}") if domain in domain_data['domain']: domain_id = d_id if inactive: logger.debug("Inactive Domain!") return False logger.debug(f"Domain ID: {domain_id}") if not domain_id: # we've not seen it before, add it logger.debug("New Domain!") insert = domains.insert().values(domain=domain_data['domain']) result = connection.execute(insert) domain_id = result.inserted_primary_key[0] logger.debug(f"Domain ID: {domain_id}") # Add mirrors if (('current_alternatives' not in domain_data) or (not domain_data['current_alternatives'])): logger.debug("Not reporting on v1 data!") return False for current_alternative in domain_data['current_alternatives']: query = db.select([mirrors]) result = connection.execute(query).fetchall() mirror_id = False for entry in result: m_id, m_url, d_id, proto, m_type, inactive = entry if current_alternative['url'] == m_url: mirror_id = m_id logger.debug(f"Mirror ID: {mirror_id}") if not mirror_id: # add it logger.debug("New Alternative!") insert = mirrors.insert().values( mirror_url=current_alternative['url'], domain_id=domain_id, mirror_type=current_alternative['type'], protocol=current_alternative['proto']) result = connection.execute(insert) mirror_id = result.inserted_primary_key[0] logger.debug(f"Mirror ID: {mirror_id}") # Make report report_data = { 'date_reported': now, 'domain_id': domain_id, 'mirror_id': mirror_id, 'user_agent': f'BC APP {mode}', 'domain_status': domain_data[domain_data['domain']], 'mirror_status': current_alternative['result'], 'ip': host_ip } insert = reports.insert().values(**report_data) result = connection.execute(insert) return True
def analyze(unzip, percent, num, daemon, range, domain): import faulthandler faulthandler.enable() # update system info last_logfile_analysis = get_sys_info(request='last_logfile_analysis', update=True) configs = get_configs() now = datetime.datetime.now() now_string = now.strftime('%d-%b-%Y:%H:%M:%S') load_dotenv() engine = db.create_engine(os.environ['DATABASE_URL']) connection = engine.connect() metadata = db.MetaData() domains = db.Table('domains', metadata, autoload=True, autoload_with=engine) domains_list = [] query = db.select([domains]) result = connection.execute(query).fetchall() for line in result: domains_list.append({ 'id': line[0], 'name': line[1], 's3_bucket': line[4], 'azure_profile': line[5] }) for dm in domains_list: if ((domain == 'all') or (dm['name'] == domain)): # First, is there an azure profile set? if ('azure_profile' in dm) and (dm['azure_profile']): logger.debug( f"Domain: {dm['name']}: Azure Profile: {dm['azure_profile']}" ) retrieve_logs(profile_name=dm['azure_profile'], range=range, s3_bucket=dm['s3_bucket']) try: s3simple = S3Simple(region_name=configs['region'], profile=configs['profile'], bucket_name=dm['s3_bucket']) except: logger.warning(f"No bucket set for domain {dm['name']}") continue # get the file list to analyze # read from S3 #logger.debug(f"Getting files from S3 bucket {dm['s3_bucket']}...") file_list = s3simple.s3_bucket_contents() if not file_list: continue logger.debug(f"File List: {file_list}") compiled_data = { 'nginx': [], 'cloudfront': [], 'fastly': [], 'azure': [] } logger.debug(f"Analyzing {dm['name']}...") for ifile in file_list: if 'LogAnalysis' in ifile: continue if (('.gz' in ifile or '.bz2' in ifile) and not unzip): continue logger.debug(f"Processing file: {ifile}") if ifile[-1] == '/': directory = configs['local_tmp'] + '/' + ifile if not os.path.isdir(directory): os.mkdir(directory) continue file_date = filter_and_get_date(ifile) if not file_date: logger.warning("Couldn't find date in logs!") continue numdays = (now - file_date).days if numdays > range: continue #download local_path = configs['local_tmp'] + '/' + ifile #logger.debug(f"Downloading ... domain: {dm['name']} to {local_path}") try: s3simple.download_file(file_name=ifile, output_file=local_path) except: continue # Add to aggregate file_parts = ifile.split('.') ext = file_parts[-1] if ext == 'bz2' or ext == 'gz': if unzip: if ext == 'bz2': raw_data = str(sh.bunzip2("-k", "-c", local_path)) else: raw_data = str(sh.gunzip("-k", "-c", local_path)) else: continue else: with open(local_path) as f: raw_data = f.read() #logger.debug(f"Files data: {raw_data}") compiled_log_data, log_type = analyze_file( raw_data, dm['name']) if not compiled_log_data: logger.warning("No Data!") continue compiled_data[log_type] += compiled_log_data #logger.debug(f"Deleting local temporary file {local_path}...") os.remove(local_path) for log_type in compiled_data: logger.debug(f"Log type: {log_type}") #logger.debug(f"Analyzed data: {compiled_data[log_type]}") if not compiled_data[log_type]: continue analyzed_log_data = analyze_data(compiled_data[log_type], log_type) (output_text, first_date, last_date, hits, home_page_hits) = output(domain=dm['name'], data=analyzed_log_data, percent=percent, num=num) logger.debug(output_text) logger.debug("Saving log analysis file...") key = 'LogAnalysis_' + dm[ 'name'] + '_' + log_type + '_' + now_string + '.json' body = str(analyzed_log_data) s3simple.put_to_s3(key=key, body=body) logger.debug("Saving output file....") key = 'LogAnalysisOutput_' + dm[ 'name'] + '_' + log_type + '_' + now_string + '.txt' s3simple.put_to_s3(key=key, body=output_text) logger.debug("Sending Report to Database...") report_save(domain=dm['name'], datetime=now, report_text=output_text, hits=hits, home_page_hits=home_page_hits, first_date_of_log=first_date, last_date_of_log=last_date, log_type=log_type) return
def move_logs(daemon, zip, recursive, range): """ Move logs from local to s3 """ configs = get_configs() now = datetime.datetime.now() now_string = now.strftime('%d-%b-%Y:%H:%M:%S') logger.debug("Reading Local Files...") if configs['paths']: # open paths file with open(configs['paths']) as pathfile: raw_path_list = pathfile.read() paths = raw_path_list.split('\n') for fpath in paths: if not fpath: continue domain, path = fpath.split('|') load_dotenv() engine = db.create_engine(os.environ['DATABASE_URL']) connection = engine.connect() metadata = db.MetaData() domains = db.Table('domains', metadata, autoload=True, autoload_with=engine) domains_list = [] query = db.select([domains]) result = connection.execute(query).fetchall() for line in result: domains_list.append({ 'id': line[0], 'name': line[1], 's3_bucket': line[4] }) domain_match = False for db_domain in domains_list: if ((db_domain['name'] == domain) and (db_domain['s3_bucket'])): domain_match = True s3simple = S3Simple(region_name=configs['region'], profile=configs['profile'], bucket_name=db_domain['s3_bucket']) if not domain_match: logger_debug("No s3 bucket match!") continue if not os.path.exists(path): logger.critical("Path doesn't exist!") return if not os.path.isdir(path): files = [path] else: files = get_list(path, recursive, range) logger.debug(f"Path: {path}") # send to S3 for file_name in files: if 'access' not in file_name: continue file_path = file_name.split('/') file_parts = file_path[-1].split('.') just_file_name = '.'.join(file_path) logger.debug(f"File Name {just_file_name}") ext = file_parts[-1] if ((ext == 'bz2' or ext == 'gz')) and not zip: continue logger.debug("sending to s3...") s3_file = 'RawLogFile_' + domain + '_' + now_string + '_' + just_file_name s3simple.send_file_to_s3(local_file=file_name, s3_file=s3_file)
def azure_add(**kwargs): configs = get_configs() domain = get_final_domain(f"http://{kwargs['domain']}") # Tenant ID for your Azure subscription TENANT_ID = configs['azure_tenant_id'] # Your service principal App ID CLIENT = configs['azure_app'] # Your service principal password KEY = configs['azure_key'] logger.debug("Authenticating...") credentials = ServicePrincipalCredentials(client_id=CLIENT, secret=KEY, tenant=TENANT_ID) endpoint_name = kwargs['domain'][0:6] + '1' endpoint_name_confirm = input( f"Azure CDN name/subdomain ({endpoint_name}.azureedge.net)?") if endpoint_name_confirm: endpoint_name = endpoint_name_confirm endpoint_full_name = endpoint_name + '.azureedge.net' my_resource_group = 'bypasscensorship' region = 'West India' tier = 'Standard_Akamai' cdn_client = CdnManagementClient(credentials, configs['azure_sub_id']) cdn_list = cdn_client.profiles.list() print("List of Azure CDNs:") count = 0 names = [] for profile in cdn_list: pf_vars = vars(profile) print(f"{count}: Name: {pf_vars['name']}") names.append(pf_vars['name']) endpoints_list = cdn_client.endpoints.list_by_profile( my_resource_group, pf_vars['name']) ep_count = 0 for endpoint in endpoints_list: ep_count += 1 print(f"Number of endpoints: {ep_count}") count += 1 cdn_choice = input(f"Which CDN to add {endpoint_full_name} to?") if not cdn_choice: return False else: cdn_name = names[int(cdn_choice)] cdn_confirm = input(f"Add to {cdn_name} (Y/n)?") if cdn_confirm.lower() == 'n': return False logger.debug("Adding...") endpoint_poller = cdn_client.endpoints.create( my_resource_group, cdn_name, endpoint_name, { "location": region, "origin_host_header": domain, "origins": [{ "name": cdn_name, "host_name": domain }] }) endpoint = endpoint_poller.result() logger.debug("Done!") return endpoint_full_name
def retrieve_logs(**kwargs): """ Retrieve logs from storage """ configs = get_configs() now = datetime.datetime.now() logger.debug("Grabbing files from Azure...") # Create a client container_name = "insights-logs-azurecdnaccesslog" # it's always called this container = ContainerClient.from_connection_string( conn_str=configs['azure_storage_conn_string'], container_name=container_name) date_match = re.compile( "y=[0-9]{4}\/m=[0-9]{2}\/d=[0-9]{2}\/h=[0-9]{2}\/m=[0-9]{2}") # List blobs blob_list = container.list_blobs() for blob in blob_list: try: log_date_string = date_match.search(blob.name).group(0) log_date = datetime.datetime.strptime(log_date_string, "y=%Y/m=%m/d=%d/h=%H/m=%M") #print(f"Date: {log_date} & {log_date_string}") except: continue # right profile? #logger.debug(f"Profile: {kwargs['profile_name']} blob.name: {blob.name}") if kwargs['profile_name'].upper() not in blob.name: #logger.debug('Not right profile') continue # TODO: Find right range of files, copy to S3 numdays = (now - log_date).days if numdays > kwargs['range']: logger.debug(f"File {blob.name} too old!") continue file_date = datetime.datetime.strftime(log_date, "%Y-%m-%d-%H-%M") s3_filename = "Azure_CDN_log_" + kwargs[ 'profile_name'] + "_" + file_date + ".json" local_path = configs['local_tmp'] + "/" + s3_filename #logger.debug(f"Local path: {local_path}") get_blob = BlobClient.from_connection_string( conn_str=configs['azure_storage_conn_string'], container_name=container_name, blob_name=blob.name) with open(local_path, "wb") as tmp_blob: blob_data = get_blob.download_blob() blob_data.readinto(tmp_blob) #upload to S3 s3simple = S3Simple(region_name=configs['region'], profile=configs['profile'], bucket_name=kwargs['s3_bucket']) s3simple.send_file_to_s3(local_file=local_path, s3_file=s3_filename) os.remove(local_path) return
""" Bypass Censorship API Allows for testing and requests for BP mirrors from an API """ import sys sys.path.insert(0, '.') from system_utilities import get_configs from app import app if __name__ == '__main__': configs = get_configs() log = configs['log_level'] logger = logging.getLogger('logger') # instantiate clogger logger.setLevel(logging.DEBUG) # pass DEBUG and higher values to handler ch = logging.StreamHandler() # use StreamHandler, which prints to stdout ch.setLevel(configs['log_level']) # ch handler uses the configura # create formatter # display the function name and logging level in columnar format if # logging mode is 'DEBUG' formatter = logging.Formatter('[%(funcName)24s] [%(levelname)8s] %(message)s') # add formatter to ch ch.setFormatter(formatter) logger.addHandler(ch)
def automation(testing, domain, test, proxy, existing, delete, domain_list, log, mirror_type, replace, remove, report, mode, generate_report, s3, ooni, missing): configs = get_configs() logger.debug(f"Repo: {configs['repo']}") if domain: if delete: delete = delete_domain(domain) if mode == 'console': if not delete: print(f"Domain {domain} not deleted from github.") else: print(f"Domain {domain} deleted from github, and {delete}") elif replace: delete_deprecated(domain) replace_mirror(domain=domain, existing=existing, replace=replace, mirror_type=mirror_type, mode=mode) elif remove: delete_deprecated(domain) removed = remove_mirror(domain=domain, remove=remove) if mode == 'console': print(removed) elif log: add_logging(domain=domain, mirror_type=mirror_type, mode=mode) elif s3: s3_storage_add = add_s3_storage(domain=domain, s3=s3) if mode == 'console': print(f"Result: {s3_storage_add}") elif mirror_type or existing: domain = strip_www(domain) delete_deprecated(domain) new_add(domain=domain, mirror_type=mirror_type, existing=existing, mode=mode) domain_testing(proxy, mode, domain) elif report: domain_reporting(domain=domain, mode=mode) elif missing: missing_mirrors(domain=domain) else: domain_data = mirror_detail(domain=domain, proxy=proxy, mode=mode, test=test) if mode == 'console': if not domain_data: print("No data returned!") return elif missing: missing_mirrors(missing=missing) elif ooni: get_ooni_data(ooni) elif testing: if mode == 'console': test = input("Test all (Y/n)?") else: test = 'y' if test.lower() != 'n': domain_testing(proxy, mode, '') elif generate_report: generate_admin_report(mode=mode, user_id=None) elif domain_list: #assuming console mode dlist = domain_listing() print(f""" List of all domains, mirrors and onions ___________________________________________________ {dlist} ___________________________________________________ """) else: click.echo("Invalid parameters! try --help") return