async def run(self): # TODO: thread issues? verify_ssl = True if bool(self.option("no_ssl_verify")): verify_ssl = False self.api_client = ApiClient(self.get_api_url(), self.get_api_token(), verify_ssl=verify_ssl) if not self.option("no_api_wait"): if not self.api_client.wait_for_up(): logger.critical( "run@http_server.py - Could not connect to api. quitting") self.exit(1) if self.option("no_sync"): logger.info("run@http_server.py - Skipping syncing api token") else: self.api_client.sync() self.boot() self.start_servers() try: count = 0 while self.is_alive(): # zones don't need to be refreshed for http server # may want to do something in the future thought count = count + 1 sleep(1) except KeyboardInterrupt: pass
async def store( form: DnsRequestCreateForm, dns_request_repo: DnsRequestRepo = Depends(DnsRequestRepo()), zone_repo: ZoneRepo = Depends(ZoneRepo()), dns_server_repo: DnsServerRepo = Depends(DnsServerRepo()), token: str = Depends(ScopedTo("dns-request:create")), ): dns_server_id = ( dns_server_repo.first_or_fail(name=form.dns_server_name.lower()).results().id ) zone = ( zone_repo.filter(literal(form.name.lower()).contains(zone_repo.label("domain"))) .first() .results() ) zone_id = zone.id if zone else None data = only( dict(form), ["name", "source_address", "source_port", "type", "protocol", "raw_request"], ) data["name"] = data["name"].lower() data["type"] = data["type"].upper() data["dns_server_id"] = dns_server_id data["zone_id"] = zone_id logger.info("[email protected] - Creating DNS Request") dns_request = dns_request_repo.create(data).data() return DnsRequestResponse(dns_request=dns_request)
def __init__( self, port=80, listen="127.0.0.1", ssl=False, ssl_key_path=None, ssl_cert_path=None, api_client=None, ): self.port = int(port) self.listen = listen self.ssl = ssl self.api_client = api_client self.logger = logger if not ssl: self.server = HttpServer((self.listen, self.port), HttpHandler, api_client, logger) else: self.server = HttpsServer((self.listen, self.port), HttpHandler, api_client, logger) logger.info( f"[email protected] - Building ssl server with certfile: {ssl_cert_path} and keyfile: {ssl_key_path}" ) self.server.socket = ssllib.wrap_socket( self.server.socket, server_side=True, certfile=ssl_cert_path, keyfile=ssl_key_path, ssl_version=ssllib.PROTOCOL_TLS, ) self.thread = None
def is_alive(self): if not self.thread: logger.info( "[email protected] - HttpServer is not alive because there is not thread" ) return False return self.thread.isAlive()
def start_thread(self): proto = "https://" if self.ssl else "http://" logger.info( f"[email protected] - Starting Http server thread for server: {proto}{self.listen}:{self.port}" ) self.thread = threading.Thread(target=self.server.serve_forever) self.thread.daemon = True self.thread.start()
def start(self): if self.ssl: logger.info("[email protected] - Starting Https server") else: logger.info("[email protected] - Starting Http server") if not self.server: raise Exception("No server on HttpServer") self.server.serve_forever()
def get_zones(self): logger.info("get_zones@api_client.py - Getting zones") zone_data = self.get( f"/dns-server/{self.dns_server_name}/zone", params={"includes": ["dns_records"]}, ) data = [ZoneData(**z) for z in zone_data["zones"]] return data
def create_or_get_dns_server(token: TokenPayload, dns_server_repo: DnsServerRepo) -> DnsServer: if not dns_server_repo.exists(name=token.payload.dns_server_name.lower()): dns_server_repo.clear() logger.info("[email protected] - Saving dns server from api token") return dns_server_repo.create( dict(name=token.payload.dns_server_name.lower())).results() else: return dns_server_repo.results()
def create_or_get_http_server(token: TokenPayload, http_server_repo: HttpServerRepo) -> HttpServer: if not http_server_repo.exists( name=token.payload.http_server_name.lower()): http_server_repo.clear() logger.info("[email protected] - Saving http server from api token") return http_server_repo.create( dict(name=token.payload.http_server_name.lower())).results() else: return http_server_repo.results()
def db_register_model_events(models): for m in models: for event_name in ORM_EVENTS: event_cb = "on_" + event_name if hasattr(m, event_cb): logger.info( "[email protected]: registering " + event_cb + " on " + str(m) ) listen(m, event_name, make_event(getattr(m, event_cb)))
def refresh_zones_if_needed(self): logger.info( "refresh_zones_if_needed@api_client.py - Checking for New Zones and Records..." ) old_zones = self.zones new_zones = self.get_zones() # TODO: fix this mess if len(old_zones) != len(new_zones): logger.warning( f"refresh_zones_if_needed@api_client.py - Zone Length mistmatch. New or Changed Zone Found: {str(old_zones)} != {str(new_zones)}. Reloading zones." ) self.load_zones() return True for nz in new_zones: # make sure new zones are in old zones is_nz_exists = False for oz in old_zones: if oz.domain == nz.domain and oz.ip == nz.ip: nz_dns_records = nz.dns_records or [] for nrec in nz_dns_records: is_rec_satisfied = False oz_dns_records = oz.dns_records or [] if len(nz_dns_records) != len(oz_dns_records): logger.warning( f"refresh_zones_if_needed@api_client.py - Zone Record Length mistmatch {str(len(nz_dns_records))} != {str(len(oz_dns_records))}. New or Changed Zone Record Found: {str(nz_dns_records)} != {str(oz_dns_records)}. Reloading zones." ) self.load_zones() return True for orec in oz_dns_records: if orec.record == nrec.record and orec.sort == nrec.sort: is_rec_satisfied = True if not is_rec_satisfied: logger.warning( f"refresh_zones_if_needed@api_client.py - New or Changed Zone Record {str(nz)}: {str(nrec)} found for server. Reloading zones." ) self.load_zones() return True is_nz_exists = True if not is_nz_exists: logger.warning( f"refresh_zones_if_needed@api_client.py - New or Changed Zone {str(nz)} found for server. Reloading zones." ) self.load_zones() return True logger.info( "refresh_zones_if_needed@api_client.py - No New Zones or Records Found. All is well" ) return False
async def run(self): if not Path(AlembicInit.migration_dir).is_dir(): logger.info("run@db_setup.py - Running alembic-init") await AlembicInit.make(self.options).run() logger.info("run@db_setup.py - Running alembic upgrade") await AlembicUpgrade.make(self.options).run() logger.info("run@db_setup.py - Running alembic migrate") await AlembicMigrate.make(self.options).run() logger.info("run@db_setup.py - Running alembic upgrade again") await AlembicUpgrade.make(self.options).run() if self.option("seed", None): logger.info("run@db_setup.py - Running db seed") await DbSeed.make(self.options).run()
async def broadcast_index(websocket: WebSocket): try: logger.info("[email protected] - Accepting websockets") await websocket.accept() except Exception as e: logger.critical(f"[email protected] - Accept: Error: {str(type(e))}") logger.critical(f"[email protected] - Accept: Trace: {str(e)}") logger.critical( f"[email protected] - Accept:Not closing or unsubscribing" ) return 1 try: while True: logger.info("[email protected] - Receiving json") data = await websocket.receive_json() logger.info("[email protected] - Received json: " + str(data)) logger.info( "[email protected] - Sending message: " + str({"message": "greetings"}) ) await websocket.send_json({"message": "greetings"}) except Exception as e: logger.critical( f"[email protected] - Receieve/Send: Error: {str(type(e))}" ) logger.critical( f"[email protected] - Receieve/Send: Trace: {str(e)}" ) logger.critical( f"[email protected] - Receieve/Send: Not closing or unsubscribing" ) return 1 await websocket.close()
def create_dns_request(self, handler, request, request_uuid): logger.info("create_dns_request@api_client.py - Creating dns request") name = str(request.q.qname) name = name.rstrip(".") data = { "name": name, "source_address": str(handler.client_address[0]), "source_port": int(handler.client_address[1]), "type": str(QTYPE[request.q.qtype]), "protocol": str(handler.protocol), "dns_server_name": str(self.dns_server_name), "raw_request": str(request), } self.post("/dns-request", data=data)
def post(self, url: str, data=None, fail=True): data = data or {} headers = self.get_default_headers() logger.info("post@api_client.py - Posting URL: " + str(self.url(url))) res = requests.post(self.url(url), json=data, headers=headers, verify=self.verify_ssl) if fail: if res.status_code != 200: logger.critical(f"Error posting API {self.url(url)}: " + str(res.json())) res.raise_for_status() return res.json()
def get(self, url: str, params=None, fail=True): params = params or {} headers = self.get_default_headers() logger.info("get@api_client.py - Getting URL: " + str(self.url(url))) res = requests.get(self.url(url), headers=headers, params=params, verify=self.verify_ssl) if fail: if res.status_code != 200: logger.critical( f"get@api_client.py - Error getting API {self.url(url)}: " + str(res.json())) res.raise_for_status() return res.json()
def create_or_get_api_token_for_dns_server( token: TokenPayload, api_token_repo: ApiTokenRepo, dns_server: DnsServer) -> ApiTokenData: scopes = token.scopes if not api_token_repo.exists(token=token.token): api_token_repo.clear() logger.info( "[email protected] - Saving api token from auth token for dns node") return api_token_repo.create( dict( token=token.token, scopes=" ".join(scopes), dns_server=dns_server, expires_at=datetime.utcfromtimestamp(float(token.exp)), )).data() else: logger.info("[email protected] - token already exists in database") return api_token_repo.loads("dns_server").includes("dns_server").data()
def wait_for_up(self): attempts = 0 while True: if attempts > 60: logger.warning("could not connect to api. api not up") return False logger.info( f"wait_for_up@api_client.py - checking for api status : {self.url('/status')}" ) try: sleep(1) self.get_status() sleep(3) return True except Exception as e: logger.info( "wait_for_up@api_client.py - api check not ready after {} attempts: {}" .format(str(attempts), str(e.__class__.__name__))) attempts = attempts + 1 sleep(1)
def boot(self): port = self.option("port") listen = self.option("listen") logger.info("boot@http_server.py - Building http server on port %d", port) self.http_server = HttpServerManager(port=port, listen=listen, ssl=False, api_client=self.api_client) if bool(self.option("enable_ssl")): ssl_port = self.option("ssl_port") ssl_listen = self.option("ssl_listen") ssl_key_path = self.option("ssl_key_path") ssl_cert_path = self.option("ssl_cert_path") logger.info( f"boot@http_server.py - Building https server manager on port {str(ssl_port)} with {ssl_cert_path} and {ssl_key_path}", ) self.https_server = HttpServerManager( port=ssl_port, listen=ssl_listen, api_client=self.api_client, ssl=True, ssl_cert_path=ssl_cert_path, ssl_key_path=ssl_key_path, ) logger.info( "boot@http_server.py - Building https server on port %d", ssl_port)
async def broadcast_auth(websocket: WebSocket): try: await websocket.accept() except Exception as e: logger.critical(f"[email protected] - Accept Error: {str(type(e))}") logger.critical(f"[email protected] - Accept Trace: {str(e)}") return 1 params = parse_qs(urlparse(str(websocket.url)).query) token = verify_jwt_token(params["ws_access_token"][0]) if not token_has_required_scopes(token, []): # TODO: check scopes later raise HTTPException(403, detail="Forbidden") user_repo = UserRepo(session()) user = await current_user(token, user_repo) subscriber, channel = await make_subscriber("auth") try: while await channel.wait_message(): logger.info("[email protected] - Waiting for message") msg = await channel.get(encoding="utf-8") logger.info("[email protected] - Received message: " + str(msg)) data = json.loads(msg) logger.info("[email protected] - Sending message: " + str(data)) await websocket.send_json(data) except Exception as e: logger.critical( f"[email protected] - Receieve/Send: Error: {str(type(e))}" ) logger.critical(f"[email protected] - Receieve/Send: Trace:{str(e)}") logger.critical( f"[email protected] - Receieve/Send: Not closing or unsubscribing" ) return 1 logger.info(f"[email protected] - Attempting to unsuscribe") await subscriber.unsubscribe("channel:auth") logger.info(f"[email protected]: Websocket - Attempting to close socket") await websocket.close()
def boot(self): port = self.get_port() listen = self.get_listen() self.resolver = Resolver(self.api_client) self.udp_server = DNSServer( self.resolver, address=listen, port=port, handler=DNSHandler, logger=DNSLogger(self.api_client), ) self.tcp_server = DNSServer( self.resolver, address=listen, port=port, tcp=True, handler=DNSHandler, logger=DNSLogger(self.api_client), ) logger.info("starting DNS server on port %d", port)
def create_http_request( self, name, path, source_address, source_port, type, protocol, raw_request, ): logger.info( "create_http_request@api_client.py - Creating http request") data = { "name": name, "path": path, "source_address": str(source_address), "source_port": int(source_port), "type": str(type), "protocol": str(protocol), "http_server_name": str(self.http_server_name), "raw_request": str(raw_request), } self.post("/http-request", data=data)
def stop(self): logger.info("[email protected] - Stopping Http server") if not self.server: raise Exception("No server on HttpServer") self.server.shutdown()
def sync(self): logger.info("sync@api_client.py - Syncing api token") return self.post("/api-token/sync", fail=True)
async def run(self): app = "boucanpy.api.main:api" kwargs = self.get_kwargs() if self.should_import_check(): logger.info("run@api_server.py - Performing import check") from boucanpy.api.main import api logger.critical( "run@api_server.py - Starting api server with options: {}".format( str(kwargs))) from boucanpy.db.checks import is_db_up, is_db_setup # alembic just destroys the loggers, it's annoying if self.should_db_check(): logger.info( "run@api_server.py - Waiting for database service to be up") db_wait_options = self._args_to_dict(self.options) await DbWait(db_wait_options).run() if self.option("db_setup"): logger.critical("run@api_server.py - Running database migration") db_setup_options = self._args_to_dict(self.options) if self.option("db_seed"): db_setup_options["seed"] = True await DbSetup(db_setup_options).run() if self.should_db_check(): logger.info( "run@api_server.py - Checking if application database is setup and configured" ) db_setup = is_db_setup() if not db_setup: logger.critical( "run@api_server.py - Database not setup error. please check logs" ) return self.exit(1) from boucanpy.broadcast import is_broadcast_up if self.should_bcast_check(): bcast_up = await is_broadcast_up() if not bcast_up: logger.critical( "run@api_server.py - Broadcast (queue) not up error. please check logs" ) return self.exit(1) if self.option("db_seed_env", False): self.seed_from_env() # taken from uvicorn/main.py:run logger.debug("run@api_server.py - Building Uvicorn Config and Server") config = UvicornConfig(app, log_config=self.get_uvicorn_logging(), **kwargs) server = UvicornServer(config=config) if self.option("force_exit"): server.force_exit = True if isinstance(app, str) and (config.debug or config.reload): logger.warning( f"run@api_server.py - Running boucanpy api in dev mode...") sock = config.bind_socket() supervisor = StatReload(config) return supervisor.run(server.run, sockets=[sock]) elif config.workers > 1: sock = config.bind_socket() supervisor = Multiprocess(config) logger.warning( f"run@api_server.py - Running boucanpy api in worker mode...") return supervisor.run(server.run, sockets=[sock]) else: sockets = None logger.warning( f"run@api_server.py - Running boucanpy api in standard mode..." ) return await server.serve(sockets=sockets)
def seed_from_env(self): from boucanpy.core.user import UserRepo from boucanpy.core.zone import ZoneRepo from boucanpy.core.dns_server import DnsServerRepo from boucanpy.db.session import _scoped_session session = _scoped_session for i in range(9): i = str(i) user_data = {} email_key = f"SEED_USER_{i}_EMAIL" email = environ.get(email_key, None) password_key = f"SEED_USER_{i}_PASSWORD" password = environ.get(password_key, None) superuser_key = f"SEED_USER_{i}_SUPERUSER" is_superuser = int(environ.get(superuser_key, 0)) if email and password: email = email.lower() hashed_password = hash_password(password) repo = UserRepo(db=session) if not repo.exists(email=email): logger.info( f"seed_from_env@api_server.py - seeding user {email}") user = factory("UserFactory", session=session).create( email=email, hashed_password=hashed_password, is_superuser=is_superuser, ) else: logger.info( f"seed_from_env@api_server.py - Seeded user {email} already exists" ) for i in range(9): i = str(i) name_key = f"SEED_DNS_SERVER_{i}_NAME" name = environ.get(name_key, None) if name: repo = DnsServerRepo(db=session) if not repo.exists(name=name): logger.info( f"seed_from_env@api_server.py - Seeding domain {name}") domain = factory("DnsServerFactory", session=session).create(name=name) for i in range(9): i = str(i) ip_key = f"SEED_ZONE_{i}_IP" domain_key = f"SEED_ZONE_{i}_DOMAIN" dns_server_name_key = f"SEED_ZONE_{i}_DNS_SERVER_NAME" ip = environ.get(ip_key, None) domain = environ.get(domain_key, None) if domain: domain = domain.lower() dns_server_name = environ.get(dns_server_name_key, None) if ip and domain: if dns_server_name: dns_server_repo = DnsServerRepo(db=session) if dns_server_repo.exists(name=dns_server_name): dns_server = dns_server_repo.results() else: logger.info( f"seed_from_env@api_server.py - Seeding dns server as zone dependency: {name}" ) dns_server = factory( "DnsServerFactory", session=session).create(name=dns_server_name) factory("ZoneFactory", session=session).create(ip=ip, domain=domain, dns_server=dns_server) else: repo = ZoneRepo(db=session) if not repo.exists(ip=ip, domain=domain): logger.info( f"seed_from_env@api_server.py - Seeding zone without dns server: {ip}, {domain}" ) factory("GlobalZoneFactory", session=session).create(ip=ip, domain=domain)
# CORS api = FastAPI(title=config.API_PROJECT_NAME, openapi_url="/api/v1/openapi.json") origins = [] # Set all CORS enabled origins if config.API_CORS_ORIGINS: origins_raw = config.API_CORS_ORIGINS.split(",") for origin in origins_raw: use_origin = origin.strip() origins.append(use_origin) logger.info(f"[email protected] - Registering cors origins {origins}") api.add_middleware( CORSMiddleware, allow_origins=origins, allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ), main_router = APIRouter() for r, ropts in routers: logger.debug(f"[email protected] - Registering router {str(r)} {str(ropts)}") main_router.include_router(r, **ropts)
async def run(self): # TODO: thread issues? verify_ssl = True if bool(self.option("no_ssl_verify")): verify_ssl = False self.api_client = ApiClient(self.get_api_url(), self.get_api_token(), verify_ssl=verify_ssl) if not self.api_client.wait_for_up(): logger.critical( "run@dns_server.py - Could not connect to api. quitting") self.exit(1) if self.option("no_sync"): logger.info("run@dns_server.py - Skipping syncing api token") else: self.api_client.sync() self.boot() self.start_servers() try: count = 0 while self.udp_server.isAlive(): if count > 0 and count % self.option("refresh_ttl") == 0: if self.api_client.refresh_zones_if_needed(): logger.critical( "run@dns_server.py - API Client found new or changed zones. Stopping servers..." ) # TODO: figure out why "stop" does not release the address self.stop_servers() sleep(1) stop_count = 0 logger.critical( "run@dns_server.py - Waiting for UDP Server to stop..." ) while self.udp_server.thread and self.udp_server.isAlive( ): if stop_count > 30: logger.critical( "run@dns_server.py - UDP Server did not stop while reloading zones" ) raise Exception( "run@dns_server.py - UDP Server threads went rogue during zone reload" ) logger.info( "run@dns_server.py - Waiting for DNS Server to stop before reloading zones" ) stop_count = stop_count + 1 sleep(1) stop_count = 0 logger.critical( "run@dns_server.py - Waiting for TCP Server to stop..." ) while self.tcp_server.thread and self.tcp_server.isAlive( ): if stop_count > 30: logger.critical( "run@dns_server.py - TCP Server did not stop while reloading zones" ) raise Exception( "run@dns_server.py - TCP Server threads went rogue during zone reload" ) logger.info( "run@dns_server.py - Waiting for DNS Server to stop before reloading zones" ) stop_count = stop_count + 1 sleep(1) logger.critical( "run@dns_server.py - Rebooting server with fresh zones..." ) self.boot() self.start_servers() count = count + 1 sleep(1) except KeyboardInterrupt: pass
async def run(self): self.db_register() if self.option("target", False) == False: self.set_option("target", "dev") if self.option("target") == "env": logger.info(f"run@db_seed.py - Seeding {self.option('target')}") raise NotImplementedError() # seed based on env vars elif self.option("target") == "dev": logger.info(f"run@db_seed.py - Seeding {self.option('target')}") logger.info("run@db_seed.py - reating superuser") super = factory("SuperUserFactory").create(email="*****@*****.**") logger.info("run@db_seed.py - Creating normal user") norm = factory("UserFactory").create(email="*****@*****.**") logger.info("run@db_seed.py - Creating dns_server") _dns_server = factory("DnsServerFactory").create( name="mydnsserver") dns_server = factory("DnsServerFactory").create() logger.info("run@db_seed.py - Creating http_server") _http_server = factory("HttpServerFactory").create( name="myhttpserver") http_server = factory("HttpServerFactory").create() logger.info("run@db_seed.py - Creating zones") zone = factory("ZoneFactory").create(domain="othersite.com", ip="127.0.1.1") zone2 = factory("ZoneFactory").create( domain="friends4life.com", ip="127.0.1.1", dns_server=dns_server, http_server=http_server, ) zone3 = factory("ZoneFactory").create(domain="differentzone.com", ip="127.0.1.1") logger.info("run@db_seed.py - Creating api_tokens") factory("ApiTokenFactory").create(dns_server=dns_server) factory("ApiTokenFactory").create(http_server=http_server) factory("ApiTokenFactory").create(dns_server=dns_server, http_server=http_server) logger.info("run@db_seed.py - Creating dns_requests") for i in range(35): factory("DnsRequestFactory").create(dns_server=dns_server, zone=zone2) for i in range(35): factory("DnsRequestFactory").create(dns_server=dns_server, zone=zone3) logger.info("run@db_seed.py - Creating dns_records") for i in range(3): factory("DnsRecordFactory").create(zone=zone) for i in range(3): factory("DnsRecordFactory").create(zone=zone2) for i in range(3): factory("DnsRecordFactory").create(zone=zone3) logger.info("run@db_seed.py - Creating http_requests") for i in range(35): factory("HttpRequestFactory").create(http_server=http_server, zone=zone2) for i in range(35): factory("HttpRequestFactory").create(http_server=http_server, zone=zone3) else: logger.critical("run@db_seed.py - invalid target set for seeder") self.exit(1)