async def run(self):
# TODO: thread issues?
verify_ssl = True
if bool(self.option("no_ssl_verify")):
verify_ssl = False
self.api_client = ApiClient(self.get_api_url(),
self.get_api_token(),
verify_ssl=verify_ssl)
if not self.option("no_api_wait"):
if not self.api_client.wait_for_up():
logger.critical(
"run@http_server.py - Could not connect to api. quitting")
self.exit(1)
if self.option("no_sync"):
logger.info("run@http_server.py - Skipping syncing api token")
else:
self.api_client.sync()
self.boot()
self.start_servers()
try:
count = 0
while self.is_alive():
# zones don't need to be refreshed for http server
# may want to do something in the future thought
count = count + 1
sleep(1)
except KeyboardInterrupt:
pass
async def store(
form: DnsRequestCreateForm,
dns_request_repo: DnsRequestRepo = Depends(DnsRequestRepo()),
zone_repo: ZoneRepo = Depends(ZoneRepo()),
dns_server_repo: DnsServerRepo = Depends(DnsServerRepo()),
token: str = Depends(ScopedTo("dns-request:create")),
):
dns_server_id = (
dns_server_repo.first_or_fail(name=form.dns_server_name.lower()).results().id
)
zone = (
zone_repo.filter(literal(form.name.lower()).contains(zone_repo.label("domain")))
.first()
.results()
)
zone_id = zone.id if zone else None
data = only(
dict(form),
["name", "source_address", "source_port", "type", "protocol", "raw_request"],
)
data["name"] = data["name"].lower()
data["type"] = data["type"].upper()
data["dns_server_id"] = dns_server_id
data["zone_id"] = zone_id
logger.info("[email protected] - Creating DNS Request")
dns_request = dns_request_repo.create(data).data()
return DnsRequestResponse(dns_request=dns_request)
def __init__(
self,
port=80,
listen="127.0.0.1",
ssl=False,
ssl_key_path=None,
ssl_cert_path=None,
api_client=None,
):
self.port = int(port)
self.listen = listen
self.ssl = ssl
self.api_client = api_client
self.logger = logger
if not ssl:
self.server = HttpServer((self.listen, self.port), HttpHandler,
api_client, logger)
else:
self.server = HttpsServer((self.listen, self.port), HttpHandler,
api_client, logger)
logger.info(
f"[email protected] - Building ssl server with certfile: {ssl_cert_path} and keyfile: {ssl_key_path}"
)
self.server.socket = ssllib.wrap_socket(
self.server.socket,
server_side=True,
certfile=ssl_cert_path,
keyfile=ssl_key_path,
ssl_version=ssllib.PROTOCOL_TLS,
)
self.thread = None
def is_alive(self):
if not self.thread:
logger.info(
"[email protected] - HttpServer is not alive because there is not thread"
)
return False
return self.thread.isAlive()
def start_thread(self):
proto = "https://" if self.ssl else "http://"
logger.info(
f"[email protected] - Starting Http server thread for server: {proto}{self.listen}:{self.port}"
)
self.thread = threading.Thread(target=self.server.serve_forever)
self.thread.daemon = True
self.thread.start()
def start(self):
if self.ssl:
logger.info("[email protected] - Starting Https server")
else:
logger.info("[email protected] - Starting Http server")
if not self.server:
raise Exception("No server on HttpServer")
self.server.serve_forever()
def get_zones(self):
logger.info("get_zones@api_client.py - Getting zones")
zone_data = self.get(
f"/dns-server/{self.dns_server_name}/zone",
params={"includes": ["dns_records"]},
)
data = [ZoneData(**z) for z in zone_data["zones"]]
return data
def create_or_get_dns_server(token: TokenPayload,
dns_server_repo: DnsServerRepo) -> DnsServer:
if not dns_server_repo.exists(name=token.payload.dns_server_name.lower()):
dns_server_repo.clear()
logger.info("[email protected] - Saving dns server from api token")
return dns_server_repo.create(
dict(name=token.payload.dns_server_name.lower())).results()
else:
return dns_server_repo.results()
def create_or_get_http_server(token: TokenPayload,
http_server_repo: HttpServerRepo) -> HttpServer:
if not http_server_repo.exists(
name=token.payload.http_server_name.lower()):
http_server_repo.clear()
logger.info("[email protected] - Saving http server from api token")
return http_server_repo.create(
dict(name=token.payload.http_server_name.lower())).results()
else:
return http_server_repo.results()
def db_register_model_events(models):
for m in models:
for event_name in ORM_EVENTS:
event_cb = "on_" + event_name
if hasattr(m, event_cb):
logger.info(
"[email protected]: registering "
+ event_cb
+ " on "
+ str(m)
)
listen(m, event_name, make_event(getattr(m, event_cb)))
def refresh_zones_if_needed(self):
logger.info(
"refresh_zones_if_needed@api_client.py - Checking for New Zones and Records..."
)
old_zones = self.zones
new_zones = self.get_zones()
# TODO: fix this mess
if len(old_zones) != len(new_zones):
logger.warning(
f"refresh_zones_if_needed@api_client.py - Zone Length mistmatch. New or Changed Zone Found: {str(old_zones)} != {str(new_zones)}. Reloading zones."
)
self.load_zones()
return True
for nz in new_zones:
# make sure new zones are in old zones
is_nz_exists = False
for oz in old_zones:
if oz.domain == nz.domain and oz.ip == nz.ip:
nz_dns_records = nz.dns_records or []
for nrec in nz_dns_records:
is_rec_satisfied = False
oz_dns_records = oz.dns_records or []
if len(nz_dns_records) != len(oz_dns_records):
logger.warning(
f"refresh_zones_if_needed@api_client.py - Zone Record Length mistmatch {str(len(nz_dns_records))} != {str(len(oz_dns_records))}. New or Changed Zone Record Found: {str(nz_dns_records)} != {str(oz_dns_records)}. Reloading zones."
)
self.load_zones()
return True
for orec in oz_dns_records:
if orec.record == nrec.record and orec.sort == nrec.sort:
is_rec_satisfied = True
if not is_rec_satisfied:
logger.warning(
f"refresh_zones_if_needed@api_client.py - New or Changed Zone Record {str(nz)}: {str(nrec)} found for server. Reloading zones."
)
self.load_zones()
return True
is_nz_exists = True
if not is_nz_exists:
logger.warning(
f"refresh_zones_if_needed@api_client.py - New or Changed Zone {str(nz)} found for server. Reloading zones."
)
self.load_zones()
return True
logger.info(
"refresh_zones_if_needed@api_client.py - No New Zones or Records Found. All is well"
)
return False
async def run(self):
if not Path(AlembicInit.migration_dir).is_dir():
logger.info("run@db_setup.py - Running alembic-init")
await AlembicInit.make(self.options).run()
logger.info("run@db_setup.py - Running alembic upgrade")
await AlembicUpgrade.make(self.options).run()
logger.info("run@db_setup.py - Running alembic migrate")
await AlembicMigrate.make(self.options).run()
logger.info("run@db_setup.py - Running alembic upgrade again")
await AlembicUpgrade.make(self.options).run()
if self.option("seed", None):
logger.info("run@db_setup.py - Running db seed")
await DbSeed.make(self.options).run()
async def broadcast_index(websocket: WebSocket):
try:
logger.info("[email protected] - Accepting websockets")
await websocket.accept()
except Exception as e:
logger.critical(f"[email protected] - Accept: Error: {str(type(e))}")
logger.critical(f"[email protected] - Accept: Trace: {str(e)}")
logger.critical(
f"[email protected] - Accept:Not closing or unsubscribing"
)
return 1
try:
while True:
logger.info("[email protected] - Receiving json")
data = await websocket.receive_json()
logger.info("[email protected] - Received json: " + str(data))
logger.info(
"[email protected] - Sending message: "
+ str({"message": "greetings"})
)
await websocket.send_json({"message": "greetings"})
except Exception as e:
logger.critical(
f"[email protected] - Receieve/Send: Error: {str(type(e))}"
)
logger.critical(
f"[email protected] - Receieve/Send: Trace: {str(e)}"
)
logger.critical(
f"[email protected] - Receieve/Send: Not closing or unsubscribing"
)
return 1
await websocket.close()
def create_dns_request(self, handler, request, request_uuid):
logger.info("create_dns_request@api_client.py - Creating dns request")
name = str(request.q.qname)
name = name.rstrip(".")
data = {
"name": name,
"source_address": str(handler.client_address[0]),
"source_port": int(handler.client_address[1]),
"type": str(QTYPE[request.q.qtype]),
"protocol": str(handler.protocol),
"dns_server_name": str(self.dns_server_name),
"raw_request": str(request),
}
self.post("/dns-request", data=data)
def post(self, url: str, data=None, fail=True):
data = data or {}
headers = self.get_default_headers()
logger.info("post@api_client.py - Posting URL: " + str(self.url(url)))
res = requests.post(self.url(url),
json=data,
headers=headers,
verify=self.verify_ssl)
if fail:
if res.status_code != 200:
logger.critical(f"Error posting API {self.url(url)}: " +
str(res.json()))
res.raise_for_status()
return res.json()
def get(self, url: str, params=None, fail=True):
params = params or {}
headers = self.get_default_headers()
logger.info("get@api_client.py - Getting URL: " + str(self.url(url)))
res = requests.get(self.url(url),
headers=headers,
params=params,
verify=self.verify_ssl)
if fail:
if res.status_code != 200:
logger.critical(
f"get@api_client.py - Error getting API {self.url(url)}: "
+ str(res.json()))
res.raise_for_status()
return res.json()
def create_or_get_api_token_for_dns_server(
token: TokenPayload, api_token_repo: ApiTokenRepo,
dns_server: DnsServer) -> ApiTokenData:
scopes = token.scopes
if not api_token_repo.exists(token=token.token):
api_token_repo.clear()
logger.info(
"[email protected] - Saving api token from auth token for dns node")
return api_token_repo.create(
dict(
token=token.token,
scopes=" ".join(scopes),
dns_server=dns_server,
expires_at=datetime.utcfromtimestamp(float(token.exp)),
)).data()
else:
logger.info("[email protected] - token already exists in database")
return api_token_repo.loads("dns_server").includes("dns_server").data()
def wait_for_up(self):
attempts = 0
while True:
if attempts > 60:
logger.warning("could not connect to api. api not up")
return False
logger.info(
f"wait_for_up@api_client.py - checking for api status : {self.url('/status')}"
)
try:
sleep(1)
self.get_status()
sleep(3)
return True
except Exception as e:
logger.info(
"wait_for_up@api_client.py - api check not ready after {} attempts: {}"
.format(str(attempts), str(e.__class__.__name__)))
attempts = attempts + 1
sleep(1)
def boot(self):
port = self.option("port")
listen = self.option("listen")
logger.info("boot@http_server.py - Building http server on port %d",
port)
self.http_server = HttpServerManager(port=port,
listen=listen,
ssl=False,
api_client=self.api_client)
if bool(self.option("enable_ssl")):
ssl_port = self.option("ssl_port")
ssl_listen = self.option("ssl_listen")
ssl_key_path = self.option("ssl_key_path")
ssl_cert_path = self.option("ssl_cert_path")
logger.info(
f"boot@http_server.py - Building https server manager on port {str(ssl_port)} with {ssl_cert_path} and {ssl_key_path}",
)
self.https_server = HttpServerManager(
port=ssl_port,
listen=ssl_listen,
api_client=self.api_client,
ssl=True,
ssl_cert_path=ssl_cert_path,
ssl_key_path=ssl_key_path,
)
logger.info(
"boot@http_server.py - Building https server on port %d",
ssl_port)
async def broadcast_auth(websocket: WebSocket):
try:
await websocket.accept()
except Exception as e:
logger.critical(f"[email protected] - Accept Error: {str(type(e))}")
logger.critical(f"[email protected] - Accept Trace: {str(e)}")
return 1
params = parse_qs(urlparse(str(websocket.url)).query)
token = verify_jwt_token(params["ws_access_token"][0])
if not token_has_required_scopes(token, []): # TODO: check scopes later
raise HTTPException(403, detail="Forbidden")
user_repo = UserRepo(session())
user = await current_user(token, user_repo)
subscriber, channel = await make_subscriber("auth")
try:
while await channel.wait_message():
logger.info("[email protected] - Waiting for message")
msg = await channel.get(encoding="utf-8")
logger.info("[email protected] - Received message: " + str(msg))
data = json.loads(msg)
logger.info("[email protected] - Sending message: " + str(data))
await websocket.send_json(data)
except Exception as e:
logger.critical(
f"[email protected] - Receieve/Send: Error: {str(type(e))}"
)
logger.critical(f"[email protected] - Receieve/Send: Trace:{str(e)}")
logger.critical(
f"[email protected] - Receieve/Send: Not closing or unsubscribing"
)
return 1
logger.info(f"[email protected] - Attempting to unsuscribe")
await subscriber.unsubscribe("channel:auth")
logger.info(f"[email protected]: Websocket - Attempting to close socket")
await websocket.close()
def boot(self):
port = self.get_port()
listen = self.get_listen()
self.resolver = Resolver(self.api_client)
self.udp_server = DNSServer(
self.resolver,
address=listen,
port=port,
handler=DNSHandler,
logger=DNSLogger(self.api_client),
)
self.tcp_server = DNSServer(
self.resolver,
address=listen,
port=port,
tcp=True,
handler=DNSHandler,
logger=DNSLogger(self.api_client),
)
logger.info("starting DNS server on port %d", port)
def create_http_request(
self,
name,
path,
source_address,
source_port,
type,
protocol,
raw_request,
):
logger.info(
"create_http_request@api_client.py - Creating http request")
data = {
"name": name,
"path": path,
"source_address": str(source_address),
"source_port": int(source_port),
"type": str(type),
"protocol": str(protocol),
"http_server_name": str(self.http_server_name),
"raw_request": str(raw_request),
}
self.post("/http-request", data=data)
def stop(self):
logger.info("[email protected] - Stopping Http server")
if not self.server:
raise Exception("No server on HttpServer")
self.server.shutdown()
def sync(self):
logger.info("sync@api_client.py - Syncing api token")
return self.post("/api-token/sync", fail=True)
async def run(self):
app = "boucanpy.api.main:api"
kwargs = self.get_kwargs()
if self.should_import_check():
logger.info("run@api_server.py - Performing import check")
from boucanpy.api.main import api
logger.critical(
"run@api_server.py - Starting api server with options: {}".format(
str(kwargs)))
from boucanpy.db.checks import is_db_up, is_db_setup
# alembic just destroys the loggers, it's annoying
if self.should_db_check():
logger.info(
"run@api_server.py - Waiting for database service to be up")
db_wait_options = self._args_to_dict(self.options)
await DbWait(db_wait_options).run()
if self.option("db_setup"):
logger.critical("run@api_server.py - Running database migration")
db_setup_options = self._args_to_dict(self.options)
if self.option("db_seed"):
db_setup_options["seed"] = True
await DbSetup(db_setup_options).run()
if self.should_db_check():
logger.info(
"run@api_server.py - Checking if application database is setup and configured"
)
db_setup = is_db_setup()
if not db_setup:
logger.critical(
"run@api_server.py - Database not setup error. please check logs"
)
return self.exit(1)
from boucanpy.broadcast import is_broadcast_up
if self.should_bcast_check():
bcast_up = await is_broadcast_up()
if not bcast_up:
logger.critical(
"run@api_server.py - Broadcast (queue) not up error. please check logs"
)
return self.exit(1)
if self.option("db_seed_env", False):
self.seed_from_env()
# taken from uvicorn/main.py:run
logger.debug("run@api_server.py - Building Uvicorn Config and Server")
config = UvicornConfig(app,
log_config=self.get_uvicorn_logging(),
**kwargs)
server = UvicornServer(config=config)
if self.option("force_exit"):
server.force_exit = True
if isinstance(app, str) and (config.debug or config.reload):
logger.warning(
f"run@api_server.py - Running boucanpy api in dev mode...")
sock = config.bind_socket()
supervisor = StatReload(config)
return supervisor.run(server.run, sockets=[sock])
elif config.workers > 1:
sock = config.bind_socket()
supervisor = Multiprocess(config)
logger.warning(
f"run@api_server.py - Running boucanpy api in worker mode...")
return supervisor.run(server.run, sockets=[sock])
else:
sockets = None
logger.warning(
f"run@api_server.py - Running boucanpy api in standard mode..."
)
return await server.serve(sockets=sockets)
def seed_from_env(self):
from boucanpy.core.user import UserRepo
from boucanpy.core.zone import ZoneRepo
from boucanpy.core.dns_server import DnsServerRepo
from boucanpy.db.session import _scoped_session
session = _scoped_session
for i in range(9):
i = str(i)
user_data = {}
email_key = f"SEED_USER_{i}_EMAIL"
email = environ.get(email_key, None)
password_key = f"SEED_USER_{i}_PASSWORD"
password = environ.get(password_key, None)
superuser_key = f"SEED_USER_{i}_SUPERUSER"
is_superuser = int(environ.get(superuser_key, 0))
if email and password:
email = email.lower()
hashed_password = hash_password(password)
repo = UserRepo(db=session)
if not repo.exists(email=email):
logger.info(
f"seed_from_env@api_server.py - seeding user {email}")
user = factory("UserFactory", session=session).create(
email=email,
hashed_password=hashed_password,
is_superuser=is_superuser,
)
else:
logger.info(
f"seed_from_env@api_server.py - Seeded user {email} already exists"
)
for i in range(9):
i = str(i)
name_key = f"SEED_DNS_SERVER_{i}_NAME"
name = environ.get(name_key, None)
if name:
repo = DnsServerRepo(db=session)
if not repo.exists(name=name):
logger.info(
f"seed_from_env@api_server.py - Seeding domain {name}")
domain = factory("DnsServerFactory",
session=session).create(name=name)
for i in range(9):
i = str(i)
ip_key = f"SEED_ZONE_{i}_IP"
domain_key = f"SEED_ZONE_{i}_DOMAIN"
dns_server_name_key = f"SEED_ZONE_{i}_DNS_SERVER_NAME"
ip = environ.get(ip_key, None)
domain = environ.get(domain_key, None)
if domain:
domain = domain.lower()
dns_server_name = environ.get(dns_server_name_key, None)
if ip and domain:
if dns_server_name:
dns_server_repo = DnsServerRepo(db=session)
if dns_server_repo.exists(name=dns_server_name):
dns_server = dns_server_repo.results()
else:
logger.info(
f"seed_from_env@api_server.py - Seeding dns server as zone dependency: {name}"
)
dns_server = factory(
"DnsServerFactory",
session=session).create(name=dns_server_name)
factory("ZoneFactory",
session=session).create(ip=ip,
domain=domain,
dns_server=dns_server)
else:
repo = ZoneRepo(db=session)
if not repo.exists(ip=ip, domain=domain):
logger.info(
f"seed_from_env@api_server.py - Seeding zone without dns server: {ip}, {domain}"
)
factory("GlobalZoneFactory",
session=session).create(ip=ip, domain=domain)
# CORS
api = FastAPI(title=config.API_PROJECT_NAME,
openapi_url="/api/v1/openapi.json")
origins = []
# Set all CORS enabled origins
if config.API_CORS_ORIGINS:
origins_raw = config.API_CORS_ORIGINS.split(",")
for origin in origins_raw:
use_origin = origin.strip()
origins.append(use_origin)
logger.info(f"[email protected] - Registering cors origins {origins}")
api.add_middleware(
CORSMiddleware,
allow_origins=origins,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
),
main_router = APIRouter()
for r, ropts in routers:
logger.debug(f"[email protected] - Registering router {str(r)} {str(ropts)}")
main_router.include_router(r, **ropts)
async def run(self):
# TODO: thread issues?
verify_ssl = True
if bool(self.option("no_ssl_verify")):
verify_ssl = False
self.api_client = ApiClient(self.get_api_url(),
self.get_api_token(),
verify_ssl=verify_ssl)
if not self.api_client.wait_for_up():
logger.critical(
"run@dns_server.py - Could not connect to api. quitting")
self.exit(1)
if self.option("no_sync"):
logger.info("run@dns_server.py - Skipping syncing api token")
else:
self.api_client.sync()
self.boot()
self.start_servers()
try:
count = 0
while self.udp_server.isAlive():
if count > 0 and count % self.option("refresh_ttl") == 0:
if self.api_client.refresh_zones_if_needed():
logger.critical(
"run@dns_server.py - API Client found new or changed zones. Stopping servers..."
)
# TODO: figure out why "stop" does not release the address
self.stop_servers()
sleep(1)
stop_count = 0
logger.critical(
"run@dns_server.py - Waiting for UDP Server to stop..."
)
while self.udp_server.thread and self.udp_server.isAlive(
):
if stop_count > 30:
logger.critical(
"run@dns_server.py - UDP Server did not stop while reloading zones"
)
raise Exception(
"run@dns_server.py - UDP Server threads went rogue during zone reload"
)
logger.info(
"run@dns_server.py - Waiting for DNS Server to stop before reloading zones"
)
stop_count = stop_count + 1
sleep(1)
stop_count = 0
logger.critical(
"run@dns_server.py - Waiting for TCP Server to stop..."
)
while self.tcp_server.thread and self.tcp_server.isAlive(
):
if stop_count > 30:
logger.critical(
"run@dns_server.py - TCP Server did not stop while reloading zones"
)
raise Exception(
"run@dns_server.py - TCP Server threads went rogue during zone reload"
)
logger.info(
"run@dns_server.py - Waiting for DNS Server to stop before reloading zones"
)
stop_count = stop_count + 1
sleep(1)
logger.critical(
"run@dns_server.py - Rebooting server with fresh zones..."
)
self.boot()
self.start_servers()
count = count + 1
sleep(1)
except KeyboardInterrupt:
pass
async def run(self):
self.db_register()
if self.option("target", False) == False:
self.set_option("target", "dev")
if self.option("target") == "env":
logger.info(f"run@db_seed.py - Seeding {self.option('target')}")
raise NotImplementedError() # seed based on env vars
elif self.option("target") == "dev":
logger.info(f"run@db_seed.py - Seeding {self.option('target')}")
logger.info("run@db_seed.py - reating superuser")
super = factory("SuperUserFactory").create(email="*****@*****.**")
logger.info("run@db_seed.py - Creating normal user")
norm = factory("UserFactory").create(email="*****@*****.**")
logger.info("run@db_seed.py - Creating dns_server")
_dns_server = factory("DnsServerFactory").create(
name="mydnsserver")
dns_server = factory("DnsServerFactory").create()
logger.info("run@db_seed.py - Creating http_server")
_http_server = factory("HttpServerFactory").create(
name="myhttpserver")
http_server = factory("HttpServerFactory").create()
logger.info("run@db_seed.py - Creating zones")
zone = factory("ZoneFactory").create(domain="othersite.com",
ip="127.0.1.1")
zone2 = factory("ZoneFactory").create(
domain="friends4life.com",
ip="127.0.1.1",
dns_server=dns_server,
http_server=http_server,
)
zone3 = factory("ZoneFactory").create(domain="differentzone.com",
ip="127.0.1.1")
logger.info("run@db_seed.py - Creating api_tokens")
factory("ApiTokenFactory").create(dns_server=dns_server)
factory("ApiTokenFactory").create(http_server=http_server)
factory("ApiTokenFactory").create(dns_server=dns_server,
http_server=http_server)
logger.info("run@db_seed.py - Creating dns_requests")
for i in range(35):
factory("DnsRequestFactory").create(dns_server=dns_server,
zone=zone2)
for i in range(35):
factory("DnsRequestFactory").create(dns_server=dns_server,
zone=zone3)
logger.info("run@db_seed.py - Creating dns_records")
for i in range(3):
factory("DnsRecordFactory").create(zone=zone)
for i in range(3):
factory("DnsRecordFactory").create(zone=zone2)
for i in range(3):
factory("DnsRecordFactory").create(zone=zone3)
logger.info("run@db_seed.py - Creating http_requests")
for i in range(35):
factory("HttpRequestFactory").create(http_server=http_server,
zone=zone2)
for i in range(35):
factory("HttpRequestFactory").create(http_server=http_server,
zone=zone3)
else:
logger.critical("run@db_seed.py - invalid target set for seeder")
self.exit(1)
