def setup_components(all_components: list):
for component in all_components:
host = component['upstream']
server = host.replace('_', '-')
name = host.split('_')[1]
path = component['path']
if os.getenv(component['env']) == '1':
# setup upstream config
c = nginx.Conf()
u = nginx.Upstream(host, nginx.Key('server', f'{server}:5000'))
c.add(u)
nginx.dumpf(c, f'upstreams/{name}.conf')
# setup enabled location config
c = nginx.Conf()
l = nginx.Location(
f'~* ^/{path}/',
nginx.Key('rewrite', f'^/{path}/(.*) /$1 break'),
nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-Real-IP $remote_addr'),
nginx.Key('proxy_pass', f'http://{host}'),
nginx.Key('proxy_pass_request_headers', 'on'),
)
c.add(l)
nginx.dumpf(c, f'locations/{name}.conf')
else:
# setup disabled location config
c = nginx.Conf()
l = nginx.Location(
f'~* ^/{path}/',
nginx.Key('return', '503'),
)
c.add(l)
nginx.dumpf(c, f'locations/{name}.conf')
def create_acme_dummy(domain):
"""
Create a dummy directory to use for serving ACME challenge data.
This function is used when no website yet exists for the desired domain.
:param str domain: Domain name to use
:returns: Path to directory for challenge data
"""
site_dir = os.path.join(config.get("websites", "site_dir"),
"acme-" + domain)
challenge_dir = os.path.join(site_dir, ".well-known/acme-challenge")
conf = nginx.Conf(
nginx.Server(
nginx.Key("listen", "80"), nginx.Key("listen", "[::]:80"),
nginx.Key("server_name", domain), nginx.Key("root", site_dir),
nginx.Location("/.well-known/acme-challenge/",
nginx.Key("root", site_dir))))
origin = os.path.join("/etc/nginx/sites-available", "acme-" + domain)
target = os.path.join("/etc/nginx/sites-enabled", "acme-" + domain)
uid = users.get_system("http").uid
nginx.dumpf(conf, origin)
if not os.path.exists(target):
os.symlink(origin, target)
if not os.path.exists(challenge_dir):
os.makedirs(challenge_dir)
os.chown(site_dir, uid, -1)
os.chown(os.path.join(site_dir, ".well-known"), uid, -1)
os.chown(challenge_dir, uid, -1)
tracked_services.register("acme", domain, domain + "(ACME Validation)",
"globe", [('tcp', 80)], 2)
nginx_reload()
return challenge_dir
def write_vhost(appinfo):
import nginx
c = nginx.Conf()
s = nginx.Server()
s.add(
nginx.Comment('SSL conf added by freessl (https://github.com/alihusnainarshad)'),
nginx.Key('listen', '443 ssl http2'),
nginx.Key('listen', '[::]:443 ssl http2'),
nginx.Key('server_name', ' '.join(appinfo.get('valid_domains'))),
nginx.Key('brotli', 'on'),
nginx.Key('brotli_static', 'off'),
nginx.Key('brotli_min_length', '100'),
nginx.Key('brotli_buffers', '16 8k'),
nginx.Key('brotli_comp_level', '5'),
nginx.Key('brotli_types', '*'),
nginx.Key('ssl', 'on'),
nginx.Key('ssl_certificate', appinfo.get('cert_path')),
nginx.Key('ssl_certificate_key', appinfo.get('key_path')),
nginx.Key('ssl_prefer_server_ciphers', 'on'),
nginx.Key('ssl_session_timeout', '5m'),
nginx.Key('ssl_protocols', 'TLSv1.1 TLSv1.2'),
nginx.Key('ssl_stapling', 'on'),
nginx.Key('ssl_stapling_verify', 'on'),
nginx.Key('resolver', '8.8.8.8 8.8.4.4 valid=86400s'),
nginx.Key('resolver_timeout', '5s'),
nginx.Key('ssl_ciphers', '"ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:!DSS"'),
nginx.Key('ssl_ecdh_curve', 'secp384r1'),
nginx.Key('ssl_session_cache', 'shared:SSL:10m'),
nginx.Key('ssl_session_tickets', 'off'),
nginx.Key('ssl_dhparam', '/etc/nginx-rc/dhparam.pem'),
nginx.Key('include', '/etc/nginx-rc/conf.d/{}.d/main.conf'.format(appinfo.get('name')))
)
c.add(s)
nginx.dumpf(c, '{}/{}-ssl.conf'.format(appinfo.get('vhostdir'), appinfo.get('name')))
def create_nginx_config_for_domain(domain, subdomains, subdomain_dir,
forward_others, use_ssl, cert_dir):
c = nginx.Conf()
c.add(nginx.Comment(generation_comment('NGINX config', domain)))
for subdomain in subdomains:
c.add(
nginx.Key('include',
str(subdomain_dir / '{}.cfg'.format(subdomain))))
if forward_others is not None:
others = nginx.Server()
others.add(
nginx.Comment('Forward remaining (sub)domains to ' +
forward_others),
nginx.Key('server_name',
'{domain} *.{domain}'.format(domain=domain)),
nginx.Key('return', '302 {}$request_uri'.format(forward_others)),
nginx.Key('listen', '80'))
if use_ssl:
others.add(
nginx.Comment('use_ssl = True'),
nginx.Key('listen', '443 ssl'), nginx.Key('ssl', 'on'),
nginx.Key('ssl_certificate',
str(cert_dir / 'certificate.crt')),
nginx.Key('ssl_certificate_key',
str(cert_dir / 'certificate.key')))
c.add(others)
return c
def nginx_add(self, site, add):
if site.path == '':
site.path = os.path.join('/srv/http/webapps/', site.name)
c = nginx.Conf()
s = nginx.Server(
nginx.Key('listen', site.port),
nginx.Key('server_name', site.addr),
nginx.Key('root', site.path),
nginx.Key('index', 'index.'+('php' if site.php else 'html'))
)
if add:
s.add(*[x for x in add])
c.add(s)
nginx.dumpf(c, os.path.join('/etc/nginx/sites-available', site.name))
# Write configuration file with info Genesis needs to know the site
f = open(os.path.join('/etc/nginx/sites-available', '.'+site.name+'.ginf'), 'w')
c = ConfigParser.SafeConfigParser()
c.add_section('website')
c.set('website', 'name', site.name)
c.set('website', 'stype', site.stype)
c.set('website', 'ssl', '')
c.set('website', 'version', site.version if site.version else 'None')
c.set('website', 'dbengine', site.dbengine if site.dbengine else '')
c.set('website', 'dbname', site.dbname if site.dbname else '')
c.set('website', 'dbuser', site.dbuser if site.dbuser else '')
c.write(f)
f.close()
def test():
return nginx.Conf(
nginx.Server(
nginx.Comment('This is a test comment'),
nginx.Key('server_name', 'localhost'),
nginx.Key('root', '/var/www'),
nginx.Location('/', nginx.Key('test', 'true'),
nginx.Key('test2', 'false'))))
def write_conf(app):
print(bcolors.OKBLUE + 'Writing NGINX vhost file for the app ' +
bcolors.BOLD + app.get('appname') + bcolors.ENDC)
appname = app.get('appname')
root = app.get('root')
username = app.get('username', 'serverpilot')
confname = vhostsdir + appname + '-ssl.conf'
domains = app.get('domains')
c = nginx.Conf()
s = nginx.Server()
s.add(
nginx.Comment(
'SSL conf added by rwssl (https://github.com/rehmatworks/serverpilot-letsencrypt)'
),
nginx.Key('listen', '443 ssl http2'),
nginx.Key('listen', '[::]:443 ssl http2'),
nginx.Key('server_name', ' '.join(domains)),
nginx.Key('ssl', 'on'),
nginx.Key('ssl_certificate',
app.get('certpath') + '/fullchain.pem'),
nginx.Key('ssl_certificate_key',
app.get('certpath') + '/privkey.pem'),
nginx.Key('root', root),
nginx.Key(
'access_log', '/srv/users/' + username + '/log/' + appname +
'/dev_nginx.access.log main'),
nginx.Key(
'error_log', '/srv/users/' + username + '/log/' + appname +
'/dev_nginx.error.log'),
nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-Real-IP $remote_addr'),
nginx.Key('proxy_set_header',
'X-Forwarded-For $proxy_add_x_forwarded_for'),
nginx.Key('proxy_set_header', 'X-Forwarded-SSL on'),
nginx.Key('proxy_set_header', 'X-Forwarded-Proto $scheme'),
nginx.Key('include',
'/etc/nginx-sp/vhosts.d/' + appname + '.d/*.conf'),
)
c.add(s)
try:
nginx.dumpf(c, confname)
print(bcolors.OKGREEN + 'Virtual host file created!' + bcolors.ENDC)
print(bcolors.OKBLUE + 'Reloading NGINX server...' + bcolors.ENDC)
reload_nginx_sp()
print(bcolors.OKGREEN +
'SSL should have been installed and activated for the app ' +
bcolors.BOLD + app.get('appname') + bcolors.ENDC)
return True
except:
print(bcolors.FAIL + 'Virtual host file cannot be created!' +
bcolors.ENDC)
return False
def certificate(self, servers):
for server_data in servers:
if server_data.get("is_ssl_certificate", False):
domain = server_data.get("domain", "")
conf = nginx.Conf()
conf.add(
nginx.Key("ssl_certificate",
f"/var/www/certificate/{domain}-cert.pem"))
conf.add(
nginx.Key("ssl_certificate_key",
f"/var/www/certificate/{domain}-key.pem"))
nginx.dumpf(
conf, f'/etc/nginx/conf.d/ssl_certificate/{domain}.conf')
def nginxConfGenerator(instances, options):
c = nginx.Conf()
for instance in instances:
s = nginx.Server()
s.add(
nginx.Key('listen', '80'),
nginx.Key('server_name',
'nxt-mq-' + instance[1] + '.ies.inventec'),
nginx.Location('/', nginx.Key('proxy_pass',
'http://' + instance[0] + ':15672')),
)
c.add(s)
nginx.dumpf(c, os.path.dirname(os.path.abspath(__file__)) + '/nginx.conf')
return
def generate_nginx_config(self):
c = nginx.Conf()
u = nginx.Upstream('loadbalancer', nginx.Key('least_conn', ''))
ip_addr = get_ip_address()
for server_idx in range(self.n_endpoints):
u.add(
nginx.Key('server', f'{ip_addr}:{self.src_port + server_idx}'))
s = nginx.Server(
nginx.Location('/', nginx.Key('proxy_pass',
'http://loadbalancer')))
loc = nginx.Location('/favicon.ico', nginx.Key('log_not_found', 'off'),
nginx.Key('access_log', 'off'))
c.add(u)
s.add(loc)
c.add(s)
nginx.dumpf(c, 'dockerfiles/loadbalancer/nginx.conf')
def nginx_add(self, site, add):
if site.path == '':
site.path = os.path.join('/srv/http/webapps/', site.name)
c = nginx.Conf()
c.add(
nginx.Comment(
'GENESIS %s %s' %
(site.stype, 'http://' + site.addr + ':' + site.port)))
s = nginx.Server(
nginx.Key('listen', site.port), nginx.Key('server_name',
site.addr),
nginx.Key('root', site.path),
nginx.Key('index', 'index.' + ('php' if site.php else 'html')))
if add:
s.add(*[x for x in add])
c.add(s)
nginx.dumpf(c, os.path.join('/etc/nginx/sites-available', site.name))
def add():
data = request.get_json()
input_port = data['input_port']
upstream = data['upstream'] + ":" + data['upstream_port']
path = "/config/" + input_port + ".conf"
c = nginx.Conf()
s = nginx.Server()
s.add(
nginx.Key('listen', input_port),
nginx.Key('proxy_pass', upstream),
nginx.Key('proxy_protocol', 'on'),
)
c.add(s)
nginx.dumpf(c, path)
return "Done!"
def create_nginx_config_for_subdomain(domain, subdomain, destination, use_ssl,
force_ssl, cert_dir):
full_domain = '{sub}.{main}'.format(main=domain, sub=subdomain)
c = nginx.Conf()
c.add(nginx.Comment(generation_comment('NGINX config', full_domain)))
if use_ssl and force_ssl:
non_ssl = nginx.Server()
non_ssl.add(nginx.Comment('force_ssl = True'),
nginx.Key('listen', '80'),
nginx.Key('server_name', full_domain),
nginx.Key('return', '301 https://$host$request_uri'))
c.add(non_ssl)
main = nginx.Server()
if not force_ssl:
main.add(nginx.Comment('force_ssl = False'), nginx.Key('listen', '80'))
proto = 'http'
if use_ssl:
proto = 'https'
main.add(
nginx.Comment('use_ssl = True'), nginx.Key('listen', '443 ssl'),
nginx.Key('ssl', 'on'),
nginx.Key('ssl_certificate', str(cert_dir / 'certificate.crt')),
nginx.Key('ssl_certificate_key',
str(cert_dir / 'certificate.key')))
main.add(
nginx.Key('server_name', full_domain),
nginx.Location(
'/', nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-Real-IP $remote_addr'),
nginx.Key('proxy_set_header',
'X-Forwarded-For $proxy_add_x_forwarded_for'),
nginx.Key('proxy_set_header', 'X-Forwarded-Proto $scheme'),
nginx.Key('proxy_set_header', 'Upgrade $http_upgrade'),
nginx.Key('proxy_set_header', 'Connection $connection_upgrade'),
nginx.Key('proxy_pass', destination),
nginx.Key('proxy_read_timeout', '90'),
nginx.Key(
'proxy_redirect',
'{dst} {proto}://{full}'.format(dst=destination,
full=full_domain,
proto=proto))))
c.add(main)
return c
def test_create():
c = nginx.Conf()
u = nginx.Upstream('php', nginx.Key('server', 'unix:/tmp/php-fcgi.socket'),
nginx.Key('server', '10.0.2.1'))
u.add(nginx.Key('server', '101.0.2.1'))
c.add(u)
s = nginx.Server()
s.add(
nginx.Key('listen', '80'),
nginx.Comment('Yes, python-nginx can read/write comments!'),
nginx.Key('server_name', 'localhost 127.0.0.1'),
nginx.Key('root', '/srv/http'), nginx.Key('index', 'index.php'),
nginx.Location('= /robots.txt', nginx.Key('allow', 'all'),
nginx.Key('log_not_found', 'off'),
nginx.Key('access_log', 'off')),
nginx.Location('~ \.php$', nginx.Key('include', 'fastcgi.conf'),
nginx.Key('fastcgi_intercept_errors', 'on'),
nginx.Key('fastcgi_pass', 'php')))
c.add(s)
nginx.dumpf(c, 'mysite')
def create_nginx_config(nginx_port, app_name):
c = nginx.Conf()
e = nginx.Events()
e.add(nginx.Key('worker_connections', '1024'))
c.add(e)
h = nginx.Http()
u = nginx.Upstream(app_name)
h.add(u)
s = nginx.Server()
s.add(
nginx.Key('listen', str(nginx_port)),
nginx.Key('server_name', app_name),
nginx.Location('/', nginx.Key('proxy_pass', 'http://' + app_name),
nginx.Key('proxy_set_header', 'Host $host')))
h.add(s)
c.add(h)
nginx.dumpf(c, nginx_configs_dir + "/" + app_name + '/nginx.conf')
def set_conf(request, proxy):
config = nginx.Conf()
serverHTTP = nginx.Server()
serverHTTPS = nginx.Server()
print(proxy.domain)
print(proxy.ssl)
print(proxy.letsencrypt)
print(proxy.rewriteHTTPS)
print(proxy.proxypass)
serverHTTP.add(
nginx.Key('listen', '*:80'),
nginx.Key('server_name', proxy.domain),
)
if proxy.rewriteHTTPS:
serverHTTP.add(
nginx.Key('return', '302 https://$server_name$request_uri'),
)
else:
serverHTTP.add(
nginx.Location(
'/',
nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-Real-IP $remote_addr'),
nginx.Key('proxy_set_header', 'X-Forwarded-For $proxy_add_x_forwarded_for'),
nginx.Key('proxy_set_header', 'Upgrade $http_upgrade'),
nginx.Key('proxy_set_header', 'Connection '"upgrade"),
nginx.Key('proxy_http_version', '1.1'),
nginx.Key('proxy_pass', proxy.proxypass),
)
)
if proxy.ssl:
letsencrypt.generate_cert(request, proxy)
serverHTTPS.add(
nginx.Key('server_name', proxy.domain),
nginx.Key('listen', '*:443 ssl'),
nginx.Key('ssl_protocols', 'TLSv1 TLSv1.1 TLSv1.2'),
nginx.Key('ssl_certificate', '/etc/letsencrypt/live/'+proxy.domain+'/fullchain.pem'),
nginx.Key('ssl_certificate_key', '/etc/letsencrypt/live/'+proxy.domain+'/privkey.pem'),
)
serverHTTPS.add(
nginx.Location(
'/',
nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-Real-IP $remote_addr'),
nginx.Key('proxy_set_header', 'X-Forwarded-For $proxy_add_x_forwarded_for'),
nginx.Key('proxy_set_header', 'Upgrade $http_upgrade'),
nginx.Key('proxy_set_header', 'Connection '"upgrade"),
nginx.Key('proxy_http_version', '1.1'),
nginx.Key('proxy_pass', proxy.proxypass),
)
)
config.add(serverHTTP)
config.add(serverHTTPS)
nginx.dumpf(config, '/etc/nginx/conf.d/' + proxy.domain + '.conf')
shell.restart_nginx(request)
return True
def __init__(self, servers):
self.config = nginx.Conf()
self.servers = servers # [{"domain": "example.com", "is_certificate": True}]
def _execute(command):
config = FileUtils.get_instance('config.ini')
nginx_conf_dir = config.get_prop("nginx-config", "nginx_conf_dir")
nginx_conf_dir_bak = config.get_prop("nginx-config", "nginx_conf_dir_bak")
os.system('mkdir -p ' + nginx_conf_dir)
os.system('cp -r ' + nginx_conf_dir + nginx_conf_dir_bak)
os.system('rm -rf ' + nginx_conf_dir + '*')
state = '0'
try:
for k1 in command:
upstreamName = k1["upstreamName"]
upstreamNode = json.loads(k1["upstreamNode"])
fileName = nginx_conf_dir + upstreamName + '.conf'
os.system('touch ' + fileName)
c = nginx.Conf()
LogUtil.info("Node信息:%s" % (upstreamNode))
t = ''
for k2 in range(len(upstreamNode)):
if (k2 == 0):
t = 'server' + ' ' + upstreamNode[k2][
"ipAdress"] + ' ' + upstreamNode[k2]["strategy"] + ';\n'
if (0 < k2 < len(upstreamNode) - 1):
t = t + ' ' + 'server' + ' ' + upstreamNode[k2][
"ipAdress"] + ' ' + upstreamNode[k2]["strategy"] + ';\n'
if (k2 == len(upstreamNode) - 1):
t = t + ' ' + 'server' + ' ' + upstreamNode[k2][
"ipAdress"] + ' ' + upstreamNode[k2]["strategy"]
ipHash = k1["ipHash"]
if (ipHash == '1'):
u = nginx.Upstream(upstreamName, nginx.Key('', t),
nginx.Key('', 'ip_hash'))
c.add(u)
if (ipHash == '0'):
u = nginx.Upstream(upstreamName, nginx.Key('', t))
c.add(u)
s = nginx.Server()
s.add(
nginx.Key('listen', k1["nginxListenPort"]),
nginx.Key('server_name', k1["nginxServerName"]),
nginx.Key('access_log', k1["accessLog"]),
nginx.Location(
'= /',
nginx.Key('proxy_pass', 'http://' + k1["upstreamName"]),
nginx.Key('proxy_redirect', 'off'),
nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-Real-IP $remote_addr'),
nginx.Key('proxy_set_header',
'X-Forwarded-For $proxy_add_x_forwarded_for')))
c.add(s)
nginx.dumpf(c, fileName)
LogUtil.info("完成Nginx配置")
output = os.popen('service nginx restart').read()
state = '1'
param = [
"{\"mac\":\"" + MachineUtil.get_mac_address() + "\",\"state\":\"" +
state + "\",\"msg\":\"" + output + "\"}"
]
os.system('rm -rf ' + nginx_conf_dir_bak)
AosServerService.route_service("ThriftApiService", "nginxLog", param)
except Exception, ex:
LogUtil.error(ex)
os.system('rm -rf ' + nginx_conf_dir)
os.system('mv ' + nginx_conf_dir_bak + nginx_conf_dir)
os.system('service nginx restart')
output = ex.message
state = '2'
param = [
"{\"mac\":\"" + MachineUtil.get_mac_address() + "\",\"state\":\"" +
state + "\",\"msg\":\"" + output + "\"}"
]
AosServerService.route_service("ThriftApiService", "nginxLog", param)
def BuildNginxConfiguration(self, server, api_services, clients,
identity_services):
print("Configuring Nginx Server")
serverConfig = server['config']
config = nginx.Conf()
# Add Root Configurations
for key, value in serverConfig.items():
if (not isinstance(value, dict)):
config.add(nginx.Key(key, value))
events = nginx.Events()
httpConf = nginx.Http()
# Add Event Configurations
if ('events' in serverConfig):
for key, value in serverConfig['events'].items():
events.add(nginx.Key(key, value))
config.add(events)
# Add Http Configurations
if ('http' in serverConfig):
for key, value in serverConfig['http'].items():
httpConf.add(nginx.Key(key, value))
# Add Services To Http
for api_service in api_services:
nginxServer = nginx.Server(
nginx.Key('listen', '80'),
nginx.Key('server_name',
str.lower(api_service['name']) + '.localhost'),
)
proxy_pass = '******' + api_service['name'] + ':' + str(
api_service['port']) + '/'
location = nginx.Location(
'/', nginx.Key('proxy_http_version', '1.1'),
nginx.Key('proxy_set_header', 'Upgrade $http_upgrade'),
nginx.Key('proxy_set_header', 'Connection keep-alive'),
nginx.Key('proxy_set_header',
'X-Forwarded-For $proxy_add_x_forwarded_for'),
nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-NginX-Proxy true'),
nginx.Key('proxy_pass', proxy_pass))
nginxServer.add(location)
httpConf.add(nginxServer)
for i_service in identity_services:
nginxServer = nginx.Server(
nginx.Key('listen', '80'),
nginx.Key('server_name',
str.lower(i_service['name']) + '.localhost'),
)
#pylint: disable-msg=E1121
proxy_pass = '******' + i_service['name'] + ':' + str(
i_service['port']) + '/'
location = nginx.Location(
'/', nginx.Key('proxy_http_version', '1.1'),
nginx.Key('proxy_set_header', 'Upgrade $http_upgrade'),
nginx.Key('proxy_set_header', 'Connection keep-alive'),
nginx.Key('proxy_set_header',
'X-Forwarded-For $proxy_add_x_forwarded_for'),
nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-NginX-Proxy true'),
nginx.Key('proxy_pass', proxy_pass))
nginxServer.add(location)
httpConf.add(nginxServer)
for client in clients:
nginxServer = nginx.Server(
nginx.Key('listen', '80'),
nginx.Key('server_name',
str.lower(client['name']) + '.localhost'),
)
#pylint: disable-msg=E1121
proxy_pass = '******' + client['name'] + ':' + str(
client['port']) + '/'
location = nginx.Location(
'/', nginx.Key('proxy_http_version', '1.1'),
nginx.Key('proxy_set_header', 'Upgrade $http_upgrade'),
nginx.Key('proxy_set_header', 'Connection keep-alive'),
nginx.Key('proxy_set_header',
'X-Forwarded-For $proxy_add_x_forwarded_for'),
nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-NginX-Proxy true'),
nginx.Key('proxy_pass', proxy_pass))
nginxServer.add(location)
httpConf.add(nginxServer)
config.add(httpConf)
return config
def _install(self, extra_vars, enable, nthread):
nthread.title = "Installing website"
msg = Notification("info", "Webs", "Preparing to install...")
nthread.update(msg)
# Make sure the chosen port is indeed open
if not tracked_services.is_open_port(self.port, self.domain):
cname = "({0})".format(self.app.id)
raise errors.InvalidConfigError(cname, nthread)\
from tracked_services.PortConflictError(self.port, self.domain)
# Set some metadata values
specialmsg, dbpasswd = "", ""
site_dir = config.get("websites", "site_dir")
path = (self.path or os.path.join(site_dir, self.id))
self.path = path
self.php = extra_vars.get("php") or self.php \
or self.app.uses_php or False
self.version = self.app.version.rsplit("-", 1)[0] \
if self.app.website_updates else None
# Classify the source package type
if not self.app.download_url:
ending = ""
elif self.app.download_url.endswith(".tar.gz"):
ending = ".tar.gz"
elif self.app.download_url.endswith(".tgz"):
ending = ".tgz"
elif self.app.download_url.endswith(".tar.bz2"):
ending = ".tar.bz2"
elif self.app.download_url.endswith(".zip"):
ending = ".zip"
elif self.app.download_url.endswith(".git"):
ending = ".git"
else:
raise errors.InvalidConfigError(
"Invalid source archive format in {0}".format(self.app.id))
msg = "Running pre-installation..."
uid, gid = users.get_system("http").uid, groups.get_system("http").gid
nthread.update(Notification("info", "Webs", msg))
# Call website type's pre-install hook
self.pre_install(extra_vars)
# If needs DB and user didn't select an engine, choose one for them
if len(self.app.database_engines) > 1 \
and extra_vars.get("dbengine", None):
self.app.selected_dbengine = extra_vars.get("dbengine")
if not getattr(self.app, "selected_dbengine", None)\
and self.app.database_engines:
self.app.selected_dbengine = self.app.database_engines[0]
# Create DB and/or DB user as necessary
if getattr(self.app, "selected_dbengine", None):
msg = "Creating database..."
nthread.update(Notification("info", "Webs", msg))
mgr = databases.get_managers(self.app.selected_dbengine)
if not mgr:
estr = "No manager found for {0}"
raise errors.InvalidConfigError(
estr.format(self.app.selected_dbengine))
# Make sure DB daemon is running if it has one
if not mgr.state:
svc = services.get(mgr.meta.database_service)
svc.restart()
self.db = mgr.add_db(self.id)
if hasattr(self.db, "path"):
os.chmod(self.db.path, 0o660)
os.chown(self.db.path, -1, gid)
# If multiuser DB type, create user
if mgr.meta.database_multiuser:
dbpasswd = random_string(16)
db_user = mgr.add_user(self.id, dbpasswd)
db_user.chperm("grant", self.db)
# Make sure the target directory exists, but is empty
pkg_path = os.path.join("/tmp", self.id + ending)
if os.path.isdir(self.path):
shutil.rmtree(self.path)
os.makedirs(self.path)
# Download and extract the source repo / package
msg = "Downloading website source..."
nthread.update(Notification("info", "Webs", msg))
if self.app.download_url and ending == ".git":
g = git.Repo.clone_from(self.app.download_url, self.path)
if hasattr(self.app, "download_at_tag"):
g = git.Git(self.path)
g.checkout(self.app.download_git_tag)
elif self.app.download_url:
download(self.app.download_url, file=pkg_path, crit=True)
# Format extraction command according to type
msg = "Extracting source..."
nthread.update(Notification("info", "Webs", msg))
if ending in [".tar.gz", ".tgz", ".tar.bz2"]:
arch = tarfile.open(pkg_path, "r:gz")
r = (x for x in arch.getnames() if re.match("^[^/]*$", x))
toplvl = next(r, None)
if not toplvl:
raise errors.OperationFailedError(
"Malformed source archive")
arch.extractall(site_dir)
os.rename(os.path.join(site_dir, toplvl), self.path)
else:
arch = zipfile.ZipFile(pkg_path)
r = (x for x in arch.namelist() if re.match("^[^/]*/$", x))
toplvl = next(r, None)
if not toplvl:
raise errors.OperationFailedError(
"Malformed source archive")
arch.extractall(site_dir)
os.rename(os.path.join(site_dir, toplvl.rstrip("/")),
self.path)
os.remove(pkg_path)
# Set proper starting permissions on source directory
os.chmod(self.path, 0o755)
os.chown(self.path, uid, gid)
for r, d, f in os.walk(self.path):
for x in d:
os.chmod(os.path.join(r, x), 0o755)
os.chown(os.path.join(r, x), uid, gid)
for x in f:
os.chmod(os.path.join(r, x), 0o644)
os.chown(os.path.join(r, x), uid, gid)
# If there is a custom path for the data directory, set it up
if getattr(self.app, "website_datapaths", None) \
and extra_vars.get("datadir"):
self.data_path = extra_vars["datadir"]
if not os.path.exists(self.data_path):
os.makedirs(self.data_path)
os.chmod(self.data_path, 0o755)
os.chown(self.data_path, uid, gid)
elif hasattr(self, "website_default_data_subdir"):
self.data_path = os.path.join(self.path,
self.website_default_data_subdir)
else:
self.data_path = self.path
# Create the nginx serverblock
addtoblock = self.addtoblock or []
if extra_vars.get("addtoblock"):
addtoblock += nginx.loads(extra_vars.get("addtoblock"), False)
default_index = "index." + ("php" if self.php else "html")
if hasattr(self.app, "website_root"):
webroot = os.path.join(self.path, self.app.website_root)
else:
webroot = self.path
block = nginx.Conf()
server = nginx.Server(
nginx.Key("listen", str(self.port)),
nginx.Key("listen", "[::]:" + str(self.port)),
nginx.Key("server_name", self.domain), nginx.Key("root", webroot),
nginx.Key(
"index",
getattr(self.app, "website_index", None) or default_index),
nginx.Location("/.well-known/acme-challenge/",
nginx.Key("root", self.path)))
if addtoblock:
server.add(*[x for x in addtoblock])
block.add(server)
nginx.dumpf(block, os.path.join("/etc/nginx/sites-available", self.id))
challenge_dir = os.path.join(self.path, ".well-known/acme-challenge/")
if not os.path.exists(challenge_dir):
os.makedirs(challenge_dir)
# Create arkOS metadata file
meta = configparser.SafeConfigParser()
meta.add_section("website")
meta.set("website", "id", self.id)
meta.set("website", "app", self.app.id)
meta.set("website", "ssl",
self.cert.id if getattr(self, "cert", None) else "None")
meta.set("website", "version", self.version or "None")
if getattr(self.app, "website_datapaths", None) \
and self.data_path:
meta.set("website", "data_path", self.data_path)
meta.set("website", "dbengine", "")
meta.set("website", "dbengine",
getattr(self.app, "selected_dbengine", ""))
with open(os.path.join(self.path, ".arkos"), "w") as f:
meta.write(f)
# Call site type's post-installation hook
msg = "Running post-installation. This may take a few minutes..."
nthread.update(Notification("info", "Webs", msg))
specialmsg = self.post_install(extra_vars, dbpasswd)
# Cleanup and reload daemons
msg = "Finishing..."
nthread.update(Notification("info", "Webs", msg))
self.installed = True
storage.websites[self.id] = self
if self.port == 80:
cleanup_acme_dummy(self.domain)
signals.emit("websites", "site_installed", self)
if enable:
self.nginx_enable()
if enable and self.php:
php.open_basedir("add", "/srv/http/")
php_reload()
msg = "{0} site installed successfully".format(self.app.name)
nthread.complete(Notification("success", "Webs", msg))
if specialmsg:
return specialmsg
content = content_file.read()
# process the template
template = jinja2.Template(content)
content = template.render(worker_connections=args.worker_connections,
worker_processes=args.worker_processes)
print content
# write back the file
f = open(nginx_file, "w")
f.write(content)
f.close()
# Build the Configuration file
c = nginx.Conf()
upstream_args = ['tasks']
for upstream in args.u.split(','):
task, label = upstream.split(':')
# example NOMAD env variables for task nginx1 with port binding http
"""
NOMAD_IP_nginx1_http=10.123.26.15
NOMAD_PORT_nginx1_http=22707
NOMAD_ADDR_nginx1_http=10.123.26.15:22707
"""
# find out the IP address of the task via env NOMAD_IP_nginx1_http
env_var = 'NOMAD_ADDR_%s_%s' % (task, label)
url = os.getenv(env_var)
def rebuild_sites(self):
""" Turns jack's site json files into useable nginx configuration files """
for uuid, site_config in self.configs.items():
maintenance_mode = 'maintenance' in site_config and site_config['maintenance']
nginx_config = nginx.Conf()
# Add some comments so anyone who looks in the nginx config
# knows what's going on
nginx_config.add(nginx.Comment('Generated by Prism CP. Any changes will be overwritten!'))
nginx_config.add(nginx.Comment('Site ID: %s' % site_config['id']))
server_block = nginx.Server()
if 'listen' in site_config:
for port in site_config['listen']:
server_block.add(nginx.Key('listen', port))
if 'hostname' in site_config:
server_block.add(nginx.Key('server_name', site_config['hostname']))
site_folder = os.path.join(self._jack_plugin.site_files_location, uuid)
# Sets the root and logs to the site's folder
server_block.add(nginx.Key('access_log', os.path.join(site_folder, 'access.log')))
server_block.add(nginx.Key('error_log', os.path.join(site_folder, 'error.log')))
if 'root' in site_config:
root_folder = os.path.join(site_folder, site_config['root'])
if not os.path.exists(root_folder):
os.makedirs(root_folder)
server_block.add(nginx.Key('root', root_folder))
if 'index' in site_config:
server_block.add(nginx.Key('index', site_config['index']))
# If the site is in maintenance mode, redirect everything to 503
if maintenance_mode:
server_block.add(nginx.Location('/',
nginx.Key('return', 503)))
else:
for path, items in site_config['locations'].items():
location_items = []
for item, content in items.items():
if isinstance(content, tuple) or isinstance(content, list):
for c in content:
location_items.append(nginx.Key(item, c))
else:
location_items.append(nginx.Key(item, content))
server_block.add(nginx.Location(path, *location_items))
# Error page blocks
server_block.add(nginx.Key('error_page', '400 /error/400.html'))
server_block.add(nginx.Key('error_page', '403 /error/403.html'))
server_block.add(nginx.Key('error_page', '404 /error/404.html'))
server_block.add(nginx.Key('error_page', '408 /error/408.html'))
server_block.add(nginx.Key('error_page', '500 /error/500.html'))
server_block.add(nginx.Key('error_page', '502 /error/502.html'))
server_block.add(nginx.Key('error_page', '503 /error/503.html'))
server_block.add(nginx.Location('^~ /error/',
nginx.Key('root', self.sites_default),
nginx.Key('internal', '')))
nginx_config.add(server_block)
# Dump to nginx's config location
nginx.dumpf(nginx_config, os.path.join(self.config_location, site_config['uuid'] + '.conf'))
# Reload nginx so it picks up the changes
prism.os_command('systemctl reload nginx.service')
def _install(self, extra_vars, enable, nthread):
# Set metadata values
site_dir = config.get("websites", "site_dir")
path = (self.path or os.path.join(site_dir, self.id))
self.path = path
if os.path.isdir(self.path):
shutil.rmtree(self.path)
os.makedirs(self.path)
# If extra data is passed in, set up the serverblock accordingly
uwsgi_block = [
nginx.Location(
extra_vars.get("lregex", "/"),
nginx.Key("{0}_pass".format(extra_vars.get("type")),
extra_vars.get("pass", "")),
nginx.Key("include",
"{0}_params".format(extra_vars.get("type"))))
]
default_block = [
nginx.Location(extra_vars.get("lregex", "/"),
nginx.Key("proxy_pass", extra_vars.get("pass", "")),
nginx.Key("proxy_redirect", "off"),
nginx.Key("proxy_buffering", "off"),
nginx.Key("proxy_set_header", "Host $host"))
]
if extra_vars:
if not extra_vars.get("type") or not extra_vars.get("pass"):
raise errors.InvalidConfigError(
"Must enter ReverseProxy type and location to pass to")
elif extra_vars.get("type") in ["fastcgi", "uwsgi"]:
self.block = uwsgi_block
else:
self.block = default_block
if extra_vars.get("xrip"):
self.block[0].add(
nginx.Key("proxy_set_header", "X-Real-IP $remote_addr"))
if extra_vars.get("xff") == "1":
xff_key = "X-Forwarded-For $proxy_add_x_forwarded_for"
self.block[0].add(nginx.Key("proxy_set_header", xff_key))
# Create the nginx serverblock and arkOS metadata files
block = nginx.Conf()
server = nginx.Server(
nginx.Key("listen", self.port),
nginx.Key("listen", "[::]:" + str(self.port)),
nginx.Key("server_name", self.domain),
nginx.Key("root", self.base_path or self.path),
nginx.Location("/.well-known/acme-challenge/",
nginx.Key("root", self.path)))
server.add(*[x for x in self.block])
block.add(server)
nginx.dumpf(block, os.path.join("/etc/nginx/sites-available", self.id))
challenge_dir = os.path.join(self.path, ".well-known/acme-challenge/")
if not os.path.exists(challenge_dir):
os.makedirs(challenge_dir)
meta = configparser.SafeConfigParser()
ssl = self.cert.id if getattr(self, "cert", None) else "None"
meta.add_section("website")
meta.set("website", "id", self.id)
meta.set("website", "app", self.app.id if self.app else "None")
meta.set("website", "version", "None")
meta.set("website", "ssl", ssl)
with open(os.path.join(self.path, ".arkos"), "w") as f:
meta.write(f)
# Track port and reload daemon
self.installed = True
storage.websites[self.id] = self
signals.emit("websites", "site_installed", self)
self.nginx_enable()
import configparser
import os
import dotenv
import nginx
# Load the environment variables from the .env file
dotenv.load_dotenv('.env')
# All app subdirectories
subdirs = [x.strip() for x in os.environ.get("SUBDIRS").split(',')]
# We need to edit the sockets in the INI files in the application directories
config = configparser.ConfigParser()
nginx_config = nginx.Conf()
for subdir in subdirs:
# Load the respective config file and get the respective env variables
file = subdir + "/" + subdir + ".ini"
host = os.environ.get(subdir + "_host")
port = os.environ.get(subdir + "_port")
# Read the INI file
config.read(file)
# Generate required socket and set it
socket = host + ":" + port
config.set('uwsgi', 'socket', socket)
# Write the updates to the file
with open(file, 'w') as configfile:
config.write(configfile)
def main():
nginx_parameter_length_limit = 4000
pythonhome_directory = ""
flask_app_file_location = ""
application_name = ""
nginx_conf_location = ""
nginx_include_location = ""
module_usage_string = """
Usage: nginx_flaskapp_whitelister -r (optional) -p <pythonenvdirectory> -f <flaskappmodule> -a <flaskapplicationname> -c <nginxconfiglocation> -n <nginxincludelocation>
Flags:
-h Help function to display functionality and guidance to use the nginx_flaskapp_whitelister module.
-r Optional: Restart Nginx to reload added configuration and for white-listing to take immediate effect.
-p <pythonenvdirectory> The directory of the python environment that the Flask app is running in. The $PYTHONHOME variable.
-f <flaskappmodule> The python module from where the Flask app is served.
-a <flaskapplicationname> The physical name of the Flask application.
-c <nginxconfiglocation> The location of the current Nginx configuration that is used to serve the Flask application.
-n <nginxincludelocation> Optional: File path to where the 'include.whitelist' file will be included from within the Nginx configuration
that is used to serve the Flask application. If no file path is provided, the default will be used as
'/etc/nginx/'.
"""
# Initial parameter/flag verification and extraction
if not len(sys.argv) > 1:
print(
'No arguments supplied. For more information run: nginx_flaskapp_whitelister -h'
)
print(module_usage_string)
sys.exit(2)
try:
opts, args = getopt.getopt(sys.argv[1:], "hrp:f:a:c:n:", [
"pythonenvdir=", "flaskappmodule=", "applicationname=",
"nginxconfiglocation=", "nginxincludelocation="
])
except getopt.GetoptError:
print(module_usage_string)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print(module_usage_string)
sys.exit()
elif opt in ("-p", "--pythonenvdir"):
pythonhome_directory = arg
elif opt in ("-f", "--flaskappmodule"):
if "/" in arg:
flask_app_file_location = arg.replace("/", ".")
if flask_app_file_location.startswith("."):
flask_app_file_location = flask_app_file_location[1:]
else:
flask_app_file_location = arg
elif opt in ("-a", "--applicationname"):
application_name = arg
elif opt in ("-c", "--nginxconfiglocation"):
nginx_conf_location = arg
elif opt in ("-n", "--nginxincludelocation"):
nginx_include_location = arg
required_arguments = [{
'pythonenvdir': pythonhome_directory
}, {
'flaskappmodule': flask_app_file_location
}, {
'applicationname': application_name
}, {
'nginxconfiglocation': nginx_conf_location
}]
missing_arg = ""
for argument in required_arguments:
if argument.values()[0] == "":
if missing_arg == "":
missing_arg = argument.keys()[0]
else:
missing_arg = missing_arg + ", " + argument.keys()[0]
if missing_arg != "":
print(
'Required arguments (' + missing_arg +
') are missing. For more information run: nginx_flaskapp_whitelister -h'
)
print(module_usage_string)
sys.exit(2)
# Setup the PYTHONPATH variable to point to the python home and import the requested Flask app from the specified module
sys.path.append(pythonhome_directory)
project = importlib.import_module(flask_app_file_location)
app = getattr(project, application_name)
# Extract all exposed flask endpoints from the specified app
links = []
for rule in app.url_map._rules:
links.append(rule.rule)
# Consolidate the retrieved rules from the flask app url map to only contain a list of allowed endpoints and strip
# parameter sections (< >) from endpoints
allowed_locations = []
for endpoint in links:
if endpoint != "/":
allowed_locations.append(re.sub('<[^>]+>', '', endpoint))
# Load the current NginX config used to serve the application into a config object
config = nginx.loadf(nginx_conf_location)
# Extract the keys and configuration from the config object that refers to the location directive that allows the access
# to "location / {}" (presumed to be the most likely allowed full access route configured)
original_location_object = []
original_server_list = list(config.filter('Http')[0].filter('Server'))
for server in original_server_list:
if server.locations != []:
original_location_object.extend(server.locations)
current_locations = []
for location in original_location_object:
current_locations.append(location.as_dict)
# Create a new NginX config object for the added included location directives for the whitelisted locations
allowed_location_keys = []
whitelist_location_conf = nginx.Conf()
for location in current_locations:
if location.keys()[0] == 'location /':
for key in location['location /']:
nested_key = []
if "if (" in key.keys()[0]:
for x in key[key.keys()[0]]:
nested_key.append(
nginx.Key(x.keys()[0], x[x.keys()[0]]).as_strings)
allowed_location_keys.append(
nginx.Key(key.keys()[0],
'{ ' + "".join(nested_key) + ' }'))
else:
allowed_location_keys.append(
nginx.Key(key.keys()[0], key[key.keys()[0]]))
# As Nginx has a parameter limit that caps the amount of characters allowed in a directive parameter,
# the allowed location endpoints are chunked accordingly into as big as possible chunks within the limit
dynamic_locations_write_variable = []
for location in allowed_locations:
if len("|".join(dynamic_locations_write_variable +
[location])) < nginx_parameter_length_limit:
dynamic_locations_write_variable.append(location)
else:
dynamic_locations_write_variable.append(location)
whitelist_location_conf.add(
nginx.Location(
'~ (' + '|'.join(dynamic_locations_write_variable) + ')',
nginx.Key('include', 'shared.conf')))
dynamic_locations_write_variable = []
if dynamic_locations_write_variable != []:
whitelist_location_conf.add(
nginx.Location(
'~ (' + '|'.join(dynamic_locations_write_variable) + ')',
nginx.Key('include', 'shared.conf')))
dynamic_locations_write_variable = []
whitelist_location_conf.add(
nginx.Location('= /', nginx.Key('include', 'shared.conf')))
whitelist_location_conf.add(
nginx.Location('~ /', nginx.Key('return', '404')))
# Create a shared Nginx config object containing the keys and configurations shared and re-used
# across various location directives. This shared.conf is then included per directive, instead
# of repeating code
shared_conf = nginx.Conf()
shared_conf.add(*allowed_location_keys)
shared_conf_tmp_file = tempfile.NamedTemporaryFile(dir='/tmp/').name
nginx.dumpf(shared_conf, shared_conf_tmp_file)
with open(shared_conf_tmp_file, 'r') as file:
shared_conf_data = file.readlines()
if any("if (" in string for string in shared_conf_data):
for line in shared_conf_data:
if "if (" in line:
shared_conf_data[shared_conf_data.index(
line)] = line.replace(') "{', ') {')
if '}";' in line:
shared_conf_data[shared_conf_data.index(
line)] = line.replace('}";', '}')
with open(shared_conf_tmp_file, 'w') as file:
file.writelines(shared_conf_data)
whitelist_location_conf_tmp_file = tempfile.NamedTemporaryFile(
dir='/tmp/').name
nginx.dumpf(whitelist_location_conf, whitelist_location_conf_tmp_file)
# If a specific Nginx include location is specified, the generated configuration are moved to this location,
# otherwise it is moved to a default location of (/etc/nginx/). This filepath should match the path of the
# include directive inserted into the main nginx.conf for enabling the nginx whitelisting.
if nginx_include_location != "":
os.rename(whitelist_location_conf_tmp_file,
nginx_include_location + "/include.whitelist")
os.rename(shared_conf_tmp_file,
nginx_include_location + "/shared.conf")
else:
os.rename(whitelist_location_conf_tmp_file,
"/etc/nginx/include.whitelist")
os.rename(shared_conf_tmp_file, "/etc/nginx/shared.conf")
# For the newly generated whitelisting config to take effect, the Nginx config should be reloaded and the
# process restarted. If the restart flag was set by the user, Nginx will be attempted to restart as part
# of the tool implementation
for opt, arg in opts:
if opt == '-r':
nginx_restart = subprocess.check_output(
["service", "nginx", "restart"])
print(nginx_restart)
