def messy():
return nginx.loads("""
# This is an example of a messy config
upstream php { server unix:/tmp/php-cgi.socket; }
server { server_name localhost; #this is the server server_name
location /{ test_key test_value; }}
""")
def test_default_server(host):
f = host.file('/etc/nginx/conf.d/default.conf')
c = nginx.loads(f.content_string)
lf = c.server.filter('Location', '/')
assert len(lf) == 1
lb = c.server.filter('Location', '/backend')
assert len(lb) == 1
def test_semicolon_in_second_key_value(self):
inp_data = nginx.loads(TESTBLOCK_CASE_9)
self.assertEqual(len(inp_data.filter("Location")), 1)
location_children = inp_data.filter("Location")[0].children
self.assertEqual(len(location_children), 1)
self.assertEqual(location_children[0].name, "add_header")
self.assertEqual(location_children[0].value, 'X-XSS-Protection "1;mode-block"')
def test_update_allowed(self, registered_host_mock):
registered_host_mock.query.values.return_value = []
def check_ips(location):
ips = [key.value for key in location.keys if key.name == 'allow']
self.assertEqual(self.accept_ips, ips)
self.assertEqual(location.keys[-1].as_dict, {'deny': 'all'})
conf = nginx.loads(self.conf)
update_allowed(self.accept_ips, conf)
servers = conf.filter('Server')
location = servers[0].filter('Location')[0]
self.assertFalse(
any([key.name in ('allow', 'deny') for key in location.keys]))
location = servers[0].filter('Location')[1]
self.assertFalse(
any([key.name in ('allow', 'deny') for key in location.keys]))
location = servers[0].filter('Location')[2]
check_ips(location)
location = servers[1].filter('Location')[0]
self.assertFalse(
any([key.name in ('allow', 'deny') for key in location.keys]))
location = servers[1].filter('Location')[1]
check_ips(location)
location = servers[1].filter('Location')[2]
check_ips(location)
def messy():
return nginx.loads("""
# This is an example of a messy config
upstream php { server unix:/tmp/php-cgi.socket; }
server { server_name localhost; #this is the server server_name
location /{ test_key test_value; }}
""")
def test_types_block(self):
inp_data = nginx.loads(TESTBLOCK_CASE_10)
self.assertEqual(len(inp_data.filter("Types")), 1)
self.assertEqual(len(inp_data.filter("Types")[0].children), 4)
self.assertEqual(len(inp_data.filter("Types")[0].filter("Key")), 4)
data_type = inp_data.filter("Types")[0].filter("Key")[0]
self.assertEqual(data_type.value, "cea")
def test_key_parse(self):
data = nginx.loads(TESTBLOCK)
self.assertEqual(len(data.server.keys), 5)
firstKey = data.server.keys[0]
thirdKey = data.server.keys[3]
self.assertEqual(firstKey.name, 'listen')
self.assertEqual(firstKey.value, '80')
self.assertEqual(thirdKey.name, 'mykey')
self.assertEqual(thirdKey.value, '"myvalue; #notme myothervalue"')
def test_key_parse(self):
data = nginx.loads(TESTBLOCK_CASE_1)
self.assertEqual(len(data.server.keys), 5)
firstKey = data.server.keys[0]
thirdKey = data.server.keys[3]
self.assertEqual(firstKey.name, 'listen')
self.assertEqual(firstKey.value, '80')
self.assertEqual(thirdKey.name, 'mykey')
self.assertEqual(thirdKey.value, '"myvalue; #notme myothervalue"')
def test_client_max_body_size(host):
f = host.file('/etc/nginx/conf.d/default.conf')
c = nginx.loads(f.content_string)
vs = c.server.filter('Key', 'client_max_body_size')
assert len(vs) == 1
assert vs[0].value == '512k'
lc = c.server.filter('Location', '/')
vl = lc[0].filter('Key', 'client_max_body_size')
assert len(vl) == 1
assert vl[0].value == '5m'
def test_limit_expect(self):
data = nginx.loads(TESTBLOCK_CASE_8)
self.assertEqual(len(data.filter("Location")), 1)
self.assertEqual(len(data.filter("Location")[0].children), 2)
self.assertEqual(len(data.filter("Location")[0].filter("LimitExcept")), 1)
limit_except = data.filter("Location")[0].filter("LimitExcept")[0]
self.assertEqual(limit_except.value, "GET POST")
self.assertEqual(len(limit_except.children), 1)
first_key = limit_except.filter("Key")[0]
self.assertEqual(first_key.name, "deny")
self.assertEqual(first_key.value, "all")
def test_key_parse_complex(self):
data = nginx.loads(SECONDTESTBLOCK)
self.assertEqual(len(data.server.keys), 5)
firstKey = data.server.keys[0]
thirdKey = data.server.keys[3]
self.assertEqual(firstKey.name, 'listen')
self.assertEqual(firstKey.value, '80')
self.assertEqual(thirdKey.name, 'mykey')
self.assertEqual(thirdKey.value, '"myvalue; #notme myothervalue"')
self.assertEqual(
data.server.locations[-1].keys[0].value,
"301 $scheme://$host:$server_port${request_uri}bitbucket/")
def test_semicolon_in_second_key_value(self):
inp_data = nginx.loads(TESTBLOCK_CASE_9)
self.assertEqual(len(inp_data.filter("Location")), 1)
location_children = inp_data.filter("Location")[0].children
self.assertEqual(len(location_children), 1)
self.assertEqual(location_children[0].name, "add_header")
self.assertEqual(location_children[0].value,
'X-XSS-Protection "1;mode-block"')
self.assertEqual(len(inp_data.filter("If")), 1)
self.assertEqual(
inp_data.filter("If")[0].value,
"( $http_user_agent = \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)\" )"
)
def server_submit():
server_name=request.POST.get('server_name', '')
server_value=request.POST.get('server_value', '')
path_file_name=request.POST.get("path_file_name","")
c = nginx.loadf(path_file_name)
servers = c.filter("Server")
for i in servers:
if server_name == i.filter("key", "server_name")[0].value:
c.remove(i)
new_c=nginx.loads(server_value)
new_server=new_c.filter('Server')[0]
c.add(new_server)
# print "remove ok"
# c.add(myserver)
nginx.dumpf(c, path_file_name)
# print myserver
return server_value
def test_messy_load(self):
data = nginx.loads(TESTBLOCK_CASE_4)
self.assertTrue(data is not None)
self.assertTrue(len(data.server.comments), 1)
self.assertTrue(len(data.server.locations), 1)
def test_comment_parse(self):
data = nginx.loads(TESTBLOCK_CASE_1)
self.assertEqual(len(data.server.comments), 4)
self.assertEqual(data.server.comments[2].comment, 'And also this one')
def _install(self, extra_vars, enable, nthread):
nthread.title = "Installing website"
msg = Notification("info", "Webs", "Preparing to install...")
nthread.update(msg)
# Make sure the chosen port is indeed open
if not tracked_services.is_open_port(self.port, self.domain):
cname = "({0})".format(self.app.id)
raise errors.InvalidConfigError(cname, nthread)\
from tracked_services.PortConflictError(self.port, self.domain)
# Set some metadata values
specialmsg, dbpasswd = "", ""
site_dir = config.get("websites", "site_dir")
path = (self.path or os.path.join(site_dir, self.id))
self.path = path
self.php = extra_vars.get("php") or self.php \
or self.app.uses_php or False
self.version = self.app.version.rsplit("-", 1)[0] \
if self.app.website_updates else None
# Classify the source package type
if not self.app.download_url:
ending = ""
elif self.app.download_url.endswith(".tar.gz"):
ending = ".tar.gz"
elif self.app.download_url.endswith(".tgz"):
ending = ".tgz"
elif self.app.download_url.endswith(".tar.bz2"):
ending = ".tar.bz2"
elif self.app.download_url.endswith(".zip"):
ending = ".zip"
elif self.app.download_url.endswith(".git"):
ending = ".git"
else:
raise errors.InvalidConfigError(
"Invalid source archive format in {0}".format(self.app.id))
msg = "Running pre-installation..."
uid, gid = users.get_system("http").uid, groups.get_system("http").gid
nthread.update(Notification("info", "Webs", msg))
# Call website type's pre-install hook
self.pre_install(extra_vars)
# If needs DB and user didn't select an engine, choose one for them
if len(self.app.database_engines) > 1 \
and extra_vars.get("dbengine", None):
self.app.selected_dbengine = extra_vars.get("dbengine")
if not getattr(self.app, "selected_dbengine", None)\
and self.app.database_engines:
self.app.selected_dbengine = self.app.database_engines[0]
# Create DB and/or DB user as necessary
if getattr(self.app, "selected_dbengine", None):
msg = "Creating database..."
nthread.update(Notification("info", "Webs", msg))
mgr = databases.get_managers(self.app.selected_dbengine)
if not mgr:
estr = "No manager found for {0}"
raise errors.InvalidConfigError(
estr.format(self.app.selected_dbengine))
# Make sure DB daemon is running if it has one
if not mgr.state:
svc = services.get(mgr.meta.database_service)
svc.restart()
self.db = mgr.add_db(self.id)
if hasattr(self.db, "path"):
os.chmod(self.db.path, 0o660)
os.chown(self.db.path, -1, gid)
# If multiuser DB type, create user
if mgr.meta.database_multiuser:
dbpasswd = random_string(16)
db_user = mgr.add_user(self.id, dbpasswd)
db_user.chperm("grant", self.db)
# Make sure the target directory exists, but is empty
pkg_path = os.path.join("/tmp", self.id + ending)
if os.path.isdir(self.path):
shutil.rmtree(self.path)
os.makedirs(self.path)
# Download and extract the source repo / package
msg = "Downloading website source..."
nthread.update(Notification("info", "Webs", msg))
if self.app.download_url and ending == ".git":
g = git.Repo.clone_from(self.app.download_url, self.path)
if hasattr(self.app, "download_at_tag"):
g = git.Git(self.path)
g.checkout(self.app.download_git_tag)
elif self.app.download_url:
download(self.app.download_url, file=pkg_path, crit=True)
# Format extraction command according to type
msg = "Extracting source..."
nthread.update(Notification("info", "Webs", msg))
if ending in [".tar.gz", ".tgz", ".tar.bz2"]:
arch = tarfile.open(pkg_path, "r:gz")
r = (x for x in arch.getnames() if re.match("^[^/]*$", x))
toplvl = next(r, None)
if not toplvl:
raise errors.OperationFailedError(
"Malformed source archive")
arch.extractall(site_dir)
os.rename(os.path.join(site_dir, toplvl), self.path)
else:
arch = zipfile.ZipFile(pkg_path)
r = (x for x in arch.namelist() if re.match("^[^/]*/$", x))
toplvl = next(r, None)
if not toplvl:
raise errors.OperationFailedError(
"Malformed source archive")
arch.extractall(site_dir)
os.rename(os.path.join(site_dir, toplvl.rstrip("/")),
self.path)
os.remove(pkg_path)
# Set proper starting permissions on source directory
os.chmod(self.path, 0o755)
os.chown(self.path, uid, gid)
for r, d, f in os.walk(self.path):
for x in d:
os.chmod(os.path.join(r, x), 0o755)
os.chown(os.path.join(r, x), uid, gid)
for x in f:
os.chmod(os.path.join(r, x), 0o644)
os.chown(os.path.join(r, x), uid, gid)
# If there is a custom path for the data directory, set it up
if getattr(self.app, "website_datapaths", None) \
and extra_vars.get("datadir"):
self.data_path = extra_vars["datadir"]
if not os.path.exists(self.data_path):
os.makedirs(self.data_path)
os.chmod(self.data_path, 0o755)
os.chown(self.data_path, uid, gid)
elif hasattr(self, "website_default_data_subdir"):
self.data_path = os.path.join(self.path,
self.website_default_data_subdir)
else:
self.data_path = self.path
# Create the nginx serverblock
addtoblock = self.addtoblock or []
if extra_vars.get("addtoblock"):
addtoblock += nginx.loads(extra_vars.get("addtoblock"), False)
default_index = "index." + ("php" if self.php else "html")
if hasattr(self.app, "website_root"):
webroot = os.path.join(self.path, self.app.website_root)
else:
webroot = self.path
block = nginx.Conf()
server = nginx.Server(
nginx.Key("listen", str(self.port)),
nginx.Key("listen", "[::]:" + str(self.port)),
nginx.Key("server_name", self.domain), nginx.Key("root", webroot),
nginx.Key(
"index",
getattr(self.app, "website_index", None) or default_index),
nginx.Location("/.well-known/acme-challenge/",
nginx.Key("root", self.path)))
if addtoblock:
server.add(*[x for x in addtoblock])
block.add(server)
nginx.dumpf(block, os.path.join("/etc/nginx/sites-available", self.id))
challenge_dir = os.path.join(self.path, ".well-known/acme-challenge/")
if not os.path.exists(challenge_dir):
os.makedirs(challenge_dir)
# Create arkOS metadata file
meta = configparser.SafeConfigParser()
meta.add_section("website")
meta.set("website", "id", self.id)
meta.set("website", "app", self.app.id)
meta.set("website", "ssl",
self.cert.id if getattr(self, "cert", None) else "None")
meta.set("website", "version", self.version or "None")
if getattr(self.app, "website_datapaths", None) \
and self.data_path:
meta.set("website", "data_path", self.data_path)
meta.set("website", "dbengine", "")
meta.set("website", "dbengine",
getattr(self.app, "selected_dbengine", ""))
with open(os.path.join(self.path, ".arkos"), "w") as f:
meta.write(f)
# Call site type's post-installation hook
msg = "Running post-installation. This may take a few minutes..."
nthread.update(Notification("info", "Webs", msg))
specialmsg = self.post_install(extra_vars, dbpasswd)
# Cleanup and reload daemons
msg = "Finishing..."
nthread.update(Notification("info", "Webs", msg))
self.installed = True
storage.websites[self.id] = self
if self.port == 80:
cleanup_acme_dummy(self.domain)
signals.emit("websites", "site_installed", self)
if enable:
self.nginx_enable()
if enable and self.php:
php.open_basedir("add", "/srv/http/")
php_reload()
msg = "{0} site installed successfully".format(self.app.name)
nthread.complete(Notification("success", "Webs", msg))
if specialmsg:
return specialmsg
def test_basic_load(self):
self.assertTrue(nginx.loads(TESTBLOCK_CASE_1) is not None)
def test_complex_upstream(self):
inp_data = nginx.loads(TESTBLOCK_CASE_6)
out_data = '\n' + nginx.dumps(inp_data)
self.assertEqual(TESTBLOCK_CASE_6, out_data)
def test_filtering(self):
data = nginx.loads(TESTBLOCK_CASE_1)
self.assertEqual(len(data.server.filter('Key', 'mykey')), 1)
self.assertEqual(data.server.filter('Key', 'nothere'), [])
def test_brace_position(self):
data = nginx.loads(TESTBLOCK_CASE_3)
self.assertEqual(len(data.filter('Upstream')), 3)
def test_reflection(self):
inp_data = nginx.loads(TESTBLOCK_CASE_1)
out_data = '\n' + nginx.dumps(inp_data)
self.assertEqual(TESTBLOCK_CASE_1, out_data)
def test_location_parse(self):
data = nginx.loads(TESTBLOCK)
self.assertEqual(len(data.server.locations), 1)
firstLoc = data.server.locations[0]
self.assertEqual(firstLoc.value, '~ \.php(?:$|/)')
self.assertEqual(len(firstLoc.keys), 1)
def addproxy(filepath):
if not os.path.isfile(filepath):
exit(1)
with open(filepath) as file:
vhost_config = file.read()
conf_root = nginx.loads(vhost_config)
for server in conf_root.query('server'):
node = server.query('ssl', first=True)
if node != False and 'on' == str(node):
server_name = str(server.query('server_name', first=True))
ssl_certificate = str(server.query('ssl_certificate',
first=True)).replace('"', '')
ssl_certificate_key = str(
server.query('ssl_certificate_key',
first=True)).replace('"', '')
ssl_ciphers = str(server.query('ssl_ciphers', first=True))
ssl_prefer_server_ciphers = str(
server.query('ssl_prefer_server_ciphers', first=True))
ssl_protocols = str(server.query('ssl_protocols', first=True))
ssl_certificate_path_parts = ssl_certificate.split('/')
ssl_certificate_filename = ssl_certificate_path_parts[
len(ssl_certificate_path_parts) - 1]
remote_ssl_certificate_filepath = REMOTE_CERTS_DIR + '/' + ssl_certificate_filename
ssl_certificate_key_path_parts = ssl_certificate_key.split('/')
ssl_certificate_key_filename = ssl_certificate_key_path_parts[
len(ssl_certificate_key_path_parts) - 1]
remote_ssl_certificate_key_filepath = REMOTE_KEYS_DIR + '/' + ssl_certificate_key_filename
path_parts = filepath.split('/')
filename = path_parts[len(path_parts) - 1]
if re.match(r'.*\.conf$', filename) is None:
return
username = path_parts[len(path_parts) - 2]
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(SSH_HOST_ADDR)
sftp = ssh.open_sftp()
proxy_config = PROXY_CONFIG_TMPL.format(
server_name, ssl_certificate_filename,
ssl_certificate_key_filename, ssl_ciphers,
ssl_prefer_server_ciphers, ssl_protocols)
with sftp.open("{0}/{1}".format(REMOTE_VHOSTS_DIR, filename),
'w') as remote_conf_file:
remote_conf_file.write(proxy_config)
sftp.put(ssl_certificate, remote_ssl_certificate_filepath)
sftp.put(ssl_certificate_key, remote_ssl_certificate_key_filepath)
sftp.chmod(remote_ssl_certificate_key_filepath, 0o600)
ssh.exec_command(NGINX_RELOAD_COMMAND)
ssh.close()
return
def test_missing_semi_colon(self):
with pytest.raises(nginx.ParseError) as e:
nginx.loads(TESTBLOCK_CASE_11)
self.assertEqual(str(e.value),
"Config syntax, missing ';' at index: 189")
def test_comment_parse(self):
data = nginx.loads(TESTBLOCK)
self.assertEqual(len(data.server.comments), 4)
self.assertEqual(data.server.comments[2].comment, 'And also this one')
def test_messy_load(self):
data = nginx.loads(MESSYBLOCK)
self.assertTrue(data is not None)
self.assertTrue(len(data.server.comments), 1)
self.assertTrue(len(data.server.locations), 1)
def test_basic_load(self):
self.assertTrue(nginx.loads(TESTBLOCK) is not None)
def test_brace_inside_block_param(self):
inp_data = nginx.loads(TESTBLOCK_CASE_12)
self.assertEqual(len(inp_data.server.filter("Location")), 1)
self.assertEqual(
inp_data.server.filter("Location")[0].value,
"~ \"^/(test|[0-9a-zA-Z]{6})$\"")
def test_location_parse(self):
data = nginx.loads(TESTBLOCK_CASE_1)
self.assertEqual(len(data.server.locations), 1)
firstLoc = data.server.locations[0]
self.assertEqual(firstLoc.value, '~ \.php(?:$|/)')
self.assertEqual(len(firstLoc.keys), 1)
def test_filtering(self):
data = nginx.loads(TESTBLOCK)
self.assertEqual(len(data.server.filter('Key', 'mykey')), 1)
self.assertEqual(data.server.filter('Key', 'nothere'), [])
def test_single_value_keys(self):
data = nginx.loads(TESTBLOCK_CASE_3)
single_value_key = data.filter('Upstream')[0].keys[0]
self.assertEqual(single_value_key.name, 'ip_hash')
self.assertEqual(single_value_key.value, '')
if hasattr(self.meta, "website_datapaths") and self.meta.website_datapaths \
and extra_vars.get("datadir"):
self.data_path = extra_vars["datadir"]
if not os.path.exists(self.data_path):
os.makedirs(self.data_path)
os.chmod(self.data_path, 0755)
os.chown(self.data_path, uid, gid)
elif hasattr(self, "website_default_data_subdir"):
self.data_path = os.path.join(self.path, self.website_default_data_subdir)
else:
self.data_path = self.path
# Create the nginx serverblock
addtoblock = self.addtoblock or []
if extra_vars.get("addtoblock"):
addtoblock += nginx.loads(extra_vars.get("addtoblock"), False)
try:
block = nginx.Conf()
server = nginx.Server(
nginx.Key("listen", str(self.port)),
nginx.Key("server_name", self.addr),
nginx.Key("root", self.path),
nginx.Key("index", "index."+("php" if self.php else "html"))
)
if addtoblock:
server.add(*[x for x in addtoblock])
block.add(server)
nginx.dumpf(block, os.path.join("/etc/nginx/sites-available", self.id))
except Exception, e:
raise Exception("nginx serverblock couldn't be written - "+str(e))
def test_quoted_key_value(self):
data = nginx.loads(TESTBLOCK_CASE_5)
out_data = '\n' + nginx.dumps(data)
self.assertEqual(out_data, TESTBLOCK_CASE_5)
extract_cmd = 'tar '
extract_cmd += 'xzf' if ending is '.tar.gz' else 'xjf'
extract_cmd += ' /tmp/%s -C %s --strip 1' % (name+ending, target_path)
else:
extract_cmd = 'unzip -d %s /tmp/%s' % (target_path, name+ending)
status = shell_cs(extract_cmd, stderr=True)
if status[0] >= 1:
raise InstallError(status[1])
os.remove(pkg_path)
php = vars.getvalue('php', '')
addtoblock = vars.getvalue('addtoblock', '')
if addtoblock:
addtoblock = nginx.loads(addtoblock, False)
else:
addtoblock = []
if wa.wa_plugin == 'Website' and php == '1' and addtoblock:
addtoblock.extend(x for x in webapp.phpblock)
elif wa.wa_plugin == 'Website' and php == '1':
addtoblock = webapp.phpblock
# Setup the webapp and create an nginx serverblock
try:
w = Webapp()
w.name = name
w.stype = wa.wa_plugin
w.path = target_path
w.addr = vars.getvalue('addr', 'localhost')
w.port = vars.getvalue('port', '80')
def test_session_sticky(self):
inp_data = nginx.loads(TESTBLOCK_CASE_7)
out_data = '\n' + nginx.dumps(inp_data)
self.assertEqual(TESTBLOCK_CASE_7, out_data)
extract_cmd = 'tar '
extract_cmd += 'xzf' if ending is '.tar.gz' else 'xjf'
extract_cmd += ' /tmp/%s -C %s --strip 1' % (name+ending, target_path)
else:
extract_cmd = 'unzip -d %s /tmp/%s' % (target_path, name+ending)
status = shell_cs(extract_cmd, stderr=True)
if status[0] >= 1:
raise InstallError(status[1])
os.remove(pkg_path)
php = vars.getvalue('php', '')
addtoblock = vars.getvalue('addtoblock', '')
if addtoblock:
addtoblock = nginx.loads(addtoblock, False)
else:
addtoblock = []
if wa.wa_plugin == 'Website' and php == '1' and addtoblock:
addtoblock.extend(x for x in webapp.phpblock)
elif wa.wa_plugin == 'Website' and php == '1':
addtoblock = webapp.phpblock
# Setup the webapp and create an nginx serverblock
try:
w = Webapp()
w.name = name
w.stype = wa.wa_plugin
w.path = target_path
w.addr = vars.getvalue('addr', 'localhost')
w.port = vars.getvalue('port', '80')
def test_reflection(self):
inp_data = nginx.loads(TESTBLOCK)
out_data = '\n' + nginx.dumps(inp_data)
self.assertEqual(TESTBLOCK, out_data)
def test_server_without_last_linebreak(self):
self.assertTrue(nginx.loads(TESTBLOCK_CASE_13) is not None)