def parse_nginx_config(): queue_rates = open(QUEUE_RATE_FILE).read() zones = open(ZONES_FILE).read() if len(queue_rates) == 0 or len(zones) == 0: return '' queue_rates = loads(queue_rates) zones = loads(zones) configs = {} zone_map = {} for z in zones: zstr = z[1] regex = re.search(ZONE_PATTERN, zstr) if regex: zone_map[regex.group(1)] = regex.group(2) for r in queue_rates: queue_name = r[0][-1] queue_name = re.search('\w+', queue_name).group(0) qoptions = r[1][0][-1] zone = re.search(QUEUE_ZONE_PATTERN, qoptions) burst = re.search(BURST_PATTERN, qoptions) nodelay = 'nodelay' in qoptions delay = re.search(DELAY_PATTERN, qoptions) zone_rate = zone_map.get(zone.group(1)) if zone_rate is None: exit('Invalid zone define %s' % zone) configs[queue_name] = {'rate': zone_rate} if burst: configs[queue_name]['burst'] = int(burst.group(1)) if nodelay: configs[queue_name]['nodelay'] = True if delay: configs[queue_name]['delay'] = int(delay.group(1)) return yaml.dump(configs)
def setUp(self): # api_proxy_block self.location = "/api0" self.proxy_pass = "******" self.parsed_api_block = loads( api_proxy_block(location=self.location, proxy_pass=self.proxy_pass)) # server_block_no_rest self.server_name = "offscale.io" self.listen = "443" self.parsed_server_block_no_rest = loads( server_block(server_name=self.server_name, listen=self.listen)) d = "/etc/letsencrypt/live/{server_name}".format( server_name=self.server_name) self.ssl_certificate = "{d}/fullchain.pem".format(d=d) self.ssl_certificate_key = "{d}/privkey.pem".format(d=d) self.nginx = configs_dir("nginx.conf") self.two_roots = configs_dir("two_roots.conf") self.one_root = configs_dir("one_root.conf") with open(self.one_root, "rt") as f: self.one_root_content = f.read() with open(self.two_roots, "rt") as f: self.two_roots_content = f.read()
def test_server_block_rest(self): server_name = "offscale.io" listen = "443" comment = None rest = """ goodbye cruel world """ self.assertEqual( loads( server_block(server_name=server_name, listen=listen, comment=comment, rest=rest)), [[ ["server"], [ ["# Emitted by nginx_parse_emit.emit.server_block", "\n"], ["server_name", server_name], ["listen", listen], ["goodbye"], ["cruel"], ["world"], ], ]], )
def ssl_enable(self, data, cpath, kpath): name, stype = data['name'], data['type'] n = nginxparser.loads( open('/etc/nginx/sites-available/' + name, 'r').read()) port = '443' for l in n: if l[0] == ['server']: for x in l[1]: if x[0] == 'listen': if x[1] == '80': x[1] = '443 ssl' port = '443' else: port = x[1] x[1] = x[1] + ' ssl' l[1].append(['ssl_certificate', cpath]) l[1].append(['ssl_certificate_key', kpath]) l[1].append(['ssl_protocols', 'TLSv1 TLSv1.1 TLSv1.2']) l[1].append(['ssl_ciphers', 'HIGH:!aNULL:!MD5']) comline = '# GENESIS ' + stype + ' https://' + data[ 'addr'] + ':' + port + '\n' open('/etc/nginx/sites-available/' + name, 'w').write(comline + nginxparser.dumps(n)) apis.webapps(self.app).get_interface(stype).ssl_enable( os.path.join('/srv/http/webapps', name), cpath, kpath) self.nginx_reload()
def configure_riot_nginx5(root="/var/www/riot", **kwargs): kwargs.setdefault("LISTEN_PORT", 80) server_block = utils.merge_into( emit.server_block( server_name=kwargs["SERVER_NAME"], listen=kwargs["LISTEN_PORT"] ), emit.html5_block("/", root), ) sio = StringIO() sio.write( dumps( loads(emit.redirect_block(server_name=kwargs["SERVER_NAME"], port=80)) + server_block if kwargs["LISTEN_PORT"] == 443 else server_block ) ) put( sio, "/etc/nginx/sites-enabled/{server_name}".format( server_name=kwargs["SERVER_NAME"] ), use_sudo=True, ) return restart_systemd("nginx")
def ssl_enable(self, data, cpath, kpath): name, stype = data['name'], data['type'] n = nginxparser.loads( open('/etc/nginx/sites-available/'+name, 'r').read()) port = '443' for l in n: if l[0] == ['server']: for x in l[1]: if x[0] == 'listen': if x[1] == '80': x[1] = '443 ssl' port = '443' else: port = x[1] x[1] = x[1] + ' ssl' l[1].append(['ssl_certificate', cpath]) l[1].append(['ssl_certificate_key', kpath]) l[1].append(['ssl_protocols', 'TLSv1 TLSv1.1 TLSv1.2']) l[1].append(['ssl_ciphers', 'HIGH:!aNULL:!MD5']) comline = '# GENESIS '+stype+' https://'+data['addr']+':'+port+'\n' open('/etc/nginx/sites-available/'+name, 'w').write( comline+nginxparser.dumps(n)) apis.webapps(self.app).get_interface(stype).ssl_enable( os.path.join('/srv/http/webapps', name), cpath, kpath) self.nginx_reload()
def test_merge_roots(self): server_name = "foo" self.assertEqual( """server { server_name foo; listen 80; return 301 https://$server_name$request_uri; } server { # Emitted by nginx_parse_emit.emit.server_block server_name foo; listen 443; }""", dumps( loads(redirect_block(server_name=server_name, port="80")) + loads(server_block(server_name=server_name, listen="443"))), )
def setUp(self): # api_proxy_block self.location = "/api0" self.proxy_pass = "******" self.parsed_api_block = loads( api_proxy_block(location=self.location, proxy_pass=self.proxy_pass)) # server_block_no_rest self.server_name = "offscale.io" self.listen = "443" self.parsed_server_block_no_rest = loads( server_block(server_name=self.server_name, listen=self.listen)) d = "/etc/letsencrypt/live/{server_name}".format( server_name=self.server_name) self.ssl_certificate = "{d}/fullchain.pem".format(d=d) self.ssl_certificate_key = "{d}/privkey.pem".format(d=d) self.two_roots = loads(""" server { # Emitted by nginx_parse_emit.emit.server_block server_name offscale.io; listen 80; } server { # Emitted by nginx_parse_emit.emit.server_block server_name offscale.io; listen 443; location /api0 { proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Scheme $scheme; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_pass http://127.0.0.1:5000/awesome; proxy_redirect off; } } """)
def ensure_nginxparser_instance(conf_file): # type: (str) -> [[[str]]] if isinstance(conf_file, list): return conf_file elif hasattr(conf_file, "read"): return load(conf_file) elif path.isfile(conf_file): with open(conf_file, "rt") as f: return load(f) else: return loads(conf_file)
def ssl_disable(self, data): name, stype = data['name'], data['type'] n = nginxparser.loads( open('/etc/nginx/sites-available/' + name, 'r').read()) port = '80' # Three passes - list loop bug omits entries on some systems for l in n: if l[0] == ['server']: for x in l[1]: if x[0] == 'listen': if x[1] == '443 ssl': x[1] = '80' port = '80' print True else: x[1] = x[1].rstrip(' ssl') print x[1] port = x[1] elif x[0] == 'ssl_certificate': l[1].remove(x) elif x[0] == 'ssl_certificate_key': l[1].remove(x) elif x[0] == 'ssl_protocols': l[1].remove(x) elif x[0] == 'ssl_ciphers': l[1].remove(x) for l in n: if l[0] == ['server']: for x in l[1]: if x[0] == 'ssl_certificate': l[1].remove(x) elif x[0] == 'ssl_certificate_key': l[1].remove(x) elif x[0] == 'ssl_protocols': l[1].remove(x) elif x[0] == 'ssl_ciphers': l[1].remove(x) for l in n: if l[0] == ['server']: for x in l[1]: if x[0] == 'ssl_certificate': l[1].remove(x) elif x[0] == 'ssl_certificate_key': l[1].remove(x) elif x[0] == 'ssl_protocols': l[1].remove(x) elif x[0] == 'ssl_ciphers': l[1].remove(x) comline = '# GENESIS ' + stype + ' http://' + data[ 'addr'] + ':' + port + '\n' open('/etc/nginx/sites-available/' + name, 'w').write(comline + nginxparser.dumps(n)) apis.webapps(self.app).get_interface(stype).ssl_disable( os.path.join('/srv/http/webapps', name)) self.nginx_reload()
def ssl_disable(self, data): name, stype = data['name'], data['type'] n = nginxparser.loads( open('/etc/nginx/sites-available/'+name, 'r').read()) port = '80' # Three passes - list loop bug omits entries on some systems for l in n: if l[0] == ['server']: for x in l[1]: if x[0] == 'listen': if x[1] == '443 ssl': x[1] = '80' port = '80' print True else: x[1] = x[1].rstrip(' ssl') print x[1] port = x[1] elif x[0] == 'ssl_certificate': l[1].remove(x) elif x[0] == 'ssl_certificate_key': l[1].remove(x) elif x[0] == 'ssl_protocols': l[1].remove(x) elif x[0] == 'ssl_ciphers': l[1].remove(x) for l in n: if l[0] == ['server']: for x in l[1]: if x[0] == 'ssl_certificate': l[1].remove(x) elif x[0] == 'ssl_certificate_key': l[1].remove(x) elif x[0] == 'ssl_protocols': l[1].remove(x) elif x[0] == 'ssl_ciphers': l[1].remove(x) for l in n: if l[0] == ['server']: for x in l[1]: if x[0] == 'ssl_certificate': l[1].remove(x) elif x[0] == 'ssl_certificate_key': l[1].remove(x) elif x[0] == 'ssl_protocols': l[1].remove(x) elif x[0] == 'ssl_ciphers': l[1].remove(x) comline = '# GENESIS '+stype+' http://'+data['addr']+':'+port+'\n' open('/etc/nginx/sites-available/'+name, 'w').write( comline+nginxparser.dumps(n)) apis.webapps(self.app).get_interface(stype).ssl_disable( os.path.join('/srv/http/webapps', name)) self.nginx_reload()
def __init__(self, content=None): self.http = None self.events = None import nginxparser if content: config = nginxparser.loads(content) else: config = [] super(NginxConfig, self).__init__(config) if self.http is None: self.http = NginxHTTP([]) if self.events is None: self.events = NginxEvents([])
def ssl_disable(self, data): name, stype = data["name"], data["type"] n = nginxparser.loads(open("/etc/nginx/sites-available/" + name, "r").read()) port = "80" # Three passes - list loop bug omits entries on some systems for l in n: if l[0] == ["server"]: for x in l[1]: if x[0] == "listen": if x[1] == "443 ssl": x[1] = "80" port = "80" print True else: x[1] = x[1].rstrip(" ssl") print x[1] port = x[1] elif x[0] == "ssl_certificate": l[1].remove(x) elif x[0] == "ssl_certificate_key": l[1].remove(x) elif x[0] == "ssl_protocols": l[1].remove(x) elif x[0] == "ssl_ciphers": l[1].remove(x) for l in n: if l[0] == ["server"]: for x in l[1]: if x[0] == "ssl_certificate": l[1].remove(x) elif x[0] == "ssl_certificate_key": l[1].remove(x) elif x[0] == "ssl_protocols": l[1].remove(x) elif x[0] == "ssl_ciphers": l[1].remove(x) for l in n: if l[0] == ["server"]: for x in l[1]: if x[0] == "ssl_certificate": l[1].remove(x) elif x[0] == "ssl_certificate_key": l[1].remove(x) elif x[0] == "ssl_protocols": l[1].remove(x) elif x[0] == "ssl_ciphers": l[1].remove(x) comline = "# GENESIS " + stype + " http://" + data["addr"] + ":" + port + "\n" open("/etc/nginx/sites-available/" + name, "w").write(comline + nginxparser.dumps(n)) apis.webapps(self.app).get_interface(stype).ssl_disable(os.path.join("/srv/http/webapps", name)) self.nginx_reload()
def configure_nginx1(*args, **kwargs): new_conf = lambda orig_conf: upsert_by_location( "/hooks", orig_conf, loads( api_proxy_block( "/hooks", "{protocol}://{host}:{port}/{urlprefix}".format( protocol="https" if kwargs.get("HOOK_SECURE") else "http", host=kwargs.get("HOOK_IP", "0.0.0.0"), port=kwargs.get("HOOK_PORT", 9000), urlprefix=kwargs.get("HOOK_URLPREFIX", "hooks"), ), )), ) upsert_upload(new_conf, name="hook") return restart_systemd("nginx")
def configure_nginx2(*args, **kwargs): kwargs.setdefault("LISTEN_PORT", 80) nginx.install_nginx0() nginx.setup_nginx_init1() if kwargs.get("self_signed", False): certificate.self_signed0(use_sudo=True, **kwargs) server_block = utils.merge_into( ( lambda server_block: utils.apply_attributes( server_block, emit.secure_attr(kwargs["SSL_CERTOUT"], kwargs["SSL_KEYOUT"]), ) if kwargs["LISTEN_PORT"] == 443 else server_block )( emit.server_block( server_name=kwargs["MATRIX_SERVER_NAME"], listen=kwargs["LISTEN_PORT"] ) ), emit.api_proxy_block("/_matrix", "https://127.0.0.1:8008"), ) sio = StringIO() sio.write( dumps( loads( emit.redirect_block(server_name=kwargs["MATRIX_SERVER_NAME"], port=80) ) + server_block if kwargs["LISTEN_PORT"] == 443 else server_block ) ) put( sio, "/etc/nginx/sites-enabled/{matrix_server_name}".format( matrix_server_name=kwargs["MATRIX_SERVER_NAME"] ), use_sudo=True, ) return restart_systemd("nginx")
def configure_nginx3(*args, **kwargs): nginx.install_nginx0() nginx.setup_nginx_init1() if kwargs.get("self_signed"): self_signed0(use_sudo=True, **kwargs) server_block = utils.merge_into( ( lambda server_block: utils.apply_attributes( server_block, emit.secure_attr(kwargs["SSL_CERTOUT"], kwargs["SSL_KEYOUT"]), ) if kwargs["LISTEN_PORT"] == 443 else server_block )( emit.server_block( server_name=kwargs["SERVER_NAME"], listen=kwargs["LISTEN_PORT"] ) ), emit.api_proxy_block("/go", "https://localhost:8154"), ) sio = StringIO() sio.write( dumps( loads(emit.redirect_block(server_name=kwargs["SERVER_NAME"], port=80)) + server_block if kwargs["LISTEN_PORT"] == 443 else server_block ) ) put( sio, "/etc/nginx/sites-enabled/{server_name}".format( server_name=kwargs["SERVER_NAME"] ), use_sudo=True, ) return restart_systemd("nginx")
def merge_into(server_name, parent_block, *child_blocks): # type: (str, str or list, *list) -> list parent_block = _copy_or_marshal(parent_block) server_name_idx = -1 indicies = set() break_ = False for i, tier in enumerate(parent_block): for j, statement in enumerate(tier): for k, stm in enumerate(statement): if statement[k][0] == "server_name" and statement[k][ 1] == server_name: server_name_idx = i indicies.add(k) if break_: break elif statement[k][0] == "listen" and statement[k][ 1].startswith("443"): break_ = True if k in indicies: break server_name_idx += 1 if not len(indicies): return parent_block length = len(parent_block[-1]) if server_name_idx >= length: server_name_idx = length - 1 parent_block[-1][server_name_idx] += list( map( lambda child_block: child_block[0] if isinstance(child_block[0], list) else loads(child_block)[0], child_blocks, )) parent_block[-1][server_name_idx] = list( reversed(uniq(reversed(parent_block[-1][-1]), itemgetter(0)))) return parent_block
def upsert_redirect_to_443_block(conf_file, server_name): # type: (str, str) -> [] conf = ensure_nginxparser_instance(conf_file) server_name_idx = None found = False ListenStmIdx = namedtuple("ListenStmIdx", ("return_stm", "i", "j", "k")) listen_stm_idx = ListenStmIdx(False, None, None, None) for i, tier in enumerate(conf): for j, statement in enumerate(tier): for k, stm in enumerate(statement): if statement[k][0] == "server_name" and statement[k][ 1] == server_name: server_name_idx = i elif statement[k][0] == "listen": if not listen_stm_idx.return_stm: listen_stm_idx = ListenStmIdx(return_stm=False, i=i, j=j, k=k) if str(statement[k][1]).startswith("443"): found = True else: statement[k][1] = "443" elif statement[k][0] == "return": listen_stm_idx = ListenStmIdx( return_stm=True, i=listen_stm_idx.i, j=listen_stm_idx.j, k=listen_stm_idx.k, ) if listen_stm_idx.return_stm: conf[listen_stm_idx.i][listen_stm_idx.j][listen_stm_idx.k][1] = "80" elif not found and server_name_idx is not None: conf.insert( server_name_idx, loads(redirect_block(server_name=server_name, port="80"))[0], ) return conf
def ssl_enable(self, data, cpath, kpath): name, stype = data["name"], data["type"] n = nginxparser.loads(open("/etc/nginx/sites-available/" + name, "r").read()) port = "443" for l in n: if l[0] == ["server"]: for x in l[1]: if x[0] == "listen": if x[1] == "80": x[1] = "443 ssl" port = "443" else: port = x[1] x[1] = x[1] + " ssl" l[1].append(["ssl_certificate", cpath]) l[1].append(["ssl_certificate_key", kpath]) l[1].append(["ssl_protocols", "TLSv1 TLSv1.1 TLSv1.2"]) l[1].append(["ssl_ciphers", "HIGH:!aNULL:!MD5"]) comline = "# GENESIS " + stype + " https://" + data["addr"] + ":" + port + "\n" open("/etc/nginx/sites-available/" + name, "w").write(comline + nginxparser.dumps(n)) apis.webapps(self.app).get_interface(stype).ssl_enable(os.path.join("/srv/http/webapps", name), cpath, kpath) self.nginx_reload()
def read_nginxconfig(connection, filename = '/etc/nginx/nginx.conf'): command = "if [ -f '" + filename + "' ]; then cat " + filename + "; elif [ -h '" + filename + "' ]; then cat `file " + filename + " | awk '{print $5}'`; else echo 'not_found'; fi" #print "Command = " #print command #print "Before running the command" stdin, stdout, stderr = connection.exec_command(command) #print "Right after running the command" output = stdout.readlines() #print "Finishing running command to get nginx file content" #print "Output = " #print output if output[0].strip('\n') == 'not_found': return False else: import os, sys lib_path = os.path.abspath(os.path.join("./nginxparser")) sys.path.append(lib_path) import nginxparser #print "Result of the file READ" #print output #print "remove line with hash" output = [x.strip() for x in output] output = [x for x in output if not normailizing_unicode(x).startswith('#')] #print "remove empty lines" #output = [x for x in output if not normailizing_unicode(x).startswith('\n')] #print "remove empty value" output = [x for x in output if normailizing_unicode(x)] #print "Replacing all tabs" #output = [x for x in output if not normailizing_unicode(x).replace("\t", " ")] #print "Removing empty field" #output = [x for x in output if not normailizing_unicode(x)] #print "Printing output again" #print output #print "Output become string" #print ''.join(output) #print "Type of output" #print type(output) #print "Length of output" #print len(output) if not output: return False #print "Output = " #print output #print "Current file name = ", #print filename config_value = {} try: config_value = nginxparser.loads(''.join(output)) #print "Config value = " #print config_value except: print "Error:" + filename #print "Print result after importing by library++++++++++++++++++++++++++++++++++++++" #print config_value #config_value = nginxparser.loads(''.join(output)) #print "********************************************************************" nginxconf = transform_config(config_value) # nginxsite = read_nginxsite(nginxconf, connection) #print "NGINX Config =====================================================" #print nginxconf #print "\n" # print "NGINX Sites =====================================================" # print nginxsite return nginxconf
def _copy_or_marshal(block): # type: (str or list) -> list return copy(block) if isinstance(block, list) else loads(block)
def install0(*args, **kwargs): kwargs.setdefault("virtual_env", "/opt/venvs/mlflow") if not kwargs.get("skip_virtualenv", False): venv0_kwargs = { "virtual_env": kwargs["virtual_env"], "python3": True, "pip_version": "19.2.3", "use_sudo": True, "remote_user": "******", "PACKAGES": ["mlflow[extras]"], } venv0_kwargs.update(kwargs) python_mod.install_venv0(**venv0_kwargs) circus0_kwargs = { "APP_NAME": "mlflow", "APP_PORT": 5000, "CMD": "{virtual_env}/bin/mlflow".format(virtual_env=kwargs["virtual_env"]), "CMD_ARGS": "ui", "WSGI_FILE": None, } circus0_kwargs.update(kwargs) circus_mod.install_circus0(**circus0_kwargs) kwargs.setdefault("skip_nginx_restart", True) kwargs.setdefault( "conf_remote_filename", "/etc/nginx/sites-enabled/{}.conf".format(kwargs["SERVER_NAME"]), ) kwargs.update({ "nginx_conf": "proxy-pass.conf", "NAME_OF_BLOCK": "mlflow", "SERVER_LOCATION": "localhost:{port}".format(port=circus0_kwargs["APP_PORT"]), "LISTEN_PORT": 80, "LOCATION": "/", }) if exists(kwargs["conf_remote_filename"]): parsed_remote_conf = get_parsed_remote_conf( kwargs["conf_remote_filename"]) parsed_api_block = loads( api_proxy_block( location=kwargs["LOCATION"], proxy_pass="******".format(kwargs["SERVER_LOCATION"]), )) sio = StringIO() sio.write( dumps( merge_into(kwargs["SERVER_NAME"], parsed_remote_conf, parsed_api_block))) sio.seek(0) put(sio, kwargs["conf_remote_filename"], use_sudo=True) else: nginx_static.setup_conf0(**kwargs) # with shell_env(VIRTUAL_ENV=kwargs['virtual_env'], PATH="{}/bin:$PATH".format(kwargs['virtual_env'])):sudo('mlflow initdb') restart_systemd("circusd")
def test_upsert_by_location(self): # upsert_by_name('/api0', self.parsed_server_block_no_rest, self.parsed_api_block) self.assertEqual( """server { # Emitted by nginx_parse_emit.emit.server_block server_name offscale.io; listen 80; } server { # Emitted by nginx_parse_emit.emit.server_block server_name offscale.io; listen 443; location /api0 { proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Scheme $scheme; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_pass http://127.0.0.1:5000/awesome; proxy_redirect off; } }""", dumps( upsert_by_location("/api0", self.two_roots, self.parsed_api_block)), ) self.assertEqual( """server { # Emitted by nginx_parse_emit.emit.server_block server_name offscale.io; listen 443; location /api0 { proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Scheme $scheme; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_pass http://127.0.0.1:5000/awesome; proxy_redirect off; } }""", dumps( upsert_by_location( self.server_name, "/api0", merge_into( self.server_name, self.parsed_server_block_no_rest, self.parsed_api_block, ), self.parsed_api_block, )), ) self.assertEqual( """server { # Emitted by nginx_parse_emit.emit.server_block server_name offscale.io; listen 443; location /api0 { proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Scheme $scheme; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_pass WRONG_DOMAIN; proxy_redirect off; } }""", dumps( upsert_by_location( self.server_name, "/api0", merge_into( self.server_name, self.parsed_server_block_no_rest, self.parsed_api_block, ), loads( api_proxy_block(location=self.location, proxy_pass="******")), )), )