def upstream_submit():
upstream_value=request.POST.get('upstream_value', '')
upstream_name=request.POST.get('upstream_name', '')
path_file_name = request.POST.get("path_file_name", "")
c = nginx.loadf(path_file_name)
search_upstream=c.filter(btype="Upstream", name=upstream_name)
if len(search_upstream):
u=search_upstream[0]
c.remove(u)
new_u = nginx.Upstream(upstream_name, )
for line in upstream_value.split("\n"):
if len(line.split(" "))>= 2:
# print line.split(" ")
new_u.add(nginx.Key(line.split(" ")[0], line.split(" ")[1]))
else:
new_u = nginx.Upstream(upstream_name, )
for line in upstream_value.split("\n"):
if len(line.split(" ")) >= 2:
# print line.split(" ")
new_u.add(nginx.Key(line.split(" ")[0], line.split(" ")[1]))
c.add(new_u)
nginx.dumpf(c, path_file_name)
print type(upstream_value),path_file_name,upstream_name
return upstream_value
def setup_components(all_components: list):
for component in all_components:
host = component['upstream']
server = host.replace('_', '-')
name = host.split('_')[1]
path = component['path']
if os.getenv(component['env']) == '1':
# setup upstream config
c = nginx.Conf()
u = nginx.Upstream(host, nginx.Key('server', f'{server}:5000'))
c.add(u)
nginx.dumpf(c, f'upstreams/{name}.conf')
# setup enabled location config
c = nginx.Conf()
l = nginx.Location(
f'~* ^/{path}/',
nginx.Key('rewrite', f'^/{path}/(.*) /$1 break'),
nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-Real-IP $remote_addr'),
nginx.Key('proxy_pass', f'http://{host}'),
nginx.Key('proxy_pass_request_headers', 'on'),
)
c.add(l)
nginx.dumpf(c, f'locations/{name}.conf')
else:
# setup disabled location config
c = nginx.Conf()
l = nginx.Location(
f'~* ^/{path}/',
nginx.Key('return', '503'),
)
c.add(l)
nginx.dumpf(c, f'locations/{name}.conf')
def _edit_nginx_entry(project_root_dir, rev_proxy_container, model_name, hostname, ip_port, old_hostname = None):
conf_dir = _copy_down_nginx_conf(project_root_dir, rev_proxy_container)
try:
conf_file = _build_relative_path(conf_dir,'nginx.conf')
c = _nginx.loadf(conf_file)
http = c.filter('Http')[0]
endpoint_url = '/{}/'.format(model_name)
# check for existing upstream entry for item, edit as needed
if old_hostname is not None:
for ups in http.filter('Upstream'):
if ups.value == old_hostname:
http.remove(ups)
# create new hostname entry
upstream = _nginx.Upstream(hostname)
upstream.add(_nginx.Key('server', ip_port))
http.add(
upstream
)
# check for existing location entry and remove if present
servers = http.filter('Server')
add2http = False
if len(servers) > 0:
server = servers[0]
for loc in server.filter('Location'):
if loc.value == endpoint_url:
server.remove(loc)
else:
add2http = True
server = _nginx.Server()
server.add(_nginx.Key('listen', '5000'))
location = _nginx.Location(endpoint_url)
location.add(
_nginx.Key('proxy_pass', 'http://{}/'.format(hostname)),
_nginx.Key('proxy_redirect', 'off'),
_nginx.Key('proxy_set_header', 'Host $host'),
_nginx.Key('proxy_set_header', 'X-Real-IP $remote_addr'),
_nginx.Key('proxy_set_header', 'X-Forwarded-For $proxy_add_x_forwarded_for'),
_nginx.Key('proxy_set_header', 'X-Forwarded-Host $server_name')
)
server.add(location)
if add2http:
http.add(server)
_nginx.dumpf(c, conf_file)
_copy_up_nginx_conf(project_root_dir, conf_dir, rev_proxy_container)
# reload nginx on server
rev_proxy_container.exec_run('/usr/sbin/nginx', detach = True)
rev_proxy_container.exec_run('/usr/sbin/nginx -s reload', detach = True)
finally:
_shutil.rmtree(conf_dir, ignore_errors=True)
def generate_nginx_config(self):
c = nginx.Conf()
u = nginx.Upstream('loadbalancer', nginx.Key('least_conn', ''))
ip_addr = get_ip_address()
for server_idx in range(self.n_endpoints):
u.add(
nginx.Key('server', f'{ip_addr}:{self.src_port + server_idx}'))
s = nginx.Server(
nginx.Location('/', nginx.Key('proxy_pass',
'http://loadbalancer')))
loc = nginx.Location('/favicon.ico', nginx.Key('log_not_found', 'off'),
nginx.Key('access_log', 'off'))
c.add(u)
s.add(loc)
c.add(s)
nginx.dumpf(c, 'dockerfiles/loadbalancer/nginx.conf')
def test_create():
c = nginx.Conf()
u = nginx.Upstream('php', nginx.Key('server', 'unix:/tmp/php-fcgi.socket'),
nginx.Key('server', '10.0.2.1'))
u.add(nginx.Key('server', '101.0.2.1'))
c.add(u)
s = nginx.Server()
s.add(
nginx.Key('listen', '80'),
nginx.Comment('Yes, python-nginx can read/write comments!'),
nginx.Key('server_name', 'localhost 127.0.0.1'),
nginx.Key('root', '/srv/http'), nginx.Key('index', 'index.php'),
nginx.Location('= /robots.txt', nginx.Key('allow', 'all'),
nginx.Key('log_not_found', 'off'),
nginx.Key('access_log', 'off')),
nginx.Location('~ \.php$', nginx.Key('include', 'fastcgi.conf'),
nginx.Key('fastcgi_intercept_errors', 'on'),
nginx.Key('fastcgi_pass', 'php')))
c.add(s)
nginx.dumpf(c, 'mysite')
def remove_server(app_name, app_server_ip_addr):
c = nginx.loadf(nginx_configs_dir + "/" + app_name + '/nginx.conf')
h = c.filter('Http')[0]
c.remove(h)
u = h.filter('Upstream')[0]
h.remove(u)
u_upd = nginx.Upstream(app_name)
for k in u.filter('Key'):
if not k.value == str(app_server_ip_addr) + ':3000':
u_upd.add(k)
h.add(u_upd)
c.add(h)
nginx.dumpf(c, nginx_configs_dir + "/" + app_name + '/nginx.conf')
def remove_server(app_name, app_server_ip_addr):
#creating a new config file everytime
c = nginx.loadf(CONFIG_DIR + app_name + '/nginx.conf')
h = c.filter('Http')[0]
c.remove(h)
u = h.filter('Upstream')[0]
h.remove(u)
u_upd = nginx.Upstream(app_name)
for k in u.filter('Key'):
if not k.value == str(app_server_ip_addr) + ':3000':
u_upd.add(k)
h.add(u_upd)
c.add(h)
nginx.dumpf(c, CONFIG_DIR + app_name + '/nginx.conf')
def create_nginx_config(nginx_port, app_name):
c = nginx.Conf()
e = nginx.Events()
e.add(nginx.Key('worker_connections', '1024'))
c.add(e)
h = nginx.Http()
u = nginx.Upstream(app_name)
h.add(u)
s = nginx.Server()
s.add(
nginx.Key('listen', str(nginx_port)),
nginx.Key('server_name', app_name),
nginx.Location('/', nginx.Key('proxy_pass', 'http://' + app_name),
nginx.Key('proxy_set_header', 'Host $host')))
h.add(s)
c.add(h)
nginx.dumpf(c, nginx_configs_dir + "/" + app_name + '/nginx.conf')
"""
NOMAD_IP_nginx1_http=10.123.26.15
NOMAD_PORT_nginx1_http=22707
NOMAD_ADDR_nginx1_http=10.123.26.15:22707
"""
# find out the IP address of the task via env NOMAD_IP_nginx1_http
env_var = 'NOMAD_ADDR_%s_%s' % (task, label)
url = os.getenv(env_var)
if not url:
print "ERROR: Environment variable %s does not exist, exiting..." % env_var
sys.exit(1)
upstream_args.append(nginx.Key('server', url))
u = nginx.Upstream(*upstream_args)
c.add(u)
s = nginx.Server()
s.add(
nginx.Key('listen', '80'),
nginx.Comment(
'Autogenerated configuration from build_config_run_nginx.py'),
nginx.Key('server_name', 'localhost'),
nginx.Key('root', '/usr/share/nginx/html'),
nginx.Key('index', 'index.html'),
nginx.Location('/', nginx.Key('proxy_pass', 'http://tasks')))
c.add(s)
print nginx.dumps(c)
def upstream(self):
for upstream_name, upstream_url in BASE_UPSTREAM.items():
upstream_obj = nginx.Upstream(upstream_name,
nginx.Key("server", upstream_url))
self.config.add(upstream_obj)
def _execute(command):
config = FileUtils.get_instance('config.ini')
nginx_conf_dir = config.get_prop("nginx-config", "nginx_conf_dir")
nginx_conf_dir_bak = config.get_prop("nginx-config", "nginx_conf_dir_bak")
os.system('mkdir -p ' + nginx_conf_dir)
os.system('cp -r ' + nginx_conf_dir + nginx_conf_dir_bak)
os.system('rm -rf ' + nginx_conf_dir + '*')
state = '0'
try:
for k1 in command:
upstreamName = k1["upstreamName"]
upstreamNode = json.loads(k1["upstreamNode"])
fileName = nginx_conf_dir + upstreamName + '.conf'
os.system('touch ' + fileName)
c = nginx.Conf()
LogUtil.info("Node信息:%s" % (upstreamNode))
t = ''
for k2 in range(len(upstreamNode)):
if (k2 == 0):
t = 'server' + ' ' + upstreamNode[k2][
"ipAdress"] + ' ' + upstreamNode[k2]["strategy"] + ';\n'
if (0 < k2 < len(upstreamNode) - 1):
t = t + ' ' + 'server' + ' ' + upstreamNode[k2][
"ipAdress"] + ' ' + upstreamNode[k2]["strategy"] + ';\n'
if (k2 == len(upstreamNode) - 1):
t = t + ' ' + 'server' + ' ' + upstreamNode[k2][
"ipAdress"] + ' ' + upstreamNode[k2]["strategy"]
ipHash = k1["ipHash"]
if (ipHash == '1'):
u = nginx.Upstream(upstreamName, nginx.Key('', t),
nginx.Key('', 'ip_hash'))
c.add(u)
if (ipHash == '0'):
u = nginx.Upstream(upstreamName, nginx.Key('', t))
c.add(u)
s = nginx.Server()
s.add(
nginx.Key('listen', k1["nginxListenPort"]),
nginx.Key('server_name', k1["nginxServerName"]),
nginx.Key('access_log', k1["accessLog"]),
nginx.Location(
'= /',
nginx.Key('proxy_pass', 'http://' + k1["upstreamName"]),
nginx.Key('proxy_redirect', 'off'),
nginx.Key('proxy_set_header', 'Host $host'),
nginx.Key('proxy_set_header', 'X-Real-IP $remote_addr'),
nginx.Key('proxy_set_header',
'X-Forwarded-For $proxy_add_x_forwarded_for')))
c.add(s)
nginx.dumpf(c, fileName)
LogUtil.info("完成Nginx配置")
output = os.popen('service nginx restart').read()
state = '1'
param = [
"{\"mac\":\"" + MachineUtil.get_mac_address() + "\",\"state\":\"" +
state + "\",\"msg\":\"" + output + "\"}"
]
os.system('rm -rf ' + nginx_conf_dir_bak)
AosServerService.route_service("ThriftApiService", "nginxLog", param)
except Exception, ex:
LogUtil.error(ex)
os.system('rm -rf ' + nginx_conf_dir)
os.system('mv ' + nginx_conf_dir_bak + nginx_conf_dir)
os.system('service nginx restart')
output = ex.message
state = '2'
param = [
"{\"mac\":\"" + MachineUtil.get_mac_address() + "\",\"state\":\"" +
state + "\",\"msg\":\"" + output + "\"}"
]
AosServerService.route_service("ThriftApiService", "nginxLog", param)
def run(args):
global conPicDict
global workerCounter
global firstTimeNginx
global firstTimeNetcat
global path
#args[1] is image name
#args[2] is container name
#args[3]+ is the commands
processDocker = subprocess.Popen(["docker","run","--name", "worker"+str(workerCounter)] + args[1:], stdout = subprocess.PIPE)
output = processDocker.communicate()
#connect container to network bridge conNet
processNetConnect = subprocess.Popen(["docker","network","connect", "conNet","worker"+str(workerCounter)], stdout = subprocess.PIPE)
output = processNetConnect.communicate()
#get the image_name and the Cmds
processCmd = subprocess.Popen(["docker", "inspect" ,"-f","'{{.Config.Cmd}}'", "worker"+str(workerCounter)], stdout = subprocess.PIPE)
cmd = processCmd.communicate()[0]
cmd = cmd.split("'")[1]
processImageName = subprocess.Popen(["docker", "inspect" ,"-f","'{{.Config.Image}}'", "worker"+str(workerCounter)], stdout = subprocess.PIPE)
imageName = processImageName.communicate()[0]
imageName = imageName.split("'")[1]
conPicDict["worker"+str(workerCounter)] = [imageName,cmd] + ["worker"+str(workerCounter)]
conPicDict['count'] = workerCounter
#everytime we run a container we add it to the load balancer by editing
#the nginx.conf file then we have to kill, rm, rebuild and rerun loadBal
#checks if its nginx or netcat(anything else)
isNginx = False
if(imageName == 'nginx:alpine'):
isNginx = True
#if both firstimes are false we add the server to both upstreams so it runs
c = nginx.loadf(path+'/load-balancer/nginx.conf')
HttpFilter = ((c.filter('Http')[0]))
if(isNginx):
if(firstTimeNginx == False):
#make upstream make worker add worker to upstream add upstream
#to the http
nginxUpstream = nginx.Upstream('nginx')
worker = nginx.Key('server', "worker"+str(workerCounter)+":80")
nginxUpstream.add(worker)
HttpFilter.add(nginxUpstream)
#when we make upstream for first time we also make server server
s = nginx.Server()
s.add(nginx.Key('listen','8100'),nginx.Location('/' , nginx.Key('proxy_pass','http://nginx'), nginx.Key('proxy_redirect','off'),nginx.Key('proxy_set_header','Host $host'), nginx.Key('proxy_set_header','X-Real_IP $remote_addr'), nginx.Key('proxy_set_header','X-Forwarded-For $proxy_add_x_forwarded_for'), nginx.Key('proxy_set_header','X-Forwarded-Host $server_name')))
HttpFilter.add(s)
#dumping
firstTimeNginx = True
nginx.dumpf(c, path+'/load-balancer/nginx.conf')
else:
upstreamSearch = HttpFilter.filter('Upstream')
j = 0
found = False
for key in upstreamSearch:
if(((key.as_dict).keys())[0] == 'upstream nginx'):
found = True
break
j = j + 1
worker = nginx.Key('server', "worker"+str(workerCounter)+":80")
if(found):
upstreamSearch[j].add(worker)
nginx.dumpf(c, path+'/load-balancer/nginx.conf')
else:
if(firstTimeNetcat == False):
#make upstream make worker add worker to upstream add upstream
#to the http
nginxUpstream = nginx.Upstream('netcat')
worker = nginx.Key('server', "worker"+str(workerCounter)+":80")
nginxUpstream.add(worker)
HttpFilter.add(nginxUpstream)
#when we make upstream for first time we also make server server
s = nginx.Server()
s.add(nginx.Key('listen','8101'),nginx.Location('/' , nginx.Key('proxy_pass','http://netcat'), nginx.Key('proxy_redirect','off'),nginx.Key('proxy_set_header','Host $host'), nginx.Key('proxy_set_header','X-Real_IP $remote_addr'), nginx.Key('proxy_set_header','X-Forwarded-For $proxy_add_x_forwarded_for'), nginx.Key('proxy_set_header','X-Forwarded-Host $server_name')))
HttpFilter.add(s)
#dumping
firstTimeNetcat = True
nginx.dumpf(c, path+'/load-balancer/nginx.conf')
else:
upstreamSearch = HttpFilter.filter('Upstream')
j = 0
found = False
for key in upstreamSearch:
if(((key.as_dict).keys())[0] == 'upstream netcat'):
found = True
break
j = j + 1
worker = nginx.Key('server', "worker"+str(workerCounter)+":80")
if(found):
upstreamSearch[j].add(worker)
nginx.dumpf(c, path+'/load-balancer/nginx.conf')
#now kill,rm,build,run
loadBalReset()
#increment workerCounter
workerCounter = workerCounter + 1
def start(args):
global path
global conPicDict
global firstTimeNginx
global firstTimeNetcat
processDocker = subprocess.Popen(["docker", "start"]+ [args[1]], stdout = subprocess.PIPE)
output = processDocker.communicate()
#get the image_name and the Cmds
processCmd = subprocess.Popen(["docker", "inspect" ,"-f","'{{.Config.Cmd}}'", ""+ args[1]], stdout = subprocess.PIPE)
cmd = processCmd.communicate()[0]
cmd = cmd.split("'")[1]
processImageName = subprocess.Popen(["docker", "inspect" ,"-f","'{{.Config.Image}}'", ""+ args[1]], stdout = subprocess.PIPE)
imageName = processImageName.communicate()[0]
imageName = imageName.split("'")[1]
#conPicDict[args[1]] = ["docker","start"] + [args[1]]
conPicDict[args[1]] = [imageName,cmd] + [args[1]]
#args[1] is gonna be the worker name, ex: worker1
isNginx = False
if(imageName == 'nginx:alpine'):
isNginx = True
#if both firstimes are false we add the server to both upstreams so it runs
c = nginx.loadf(path+'/load-balancer/nginx.conf')
HttpFilter = ((c.filter('Http')[0]))
if(isNginx):
if(firstTimeNginx == False):
#make upstream make worker add worker to upstream add upstream
#to the http
nginxUpstream = nginx.Upstream('nginx')
worker = nginx.Key('server', args[1]+":80")
nginxUpstream.add(worker)
HttpFilter.add(nginxUpstream)
#when we make upstream for first time we also make server server
s = nginx.Server()
s.add(nginx.Key('listen','8100'),nginx.Location('/' , nginx.Key('proxy_pass','http://nginx'), nginx.Key('proxy_redirect','off'),nginx.Key('proxy_set_header','Host $host'), nginx.Key('proxy_set_header','X-Real_IP $remote_addr'), nginx.Key('proxy_set_header','X-Forwarded-For $proxy_add_x_forwarded_for'), nginx.Key('proxy_set_header','X-Forwarded-Host $server_name')))
HttpFilter.add(s)
#dumping
firstTimeNginx = True
nginx.dumpf(c, path+'/load-balancer/nginx.conf')
else:
upstreamSearch = HttpFilter.filter('Upstream')
j = 0
found = False
for key in upstreamSearch:
if(((key.as_dict).keys())[0] == 'upstream nginx'):
found = True
break
j = j + 1
worker = nginx.Key('server', args[1]+":80")
if(found):
upstreamSearch[j].add(worker)
nginx.dumpf(c, path+'/load-balancer/nginx.conf')
else:
if(firstTimeNetcat == False):
#make upstream make worker add worker to upstream add upstream
#to the http
nginxUpstream = nginx.Upstream('netcat')
worker = nginx.Key('server', args[1]+":80")
nginxUpstream.add(worker)
HttpFilter.add(nginxUpstream)
#when we make upstream for first time we also make server server
s = nginx.Server()
s.add(nginx.Key('listen','8101'),nginx.Location('/' , nginx.Key('proxy_pass','http://netcat'), nginx.Key('proxy_redirect','off'),nginx.Key('proxy_set_header','Host $host'), nginx.Key('proxy_set_header','X-Real_IP $remote_addr'), nginx.Key('proxy_set_header','X-Forwarded-For $proxy_add_x_forwarded_for'), nginx.Key('proxy_set_header','X-Forwarded-Host $server_name')))
HttpFilter.add(s)
#dumping
firstTimeNetcat = True
nginx.dumpf(c, path+'/load-balancer/nginx.conf')
else:
upstreamSearch = HttpFilter.filter('Upstream')
j = 0
found = False
for key in upstreamSearch:
if(((key.as_dict).keys())[0] == 'upstream netcat'):
found = True
break
j = j + 1
worker = nginx.Key('server', args[1]+":80")
if(found):
upstreamSearch[j].add(worker)
nginx.dumpf(c, path+'/load-balancer/nginx.conf')
#now kill,rm,build,run
loadBalReset()