def main(role: str, env: str, timeout: int = 30):
hosts = []
if role == 'api3':
for api3_role in API3_ROLES:
hosts += get_hosts(env, api3_role)
elif role == 'classifier':
for classifier_role in CLASSIFIER_ROLES:
hosts += get_hosts(env, classifier_role)
else:
hosts = get_hosts(env, role)
hosts = list(hosts)
checked_hosts = check_hosts(hosts)
command = get_command(role)
asyncio.run(run_all_hosts(checked_hosts, command, timeout=timeout))
def get_physical_hosts(self):
from utils import get_hosts
host_data = get_hosts(self.controller_IP, "8080", "")
if host_data:
count = 1
print "Receiving physical network data from SDN controller ..."
for host in host_data['hostConfig']:
IP = host['networkAddress']
MAC = host['dataLayerAddress']
Service = 'Streaming_' + str(count)
Status = 'Active'
physical = {
'MAC': MAC,
'IP': IP,
'Service': Service,
'Status': Status
}
self.physical_hosts.append(physical)
print "Get physical host:" + str(physical)
count = count + 1
else:
print "No physical network info. Check SDN controller"
from utils import config_json_read
data = config_json_read(host_path)
data['Physical_hosts'] = self.physical_hosts
#write hosts information in host.json file
from utils import config_json_write
config_json_write(host_path, data)
print "Received host information"
def get_physical_hosts(self):
from utils import get_hosts
host_data = get_hosts(self.controller_IP, "8080","")
if host_data:
count = 1
print "Receiving physical network data from SDN controller ..."
for host in host_data['hostConfig']:
IP = host['networkAddress']
MAC = host['dataLayerAddress']
Service = 'Streaming_'+str(count)
Status = 'Active'
physical ={'MAC':MAC, 'IP':IP, 'Service':Service, 'Status':Status}
self.physical_hosts.append(physical)
print "Get physical host:"+str(physical)
count = count + 1
else:
print "No physical network info. Check SDN controller"
from utils import config_json_read
data = config_json_read(host_path)
data['Physical_hosts']=self.physical_hosts
#write hosts information in host.json file
from utils import config_json_write
config_json_write(host_path,data)
print "Received host information"
def main(r_parser):
"""
:param r_parser: 传入的参数
:return:
"""
r_parser.add_argument("-m", "--module", help="module", required=True)
r_parser.add_argument("-u",
"--user",
help="remote machine user",
required=True)
r_parser.add_argument("-p", "--password", help="remote machine password")
r_parser.add_argument("-d",
"--destination",
help="destination hosts group",
required=True)
r_parser.add_argument("-c",
"--command",
help="command or shell scripts",
required=True)
args = r_parser.parse_args()
module = args.module
user = args.user
password = args.password
dest = args.destination
command = args.command
config = utils.read_config('./rcm.conf')
hosts_lst = utils.get_hosts(config, dest)
_exe_command(hosts_lst, module, user, password, command)
def status_host(request):
c = {}
c['messages'] = []
c['errors'] = []
c['hosts'] = utils.get_hosts(request, **request.GET)
c['host_name'] = request.GET.get('detail', None)
return render_to_response('status_host.html',
c,
context_instance=RequestContext(request))
def __config_load(self, config={}):
"""
Loading config from config dict
:param config: dictionary of settings
:return: None
"""
logger.info("Loading config...")
print(config)
self.gw = config.get('gateway', None)
if not self.gw:
logger.critical("IP of gateway not specified! Using default one: {}".format(DEFAULT_GATEWAY))
self.gw = DEFAULT_GATEWAY
mac = utils.arpping(self.gw)
while mac is None:
logger.debug('Cannot obtain MAC addr of the gateway. Make sure {} is correct!'.format(self.gw))
mac = utils.arpping(self.gw)
self.ip_mac[self.gw] = mac
logger.debug('Gateway MAC addr: {}'.format(mac))
self.network = config.get('network', None)
if self.network is None:
logger.debug('Using default network! {}'.format(DEFAULT_NETWORK))
self.network = DEFAULT_NETWORK
self.targets = config.get('target', None)
if not self.targets:
logger.debug('No explicit targets found! Calculating from the net!')
self.targets = utils.get_hosts(self.network)
logger.debug('Targets: {}'.format(self.targets))
for t in self.targets:
mac = utils.arpping(t)
cc = 0
while mac is None:
logger.debug('Target {} has MAC: {}'.format(t, mac))
cc += 1
mac = utils.arpping(t)
if cc > 3 and mac is None:
logger.critical('Cannot determine MAC addr of {}'.format(t))
raise ValueError('Check if {} addr is correct!'.format(t))
if mac is not None:
self.ip_mac[t] = mac
self.iface = config['iface']
self.iface_mac = utils.getHwAddr(self.iface)
def dashboard(request):
# Get statistics
c = adagios.status.utils.get_statistics(request)
c['messages'] = []
c['errors'] = []
all_down_hosts = utils.get_hosts(request, state__isnot='0', **request.GET)
hostnames_that_are_down = map(lambda x: x.get('name'), all_down_hosts)
# Discover network outages,
# Remove acknowledgements and also all hosts where all parent hosts are down
c['host_problems'] = [] # Unhandled host problems
c['network_problems'] = [] # Network outages
for i in all_down_hosts:
if i.get('acknowledged') != 0:
continue
if i.get('scheduled_downtime_depth') != 0:
continue
# Do nothing if parent of this host is also down
for parent in i.get('parents'):
if parent in hostnames_that_are_down:
parent_is_down = True
continue
else:
parent_is_down = False
if parent_is_down == True:
continue
# If host has network childs, put them in the network outages box
if i.get('childs') == []:
c['host_problems'].append(i)
else:
c['network_problems'].append(i)
#
c['hosts'] = c['network_problems'] + c['host_problems']
# Service problems
c['service_problems'] = utils.get_services(request,
state__isnot="0",
acknowledged="0",
scheduled_downtime_depth="0",
host_state="0",
**request.GET)
# Sort problems by state and last_check as secondary sort field
c['service_problems'].sort(
reverse=True, cmp=lambda a, b: cmp(a['last_check'], b['last_check']))
c['service_problems'].sort(reverse=True,
cmp=lambda a, b: cmp(a['state'], b['state']))
return render_to_response('status_dashboard.html',
c,
context_instance=RequestContext(request))
def rm(c, name):
sk, sv = utils.get_service(env.services, name)
hosts = utils.get_hosts(env.hosts, sv["label"])
if sv["label"] == "zookeeper":
zid = 0
for k, v in hosts.iteritems():
zid += 1
cmd = "docker rm -f {}-{}".format(sk, zid)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
else:
_docker_rm(c, hosts, sk)
def test_get_host(self):
self.assertIn(
'10.xx.xx.231',
utils.get_hosts(
{
'logger_test': ['10.xx.xx.231', '10.xx.xx.232'],
'logger': ['10.xx.xx.84:8081', '10.xx.xx.85:8081']
}, 'logger_test'))
self.assertIn(
'10.xx.xx.231',
utils.get_hosts(
{
'logger_test': ['10.xx.xx.231', '10.xx.xx.232'],
'logger': ['10.xx.xx.84:8081', '10.xx.xx.85:8081']
}, 'logger_test'))
self.assertIn(
'10.xx.xx.232',
utils.get_hosts(
{
'logger_test': ['10.xx.xx.231', '10.xx.xx.232'],
'logger': ['10.xx.xx.84:8081', '10.xx.xx.85:8081']
}, 'logger_test'))
self.assertIn(
'10.xx.xx.232',
utils.get_hosts(
{
'logger_test': ['10.xx.xx.231', '10.xx.xx.232'],
'logger': ['10.xx.xx.84:8081', '10.xx.xx.85:8081']
}, 'logger'))
self.assertIn(
'10.xx.xx.84:8081',
utils.get_hosts(
{
'logger_test': ['10.xx.xx.231', '10.xx.xx.232'],
'logger': ['10.xx.xx.84:8081', '10.xx.xx.85:8081']
}, 'logger'))
def run(c, name):
sk, sv = utils.get_service(env.services, name)
hosts = utils.get_hosts(env.hosts, sv["label"])
cmd = ""
if sv["label"] == "registry":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{}".format(sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "jupyter":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "spark-master":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "spark-worker":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "mysql":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{} --default-authentication-plugin=mysql_native_password ".format(
sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "pgsql":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{} ".format(sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "airflow":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "portainer":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "portainer-agent":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{}".format(sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "zookeeper":
zid = 0
for k, v in hosts.iteritems():
zid += 1
cmd = "docker rm -f {}-{}".format(sk, zid)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
with open('/tmp/zoo.cfg', 'w') as f:
lines = [
'tickTime=2000', 'dataDir=/opt/zookeeper/data',
'clientPort=2181', 'initLimit=5', 'syncLimit=2'
]
zid = 0
for k, v in hosts.iteritems():
zid += 1
lines.append("server.{}={}-{}:2888:3888".format(zid, sk, zid))
for line in lines:
f.write("{}\n".format(line))
zid = 0
for k, v in hosts.iteritems():
zid += 1
with open('/tmp/myid', 'w') as f:
f.write("{}\n".format(zid))
path_z = os.path.expandvars(sv["path"])
path_conf = "{}/conf".format(path_z)
cmd = "sudo mkdir -p {} ".format(path_conf)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
path_data = "{}/data".format(path_z)
cmd = "sudo mkdir -p {} ".format(path_data)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
cmd = "docker-machine scp /tmp/zoo.cfg {}:/tmp/zoo.cfg ".format(k)
utils.run_with_exit(c, cmd)
cmd = "docker-machine ssh {} sudo cp /tmp/zoo.cfg {} ".format(
k, path_conf)
utils.run_with_exit(c, cmd)
cmd = "docker-machine scp /tmp/myid {}:/tmp/myid ".format(k)
utils.run_with_exit(c, cmd)
cmd = "docker-machine ssh {} sudo cp /tmp/myid {} ".format(
k, path_data)
utils.run_with_exit(c, cmd)
cmd = "docker run -i -t -d --privileged --restart=always --name {}-{} ".format(
sk, zid)
cmd += _get_opts(sv)
cmd += "{}".format(sv["image"])
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
elif sv["label"] == "kafka":
_docker_rm(c, hosts, sk)
zsk, zsv = utils.get_service(env.services, sv["zookeeper"])
zhosts = utils.get_hosts(env.hosts, "zookeeper")
zid = 0
zconns = []
for zk, zv in zhosts.iteritems():
zid += 1
zconns.append("{}-{}:2181".format(zsk, zid))
zconn = ",".join(zconns)
print("Zookeeper Conn: {}".format(zconn))
bid = 0
for k, v in hosts.iteritems():
bid += 1
cmd = "docker-machine ip {} ".format(k)
res = c.run(cmd)
ip = res.stdout.strip()
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += '-e KAFKA_BROKER_ID="{}" '.format(bid)
cmd += '-e KAFKA_ADVERTISED_HOST_NAME="{}" '.format(k)
cmd += '-e KAFKA_ZOOKEEPER_CONNECT="{}" '.format(zconn)
cmd += '-e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP="INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" '
cmd += '-e KAFKA_ADVERTISED_LISTENERS="INSIDE://:9092,OUTSIDE://{}:9094" '.format(
k)
cmd += '-e KAFKA_LISTENERS="INSIDE://:9092,OUTSIDE://:9094" '
cmd += '-e KAFKA_INTER_BROKER_LISTENER_NAME="INSIDE" '
cmd += '-e KAFKA_ADVERTISED_PORT="9092" '
cmd += "{}".format(sv["image"])
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_with_exit(c, rcmd)
else:
print("Unkown Label: {}".format(cv["label"]))
sys.exit(-1)
def __config_load(self, config={}):
"""
Loading config from config dict
:param config: dictionary of settings
:return: None
"""
logger.info("Loading config...")
print(config)
self.gw = config.get('gateway', None)
if not self.gw:
logger.critical("IP of gateway not specified! Using default one: {}".format(DEFAULT_GATEWAY))
self.gw = DEFAULT_GATEWAY
mac = utils.arpping(self.gw)
while mac is None:
logger.debug('Cannot obtain MAC addr of the gateway. Make sure {} is correct!'.format(self.gw))
mac = utils.arpping(self.gw)
self.ip_mac[self.gw] = mac
logger.debug('Gateway MAC addr: {}'.format(mac))
self.network = config.get('network', None)
if self.network is None:
logger.debug('Using default network! {}'.format(DEFAULT_NETWORK))
self.network = DEFAULT_NETWORK
self.targets = config.get('target', None)
if not self.targets:
logger.debug('No explicit targets found! Calculating from the net!')
self.targets = utils.get_hosts(self.network)
logger.debug('Targets: {}'.format(self.targets))
for t in self.targets:
mac = utils.arpping(t)
cc = 0
while mac is None:
logger.debug('Target {} has MAC: {}'.format(t, mac))
cc += 1
mac = utils.arpping(t)
if cc > 3 and mac is None:
logger.critical('Cannot determine MAC addr of {}'.format(t))
raise ValueError('Check if {} addr is correct!'.format(t))
if mac is not None:
self.ip_mac[t] = mac
self.iface = config['iface']
self.iface_mac = utils.getHwAddr(self.iface)
if 'no_rearp' in config.keys():
self.rearp = not config['no_rearp']
if 'in_docker' in config.keys():
self.in_docker = config['in_docer']
def get_hosts():
hosts = utils.get_hosts()
return render_template('get_hosts.html', hosts=hosts)
def global_variables():
hosts = utils.get_hosts()
return dict(hosts=hosts)
env.password = '******'
#TOOL
def get_hosts(file):
"""
Return the hosts file through the file
"""
ip_list = []
#print ip_list
with open(file) as ip_file:
for i in ip_file.readlines():
#print i
ip_list.append(i.strip())
return ip_list
hosts_ip = utils.get_hosts("ip.alist")
print "the real addresss is ",hosts_ip
env.hosts = hosts_ip
env.password='******'
FORMAT = '%(asctime)-15s %(clientip)s %(user)-82 %(message)s'
logging.basicConfig(filename="apis.mapabc.com_fab.log", format=FORMAT)
logging.warning("Warning information!") #will print the loging information to the console
logging.info("The Information!") # will not print the message
###Usage ##########################
##################################
def add_key():
"""
def get_hosts():
hosts = utils.get_hosts()
return render_template('get_hosts.html', hosts=hosts)
def global_variables():
hosts = utils.get_hosts()
return dict(hosts=hosts)