def _inv_add_ports_ipmi(self):
log = logger.getlogger()
from lib.inventory import Inventory
inv = Inventory(cfg_file=self.config_file_path)
if (inv.check_all_nodes_ipmi_macs() and
inv.check_all_nodes_ipmi_ipaddrs()):
log.info("IPMI ports MAC and IP addresses already in inventory")
return
dhcp_lease_file = '/var/lib/misc/dnsmasq.leases'
from lib.container import Container
cont = Container(self.config_file_path)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(os.path.join(
gen.get_container_python_path(), 'inv_add_ports.py'))
cmd.append(dhcp_lease_file)
cmd.append('ipmi')
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd)
except UserException as exc:
print('Fail:', exc.message, file=sys.stderr)
sys.exit(1)
print('IPMI ports added to inventory')
def _create_container(self):
print(COL.scroll_ten, COL.up_ten)
print('{}Creating container for running the POWER-Up '
'software{}\n'.format(COL.header1, COL.endc))
from lib.container import Container
cont = Container(self.config_file_path)
print('created container object')
try:
cont.check_permissions(getpass.getuser())
except UserException as exc:
print('Fail:', exc, file=sys.stderr)
sys.exit(1)
try:
conf = lxc_conf.LxcConf(self.config_file_path)
conf.create()
except Exception as exc:
print("Fail:", exc, file=sys.stderr)
sys.exit(1)
try:
print('create it')
cont.create()
except UserException as exc:
print('Fail:', exc, file=sys.stderr)
sys.exit(1)
print('Success: Created container')
def __init__(self, idd, pinned_cpus, mem_ratio=1):
self.idd = idd # a unique uint id for this workload
# process handling
self.thread = None
self.popen = None
self.stdout = None
self.stderr = None
# Container creation
self.mem_ratio = mem_ratio
self.container = Container(self.get_name(), self.ideal_mem,
self.mem_ratio)
self.container.create()
# Pin CPUs
self.pinned_cpus = pinned_cpus
# Get shell command
procs_path = self.container.get_procs_path()
self.cmdline = self.get_cmdline(procs_path, pinned_cpus)
# task timings
self.ts_start = 0
self.ts_finish = 0
# Getting gradient coeffs ready
self.percent = 0
self.ratio = 1
self.get_gradient()
def _gather_mac_addr(self):
from lib.container import Container
cont = Container(self.config_file_path)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(os.path.join(
gen.get_container_python_path(), 'clear_port_macs.py'))
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd)
except UserException as exc:
print('Fail:', exc.message, file=sys.stderr)
sys.exit(1)
_run_playbook("activate_client_interfaces.yml", self.config_file_path)
cmd[-2] = os.path.join(
gen.get_container_python_path(), 'set_port_macs.py')
try:
cont.run_command(cmd)
except UserException as exc:
print('Fail:', exc.message, file=sys.stderr)
sys.exit(1)
else:
print('Success: Gathered Client MAC addresses')
def _inv_add_ports_pxe(self):
log = logger.getlogger()
from lib.inventory import Inventory
inv = Inventory(cfg_file=self.config_file_path)
if (inv.check_all_nodes_pxe_macs() and
inv.check_all_nodes_pxe_ipaddrs()):
log.info("PXE ports MAC and IP addresses already in inventory")
return
power_wait = gen.get_power_wait()
ipmi_set_power('off', self.config_file_path, wait=power_wait)
# set boot dev to bios, to avoid situations where some node types can skip
# past pxe boot or attempt to boot from disk if pxe does not respond in time
ipmi_set_bootdev('setup', False, self.config_file_path)
ipmi_set_power('on', self.config_file_path, wait=power_wait)
dhcp_lease_file = '/var/lib/misc/dnsmasq.leases'
from lib.container import Container
cont = Container(self.config_file_path, self.args.inv_add_ports_pxe)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(os.path.join(
gen.get_container_python_path(), 'inv_add_ports.py'))
cmd.append(dhcp_lease_file)
cmd.append('pxe')
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd)
except UserException as exc:
print('Fail:', exc.message, file=sys.stderr)
sys.exit(1)
print('PXE ports added to inventory')
def _config_data_switches(self):
import configure_data_switches
print(COL.scroll_ten, COL.up_ten)
print('{}Configuring data switches{}\n'.
format(COL.header1, COL.endc))
print('This may take a few minutes depending on the size'
' of the cluster')
if gen.is_container_running():
from lib.container import Container
cont = Container(self.config_file_path)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(os.path.join(
gen.get_container_python_path(), 'configure_data_switches.py'))
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd)
except UserException as exc:
print('\n{}Fail: {}{}'.format(COL.red, exc.message, COL.endc),
file=sys.stderr)
else:
print('\nSuccesfully configured data switches')
else:
try:
configure_data_switches.configure_data_switch(self.args.config_file_name)
except UserException as exc:
print('\n{}Fail: {}{}'.format(COL.red, exc.message, COL.endc),
file=sys.stderr)
except SwitchException as exc:
print('\n{}Fail (switch error): {}{}'.format(
COL.red, exc.message, COL.endc), file=sys.stderr)
else:
print('\nSuccesfully configured data switches')
def _create_inventory(self):
# from lib.inventory import Inventory
# log = logger.getlogger()
# inv = Inventory(cfg_file=self.config_file_path)
# node_count = len(inv.inv['nodes'])
# if node_count > 0:
# log.info("Inventory already exists!")
# print("\nInventory already exists with {} nodes defined."
# "".format(node_count))
# print("Press enter to continue using the existing inventory.")
# print("Type 'C' to continue creating a new inventory. "
# "WARNING: Contents of current file will be overwritten!")
# resp = raw_input("Type 'T' to terminate Cluster Genesis ")
# if resp == 'T':
# sys.exit('POWER-Up stopped at user request')
# elif resp == 'C':
# log.info("'{}' entered. Creating new inventory file."
# "".format(resp))
# else:
# log.info("Continuing with existing inventory.")
# return
from lib.container import Container
log = logger.getlogger()
cont = Container(self.config_file_path, self.args.create_inventory)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(os.path.join(
gen.get_container_python_path(), 'inv_create.py'))
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd)
except UserException as exc:
print('Fail:', exc.message, file=sys.stderr)
sys.exit(1)
deployer_inv_file = gen.get_symlink_realpath(self.config_file_path)
# If inventory file symlink is broken link remove it
symlink_path = gen.get_symlink_path(self.config_file_path)
if os.path.islink(symlink_path):
if not os.path.exists(os.readlink(symlink_path)):
os.unlink(symlink_path)
# If inventory is an empty file delete it
if (os.path.isfile(deployer_inv_file) and
os.stat(deployer_inv_file).st_size == 0):
os.remove(deployer_inv_file)
# Create a sym link on deployer to inventory inside container
if not os.path.isfile(deployer_inv_file):
cont_inv_file = os.path.join(gen.LXC_DIR, cont.name, 'rootfs',
gen.CONTAINER_PACKAGE_PATH[1:],
gen.INV_FILE_NAME)
log.debug("Creating symlink on deployer to container inventory: "
"{} -> {}".format(deployer_inv_file, cont_inv_file))
os.symlink(cont_inv_file, deployer_inv_file)
print('Success: Created inventory file')
def testContainer():
c = Container(x=5)
c.y = 8
c.z = 9
c.w = 10
c.foo = 5
print (c)
def get_blog_list_container_status(blog_id_list, uid):
"""
获取blog容器状态
:param blog_id_list:
:param uid:
:return:
"""
blog_container_obj = Container('blog')
return blog_container_obj.get_item_list_container_status(blog_id_list, uid)
def add_blog_stat_item(stat_type, blog_id, uid):
"""
添加blog统计明细
:param stat_type:
:param blog_id:
:param uid:
:return:
"""
blog_container_obj = Container('blog')
return blog_container_obj.add_item(stat_type, blog_id, uid)
def add_user_stat_item(stat_type, uid, blog_id):
"""
添加user统计明细
:param stat_type:
:param blog_id:
:param uid:
:return:
"""
blog_container_obj = Container('user')
return blog_container_obj.add_item(stat_type, uid, blog_id)
def _gather_mac_addr(self):
from lib.container import Container
from lib.inventory import Inventory
yellow = '\033[93m'
endc = '\033[0m'
log = logger.getlogger()
cont = Container(self.config_file_path)
found_all = False
while found_all is not True:
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(
os.path.join(gen.get_container_python_path(),
'clear_port_macs.py'))
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd, interactive=True)
except UserException as exc:
print('Fail:', str(exc), file=sys.stderr)
sys.exit(1)
_run_playbook("activate_client_interfaces.yml",
self.config_file_path)
cmd[-2] = os.path.join(gen.get_container_python_path(),
'set_port_macs.py')
try:
cont.run_command(cmd, interactive=True)
except UserException as exc:
print('Fail:', str(exc), file=sys.stderr)
sys.exit(1)
inv = Inventory(cfg_file=self.config_file_path)
if inv.check_data_interfaces_macs():
found_all = True
else:
print()
msg = 'Some data interface MAC addresses were not found!'
log.warning(msg)
print(f'{yellow}' + ('-' * (len(msg) + 10)) + f'{endc}')
print("\nPress enter to retry")
resp = input(
"Enter C to continue POWER-Up or 'T' to terminate ")
if resp == 'T':
log.info(
"'{}' entered. Terminating POWER-Up at user request".
format(resp))
sys.exit(1)
elif resp == 'C':
log.info("'{}' entered. Continuing POWER-Up".format(resp))
found_all = True
print('Success: Gathered Client MAC addresses')
def _create_inventory(self):
# from lib.inventory import Inventory
# log = logger.getlogger()
# inv = Inventory(cfg_file=self.config_file_path)
# node_count = len(inv.inv['nodes'])
# if node_count > 0:
# log.info("Inventory already exists!")
# print("\nInventory already exists with {} nodes defined."
# "".format(node_count))
# print("Press enter to continue using the existing inventory.")
# print("Type 'C' to continue creating a new inventory. "
# "WARNING: Contents of current file will be overwritten!")
# resp = input("Type 'T' to terminate POWER-Up ")
# if resp == 'T':
# sys.exit('POWER-Up stopped at user request')
# elif resp == 'C':
# log.info("'{}' entered. Creating new inventory file."
# "".format(resp))
# else:
# log.info("Continuing with existing inventory.")
# return
from lib.container import Container
from shutil import copy2
from lib.utilities import timestamp
log = logger.getlogger()
cont = Container(self.config_file_path, self.args.create_inventory)
cont.copy(self.config_file_path, self.cont_config_file_path)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(
os.path.join(gen.get_container_python_path(), 'inv_create.py'))
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd, interactive=True)
except UserException as exc:
print('Fail:', str(exc), file=sys.stderr)
sys.exit(1)
log.info('Success: Created inventory file')
name = os.path.basename(self.config_file_path)
parts = name.rpartition('.')
bakupname = parts[0] + '-' + timestamp() + '.' + parts[2]
bakuppath = os.path.join(gen.get_logs_path(), 'config-files')
if not os.path.exists(bakuppath):
os.mkdir(bakuppath)
try:
copy2(self.config_file_path, os.path.join(bakuppath, bakupname))
except FileNotFoundError as exc:
log.debug(f'Unable to create copy of config file. {exc}')
def teardown_deployer_container(config_path):
"""Teardown the POWER-Up container on the deployer.
This function is idempotent.
"""
log = logger.getlogger()
container = Container(config_path)
if container.cont is None:
log.info(f'container name: {container.name} does not exist.')
else:
log.info(f'Destroying container: {container.name}')
container.cont.remove(force=True)
def _create_container(self):
print(COL.scroll_ten, COL.up_ten)
print('{}Creating container for running the POWER-Up '
'software{}\n'.format(COL.header1, COL.endc))
from lib.container import Container
cont = Container(self.config_file_path)
try:
cont.create()
except UserException as exc:
print('Fail:', exc, file=sys.stderr)
sys.exit(1)
print('Success: Created container')
def _add_cobbler_systems(self):
from lib.container import Container
cont = Container(self.config_file_path)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(os.path.join(
gen.get_container_python_path(), 'cobbler_add_systems.py'))
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd)
except UserException as exc:
print('Fail:', exc.message, file=sys.stderr)
sys.exit(1)
print('Success: Cobbler systems added')
def _reserve_ipmi_pxe_ips(self):
from lib.container import Container
cont = Container(self.config_file_path)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(os.path.join(
gen.get_container_python_path(), 'inv_reserve_ipmi_pxe_ips.py'))
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd)
except UserException as exc:
print('Fail:', exc.message, file=sys.stderr)
sys.exit(1)
print('Success: IPMI and PXE IP Addresses Reserved')
def _add_cobbler_distros(self):
from lib.container import Container
cont = Container(self.config_file_path)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(
os.path.join(gen.get_container_python_path(),
'cobbler_add_distros.py'))
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd, interactive=True)
except UserException as exc:
print('Fail:', str(exc), file=sys.stderr)
sys.exit(1)
print('Success: Cobbler distros and profiles added')
def _download_os_images(self):
from lib.container import Container
try:
download_os_images.download_os_images(self.config_file_path)
except UserException as exc:
print('Fail:', exc.message, file=sys.stderr)
sys.exit(1)
cont = Container(self.config_file_path)
local_os_images = gen.get_os_images_path()
cont_os_images = gen.get_container_os_images_path()
try:
cont.copy_dir_to_container(local_os_images, cont_os_images)
except UserException as exc:
print('Fail:', exc.message, file=sys.stderr)
sys.exit(1)
print('Success: OS images downloaded and copied into container')
def _install_client_os(self):
from lib.container import Container
remove_client_host_keys.remove_client_host_keys(self.config_file_path)
cont = Container(self.config_file_path)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(os.path.join(
gen.get_container_python_path(), 'install_client_os.py'))
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd)
except UserException as exc:
print('Fail:', exc.message, file=sys.stderr)
sys.exit(1)
_run_playbook("wait_for_clients_ping.yml", self.config_file_path)
print('Success: Client OS installaion complete')
def _download_os_images(self):
from lib.container import Container
try:
download_os_images.download_os_images(self.config_file_path)
except UserException as exc:
print('Fail:', str(exc), file=sys.stderr)
sys.exit(1)
cont = Container(self.config_file_path)
local_os_images = gen.get_os_images_path()
cont_os_images = gen.get_container_os_images_path()
for item in os.listdir(local_os_images):
try:
cont.copy(os.path.join(local_os_images, item),
os.path.join(cont_os_images, item))
except UserException as exc:
print('Fail:', str(exc), file=sys.stderr)
sys.exit(1)
print('Success: OS images downloaded and copied into container')
def _create_inventory(self):
# from lib.inventory import Inventory
# log = logger.getlogger()
# inv = Inventory(cfg_file=self.config_file_path)
# node_count = len(inv.inv['nodes'])
# if node_count > 0:
# log.info("Inventory already exists!")
# print("\nInventory already exists with {} nodes defined."
# "".format(node_count))
# print("Press enter to continue using the existing inventory.")
# print("Type 'C' to continue creating a new inventory. "
# "WARNING: Contents of current file will be overwritten!")
# resp = input("Type 'T' to terminate Cluster Genesis ")
# if resp == 'T':
# sys.exit('POWER-Up stopped at user request')
# elif resp == 'C':
# log.info("'{}' entered. Creating new inventory file."
# "".format(resp))
# else:
# log.info("Continuing with existing inventory.")
# return
from lib.container import Container
cont = Container(self.config_file_path, self.args.create_inventory)
cont.copy(self.config_file_path, self.cont_config_file_path)
cmd = []
cmd.append(gen.get_container_venv_python_exe())
cmd.append(
os.path.join(gen.get_container_python_path(), 'inv_create.py'))
cmd.append(self.cont_config_file_path)
try:
cont.run_command(cmd, interactive=True)
except UserException as exc:
print('Fail:', str(exc), file=sys.stderr)
sys.exit(1)
print('Success: Created inventory file')
def _remove_docker_networks(cfg):
try:
container = Container(cfg.config_path)
container.create_networks(remove=True)
except errors.NotFound:
LOG.info('Container not found')
def _remove_docker_networks(cfg):
container = Container(cfg.config_path)
container.create_networks(remove=True)
def cmd_main(self):
return Container.cmd_from_list(
[self.cmd_next_seed(), self.cmd_assign()])