def _copytool_vars(id):
settings = config.get("settings", "agent")
copytool = Copytool(**config.get("copytools", id))
ct = copytool.as_dict()
ct["event_fifo"] = copytool.event_fifo
ct["report_interval"] = COPYTOOL_PROGRESS_INTERVAL
return {
"id": id,
"ct_path": ct["bin_path"],
"ct_arguments": settings["copytool_template"] % ct,
}
def _copytool_vars(id):
settings = config.get('settings', 'agent')
copytool = Copytool(**config.get('copytools', id))
ct = copytool.as_dict()
ct['event_fifo'] = copytool.event_fifo
ct['report_interval'] = COPYTOOL_PROGRESS_INTERVAL
return {
'id': id,
'ct_path': ct['bin_path'],
'ct_arguments': settings['copytool_template'] % ct
}
def _convert_agentstore_config():
server_conf_path = os.path.join(config.path, "server_conf")
if os.path.exists(server_conf_path):
with open(server_conf_path) as f:
old_server_conf = json.load(f)
set_server_url(old_server_conf.get("url").replace("/agent/", "/"))
os.unlink(server_conf_path)
else:
try:
url = config.get("settings",
"server").get("url").replace("/agent/", "/")
set_server_url(url)
config.delete("settings", "server")
except (KeyError, TypeError):
pass
crypto_files = map(
lambda x: (os.path.join(config.path, x), os.path.join(ENV_PATH, x)),
["authority.crt", "private.pem", "self.crt"],
)
map(lambda x: migrate_file(*x), crypto_files)
uuid_re = re.compile(
r"[0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{12}")
for entry in os.listdir(config.path):
if uuid_re.match(entry):
target_conf_path = os.path.join(config.path, entry)
with open(target_conf_path) as f:
old_target_conf = json.load(f)
config.set("targets", entry, old_target_conf)
os.unlink(target_conf_path)
def _update_copytool(id, **kwargs):
ct = config.get("copytools", id)
new_ct = copy.deepcopy(ct)
for key, val in kwargs.items():
if val:
new_ct[key] = val
# Don't bother doing anything if no values changed.
if ct == new_ct:
return id
# stop/unconfigure the old instance
unconfigure_copytool(id)
# register/start the new instance
configure_copytool(
new_ct["id"],
new_ct["index"],
new_ct["bin_path"],
new_ct["archive_number"],
new_ct["filesystem"],
new_ct["mountpoint"],
new_ct["hsm_arguments"],
)
start_monitored_copytool(id)
return id
def _get_target_config(uuid):
info = config.get('targets', uuid)
# Some history, previously the backfstype, device_type was not stored so if not present presume ldiskfs/linux
if ('backfstype' not in info) or ('device_type' not in info):
info['backfstype'] = info.get('backfstype', 'ldiskfs')
info['device_type'] = info.get('device_type', 'linux')
config.update('targets', uuid, info)
return info
def _get_target_config(uuid):
info = config.get("targets", uuid)
# Some history, previously the backfstype, device_type was not stored so if
# not present presume ldiskfs/linux
if ("backfstype" not in info) or ("device_type" not in info):
info["backfstype"] = info.get("backfstype", "ldiskfs")
info["device_type"] = info.get("device_type", "linux")
config.update("targets", uuid, info)
return info
def _full_scan(self):
# If we are a worker node then return nothing because our devices are not of interest. This is a short term
# solution for HYD-3140. This plugin should really be loaded if it is not needed but for now this sorts out
# and issue with PluginAgentResources being in the linux plugin.
if config.get('settings', 'profile')['worker']:
return {}
# Before we do anything do a partprobe, this will ensure that everything gets an up to date view of the
# device partitions. partprobe might throw errors so ignore return value
AgentShell.run(["partprobe"])
# Map of block devices major:minors to /dev/ path.
block_devices = BlockDevices()
# Devicemapper: LVM and Multipath
dmsetup = DmsetupTable(block_devices)
# Software RAID
mds = MdRaid(block_devices).all()
# _zpools
zfs_devices = ZfsDevices()
zfs_devices.full_scan(block_devices)
# EMCPower Devices
emcpowers = EMCPower(block_devices).all()
# Local filesystems (not lustre) in /etc/fstab or /proc/mounts
local_fs = LocalFilesystems(block_devices).all()
# We have scan devices, so set the devices scanned flags.
LinuxDevicePlugin.devices_scanned = True
return {
"vgs": dmsetup.vgs,
"lvs": dmsetup.lvs,
"zfspools": zfs_devices.zpools,
"zfsdatasets": zfs_devices.datasets,
"zfsvols": zfs_devices.zvols,
"mpath": dmsetup.mpaths,
"devs": block_devices.block_device_nodes,
"local_fs": local_fs,
'emcpower': emcpowers,
'mds': mds
}
def detect_scan(target_devices=None):
"""Look for Lustre on possible devices
Save the input devices when possible. Then future calls will
not need to specify the target_devices
"""
right_now = str(datetime.now())
if target_devices is not None:
target_devices_time_stamped = dict(timestamp=right_now,
target_devices=target_devices)
config.update('settings', 'last_detect_scan_target_devices',
target_devices_time_stamped)
try:
# Recall the last target_devices used in this method
settings = config.get('settings', 'last_detect_scan_target_devices')
except KeyError:
# This method was never called with a non-null target_devices
# or the setting file holding the device record is not found.
daemon_log.warn("detect_scan improperly called without target_devices "
"and without a previous call's target_devices to use.")
# TODO: Consider an exception here. But, since this is a rare case, it seems reasonable to return emptiness
# TODO: If this raised an exception, it should be a handled one in any client, and that seems too heavy
local_targets = LocalTargets([])
timestamp = right_now
else:
# Have target devices, so process them
timestamp = settings['timestamp']
daemon_log.info(
"detect_scan called at %s with target_devices saved on %s" %
(str(datetime.now()), timestamp))
local_targets = LocalTargets(settings['target_devices'])
# Return the discovered Lustre components on the target devices, may return emptiness.
mgs_targets = MgsTargets(local_targets.targets)
return {
"target_devices_saved_timestamp": timestamp,
"local_targets": local_targets.targets,
"mgs_targets": mgs_targets.filesystems
}
def get_ring0():
# ring0 will always be on the interface used for agent->manager comms
from urlparse import urlparse
server_url = config.get('settings', 'server')['url']
manager_address = socket.gethostbyname(urlparse(server_url).hostname)
out = AgentShell.try_run(['/sbin/ip', 'route', 'get', manager_address])
match = re.search(r'dev\s+([^\s]+)', out)
if match:
manager_dev = match.groups()[0]
else:
raise RuntimeError("Unable to find ring0 dev in %s" % out)
console_log.info("Chose %s for corosync ring0" % manager_dev)
ring0 = CorosyncRingInterface(manager_dev)
if ring0.ipv4_prefixlen < 9:
raise RuntimeError("%s subnet is too large (/%s)" %
(ring0.name, ring0.ipv4_prefixlen))
return ring0
def _update_copytool(id, **kwargs):
ct = config.get('copytools', id)
new_ct = copy.deepcopy(ct)
for key, val in kwargs.items():
if val:
new_ct[key] = val
# Don't bother doing anything if no values changed.
if ct == new_ct:
return id
# stop/unconfigure the old instance
unconfigure_copytool(id)
# register/start the new instance
configure_copytool(new_ct['id'], new_ct['index'], new_ct['bin_path'],
new_ct['archive_number'], new_ct['filesystem'],
new_ct['mountpoint'], new_ct['hsm_arguments'])
start_monitored_copytool(id)
return id
def main():
parser = ArgumentParser(
description="Intel® Manager for Lustre* software Copytool Monitor")
parser.add_argument("copytool_id", action=GetCopytoolAction)
args = parser.parse_args()
copytool_log_setup()
try:
manager_url = config.get('settings',
'server')['url'] + "copytool_event/"
except KeyError:
copytool_log.error(
"No configuration found (must be configured before starting a copytool monitor)"
)
sys.exit(1)
client = CryptoClient(manager_url, Crypto(config.path))
monitor = CopytoolMonitor(client, args.copytool)
def teardown_callback(*args, **kwargs):
monitor.stop()
signal.signal(signal.SIGTERM, teardown_callback)
signal.signal(signal.SIGINT, teardown_callback)
signal.signal(signal.SIGUSR1, decrease_loglevel)
signal.signal(signal.SIGUSR2, increase_loglevel)
try:
monitor.start()
while not monitor.stopping.is_set():
monitor.stopping.wait(timeout=10)
monitor.join()
except Exception as e:
copytool_log.exception()
sys.stderr.write("Unhandled exception: %s\n" % e)
sys.exit(1)
copytool_log.info("Terminating")
def update_profile(profile):
'''
Sets the profile to the profile_name by fetching the profile from the manager
:param profile_name:
:return: error or result OK
'''
old_profile = config.get('settings', 'profile')
'''
This is an incomplete solution but the incompleteness is at the bottom of the stack and we need this as a fix up
for 2.2 release.
What really needs to happen here is that the profile contains the name of the packages to install and then this
code would diff the old list and the new list and remove and add appropriately. For now we are just going to do that
in a hard coded way using the managed property.
To do this properly the profile needs to contain the packages and the endpoint needs to return them. We are going to
need it and when we do this function and profiles will need to be extended.
This code might want to use the update_pacakges as well but it's not clear and we are in a pickle here. This code is
not bad and doesn't have bad knock on effects.
'''
if old_profile['managed'] != profile['managed']:
if profile['managed']:
action = 'install'
else:
action = 'remove'
try:
yum_util(action,
enablerepo=["iml-agent"],
packages=['chroma-agent-management'])
except AgentShell.CommandExecutionError as cee:
return agent_error(
"Unable to set profile because yum returned %s" %
cee.result.stdout)
config.update('settings', 'profile', profile)
return agent_result_ok
def __call__(self, parser, namespace, values, option_string=None):
try:
setattr(namespace, 'copytool',
Copytool(**config.get('copytools', values)))
except KeyError:
raise ArgumentError(self, "Unknown copytool id '%s'" % values)
def event_fifo(self):
fifo_dir = config.get('settings', 'agent')['copytool_fifo_directory']
return os.path.join(fifo_dir, "%s-events" % self)
def set_agent_config(key, val):
agent_settings = config.get('settings', 'agent')
agent_settings[key] = val
config.update('settings', 'agent', agent_settings)
def get_agent_config(key):
return config.get("settings", "agent")[key]
def set_agent_config(key, val):
agent_settings = config.get("settings", "agent")
agent_settings[key] = val
config.update("settings", "agent", agent_settings)
def main():
"""handle unexpected exceptions"""
parser = argparse.ArgumentParser(
description="Intel® Manager for Lustre* software Agent")
parser.add_argument("--publish-zconf", action="store_true")
args = parser.parse_args()
signal.signal(signal.SIGHUP, signal.SIG_IGN)
daemon_log_setup()
console_log_setup()
daemon_log.info("Starting")
try:
daemon_log.info("Entering main loop")
try:
conf = config.get('settings', 'server')
except (KeyError, TypeError) as e:
daemon_log.error(
"No configuration found (must be registered before running the agent service), "
"details: %s" % e)
return
if config.profile_managed is False:
# This is kind of terrible. The design of DevicePluginManager is
# such that it can be called with either class methods or
# instantiated and then called with instance methods. As such,
# we can't pass in a list of excluded plugins to the instance
# constructor. Well, we could, but it would only work some
# of the time and that would be even more awful.
import chroma_agent.plugin_manager
chroma_agent.plugin_manager.EXCLUDED_PLUGINS += ['corosync']
agent_client = AgentClient(conf['url'] + "message/",
ActionPluginManager(),
DevicePluginManager(), ServerProperties(),
Crypto(config.path))
def teardown_callback(*args, **kwargs):
agent_client.stop()
agent_client.join()
[function() for function in agent_daemon_teardown_functions]
signal.signal(signal.SIGINT, teardown_callback)
signal.signal(signal.SIGTERM, teardown_callback)
signal.signal(signal.SIGUSR1, decrease_loglevel)
signal.signal(signal.SIGUSR2, increase_loglevel)
# Call any agent daemon startup methods that were registered.
[function() for function in agent_daemon_startup_functions]
agent_client.start()
# Waking-wait to pick up signals
while not agent_client.stopped.is_set():
agent_client.stopped.wait(timeout=10)
agent_client.join()
except Exception, e:
backtrace = '\n'.join(traceback.format_exception(*(sys.exc_info())))
daemon_log.error("Unhandled exception: %s" % backtrace)
def get_agent_config(key):
return config.get('settings', 'agent')[key]
context.open()
daemon_log_setup()
console_log_setup()
daemon_log.info("Starting in the background")
else:
context = None
daemon_log_setup()
daemon_log.addHandler(logging.StreamHandler())
console_log_setup()
try:
daemon_log.info("Entering main loop")
try:
conf = config.get('settings', 'server')
except (KeyError, TypeError) as e:
daemon_log.error(
"No configuration found (must be registered before running the agent service), "
"details: %s" % e)
return
if config.profile_managed is False:
# This is kind of terrible. The design of DevicePluginManager is
# such that it can be called with either class methods or
# instantiated and then called with instance methods. As such,
# we can't pass in a list of excluded plugins to the instance
# constructor. Well, we could, but it would only work some
# of the time and that would be even more awful.
import chroma_agent.plugin_manager
chroma_agent.plugin_manager.EXCLUDED_PLUGINS += ['corosync']
def event_fifo(self):
fifo_dir = config.get("settings", "agent")["copytool_fifo_directory"]
return os.path.join(fifo_dir, "%s-events" % self)
