def __init__(self,
log_file,
config_dir=None,
profile=None,
user_profile=None):
# In-memory log object to save system settings
self.log_object = None
# Dictionary of types of tunables and their service functions
self.tunables = { # tunable : service functions
SERVICE: {
"get": self.get_service_state,
"set": self.set_service_state
},
SYSCTL: {
"get": self.get_sysctl_value,
"set": self.set_sysctl_value
},
SYSFS: {
"get": self.get_sysfs_value,
"set": self.set_sysfs_value
},
CPU: {
"get": self.get_cpu_value,
"set": self.set_cpu_value
}
}
'''
Recovery instance do not need config_dir and profiles.
However if this instance is going to configure tunables,
then log file, config_dir, profile and user_profile
need to be passed in. Validate passed in params.
'''
self.recovery_instance = (config_dir is None and profile is None
and user_profile is None)
if (self.recovery_instance is False): # Instance to configure tunable
# Bail out if auto generated profile doesn't exist
self.auto_profile = "{0}/{1}.ini".format(config_dir, profile)
if (os.path.isfile(self.auto_profile) is False):
# Fatal error, auto profile doesn't exist
raise Ec2AutotuneError(
"Auto generated tunables file {0} does not "
"exist.".format(self.auto_profile))
# Bail out if user config profile doesn't exist
self.user_profile = user_profile
if (os.path.isfile(self.user_profile) is False):
# Fatal error, user profile doesn't exist
raise Ec2AutotuneError("User customized tunables file {0} "
"does not exist.".format(
self.user_profile))
# Validate the tunables to be set (dry run)
try:
self.validate_tunables_to_set()
except Ec2AutotuneError, e:
raise Ec2AutotuneError("Validation of tunables to be set "
"failed: {0}".format(e.msg))
class Ec2AutotuneStop(object):
'''
Restore system to earlier default state
'''
def __init__(self, config="/etc/ec2sys-autotune.cfg"):
self.config = config
self.cfg_file = RawConfigParser(allow_no_value=True)
self.cfg_file.read(self.config)
return
def stop(self):
# Disallow multiple instances of the same service
try:
lock_obj = Lock(self.config)
except Ec2AutotuneError, e:
raise (e)
# State directory - mandatory option in config file to stop service
if (self.cfg_file.has_option("DEFAULT", "STATE_DIR") is False):
raise Ec2AutotuneError("Missing STATE_DIR in config file.")
STATE_DIR = self.cfg_file.get("DEFAULT", "STATE_DIR")
# Log file - mandatory option in config file to stop service
if (self.cfg_file.has_option("DEFAULT", "LOG_FILE") is False):
raise Ec2AutotuneError("Missing LOG_FILE in config file.")
LOG_FILE = self.cfg_file.get("DEFAULT", "LOG_FILE")
# Status file - mandatory option in config file to stop service
if (self.cfg_file.has_option("DEFAULT", "STATUS") is False):
raise Ec2AutotuneError("Missing STATUS in config file.")
STATUS = self.cfg_file.get("DEFAULT", "STATUS")
if (os.path.isfile(STATUS) is False):
raise Ec2AutotuneError("Autotune is not active")
try:
instance = None
# Restore the system settings to earlier state
instance = Ec2InstanceCfgEngine("{0}/{1}".format(STATE_DIR,
LOG_FILE))
instance.restore_system_settings()
# Clear the service status
os.remove(STATUS)
syslog("EC2 AWS Autotune has restored original system settings "
"after clean up.")
except Ec2AutotuneError, e:
raise (e)
def parse_tunable_output(self, output):
'''
Tunables values are in four formats (for tunables we are
working):
1: number
2: [foo] bar
3: foo
4: number1\tnumber2\tnumber2
Second format above is tricky as the value set is the one enclosed
within []. In cases like these, we need to extract the string
enclosed between [] and return it to caller.
For all format, convert the output into a list and return
'''
if (output is None or len(output) == 0):
return None
output = output.strip()
start = output.find("[")
if (start != -1):
end = output.find("]")
if (end == -1): # Should never happen
raise Ec2AutotuneError("Parsing error of {0}".format(output))
output = output[start + 1:end]
output = output.split()
if (output[0].isdigit() is True):
output = map(int, output)
return output
def exec_cmds(cmds):
'''
Execute passed in command
'''
devnull = open(os.devnull, 'wb')
for cmd in cmds:
try:
retcode = call(shlex.split(cmd),
shell=False,
stderr=devnull,
stdout=devnull)
if (retcode != 0):
raise Ec2AutotuneError("Failed to execute: {0}".format(cmd))
except OSError, e:
raise Ec2AutotuneError("Exception encountered while trying to "
"execute: {0} error: {1}".format(cmd, e))
def write_sysfs_file(sysfs_file, value):
try:
with open(sysfs_file, "wb") as fd:
fd.write(value)
return
except IOError, e:
raise Ec2AutotuneError(
"Error while trying to write {0}, error {1}.".format(
sysfs_file, e.errno))
def read_sysfs_file(sysfs_file):
try:
with open(sysfs_file) as fd:
# Strip trailing newline feed
output = fd.read()[:-1]
return output
except IOError, e:
raise Ec2AutotuneError(
"Error while trying to query {0}, error {1}.".format(
sysfs_file, e.errno))
def __init__(self, config="/etc/ec2sys-autotune.cfg", genconfigonly=False):
self.config = config
self.genconfigonly = genconfigonly
self.cfg_file = RawConfigParser(allow_no_value=True)
self.cfg_file.read(self.config)
# Read in the config directory location
if (self.cfg_file.has_option("DEFAULT", "CONFIG_DIR") is False):
raise Ec2AutotuneError("Missing CONFIG_DIR in config file.")
self.config_dir = self.cfg_file.get("DEFAULT", "CONFIG_DIR")
# Read in the user config file
if (self.cfg_file.has_option("DEFAULT", "USER_CONFIG") is False):
raise Ec2AutotuneError("Missing USER_CONFIG in config file.")
self.user_config = self.cfg_file.get("DEFAULT", "USER_CONFIG")
self.usercfg_log = RawConfigParser(allow_no_value=True)
self.usercfg_log.read("{0}/{1}".format(self.config_dir,
self.user_config))
return
def get_cmd_output(cmd):
'''
Execute passed in command and return its output
'''
try:
# Strip trailing newline feed
output = check_output(shlex.split(cmd), shell=False)[:-1]
return output
except CalledProcessError, e:
raise Ec2AutotuneError("Exception encountered while trying to execute:"
" {0} error: {1}".format(cmd, e.output))
def convert_input_value(self, value):
'''
value will be a list, return string representation
'''
if (isinstance(value, list) is False):
raise Ec2AutotuneError("input value is in an invalid format.")
length = len(value)
value = " ".join(map(str, value))
# Embed multiple values inside strings
if (length > 1):
value = "\"{0}\"".format(value)
return value
def validate_tunables_to_set(self):
'''
Function that validates the tunable before applying
'''
# Recovery instance should not be trying to configure
if (self.recovery_instance is True):
raise Ec2AutotuneError("Incorrect instantiation and object usage.")
# Validate tunables
for tunable, functions in self.tunables.items():
try:
self.configure_system(tunable, functions["get"], True)
except Ec2AutotuneError, e:
raise e
def get_piped_cmd_output(cmd1, cmd2):
try:
p1 = Popen(shlex.split(cmd1), stdout=PIPE, shell=False)
p2 = Popen(shlex.split(cmd2),
stdin=p1.stdout,
stdout=PIPE,
shell=False)
p1.stdout.close()
# Strip trailing newline feed
output = (p2.communicate()[0])[:-1]
return output
except CalledProcessError, e:
raise Ec2AutotuneError("Exception encountered while trying to execute:"
" {0} | {1} error: {2}".format(
cmd1, cmd2, e.output))
def configure_system_settings(self):
'''
Main function that configures the system
'''
self.start_recovery_logging()
# Recovery instance should not be trying to configure
if (self.recovery_instance is True):
raise Ec2AutotuneError("Incorrect instantiation and object usage.")
# Configure tunables and save their defaults
for tunable, functions in self.tunables.items():
self.configure_system(tunable, functions["set"])
self.tunables = None
self.stop_recovery_logging()
return
def get_service_state(self, service):
'''
Returns the current status of service in the system
'''
try:
output = get_piped_cmd_output(
"/bin/systemctl status {0}".format(service),
"/bin/grep Active:")
if ("running" in output):
return (["start"])
elif ("dead" in output):
return (["stop"])
else:
raise Ec2AutotuneError(
"{0} package not installed.".format(service))
except Ec2AutotuneError, e:
raise e
def restore_system_settings(self):
'''
Function called during shutdown to restore system to eariler default
'''
# Read log file for restoring to earlier state (json format)
if (os.path.isfile(self.log_file) is False):
raise Ec2AutotuneError("Recovery log file does not exist, "
"failed to revert original settings.")
log_file = open(self.log_file, "rb")
list_log = json.load(log_file)
log_file.close()
# Restore tunables to their earlier defaults
for tunable, functions in self.tunables.items():
self.restore_system(list_log, tunable, functions["set"])
self.tunables = None
return
class Lock(object):
'''
This is a blocking lock implementation. However this will block
and retry only MAX_TRIES for every SLEEP_TIME seconds before failing.
'''
def __init__(self, lock):
SLEEP_TIME = 3
MAX_TRIES = 10
for count in range(0, MAX_TRIES):
try:
self.lock = open(lock, "r+")
fcntl.lockf(self.lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
syslog("Acquired lock")
return
except IOError, e:
syslog("Retrying lock on error {0}".format(e.errno))
self.lock.close()
time.sleep(SLEEP_TIME)
continue
raise Ec2AutotuneError(
"Couldn't acquire the lock during service startup.")
class Ec2AutotuneStart(object):
'''
Generate config file for a particular profile
'''
def __init__(self, config="/etc/ec2sys-autotune.cfg", genconfigonly=False):
self.config = config
self.genconfigonly = genconfigonly
self.cfg_file = RawConfigParser(allow_no_value=True)
self.cfg_file.read(self.config)
# Read in the config directory location
if (self.cfg_file.has_option("DEFAULT", "CONFIG_DIR") is False):
raise Ec2AutotuneError("Missing CONFIG_DIR in config file.")
self.config_dir = self.cfg_file.get("DEFAULT", "CONFIG_DIR")
# Read in the user config file
if (self.cfg_file.has_option("DEFAULT", "USER_CONFIG") is False):
raise Ec2AutotuneError("Missing USER_CONFIG in config file.")
self.user_config = self.cfg_file.get("DEFAULT", "USER_CONFIG")
self.usercfg_log = RawConfigParser(allow_no_value=True)
self.usercfg_log.read("{0}/{1}".format(self.config_dir,
self.user_config))
return
def start(self):
'''
Start Autotune service
'''
# Disallow multiple instances of the same service
try:
lock_obj = Lock(self.config)
except Ec2AutotuneError, e:
raise (e)
# Profile to tune the system with
if (self.usercfg_log.has_option("profile", "PROFILE") is False):
raise Ec2AutotuneError("Missing PROFILE in user config file.")
PROFILE = self.usercfg_log.get("profile", "PROFILE")
# State dir
if (self.cfg_file.has_option("DEFAULT", "STATE_DIR") is False):
raise Ec2AutotuneError("Missing STATE_DIR in config file.")
STATE_DIR = self.cfg_file.get("DEFAULT", "STATE_DIR")
# Log file
if (self.cfg_file.has_option("DEFAULT", "LOG_FILE") is False):
raise Ec2AutotuneError("Missing LOG_FILE in config file.")
LOG_FILE = self.cfg_file.get("DEFAULT", "LOG_FILE")
# Status file
if (self.cfg_file.has_option("DEFAULT", "STATUS") is False):
raise Ec2AutotuneError("Missing STATUS in config file.")
STATUS = self.cfg_file.get("DEFAULT", "STATUS")
if (os.path.isfile(STATUS) is True and self.genconfigonly is False):
raise Ec2AutotuneError(
"Autotune profile {0} is already active".format(PROFILE))
try:
instance = None
# Instantiate appropriate object for the profile
if ("base" in PROFILE):
instance = Ec2InstanceCfgGen(self.config_dir, PROFILE)
elif ("udp-server" in PROFILE):
instance = UdpServerCfgGen(self.config_dir, PROFILE)
elif ("placement-group" in PROFILE):
instance = PlacementGroupCfgGen(self.config_dir, PROFILE)
else:
raise Ec2AutotuneError(
"Invalid role {0} specified.".format(PROFILE))
# Generate config tunables for sub systems
instance.tune()
syslog(
"Configuration {0} role has been generated.".format(PROFILE))
except Ec2AutotuneEexists, e:
# Config file exists from previous instance, reuse the same
syslog(e.msg)
def get_cpu_value(self, cpu_state):
'''
Get value of a particular CPU state
'''
try:
retcode = 0
output = None
# Query frequency governor
if (cpu_state == "p-state"):
# Work only with intel_pstate driver
driver = get_piped_cmd_output(
"/bin/cpupower frequency-info --driver",
"/bin/grep \"driver: intel_pstate\"")
if (len(driver) > 0):
# Return current governor being used
output = get_piped_cmd_output(
"/bin/cpupower frequency-info",
"/bin/grep \"The governor\"")
start = output.find("\"")
if (start == -1):
raise Ec2AutotuneError("Parsing error of current "
"frequency-info governor")
end = output.find("\"", start + 1)
if (end == -1):
raise Ec2AutotuneError("Parsing error of current "
"frequency-info governor")
return ([output[start + 1:end]])
# Query CPU idle state
elif (cpu_state == "c-state"):
# Work only with intel_idle driver
driver = get_piped_cmd_output(
"/bin/cpupower idle-info --silent",
"/bin/grep \"driver: intel_idle\"")
if (len(driver) > 0):
# Number of idle states
output = get_piped_cmd_output(
"/bin/cpupower idle-info",
"/bin/grep \"Number of idle states:\"")
max_states = int(output[(output.index("states: ") + 8):])
# Available idle states
idle_states = []
output = get_piped_cmd_output(
"/bin/cpupower idle-info",
"/bin/grep \"Available idle states:\"")
beg = 0
end = len(output)
for state in range(max_states):
try:
idx = output.rindex(" ", beg, end) + 1
except:
raise Ec2AutotuneError(
"Parsing error of available idle states")
if (idx == -1):
raise Ec2AutotuneError(
"Parsing error of available idle states")
idle_states.append(output[idx:end])
end = idx - 1
idle_states.reverse()
# Return deepest enabled state
output = get_piped_cmd_output("/bin/cpupower idle-info",
"/bin/grep DISABLED")
if (len(output) == 0):
# No state is disabled, return deepest state
return ([idle_states[max_states - 1]])
else:
index = output.index(" ")
output = output[:index]
return ([idle_states[idle_states.index(output) - 1]])
else:
# State should always be either p-state or c-state
raise Ec2AutotuneError("Invalid {0} state".format(cpu_state))
except Ec2AutotuneError, e:
raise e
'''
cmd = ["/bin/cpupower frequency-set -g {0}".format(tmp_new_value)]
elif (cpu_state == "c-state"):
'''
C-state: value has to be one of the supported idle
states by intel_idle driver
'''
idle_states = { # state : latency
"POLL": 0,
"C1": 2,
"C1E": 10,
"C3": 40,
"C6": 133
}
if (not (tmp_new_value in idle_states)):
raise Ec2AutotuneError("Invalid value for c-state = "
"{0}".format(tmp_new_value))
cmd = ([
"/bin/cpupower idle-set --enable-all",
"/bin/cpupower idle-set --disable-by-latency {0}".format(
str(idle_states[tmp_new_value] + 1))
])
else:
raise Ec2AutotuneError("Invalid CPU state {0}".format(cpu_state))
return self.set_tunable(cpu_state, cmd, self.get_cpu_value, CPU,
new_value, orig_value)
def configure_system(self, section, configure, dry_run=False):
if (dry_run is True):
fetch_configuration(self.auto_profile, self.user_profile, section,
None, None, None, configure)
else: