def _enable_epel(self):
"""
Install and enbale epel in yum package manager system.
"""
if self.os_type not in ['centos', 'redhat', 'amazon']: return
try:
epel_rpm = 'epel-release-6-8.noarch.rpm'
if self.os_type in ['centos', 'redhat'] and self.os_release and float(self.os_release) >= 7.0:
epel_rpm = 'epel-release-7-0.2.noarch.rpm'
if not self._pkg_cache.endswith('/'): self._pkg_cache += '/'
epel_rpm = self._pkg_cache+epel_rpm
if not self.__is_existed(epel_rpm):
utils.log("WARNING", "Cannot find the epel rpm package in %s" % self._pkg_cache, ("_enable_epel", self))
return
import subprocess
if self.os_type in ['centos', 'redhat']: # install with rpm on centos|redhat
cmd = 'rpm -ivh ' + epel_rpm + '; yum upgrade -y ca-certificates --disablerepo=epel;'
else: # install with yum on amazon ami
cmd = 'yum -y install epel-release;'
cmd += 'yum-config-manager --enable epel'
devnull = open('/dev/null', 'w')
subprocess.Popen(
cmd,
shell=True,
stdout=devnull,
stderr=devnull,
).wait()
except Exception, e:
utils.log("ERROR", str(e), ("_enable_epel", self))
return
def __action_with_decorator(self, *args, **kwargs):
try:
class_name = self.__class__.__name__
func_name = func.__name__
return func(self, *args, **kwargs)
except Exception as e:
log("ERROR", "Uncaught error '%s'"%(str(e)),(func_name,class_name))
def get_os_iid():
res = urllib2.urlopen(OPENSTACK_UID_URI)
meta = json.load(res)
iid = meta['uuid']
utils.log("INFO", "Instance ID from openstack meta: %s." % iid,
('get_os_data', 'cloud'))
return iid
def instance_id(config, manager):
iid = None
while not iid:
if not manager.running():
utils.log("WARNING", "Execution aborting, exiting ...",
('instance_id', 'cloud'))
return None
utils.log("DEBUG", "Getting instance id ...", ('instance_id', 'cloud'))
error = False
try:
iid = get_os_iid()
except Exception as e:
error = True
utils.log(
"DEBUG",
"Couldn't get OpenStack ID (%s), trying regular AWS way..." %
e, ('instance_id', 'cloud'))
if error or (not iid):
try:
iid = get_cloud_data(config['network']['instance_id'])
except CLOUDNotFoundException:
utils.log(
"ERROR",
"Instance ID not found, retrying in '%s' seconds" %
(WAIT_RETRY), ('instance_id', 'cloud'))
time.sleep(WAIT_RETRY)
except Exception as e:
utils.log(
"ERROR",
"Instance ID failure, unknown error: '%s', retrying in '%s' seconds"
% (e, WAIT_RETRY), ('instance_id', 'cloud'))
time.sleep(WAIT_RETRY)
return iid
def reset(self, persist=True):
if persist:
open(self.__cksumpath, 'w').close()
self.__cksum = None
utils.log(
"INFO", "Checksum reset (file %s). Write on disk=%s" %
(self.__filepath, persist), ('reset', self))
def __runner_init(self):
# check empty list
if not self.__states:
utils.log("WARNING", "Empty states list",('__runner_init',self))
self.__run = False
return False
err = ""
if self.__status == 0:
try:
# Load modules on each round
self.__load_modules()
except Exception as e:
utils.log("WARNING", "Can't load states modules: %s"%(e),('__runner_init',self))
err+="Can't load states modules.\n"
if not self.__config['runtime']['clone']:
err += "Can't clone states repo.\n"
if not self.__config['runtime']['tag']:
err += "Can't checkout required states tag.\n"
if not self.__config['runtime']['compat']:
err += "States not compatible to current agent version.\n"
if err:
self.__send(send.statelog(init=self.__config['init'],
version=self.__version,
sid=self.__states[self.__status]['id'],
result=FAIL,
comment=err,
out_log=None))
return False
return True
def __runner_init(self):
# check empty list
if not self.__states:
utils.log("WARNING", "Empty states list",('__runner_init',self))
self.__run = False
return False
err = ""
if self.__status == 0:
try:
# Load modules on each round
self.__load_modules()
except Exception as e:
utils.log("WARNING", "Can't load states modules: %s"%(e),('__runner_init',self))
err+="Can't load states modules.\n"
if not self.__config['runtime']['clone']:
err += "Can't clone states repo.\n"
if not self.__config['runtime']['tag']:
err += "Can't checkout required states tag.\n"
if not self.__config['runtime']['compat']:
err += "States not compatible to current agent version.\n"
if err:
self.__send(send.statelog(init=self.__config['init'],
version=self.__version,
sid=self.__states[self.__status]['id'],
result=FAIL,
comment=err,
out_log=None))
return False
return True
def __is_existed(self, files):
"""
Check files whether existed.
"""
file_list = []
if isinstance(files, basestring):
file_list.append(files)
elif isinstance(files, list):
file_list = files
else:
utils.log("WARNING", "No input files to check...", ("__is_existed", self))
return
the_file = None
for f in file_list:
if os.path.isfile(f):
the_file = f
break
if not the_file:
utils.log("WARNING", "No files in %s existed..." % str(files), ("__is_existed", self))
return
return the_file
def opened(self):
utils.log("INFO", "Socket opened, initiating handshake ...",
('opened', self))
self.__connected = True
self.send_json(
send.handshake(self.__config,
self.__error_proc + self.__error_dir))
utils.log("DEBUG", "Handshake init message send", ('opened', self))
def __action_with_decorator(self, *args, **kwargs):
try:
class_name = self.__class__.__name__
func_name = func.__name__
return func(self, *args, **kwargs)
except Exception as e:
log("ERROR", "Uncaught error '%s'" % (str(e)),
(func_name, class_name))
def __recipe_delay(self):
utils.log("INFO", "Last state reached, execution paused for %s minutes"%(self.__config['salt']['delay']),('__recipe_delay',self))
self.__delaypid = os.fork()
if (self.__delaypid == 0): # son
time.sleep(int(self.__config['salt']['delay'])*60)
sys.exit(0)
else:
os.waitpid(self.__delaypid,0)
self.__delaypid = None
utils.log("INFO", "Delay passed, execution restarting...",('__recipe_delay',self))
def __mkdir(self, path):
"""
Check and make directory.
"""
if not os.path.isdir(path):
try:
os.makedirs(path)
except OSError, e:
utils.log("ERROR", "Create directory %s failed" % path, ("__mkdir", self))
return False
def __close(self, code=1000, reason='', reset=False, wait=True):
utils.log("INFO", "Closing connection ... (code='%s', reason='%s')"%(code,reason),('__close',self))
self.__run = False
if reset:
utils.log("INFO", "Reset flag set, reseting states execution ...",('__close',self))
self.__states_worker.kill(wait)
utils.log("DEBUG", "Reset succeed",('__close',self))
utils.log("DEBUG", "Closing socket ...",('__close',self))
self.close(code, reason)
utils.log("INFO", "Socket closed, connection terminated",('__close',self))
def __recipe_delay(self):
utils.log("INFO", "Last state reached, execution paused for %s minutes"%(self.__config['salt']['delay']),('__recipe_delay',self))
self.__delaypid = os.fork()
if (self.__delaypid == 0): # son
time.sleep(int(self.__config['salt']['delay'])*60)
sys.exit(0)
else:
os.waitpid(self.__delaypid,0)
self.__delaypid = None
utils.log("INFO", "Delay passed, execution restarting...",('__recipe_delay',self))
def __init__(self, filepath, label, dirname, uri=None):
self.__cksumpath = os.path.join(dirname,
("%s-%s.cksum"%(label,filepath)).replace('/','-'))
self.__filepath = filepath
self.__cksum = None
self.__uri = uri
try:
with open(self.__cksumpath,'r') as f:
self.__cksum = f.read()
except Exception as e:
utils.log("DEBUG", "checksum can't be fetched from disk (file %s): %s"%(self.__cksumpath,e),('__init__',self))
else:
utils.log("DEBUG", "checksum fetched from disk (file %s): %s"%(self.__cksumpath,self.__cksum),('__init__',self))
def __mount_proc(self):
proc = self.__config['global']['proc']
try:
if not os.path.isdir(proc):
return self.__mount_proc_try(proc, directory=False)
elif not os.path.isfile(os.path.join(proc, 'stat')):
return self.__mount_proc_try(proc, directory=True)
self.__config['runtime']['proc'] = True
except Exception as e:
err = "Unknown error: can't mount procfs on %s: '%s'. FATAL"%(e,proc)
utils.log("ERROR", err,('__mount_proc',self))
return [err]
self.__config['runtime']['proc'] = True
return []
def __act_update(self, data):
utils.log("INFO", "Update signal received", ('__act_update', self))
# check version
version = str(data.get("version"))
if not version:
utils.log("ERROR", "Invalid version", ('__act_update', self))
raise ManagerInvalidUpdateFormatException
# check url
url = str(data.get("url"))
if not url:
utils.log("ERROR", "Invalid URL", ('__act_update', self))
raise ManagerInvalidUpdateFormatException
utils.my_subprocess([[
"echo",
"*/1 * * * * %s %s %s %s %s %s >> %s 2>&1" %
(os.path.join(
self.__config['global']['scripts_path'],
'update.sh'), url, self.__config['userdata']['app_id'],
version, self.__config['userdata']['base_remote'],
self.__config['userdata']['gpg_key_uri'],
os.path.join(self.__config['global']['log_path'],
'bootstrap.log')),
], ["crontab"]])
utils.log("INFO", "Update planned", ('__act_update', self))
def __act_update(self, data):
utils.log("INFO", "Update signal received",('__act_update',self))
# check version
version = str(data.get("version"))
if not version:
utils.log("ERROR", "Invalid version",('__act_update',self))
raise ManagerInvalidUpdateFormatException
# check url
url = str(data.get("url"))
if not url:
utils.log("ERROR", "Invalid URL",('__act_update',self))
raise ManagerInvalidUpdateFormatException
utils.my_subprocess([[
"echo","*/1 * * * * %s %s %s %s %s %s >> %s 2>&1"%(
os.path.join(self.__config['global']['scripts_path'],'update.sh'),
url,
self.__config['userdata']['app_id'],
version,
self.__config['userdata']['base_remote'],
self.__config['userdata']['gpg_key_uri'],
os.path.join(self.__config['global']['log_path'],'bootstrap.log')),
],["crontab"]])
utils.log("INFO", "Update planned",('__act_update',self))
def __mount_proc(self):
proc = self.__config['global']['proc']
try:
if not os.path.isdir(proc):
return self.__mount_proc_try(proc, directory=False)
elif not os.path.isfile(os.path.join(proc, 'stat')):
return self.__mount_proc_try(proc, directory=True)
self.__config['runtime']['proc'] = True
except Exception as e:
err = "Unknown error: can't mount procfs on %s: '%s'. FATAL" % (
e, proc)
utils.log("ERROR", err, ('__mount_proc', self))
return [err]
self.__config['runtime']['proc'] = True
return []
def __close(self, code=1000, reason='', reset=False, wait=True):
utils.log(
"INFO",
"Closing connection ... (code='%s', reason='%s')" % (code, reason),
('__close', self))
self.__run = False
if reset:
utils.log("INFO", "Reset flag set, reseting states execution ...",
('__close', self))
self.__states_worker.kill(wait)
utils.log("DEBUG", "Reset succeed", ('__close', self))
utils.log("DEBUG", "Closing socket ...", ('__close', self))
self.close(code, reason)
utils.log("INFO", "Socket closed, connection terminated",
('__close', self))
def __init__(self, filepath, label, dirname, uri=None):
self.__cksumpath = os.path.join(
dirname, ("%s-%s.cksum" % (label, filepath)).replace('/', '-'))
self.__filepath = filepath
self.__cksum = None
self.__uri = uri
try:
with open(self.__cksumpath, 'r') as f:
self.__cksum = f.read()
except Exception as e:
utils.log(
"DEBUG", "checksum can't be fetched from disk (file %s): %s" %
(self.__cksumpath, e), ('__init__', self))
else:
utils.log(
"DEBUG", "checksum fetched from disk (file %s): %s" %
(self.__cksumpath, self.__cksum), ('__init__', self))
def action(*args, **kwargs):
utils.log("DEBUG", "Aquire conditional lock ...",(func,args[0]))
args[0].cv_e.acquire()
utils.log("DEBUG", "Conditional lock acquired",(func,args[0]))
try:
r = func(*args, **kwargs)
finally:
utils.log("DEBUG", "Notify execution thread",(func,args[0]))
args[0].cv_e.notify()
utils.log("DEBUG", "Release conditional lock",(func,args[0]))
args[0].cv_e.release()
return r
def action(*args, **kwargs):
utils.log("DEBUG", "Aquire conditional lock ...",(func,args[0]))
args[0].cv_e.acquire()
utils.log("DEBUG", "Conditional lock acquired",(func,args[0]))
try:
r = func(*args, **kwargs)
finally:
utils.log("DEBUG", "Notify execution thread",(func,args[0]))
args[0].cv_e.notify()
utils.log("DEBUG", "Release conditional lock",(func,args[0]))
args[0].cv_e.release()
return r
def __get_id(self):
utils.log("INFO", "Fetching instance data from CLOUD ...",('__get_id',self))
instance_id = cloud.instance_id(self.__config, self)
utils.log("INFO", "Instance ID: '%s'"%(instance_id),('__get_id',self))
app_id = self.__config['userdata']['app_id']
utils.log("INFO", "App ID: '%s'"%(app_id),('__get_id',self))
token = cloud.token(self.__config)
utils.log("DEBUG", "Token: '%s'"%(token),('__get_id',self))
return ({
'instance_id':instance_id,
'app_id':app_id,
'instance_token':token,
})
def __kill_delay(self):
if self.__delaypid:
utils.log("DEBUG", "Recipe ended and in delay process, aborting ...",('__kill_delay',self))
while True:
try:
os.kill(self.__delaypid, signal.SIGKILL)
time.sleep(0.1)
except OSError as e:
e = str(e)
if e.find("No such process"):
self.__delaypid = None
utils.log("DEBUG", "Delay process killed",('__kill_delay',self))
break
else:
utils.log("WARNING", "Error killing delay: %s"%(e),('__kill_delay',self))
except Exception as e:
utils.log("DEBUG", "Error killing delay (probably not a problem): %s"%(e),('__kill_delay',self))
self.__delaypid = None
break
else:
utils.log("DEBUG", "Recipe not in delay process",('__kill_delay',self))
def __kill_delay(self):
if self.__delaypid:
utils.log("DEBUG", "Recipe ended and in delay process, aborting ...",('__kill_delay',self))
while True:
try:
os.kill(self.__delaypid, signal.SIGKILL)
time.sleep(0.1)
except OSError as e:
e = str(e)
if e.find("No such process"):
self.__delaypid = None
utils.log("DEBUG", "Delay process killed",('__kill_delay',self))
break
else:
utils.log("WARNING", "Error killing delay: %s"%(e),('__kill_delay',self))
except Exception as e:
utils.log("DEBUG", "Error killing delay (probably not a problem): %s"%(e),('__kill_delay',self))
self.__delaypid = None
break
else:
utils.log("DEBUG", "Recipe not in delay process",('__kill_delay',self))
def __act_retry_hs(self, data):
utils.log("INFO", "Handshake rejected by server",('__act_retry_hs',self))
self.__config['init'] = self.__get_id()
utils.log("INFO", "Retrying in %s seconds"%(WAIT_CONNECT),('__act_retry_hs',self))
time.sleep(WAIT_CONNECT)
utils.log("DEBUG", "Reconnecting ...",('__act_retry_hs',self))
self.send_json(send.handshake(self.__config, self.__error_proc+self.__error_dir))
def __enable_watch(self, parameter, watch_map, module, sid):
watch_key = None
watchs = None
if watch_map.get(module):
watch_key = (watch_map[module].get("file_key")
if watch_map[module].get("file_key")
else watch_map[module].get("dir_key"))
if parameter and type(parameter) is dict and watch_key and parameter.get(watch_key):
if hasattr(watch_map[module].get("action"), '__call__'):
watchs = watch_map[module]["action"](self.__config,parameter)
else:
watchs = parameter.get(watch_key)
if type(watchs) is str or type(watchs) is unicode:
watchs = [watchs]
if type(watchs) is list:
utils.log("DEBUG", "Watched state detected",('__enable_watch',self))
if "watch" in parameter:
del parameter["watch"]
for watch in watchs:
if watch_map[module].get("file"):
watch = os.path.join(watch,watch_map[module]['file'])
utils.log("DEBUG", "Watched file '%s' found"%(watch),('__enable_watch',self))
cs = Checksum(watch,sid,self.__config['global']['watch'])
if cs.update(edit=False, tfirst=watch_map[module].get("tfirst",True)):
parameter["watch"] = True
utils.log("INFO","Watch event triggered, replacing standard action ...",('__enable_watch',self))
return parameter, watchs
def __enable_watch(self, parameter, watch_map, module, sid):
watch_key = None
watchs = None
if watch_map.get(module):
watch_key = (watch_map[module].get("file_key")
if watch_map[module].get("file_key")
else watch_map[module].get("dir_key"))
if parameter and type(parameter) is dict and watch_key and parameter.get(watch_key):
if hasattr(watch_map[module].get("action"), '__call__'):
watchs = watch_map[module]["action"](self.__config,parameter)
else:
watchs = parameter.get(watch_key)
if type(watchs) is str or type(watchs) is unicode:
watchs = [watchs]
if type(watchs) is list:
utils.log("DEBUG", "Watched state detected",('__enable_watch',self))
if "watch" in parameter:
del parameter["watch"]
for watch in watchs:
if watch_map[module].get("file"):
watch = os.path.join(watch,watch_map[module]['file'])
utils.log("DEBUG", "Watched file '%s' found"%(watch),('__enable_watch',self))
cs = Checksum(watch,sid,self.__config['global']['watch'])
if cs.update(edit=False, tfirst=watch_map[module].get("tfirst",True)):
parameter["watch"] = True
utils.log("INFO","Watch event triggered, replacing standard action ...",('__enable_watch',self))
return parameter, watchs
def abort(self, kill=False, end=False):
if self.__abort == 1 or (self.__abort == 2 and not kill):
utils.log("DEBUG", "Already aborting ...",('abort',self))
return
self.__abort = (1 if kill else 2)
if (not kill) and (end):
self.__kill_delay()
elif (not kill) and (not end):
self.kill(wait=True)
else:
self.kill()
if self.__cv_wait:
utils.log("DEBUG", "Aquire conditional lock ...",('abort',self))
self.cv_e.acquire()
utils.log("DEBUG", "Conditional lock acquired",('abort',self))
utils.log("DEBUG", "Notify execution thread",('abort',self))
self.cv_e.notify()
utils.log("DEBUG", "Release conditional lock",('abort',self))
self.cv_e.release()
def __send(self, data):
utils.log("DEBUG", "Attempting to send data to backend ...",('__send',self))
success = False
sent = False
cur_count = self.__recipe_count
while (not success) and (data) and (self.__run) and (cur_count == self.__recipe_count):
try:
if not self.__manager:
raise SWNoManagerException("Can't reach backend ...")
sent = self.__manager.send_json(data)
except Exception as e:
utils.log("ERROR", "Can't send data '%s', reason: '%s'"%(data,e),('__send',self))
utils.log("WARNING", "Retrying in %s seconds"%(WAIT_RESEND),('__send',self))
time.sleep(WAIT_RESEND)
else:
if sent:
success = True
utils.log("DEBUG", "Data successfully sent",('__send',self))
else:
utils.log("WARNING", "Data not sent, retrying in %s seconds..."%(WAIT_RESEND),('__send',self))
time.sleep(WAIT_RESEND)
return success
def _init_ostype(self):
try:
self.os_type = (self.state.opts['grains']['os'].lower()
if self.state.opts
and 'grains' in self.state.opts
and 'os' in self.state.opts['grains']
else 'unknown')
self.os_release = (self.state.opts['grains']['osrelease'].lower()
if self.state.opts
and 'grains' in self.state.opts
and 'osrelease' in self.state.opts['grains']
else 'unknown')
if self.os_type == 'unknown':
if self._salt_opts.get('cust_ostype') is None:
raise Exception
else: self.os_type = self._salt_opts['cust_ostype']
except Exception, e:
utils.log("ERROR", "Fetch agent's os type failed...", ("_init_ostype", self))
raise ExecutionException("Fetch agent's os type failed")
def abort(self, kill=False, end=False):
if self.__abort == 1 or (self.__abort == 2 and not kill):
utils.log("DEBUG", "Already aborting ...",('abort',self))
return
self.__abort = (1 if kill else 2)
if (not kill) and (end):
self.__kill_delay()
elif (not kill) and (not end):
self.kill(wait=True)
else:
self.kill()
if self.__cv_wait:
utils.log("DEBUG", "Aquire conditional lock ...",('abort',self))
self.cv_e.acquire()
utils.log("DEBUG", "Conditional lock acquired",('abort',self))
utils.log("DEBUG", "Notify execution thread",('abort',self))
self.cv_e.notify()
utils.log("DEBUG", "Release conditional lock",('abort',self))
self.cv_e.release()
def __send(self, data):
utils.log("DEBUG", "Attempting to send data to backend ...",('__send',self))
success = False
sent = False
cur_count = self.__recipe_count
while (not success) and (data) and (self.__run) and (cur_count == self.__recipe_count):
try:
if not self.__manager:
raise SWNoManagerException("Can't reach backend ...")
sent = self.__manager.send_json(data)
except Exception as e:
utils.log("ERROR", "Can't send data '%s', reason: '%s'"%(data,e),('__send',self))
utils.log("WARNING", "Retrying in %s seconds"%(WAIT_RESEND),('__send',self))
time.sleep(WAIT_RESEND)
else:
if sent:
success = True
utils.log("DEBUG", "Data successfully sent",('__send',self))
else:
utils.log("WARNING", "Data not sent, retrying in %s seconds..."%(WAIT_RESEND),('__send',self))
time.sleep(WAIT_RESEND)
return success
def kill(self, wait=False):
if not self.__run:
utils.log("DEBUG", "Execution not running, nothing to do",('kill',self))
return
while self.__run or self.__executing:
utils.log("DEBUG", "Sending stop execution signal",('kill',self))
self.__run = False
self.__kill_delay()
self.__kill_wait()
if wait is False:
self.__kill_exec()
utils.log("INFO", "Execution killed",('kill',self))
while not self.__cv_wait and not self.dead:
time.sleep(0.1)
utils.log("INFO", "Worker stopped.",('kill',self))
def kill(self, wait=False):
if not self.__run:
utils.log("DEBUG", "Execution not running, nothing to do",('kill',self))
return
while self.__run or self.__executing:
utils.log("DEBUG", "Sending stop execution signal",('kill',self))
self.__run = False
self.__kill_delay()
self.__kill_wait()
if wait is False:
self.__kill_exec()
utils.log("INFO", "Execution killed",('kill',self))
while not self.__cv_wait and not self.dead:
time.sleep(0.1)
utils.log("INFO", "Worker stopped.",('kill',self))
def __get_id(self):
utils.log("INFO", "Fetching instance data from CLOUD ...",
('__get_id', self))
instance_id = cloud.instance_id(self.__config, self)
utils.log("INFO", "Instance ID: '%s'" % (instance_id),
('__get_id', self))
app_id = self.__config['userdata']['app_id']
utils.log("INFO", "App ID: '%s'" % (app_id), ('__get_id', self))
token = cloud.token(self.__config)
utils.log("DEBUG", "Token: '%s'" % (token), ('__get_id', self))
return ({
'instance_id': instance_id,
'app_id': app_id,
'instance_token': token,
})
def __reset(self, done=True):
utils.log("INFO", "reseting states status",('__reset',self))
self.__status = 0
self.__run = False
if done:
utils.log("DEBUG", "reseting wait status",('__reset',self))
self.__done[:] = []
utils.log("DEBUG", "reset done",('__reset',self))
def __reset(self, done=True):
utils.log("INFO", "reseting states status",('__reset',self))
self.__status = 0
self.__run = False
if done:
utils.log("DEBUG", "reseting wait status",('__reset',self))
self.__done[:] = []
utils.log("DEBUG", "reset done",('__reset',self))
def userdata(config, manager):
ud = None
while not ud:
if not manager.running():
utils.log("WARNING", "Execution aborting, exiting ...",('userdata','cloud'))
return None
utils.log("DEBUG", "Getting userdata ...",('userdata','cloud'))
try:
ud = get_cloud_data(config['network']['userdata'])
except CLOUDNotFoundException:
utils.log("WARNING", "Userdata not found. Retrying in %s seconds"%(WAIT_RETRY),('userdata','cloud'))
time.sleep(WAIT_RETRY)
except Exception as e:
utils.log("WARNING", "User data failure, error: '%s'. Retrying in %s seconds"%(e,WAIT_RETRY),('userdata','cloud'))
time.sleep(WAIT_RETRY)
return parse_ud(ud, ["APP_ID","WS_URI","VERSION","BASE_REMOTE","GPG_KEY_URI"])
def __init_dir(self):
dirs = [
self.__config['global']['watch'],
self.__config['salt']['srv_root'],
self.__config['salt']['extension_modules'],
self.__config['salt']['cachedir'],
]
errors = []
for directory in dirs:
try:
if not os.path.isdir(directory):
os.makedirs(directory,0700)
if not os.access(directory, os.W_OK):
raise ManagerInitDirDeniedException
except ManagerInitDirDeniedException:
err = "'%s' directory not writable. FATAL"%(directory)
utils.log("ERROR", err,('__init_dir',self))
errors.append(err)
except Exception as e:
err = "Can't create '%s' directory: '%s'. FATAL"%(directory,e)
utils.log("ERROR", err,('__init_dir',self))
errors.append(err)
return (errors)
def __kill_exec(self):
if self.__executing:
utils.log("DEBUG", "Killing execution, pgid#%s"%(self.__executing.pid),('__kill_exec',self))
brk = False
while self.__executing and not brk:
try:
os.killpg(self.__executing.pid,signal.SIGKILL)
except OSError as e:
e = str(e)
if e.find("No such process"):
utils.log("INFO", "Execution killed, pgid#%s"%(self.__executing.pid),('__kill_exec',self))
brk = True
else:
utils.log("WARNING", "Error trying to kill process: %s"%(e),('__kill_exec',self))
except Exception as e:
utils.log("WARNING", "Error killing execution (probably not a problem): %s"%(e),('__kill_exec',self))
brk = True
try:
self.__executing.terminate()
except Exception: pass
time.sleep(0.1)
else:
utils.log("DEBUG", "Execution not running",('__kill_exec',self))
def __kill_exec(self):
if self.__executing:
utils.log("DEBUG", "Killing execution, pgid#%s"%(self.__executing.pid),('__kill_exec',self))
brk = False
while self.__executing and not brk:
try:
os.killpg(self.__executing.pid,signal.SIGKILL)
except OSError as e:
e = str(e)
if e.find("No such process"):
utils.log("INFO", "Execution killed, pgid#%s"%(self.__executing.pid),('__kill_exec',self))
brk = True
else:
utils.log("WARNING", "Error trying to kill process: %s"%(e),('__kill_exec',self))
except Exception as e:
utils.log("WARNING", "Error killing execution (probably not a problem): %s"%(e),('__kill_exec',self))
brk = True
try:
self.__executing.terminate()
except Exception: pass
time.sleep(0.1)
else:
utils.log("DEBUG", "Execution not running",('__kill_exec',self))
def __init_dir(self):
dirs = [
self.__config['global']['watch'],
self.__config['salt']['srv_root'],
self.__config['salt']['extension_modules'],
self.__config['salt']['cachedir'],
]
errors = []
for directory in dirs:
try:
if not os.path.isdir(directory):
os.makedirs(directory, 0700)
if not os.access(directory, os.W_OK):
raise ManagerInitDirDeniedException
except ManagerInitDirDeniedException:
err = "'%s' directory not writable. FATAL" % (directory)
utils.log("ERROR", err, ('__init_dir', self))
errors.append(err)
except Exception as e:
err = "Can't create '%s' directory: '%s'. FATAL" % (directory,
e)
utils.log("ERROR", err, ('__init_dir', self))
errors.append(err)
return (errors)
def _init_cust_ostype(self):
try:
import subprocess
config_file = self.__is_existed(['/etc/issue', '/etc/redhat-release'])
if not config_file:
raise ExecutionException("Cannot find the system config file")
cmd = 'grep -io -E "ubuntu|debian|centos|redhat|red hat|amazon" ' + config_file
process = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = process.communicate()
if process.returncode != 0:
utils.log("ERROR", "Excute cmd %s failed..."%cmd, ("_init_cust_ostype", self))
raise ExecutionException("Excute cmd %s failed"%cmd)
self._salt_opts['cust_ostype'] = out.lower().replace(" ","")
except Exception, e:
utils.log("ERROR", "Fetch custom agent's os type failed...", ("_init_cust_ostype", self))
def __update_states(self, force=False):
if self.__config['module']['mod_tag'] == "develop":
force = True
# clone states
try:
utils.clone_repo(
self.__config,
self.__config['module']['root'],
self.__config['module']['name'],
self.__config['module']['mod_repo'],
force=force)
utils.link_repo(
self.__config,
self.__config['module']['root'],
self.__config['module']['name'])
except ManagerInvalidStatesRepoException:
self.__config['runtime']['clone'] = False
else:
self.__config['runtime']['clone'] = True
try:
utils.checkout_repo(
self.__config,
self.__config['module']['root'],
self.__config['module']['name'],
self.__config['module']['mod_tag'],
self.__config['module']['mod_repo'])
except ManagerInvalidStatesRepoException:
self.__config['runtime']['tag'] = False
else:
self.__config['runtime']['tag'] = True
if self.__config['runtime']['clone'] or self.__config['runtime']['tag']:
utils.log("INFO", "Update repo succeed.",('update_states',self))
utils.bootstrap_mod(self.__config)
return True
return False
def instance_id(config, manager):
iid = None
while not iid:
if not manager.running():
utils.log("WARNING", "Execution aborting, exiting ...",('instance_id','cloud'))
return None
utils.log("DEBUG", "Getting instance id ...",('instance_id','cloud'))
error = False
try:
iid = get_os_iid()
except Exception as e:
error = True
utils.log("DEBUG", "Couldn't get OpenStack ID (%s), trying regular AWS way..."%e,('instance_id','cloud'))
if error or (not iid):
try:
iid = get_cloud_data(config['network']['instance_id'])
except CLOUDNotFoundException:
utils.log("ERROR", "Instance ID not found, retrying in '%s' seconds"%(WAIT_RETRY),('instance_id','cloud'))
time.sleep(WAIT_RETRY)
except Exception as e:
utils.log("ERROR", "Instance ID failure, unknown error: '%s', retrying in '%s' seconds"%(e,WAIT_RETRY),('instance_id','cloud'))
time.sleep(WAIT_RETRY)
return iid
def exec_salt(self, states):
"""
Transfer and exec salt state.
return result format: (result,comment,out_log), result:True/False
"""
result = False
comment = ''
out_log = ''
# check
if not states:
out_log = "Null states"
return (result, comment, out_log)
if not states or not isinstance(states, list):
out_log = "Invalid state format %s" % str(states)
return (result, comment, out_log)
# check whether contain specail module
try:
mods = self.get_mods(states)
# whether special module
inter_mods = list(set(mods).intersection(set(self.spec_mods)))
if len(inter_mods)>0:
self._enable_epel()
# pre-installed npm
if 'npm' in inter_mods:
if self.os_type in ['redhat', 'centos'] and float(self.os_release) >= 7.0 or self.os_type == 'debian':
self.__preinstall_npm()
except Exception, e:
utils.log("WARNING", "Enable epel repo failed...",("exec_salt", self))
comment = 'Enable epel repo failed'
return (result, comment, str(e))
def __update_states(self, force=False):
if self.__config['module']['mod_tag'] == "develop":
force = True
# clone states
try:
utils.clone_repo(
self.__config,
self.__config['module']['root'],
self.__config['module']['name'],
self.__config['module']['mod_repo'],
force=force)
utils.link_repo(
self.__config,
self.__config['module']['root'],
self.__config['module']['name'])
except ManagerInvalidStatesRepoException:
self.__config['runtime']['clone'] = False
else:
self.__config['runtime']['clone'] = True
try:
utils.checkout_repo(
self.__config,
self.__config['module']['root'],
self.__config['module']['name'],
self.__config['module']['mod_tag'],
self.__config['module']['mod_repo'])
except ManagerInvalidStatesRepoException:
self.__config['runtime']['tag'] = False
else:
self.__config['runtime']['tag'] = True
if self.__config['runtime']['clone'] and self.__config['runtime']['tag']:
utils.log("INFO", "Update repo succeed.",('update_states',self))
utils.bootstrap_mod(self.__config)
return True
return False
def __act_retry_hs(self, data):
utils.log("INFO", "Handshake rejected by server",
('__act_retry_hs', self))
self.__config['init'] = self.__get_id()
utils.log("INFO", "Retrying in %s seconds" % (WAIT_CONNECT),
('__act_retry_hs', self))
time.sleep(WAIT_CONNECT)
utils.log("DEBUG", "Reconnecting ...", ('__act_retry_hs', self))
self.send_json(
send.handshake(self.__config,
self.__error_proc + self.__error_dir))
def __update_ud(self):
utils.log("DEBUG", "Updating userdata variables ...",('__update_ud',self))
ud = cloud.userdata(self.__config, self)
for key in ud:
if self.__config['userdata'].get(key) != ud[key]:
if utils.update_config_file(self.__config, key, ud[key]):
utils.log("INFO", "%s has been updated from %s to %s"%(key,self.__config['userdata'].get(key),ud[key]),
('__update_ud',self))
self.__config['userdata'][key] = ud[key]
if key in self.__config['global']['token_reset']:
utils.reset_token(self.__config)
utils.log("DEBUG", "Userdata variables updated",('__update_ud',self))
def __update_ud(self):
utils.log("DEBUG", "Updating userdata variables ...",
('__update_ud', self))
ud = cloud.userdata(self.__config, self)
for key in ud:
if self.__config['userdata'].get(key) != ud[key]:
if utils.update_config_file(self.__config, key, ud[key]):
utils.log(
"INFO", "%s has been updated from %s to %s" %
(key, self.__config['userdata'].get(key), ud[key]),
('__update_ud', self))
self.__config['userdata'][key] = ud[key]
if key in self.__config['global']['token_reset']:
utils.reset_token(self.__config)
utils.log("DEBUG", "Userdata variables updated", ('__update_ud', self))
def __mount_proc_try(self, proc, directory=False):
utils.log("WARNING", "procfs not present, attempting to mount...",('__mount_proc_try',self))
if not directory:
try:
os.makedirs(proc,0755)
except Exception as e:
err = "Can't create '%s' directory: '%s'. FATAL"%(proc,e)
utils.log("ERROR", err,('__mount_proc_try',self))
return err
p = subprocess.Popen(['mount','-t','proc','proc',proc])
if p.wait():
err = "Can't mount procfs on '%s'. FATAL"%(proc)
utils.log("ERROR", err,('__mount_proc_try',self))
return err
return None
def __mount_proc_try(self, proc, directory=False):
utils.log("WARNING", "procfs not present, attempting to mount...",
('__mount_proc_try', self))
if not directory:
try:
os.makedirs(proc, 0755)
except Exception as e:
err = "Can't create '%s' directory: '%s'. FATAL" % (proc, e)
utils.log("ERROR", err, ('__mount_proc_try', self))
return err
p = subprocess.Popen(['mount', '-t', 'proc', 'proc', proc])
if p.wait():
err = "Can't mount procfs on '%s'. FATAL" % (proc)
utils.log("ERROR", err, ('__mount_proc_try', self))
return err
return None
def load(self, version=None, states=None):
self.__version = version
self.__recipe_count = (self.__recipe_count+1 if self.__recipe_count < RECIPE_COUNT_RESET else 0)
exp = None
try:
if states:
utils.log("INFO", "Loading new states",('load',self))
tmp_s = self.__states
self.__states = copy.deepcopy(states)
del tmp_s
else:
utils.log("INFO", "No change in states",('load',self))
utils.log("DEBUG", "Allow to run",('load',self))
self.__run = True
except Exception as e:
raise OpsAgentException(e)
def load(self, version=None, states=None):
self.__version = version
self.__recipe_count = (self.__recipe_count+1 if self.__recipe_count < RECIPE_COUNT_RESET else 0)
exp = None
try:
if states:
utils.log("INFO", "Loading new states",('load',self))
tmp_s = self.__states
self.__states = copy.deepcopy(states)
del tmp_s
else:
utils.log("INFO", "No change in states",('load',self))
utils.log("DEBUG", "Allow to run",('load',self))
self.__run = True
except Exception as e:
raise OpsAgentException(e)
def userdata(config, manager):
ud = None
while not ud:
if not manager.running():
utils.log("WARNING", "Execution aborting, exiting ...",
('userdata', 'cloud'))
return None
utils.log("DEBUG", "Getting userdata ...", ('userdata', 'cloud'))
try:
ud = get_cloud_data(config['network']['userdata'])
except CLOUDNotFoundException:
utils.log(
"WARNING",
"Userdata not found. Retrying in %s seconds" % (WAIT_RETRY),
('userdata', 'cloud'))
time.sleep(WAIT_RETRY)
except Exception as e:
utils.log(
"WARNING",
"User data failure, error: '%s'. Retrying in %s seconds" %
(e, WAIT_RETRY), ('userdata', 'cloud'))
time.sleep(WAIT_RETRY)
return parse_ud(
ud, ["APP_ID", "WS_URI", "VERSION", "BASE_REMOTE", "GPG_KEY_URI"])
def token(config):
f = config['global'].get('token')
utils.log("DEBUG", "Getting token located in %s" % (f), ('token', 'cloud'))
t = ''
try:
with open(f, 'r') as f:
t = f.read()
except Exception as e:
utils.log("WARNING",
"Can't get token file (%s): %s, updating token" % (f, e),
('token', 'cloud'))
utils.reset_token(config)
try:
with open(f, 'r') as f:
t = f.read()
except Exception as e:
utils.log("ERROR", "Can't get token file (%s): %s" % (f, e),
('token', 'cloud'))
return t
def run(self):
while not self.__abort:
self.cv_e.acquire()
try:
if not self.__run and not self.__abort:
utils.log("INFO", "Waiting for recipes ...",('run',self))
self.__cv_wait = True
self.cv_e.wait()
self.__cv_wait = False
utils.log("DEBUG", "Ready to go ...",('run',self))
self.__runner()
except Exception as e:
utils.log("ERROR", "Unexpected error: %s"%(e),('run',self))
self.__reset()
self.cv_e.release()
self.dead = True
utils.log("WARNING", "Exiting...",('run',self))
if self.__manager:
utils.log("INFO", "Stopping manager...",('run',self))
self.__manager.stop()
utils.log("INFO", "Manager stopped",('run',self))
utils.log("WARNING", "Terminated",('run',self))
def __runner(self):
utils.log("INFO", "Running StatesWorker ...",('__runner',self))
while self.__run:
# Init
if not self.__runner_init():
utils.log("WARNING", "Init failed, retrying current state in %s seconds"%(WAIT_STATE_RETRY),('__runner',self))
time.sleep(WAIT_STATE_RETRY)
continue
# Execute state
(result,comment,out_log) = self.__run_state()
self.wait_event_e.set()
# Transmit results
if self.__run:
utils.log("INFO", "Execution complete, reporting logs to backend",('__runner',self))
# send result to backend
sent = self.__send(send.statelog(init=self.__config['init'],
version=self.__version,
sid=self.__states[self.__status]['id'],
result=result,
comment=comment,
out_log=out_log))
# state succeed
if result == SUCCESS and sent:
# global status iteration
self.__status += 1
if self.__status >= len(self.__states):
utils.log("INFO", "All good, last state succeed! Back to first one",('__runner',self))
# not terminating, wait before next round
if not self.__abort:
self.__recipe_delay()
self.__status = 0
# terminating ...
if self.__abort: # don't "else" as abort may happen during the delay
self.__run = False
else:
time.sleep(WAIT_STATE)
utils.log("INFO", "All good, switching to next state",('__runner',self))
# state failed
else:
self.__status = 0
if self.__abort:
self.__run = False
else:
utils.log("WARNING", "Something went wrong, retrying current state in %s seconds"%(WAIT_STATE_RETRY),('__runner',self))
time.sleep(WAIT_STATE_RETRY)
utils.log("WARNING", "Execution aborted",('__runner',self))
