def post(self, service):
""" Create a new service object """
try:
path = ServiceController.servicePath(service)
# skip if the path already exists
if os.path.exists(path) and os.path.isdir(path):
return doneResult(request, response, httpStatus = 201, result = service, controller = self)
else:
serviceutil.createServiceIfNeeded(service)
# save metadata from payload if any
if request.body:
lcm_meta = json.loads(request.body)
serviceutil.updateLcmMeta(service, lcm_meta)
return doneResult(request, response, result = service, controller = self)
except OSError:
return errorResult(request, response, Errors.SERVICE_EXISTS,
"Service(%s) already exists(or %s)" % (service, traceback.format_exc(2)),
controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when posting services %s - %s' %
(str(excep), traceback.format_exc(2)),
controller = self)
def post(self, service):
""" Create a new service object """
try:
LOG.info('Got a post request for service ' + service)
path = ServiceController.servicePath(service)
# skip if the path already exists
if os.path.exists(path) and os.path.isdir(path):
return doneResult(request,
response,
httpStatus=201,
result=service,
controller=self)
os.makedirs(path)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'installed-packages'))
os.makedirs(os.path.join(path, 'modules'))
os.makedirs(os.path.join(path, 'downloaded-packages'))
os.makedirs(os.path.join(path, '.appdata'))
os.makedirs(os.path.join(path, '.data'))
utils.rchmod(os.path.join(path, '.appdata'), '+rwx')
# verify that the path exists
if (not os.path.isdir(path)):
return errorResult(request,
response,
Errors.UNKNOWN_ERROR,
"Service(%s) was not created" % service,
controller=self)
return doneResult(request,
response,
result=service,
controller=self)
except OSError:
return errorResult(request,
response,
Errors.SERVICE_EXISTS,
"Service(%s) already exists(or %s)" %
(service, traceback.format_exc(2)),
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error when posting services %s - %s' %
(str(excep), traceback.format_exc(2)),
controller=self)
def listPackages(self):
""" list content of packages folder
"""
result = {}
pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
result['packages'] = pkgs
return doneResult(request, response, result = result, controller = self)
def activate(self, service, manifest):
""" activate manifest, if already active then skip """
from agent.lib.agent_thread.activate_manifest import ActivateManifest
LOG.info('activateManifest for service(%s) with body: %s', service, request.body)
try:
appGlobal = config['pylons.app_globals']
if manifestutil.getActiveManifest(service) == manifest:
return doneResult(request, response, controller=self)
else:
if request.body:
pushedData = json.loads(request.body)
serviceutil.updateLcmMeta(service, pushedData)
mf_path = os.path.join(manifestutil.manifestPath(service, manifest))
if (not os.path.exists(mf_path)):
return errorResult(request, response, Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' % (service, manifest),
controller=self)
LOG.debug('Manifest path exists: %s' % (mf_path))
activateThread = ActivateManifest(appGlobal.threadMgr, service, manifest,
action=ActivateManifest.ACTION_ACTIVATION)
self.injectJobCtx(activateThread)
activateThread.start()
activateThread.threadMgrEvent.wait()
return statusResult(request, response, activateThread, controller=self)
except Exception as excep:
msg = 'Unknown error for activateManifest(%s/%s) - %s - %s' % (service, manifest, str(excep), traceback.format_exc(2))
return errorResult(request, response, error=Errors.UNKNOWN_ERROR,
errorMsg=msg, controller=self)
def deployservice(self, service):
""" activate manifest, if already active then skip """
LOG.info('deploy service for service(%s) with body: %s', service, request.body)
try:
appGlobal = config['pylons.app_globals']
# parse the body
if (request.body == ""):
return errorResult(request, response, Errors.INVALID_REQUEST,
'No body found in post command',
controller = self)
requestjson = json.loads(request.body)
manifest = requestjson['manifest']
packages = requestjson['package']
skipProp = asbool(requestjson['skipProp']) if 'skipProp' in requestjson else configutil.getConfigAsBool('download_skip_prop')
skipActivation = asbool(requestjson['skipActivation']) if 'skipActivation' in requestjson else False
# activate manifest if not already activated
if manifestutil.getActiveManifest(service) == manifest:
return doneResult(request, response, controller=self)
deployServiceThread = DeployService(appGlobal.threadMgr, service, manifest, packages, skipProp = skipProp, skipActivation = skipActivation)
self.injectJobCtx(deployServiceThread)
deployServiceThread.start()
deployServiceThread.threadMgrEvent.wait()
return statusResult(request, response, deployServiceThread, controller = self)
except Exception as excep:
msg = 'Unknown error for activateManifest(%s/%s) - %s - %s' % (service, manifest, str(excep), traceback.format_exc(2))
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = msg, controller = self)
def addKey(self, key):
""" adding a new public key to agent"""
try:
pubKeyDir = os.path.join(manifestutil.appDataPath('agent'), 'secure')
pubKeyFiles = [f for f in os.listdir(pubKeyDir) if re.match(r'.*\.pub', f)]
if not key.endswith('.pub'):
return errorResult(request, response, Errors.INVALID_REQUEST, 'Key %s must end with .pub' % key, controller = self)
if key in pubKeyFiles:
return errorResult(request, response, Errors.INVALID_REQUEST, 'Key %s already exist' % key, controller = self)
if (not request.body or request.body == ""):
LOG.error('invalid body found in post command')
return errorResult(request, response, 10001, 'No body found', controller = self)
body = json.loads(request.body)
if 'content' not in body:
return errorResult(request, response, Errors.INVALID_REQUEST, 'No key content found', controller = self)
keycontent = body['content']
filepath = os.path.join(pubKeyDir, key)
utils.writeFile(filepath, keycontent)
return doneResult(request, response, controller = self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error adding key %s, %s - %s' %
(key, str(excp), traceback.format_exc(2)),
controller = self)
def getRoutes(self):
""" print all available routes """
mapper = config['routes.map']
apistr = mapper.__str__()
apis = apistr.splitlines()
result = map(lambda x: x.strip(), apis)
return doneResult(request, response, result=result, controller = self)
def delete(self, module):
""" Delete a new service object """
try:
LOG.info('Got a delete request for module ' + module)
path = manifestutil.modulePath('agent', module)
if (not os.path.exists(path) and not os.path.isdir(path)):
return doneResult(request, response, controller=self)
# start the delete thread
appGlobal = config['pylons.app_globals']
deleteThread = ModuleDelete(appGlobal.threadMgr, module)
self.injectJobCtx(deleteThread)
deleteThread.start()
deleteThread.threadMgrEvent.wait()
return statusResult(request,
response,
deleteThread,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error when deleting module(%s) %s - %s' %
(module, str(excep), traceback.format_exc(2)),
controller=self)
def get(self, module):
""" Get a new service object """
try:
# make sure the service path exists
path = manifestutil.modulePath('agent', module)
if (not os.path.exists(path)):
return errorResult(request,
response,
error=Errors.SERVICE_NOT_FOUND,
errorMsg='Unable to find module (%s)' %
module,
controller=self)
result = {}
modulePackage = readlink(path)
result['package'] = modulePackage
return doneResult(request,
response,
result=result,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error when getting module (%s) %s - %s' %
(module, str(excep), traceback.format_exc(2)),
controller=self)
def delete(self):
""" un-enroll agent """
LOG.info('unenroll agent')
try:
with self.lock:
appGlobal = config['pylons.app_globals']
if hasattr(appGlobal, 'hwPath'):
setattr(appGlobal, 'hwPath', None)
manifestutil.updateServiceMetaFile('agent', {'hwPath': None})
appGlobal.enrolled = False
appGlobal.agentMonitor.reloadMonitors()
return doneResult(request, response, controller=self)
return errorResult(
request,
response,
error=Errors.THREAD_ALREADY_ADDED,
errorMsg="Cann't acquire lock for un-enrollment",
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error for un-enroll - %s - %s' %
(str(excep), traceback.format_exc(2)),
controller=self)
def activatemanifest(self, service):
""" activate manifest, if already active then skip """
LOG.info('activateManifest for service(%s) with body: %s', service, request.body)
try:
appGlobal = config['pylons.app_globals']
manifest = ''
requestjson = json.loads(request.body)
manifest = requestjson['manifest']
force = asbool(requestjson['force']) if 'force' in requestjson else False
if not force and manifestutil.getActiveManifest(service) == manifest:
return doneResult(request, response, controller=self)
else:
mf_path = os.path.join(ManifestController.manifestPath(service, manifest))
if (not os.path.exists(mf_path)):
return errorResult(request, response, Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' % (service, manifest),
controller = self)
LOG.debug('Manifest path exists: %s' % (mf_path))
activateThread = ActivateManifest(appGlobal.threadMgr, service, manifest)
self.injectJobCtx(activateThread)
activateThread.start()
activateThread.threadMgrEvent.wait()
return statusResult(request, response, activateThread, controller = self)
except Exception as excep:
msg = 'Unknown error for activateManifest(%s/%s) - %s - %s' % (service, manifest, str(excep), traceback.format_exc(2))
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = msg, controller = self)
def getRoutes(self):
""" print all available routes """
mapper = config['routes.map']
apistr = mapper.__str__()
apis = apistr.splitlines()
result = map(lambda x: x.strip(), apis)
return doneResult(request, response, result=result, controller = self)
def reloadmonitors(self):
""" reload all monitors.
"""
appGlobal = pylons.config['pylons.app_globals']
appGlobal.agentMonitor.reloadMonitors()
return doneResult(request, response, controller = self)
def delete(self, module):
""" Delete a new service object """
try:
LOG.info("Got a delete request for module " + module)
path = manifestutil.modulePath("agent", module)
if not os.path.exists(path) and not os.path.isdir(path):
return doneResult(request, response, controller=self)
# start the delete thread
appGlobal = config["pylons.app_globals"]
deleteThread = ModuleDelete(appGlobal.threadMgr, module)
self.injectJobCtx(deleteThread)
deleteThread.start()
deleteThread.threadMgrEvent.wait()
return statusResult(request, response, deleteThread, controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg="Unknown error when deleting module(%s) %s - %s"
% (module, str(excep), traceback.format_exc(2)),
controller=self,
)
def getSysMetrics(self):
diskstr = request.params.get('disks', None)
disks = diskstr.split(',') if diskstr else []
ifcstr = request.params.get('ifcs', None)
ifcs = ifcstr.split(',') if ifcstr else ['eth0']
result = SysMetricsController.collectMetrics(disks, ifcs)
return doneResult(request, response, result = result, controller = self)
def get(self, module):
""" Get a new service object """
try:
# make sure the service path exists
path = manifestutil.modulePath("agent", module)
if not os.path.exists(path):
return errorResult(
request,
response,
error=Errors.SERVICE_NOT_FOUND,
errorMsg="Unable to find module (%s)" % module,
controller=self,
)
result = {}
modulePackage = readlink(path)
result["package"] = modulePackage
return doneResult(request, response, result=result, controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg="Unknown error when getting module (%s) %s - %s"
% (module, str(excep), traceback.format_exc(2)),
controller=self,
)
def listPackages(self):
""" list content of packages folder
"""
result = {}
pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
result['packages'] = pkgs
return doneResult(request, response, result = result, controller = self)
def updateMetadata(self, service):
""" create or update .metadata file for a service """
# now connect to state server to store metadata for the service in .metadata
metadata = None
if request.body:
# two flavor of updatemetadata call, one with body, one without
body = json.loads(request.body)
if 'metadata' in body:
metadata = body['metadata']
try:
result = {}
appGlobal = pylons.config['pylons.app_globals']
if metadata is not None and 'monitoring.metric.tags' in metadata:
appGlobal.agentMonitor.reloadMonitors()
result = manifestutil.updateServiceMetaFile(service, metadata)
return doneResult(request, response, result = result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when update service metadata (%s) %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller = self)
def reloadmonitors(self):
""" reload all monitors.
"""
appGlobal = pylons.config['pylons.app_globals']
appGlobal.agentMonitor.reloadMonitors()
return doneResult(request, response, controller = self)
def post(self):
""" enroll agent """
try:
body = json.loads(request.body)
if 'hardwarePath' not in body:
LOG.error('failed when enrolling: hardwarePath is not provided')
return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg='hardwarePath is not specified', controller = self)
with self.lock:
LOG.info('enroll agent with hardwarePath %s' % (body['hardwarePath']))
hardwarePath = body['hardwarePath']
manifestutil.updateServiceMetaFile('agent', {'hwPath': hardwarePath,})
appGlobal = config['pylons.app_globals']
appGlobal.hwPath = hardwarePath
appGlobal.enrolled = True
appGlobal.agentMonitor.reloadMonitors()
return doneResult(request, response, controller = self)
return errorResult(request, response,
error = Errors.THREAD_ALREADY_ADDED,
errorMsg = "Cann't acquire lock for enrollment",
controller = self )
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error for enroll - %s - %s' %
(str(excep), traceback.format_exc(2)),
controller = self)
def post(self, service, manifest):
""" Create a new service object """
from agent.lib.agent_thread.manifest_create import ManifestCreate
try:
LOG.info('Post for service (%s) and manifest (%s) with body: %s',
service, manifest, request.body)
# check to see if the manifest already exists
path = ManifestController.manifestPath(service, manifest)
if (os.path.isdir(path)):
return doneResult(request, response, httpStatus = 201, controller = self)
# parse the body
if (request.body == ""):
return errorResult(request, response, Errors.MANIFEST_PACKAGE_PARSING_ERROR,
'No body found in post command',
controller = self)
body = json.loads(request.body)
packages = body['package']
forcedPackages = body['forcePackageName'] if 'forcePackageName' in body else None
skipProp = asbool(body['skipProp']) if 'skipProp' in body else configutil.getConfigAsBool('download_skip_prop')
LOG.debug('pkgs = %s, %s', packages, forcedPackages)
# parse the package list
for idx, package in enumerate(packages):
# to support reuse of an package from an existing manifest (active if possible)
# without sending the complete package location in request body
if package.startswith('/'):
packageRef = package
tokens = package.split('/')
pkgnamePrefix = tokens[-1].rstrip()
fullPkgLoc = manifestutil.getPackageByName(service, manifest = None, pkgnamePrefix = pkgnamePrefix)
if fullPkgLoc is None:
return errorResult(request, response, Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'manifest (%s/%s) package (%s) does not exist' %
(service, manifest, packages), controller = self)
else:
LOG.info('expanding package reuse ref %s with full package location %s' % (packageRef, fullPkgLoc))
packages[idx] = fullPkgLoc
appGlobal = config['pylons.app_globals']
# start a thread to create the package
manThread = ManifestCreate(appGlobal.threadMgr, service, manifest, packages, forcePackages = forcedPackages, skipProp = skipProp)
self.injectJobCtx(manThread)
manThread.start()
manThread.threadMgrEvent.wait()
return statusResult(request, response, manThread, controller = self)
except AgentException as excep:
return errorResult(request, response, error = excep.getCode(), errorMsg = excep.getMsg(), controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error for activateManifest(%s/%s) - %s - %s' %
(service, manifest, str(excep), traceback.format_exc(2)),
controller = self)
def post(self, service):
""" Create a new service object """
try:
LOG.info("Got a post request for service " + service)
path = ServiceController.servicePath(service)
# skip if the path already exists
if os.path.exists(path) and os.path.isdir(path):
return doneResult(request, response, httpStatus=201, result=service, controller=self)
os.makedirs(path)
os.makedirs(os.path.join(path, "manifests"))
os.makedirs(os.path.join(path, "installed-packages"))
os.makedirs(os.path.join(path, "modules"))
os.makedirs(os.path.join(path, "downloaded-packages"))
os.makedirs(os.path.join(path, ".appdata"))
os.makedirs(os.path.join(path, ".data"))
utils.rchmod(os.path.join(path, ".appdata"), "+rwx")
# verify that the path exists
if not os.path.isdir(path):
return errorResult(
request, response, Errors.UNKNOWN_ERROR, "Service(%s) was not created" % service, controller=self
)
return doneResult(request, response, result=service, controller=self)
except OSError:
return errorResult(
request,
response,
Errors.SERVICE_EXISTS,
"Service(%s) already exists(or %s)" % (service, traceback.format_exc(2)),
controller=self,
)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg="Unknown error when posting services %s - %s" % (str(excep), traceback.format_exc(2)),
controller=self,
)
def getServicesSummary(self):
""" service summary, all about service in one call """
try:
services_data = serviceutil.getServiceSummary()
return doneResult(request, response, result = services_data, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when get services summary %s - %s' %
(str(excep), traceback.format_exc(2)),
controller = self)
def listServices(self):
""" list all the services in the agent """
try:
services = ServiceController.getServices()
return doneResult(request, response, result = services, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when listing services %s - %s' %
(str(excep), traceback.format_exc(2)),
controller = self)
def getMetadata(self, service):
""" get .metadata file for a service """
try:
result = manifestutil.readJsonServiceMeta(service)
return doneResult(request, response, result = result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error get service metadata (%s) %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller = self)
def getSizeOfMgrs(self):
""" get size of object """
appGlobal = config['pylons.app_globals']
result = {}
result['threadmgr'] = asizeof(appGlobal.threadMgr)
result['packagemgr'] = asizeof(appGlobal.packageMgr)
result['montior'] = asizeof(appGlobal.agentMonitor)
result['all'] = asizeof(appGlobal)
return doneResult(request, response, result = result, controller = self)
def getSizeOfMgrs(self):
""" get size of object """
appGlobal = config['pylons.app_globals']
result = {}
result['threadmgr'] = asizeof(appGlobal.threadMgr)
result['packagemgr'] = asizeof(appGlobal.packageMgr)
result['montior'] = asizeof(appGlobal.agentMonitor)
result['all'] = asizeof(appGlobal)
return doneResult(request, response, result = result, controller = self)
def getConfig(self):
""" get config overrides """
LOG.info('get config overrides for agent')
result = {}
try:
result = manifestutil.readJsonServiceMeta('agent', ['configs'])
return doneResult(request, response, result = result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when clean agent configs %s - %s' %
(str(excep), traceback.format_exc(2)), controller = self)
def getConfig(self):
""" get config overrides """
LOG.info('get config overrides for agent')
result = {}
try:
result = manifestutil.readJsonServiceMeta('agent', ['configs'])
return doneResult(request, response, result = result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when clean agent configs %s - %s' %
(str(excep), traceback.format_exc(2)), controller = self)
def cancelSelfUpdate(self):
""" cancel an in-flight agent selfupdate
"""
appGlobal = config['pylons.app_globals']
catThreads = appGlobal.threadMgr.getThreadByCat(AgentUpdate.THREAD_NAME, fastbreak = False)
result = {}
for catThread in catThreads:
catThread.stop()
result['uuid'] = catThread.getUuid()
return doneResult(request, response, result = result, controller = self)
def cancelSelfUpdate(self):
""" cancel an in-flight agent selfupdate
"""
appGlobal = config['pylons.app_globals']
catThreads = appGlobal.threadMgr.getThreadByCat(AgentUpdate.THREAD_NAME, fastbreak = False)
result = {}
for catThread in catThreads:
catThread.stop()
result['uuid'] = catThread.getUuid()
return doneResult(request, response, result = result, controller = self)
def listKeys(self):
""" list existing keys in agent """
try:
pubKeyDir = os.path.join(manifestutil.appDataPath('agent'), 'secure')
pubKeyFiles = [f for f in os.listdir(pubKeyDir) if re.match(r'.*\.pub', f)]
return doneResult(request, response, result = pubKeyFiles, controller = self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error listing key %s - %s' %
(str(excp), traceback.format_exc(2)),
controller = self)
def listModules(self):
""" list all the modules in the agent """
try:
LOG.info('Got a list service request')
modules = manifestutil.getModules()
return doneResult(request, response, result = modules, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when listing modules %s - %s' %
(str(excep), traceback.format_exc(2)),
controller = self)
def deployservice(self, service):
""" activate manifest, if already active then skip """
LOG.info('deploy service for service(%s) with body: %s', service,
request.body)
try:
appGlobal = config['pylons.app_globals']
# parse the body
if (request.body == ""):
return errorResult(request,
response,
Errors.INVALID_REQUEST,
'No body found in post command',
controller=self)
requestjson = json.loads(request.body)
manifest = requestjson['manifest']
packages = requestjson['package']
skipProp = asbool(
requestjson['skipProp']
) if 'skipProp' in requestjson else configutil.getConfigAsBool(
'download_skip_prop')
skipActivation = asbool(
requestjson['skipActivation']
) if 'skipActivation' in requestjson else False
# activate manifest if not already activated
if manifestutil.getActiveManifest(service) == manifest:
return doneResult(request, response, controller=self)
deployServiceThread = DeployService(appGlobal.threadMgr,
service,
manifest,
packages,
skipProp=skipProp,
skipActivation=skipActivation)
self.injectJobCtx(deployServiceThread)
deployServiceThread.start()
deployServiceThread.threadMgrEvent.wait()
return statusResult(request,
response,
deployServiceThread,
controller=self)
except Exception as excep:
msg = 'Unknown error for activateManifest(%s/%s) - %s - %s' % (
service, manifest, str(excep), traceback.format_exc(2))
return errorResult(request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg=msg,
controller=self)
def get(self, service):
""" Get a new service object """
try:
from agent.lib.agent_thread.manifest_create import ManifestCreate
# make sure the service path exists
path = ServiceController.servicePath(service)
if (not os.path.exists(path)):
return errorResult(request,
response,
error=Errors.SERVICE_NOT_FOUND,
errorMsg='Unable to find service (%s)' %
service,
controller=self)
path = ServiceController.manifestPath(service)
activeManifest = None
manifestList = []
for manifest in os.listdir(path):
if (ManifestCreate.isInProgress(manifest)):
continue
manifestPath = os.path.join(path, manifest)
if (manifest == 'active'):
activeLink = readlink(manifestPath)
if (activeLink == None):
manifestList.append(manifest)
else:
activeManifest = os.path.basename(activeLink)
else:
manifestList.append(manifest)
result = {}
manifestList.sort()
result['manifest'] = manifestList
result['activemanifest'] = activeManifest
return doneResult(request,
response,
result=result,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error when getting service (%s) %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller=self)
def get(self, service):
""" Get a new service object """
try:
from agent.lib.agent_thread.manifest_create import ManifestCreate
# make sure the service path exists
path = ServiceController.servicePath(service)
if not os.path.exists(path):
return errorResult(
request,
response,
error=Errors.SERVICE_NOT_FOUND,
errorMsg="Unable to find service (%s)" % service,
controller=self,
)
path = ServiceController.manifestPath(service)
activeManifest = None
manifestList = []
for manifest in os.listdir(path):
if ManifestCreate.isInProgress(manifest):
continue
manifestPath = os.path.join(path, manifest)
if manifest == "active":
activeLink = readlink(manifestPath)
if activeLink == None:
manifestList.append(manifest)
else:
activeManifest = os.path.basename(activeLink)
else:
manifestList.append(manifest)
result = {}
manifestList.sort()
result["manifest"] = manifestList
result["activemanifest"] = activeManifest
return doneResult(request, response, result=result, controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg="Unknown error when getting service (%s) %s - %s"
% (service, str(excep), traceback.format_exc(2)),
controller=self,
)
def listPackages(self, service):
""" list installed packages in this service"""
result = {}
installedRoot = manifestutil.installedPkgRootPath(service)
pkgs = [ packageDir for packageDir in os.listdir(installedRoot) ]
pkgsVersions = []
for pkg in pkgs:
pkgVersions = {}
pkgVersions['pkgName'] = os.path.basename(pkg)
pkgVersions['pkgVersion'] = [ os.path.basename(verDir) for verDir in os.listdir(os.path.join(installedRoot, pkg)) ]
pkgsVersions.append(pkgVersions)
result['packages'] = pkgsVersions
return doneResult(request, response, result = result, controller = self)
def dumpMemory(self):
""" dump what's in the class in memory """
dumpFile = os.path.join(manifestutil.manifestPath('agent'), 'agent', 'logs', 'memory.pickle')
with open(dumpFile, 'w') as dump:
for obj in gc.get_objects():
i = id(obj)
size = sys.getsizeof(obj, 0)
referents = [id(o) for o in gc.get_referents(obj) if hasattr(o, '__class__')]
if hasattr(obj, '__class__'):
cls = str(obj.__class__)
cPickle.dump({'id': i, 'class': cls, 'size': size, 'referents': referents}, dump)
return doneResult(request, response, controller = self)
def dumpMemory(self):
""" dump what's in the class in memory """
dumpFile = os.path.join(manifestutil.manifestPath('agent'), 'agent', 'logs', 'memory.pickle')
with open(dumpFile, 'w') as dump:
for obj in gc.get_objects():
i = id(obj)
size = sys.getsizeof(obj, 0)
referents = [id(o) for o in gc.get_referents(obj) if hasattr(o, '__class__')]
if hasattr(obj, '__class__'):
cls = str(obj.__class__)
cPickle.dump({'id': i, 'class': cls, 'size': size, 'referents': referents}, dump)
return doneResult(request, response, controller = self)
def cleanConfig(self):
""" clean all configs """
LOG.info('clean agent config overrides')
result = {}
try:
result = manifestutil.updateServiceMetaFile('agent', {'configs': None})
return doneResult(request, response, result = result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when clean agent configs %s - %s' %
(str(excep), traceback.format_exc(2)), controller = self)
finally:
# reload config overrides
configutil.loadConfigOverrides()
def cleanConfig(self):
""" clean all configs """
LOG.info('clean agent config overrides')
result = {}
try:
result = manifestutil.updateServiceCatMeta('agent', manifestutil.CFG_META, None)
return doneResult(request, response, result = result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when clean agent configs %s - %s' %
(str(excep), traceback.format_exc(2)), controller = self)
finally:
# reload config overrides
configutil.loadConfigOverrides()
def deletePackage(self, package):
""" secret !!hack!! API to delete one cronus package in the packages folder. """
try:
LOG.info('secret delete called for package %s' % package)
packagesPath = PackageMgr.packagePath()
thisPkgPath = os.path.join(packagesPath, package + '.cronus')
thisPropPath = os.path.join(packagesPath, package + '.cronus.prop')
thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath)
PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath)
return doneResult(request, response, controller=self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when deleting package %s, %s - %s' %
(package, str(excp), traceback.format_exc(2)),
controller = self)
def deletePackage(self, package):
""" secret !!hack!! API to delete one cronus package in the packages folder. """
try:
LOG.info('secret delete called for package %s' % package)
packagesPath = PackageMgr.packagePath()
thisPkgPath = os.path.join(packagesPath, package + '.cronus')
thisPropPath = os.path.join(packagesPath, package + '.cronus.prop')
thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath)
PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath)
return doneResult(request, response, controller=self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when deleting package %s, %s - %s' %
(package, str(excp), traceback.format_exc(2)),
controller = self)
def getMetadata(self, service):
""" get .metadata file for a service """
try:
result = manifestutil.readJsonServiceMeta(service)
return doneResult(request,
response,
result=result,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error get service metadata (%s) %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller=self)
def listPackages(self, service):
""" list installed packages in this service"""
result = {}
installedRoot = manifestutil.installedPkgRootPath(service)
pkgs = [packageDir for packageDir in os.listdir(installedRoot)]
pkgsVersions = []
for pkg in pkgs:
pkgVersions = {}
pkgVersions['pkgName'] = os.path.basename(pkg)
pkgVersions['pkgVersion'] = [
os.path.basename(verDir)
for verDir in os.listdir(os.path.join(installedRoot, pkg))
]
pkgsVersions.append(pkgVersions)
result['packages'] = pkgsVersions
return doneResult(request, response, result=result, controller=self)
def listServices(self):
""" list all the services in the agent """
try:
LOG.info('Got a list service request')
services = ServiceController.getServices()
return doneResult(request,
response,
result=services,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error when listing services %s - %s' %
(str(excep), traceback.format_exc(2)),
controller=self)
def activatemanifest(self, service):
""" activate manifest, if already active then skip """
LOG.info('activateManifest for service(%s) with body: %s', service,
request.body)
try:
appGlobal = config['pylons.app_globals']
manifest = ''
requestjson = json.loads(request.body)
manifest = requestjson['manifest']
force = asbool(
requestjson['force']) if 'force' in requestjson else False
if not force and manifestutil.getActiveManifest(
service) == manifest:
return doneResult(request, response, controller=self)
else:
mf_path = os.path.join(
ManifestController.manifestPath(service, manifest))
if (not os.path.exists(mf_path)):
return errorResult(request,
response,
Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' %
(service, manifest),
controller=self)
LOG.debug('Manifest path exists: %s' % (mf_path))
activateThread = ActivateManifest(appGlobal.threadMgr, service,
manifest)
self.injectJobCtx(activateThread)
activateThread.start()
activateThread.threadMgrEvent.wait()
return statusResult(request,
response,
activateThread,
controller=self)
except Exception as excep:
msg = 'Unknown error for activateManifest(%s/%s) - %s - %s' % (
service, manifest, str(excep), traceback.format_exc(2))
return errorResult(request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg=msg,
controller=self)
def get(self, service, manifest):
""" Get a new service object """
LOG.info('Get for service (%s) and manifest (%s)', service, manifest)
try:
# first check that the manifest directory exists
path = ManifestController.manifestPath(service, manifest)
if (not os.path.isdir(path)):
return errorResult(request,
response,
Errors.MANIFEST_NOT_FOUND,
'manifest (%s/%s) missing service' %
(service, manifest),
controller=self)
# now go through the list of packages in the manifest
packages = []
packageLinkNames = glob.glob(
os.path.join(self.manifestPath(service, manifest), '*'))
for packageLink in packageLinkNames:
package = readlink(packageLink)
LOG.debug('Get package (%s) in manifest (%s)', package,
manifest)
# deal with the case where the package has a / or \ (for windoz) at the end
package = package.rstrip('/')
package = package.rstrip('\\')
# the name of the package can be constructed from the last two path components
(head, version) = os.path.split(package)
(head, name) = os.path.split(head)
LOG.debug('Add package %s-%s.cronus' % (name, version))
packages.append('%s-%s.cronus' % (name, version))
except OSError as excp:
return errorResult(request,
response,
Errors.MANIFEST_PATH_ERROR,
'Manifest(%s, %s) path error: %s' %
(service, manifest, str(excp)),
controller=self)
return doneResult(request, response, result=packages, controller=self)
def deployservice(self, service):
""" activate manifest, if already active then skip """
LOG.info('deploy service for service(%s) with body: %s', service, request.body)
manifest = None
try:
appGlobal = config['pylons.app_globals']
# parse the body
if (request.body == ""):
return errorResult(request, response, Errors.INVALID_REQUEST,
'No body found in post command',
controller = self)
requestjson = json.loads(request.body)
packages = requestjson['package']
if 'manifest' in requestjson:
manifest = requestjson['manifest']
else:
manifest = PackageUtil.getPackageVersion(packages[-1])
serviceutil.createServiceIfNeeded(service)
# activate manifest if not already activated
if manifestutil.getActiveManifest(service) == manifest:
return doneResult(request, response, controller=self)
else:
# save metadata from payload
pushedData = {}
pushedData.update(requestjson)
for key in ['manifest', 'package']:
if key in pushedData:
del pushedData[key]
serviceutil.updateLcmMeta(service, pushedData)
# deploy
deployServiceThread = DeployService(appGlobal.threadMgr, service, manifest, packages)
self.injectJobCtx(deployServiceThread)
deployServiceThread.start()
deployServiceThread.threadMgrEvent.wait()
return statusResult(request, response, deployServiceThread, controller = self)
except Exception as excep:
msg = 'Unknown error for deployService(%s/%s) - %s - %s' % (service, manifest, str(excep), traceback.format_exc(2))
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = msg, controller = self)
def post(self):
""" enroll agent """
try:
body = json.loads(request.body)
if 'hardwarePath' not in body:
LOG.error(
'failed when enrolling: hardwarePath is not provided')
return errorResult(request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='hardwarePath is not specified',
controller=self)
with self.lock:
LOG.info('enroll agent with hardwarePath %s' %
(body['hardwarePath']))
hardwarePath = body['hardwarePath']
manifestutil.updateServiceMetaFile('agent', {
'hwPath': hardwarePath,
})
appGlobal = config['pylons.app_globals']
appGlobal.hwPath = hardwarePath
appGlobal.enrolled = True
appGlobal.agentMonitor.reloadMonitors()
return doneResult(request, response, controller=self)
return errorResult(request,
response,
error=Errors.THREAD_ALREADY_ADDED,
errorMsg="Cann't acquire lock for enrollment",
controller=self)
except Exception as excep:
return errorResult(request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error for enroll - %s - %s' %
(str(excep), traceback.format_exc(2)),
controller=self)
def getTaskSlaReport(self, task, threshold=0, starttime=0, fmt='raw'):
""" get task SLA report """
LOG.info('generating task SLA report')
try:
script = os.path.join(manifestutil.manifestPath('agent'), 'agent', 'cronus', 'scripts', 'perfmetric')
LOG.info('task sla report script %s' % script)
if not task or task == '':
raise AgentException(Errors.INVALID_REQUEST, 'task name cannot be empty')
tmp = [script, task, str(threshold), str(starttime), fmt]
cmds = []
for cmd in tmp:
cmds.append(cmd.encode('ascii', 'ignore'))
cmdout = utils.runsyscmdwstdout(cmds)
result = json.loads(cmdout)
return doneResult(request, response, result=result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Error when getting task sla report %s - %s' %
(str(excep), traceback.format_exc(2)), controller = self)
def getTaskSlaReport(self, task, threshold=0, starttime=0, fmt='raw'):
""" get task SLA report """
LOG.info('generating task SLA report')
try:
script = os.path.join(manifestutil.manifestPath('agent'), 'agent', 'cronus', 'scripts', 'perfmetric')
LOG.info('task sla report script %s' % script)
if not task or task == '':
raise AgentException(Errors.INVALID_REQUEST, 'task name cannot be empty')
tmp = [script, task, str(threshold), str(starttime), fmt]
cmds = []
for cmd in tmp:
cmds.append(cmd.encode('ascii', 'ignore'))
cmdout = utils.runsyscmdwstdout(cmds)
result = json.loads(cmdout)
return doneResult(request, response, result=result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Error when getting task sla report %s - %s' %
(str(excep), traceback.format_exc(2)), controller = self)
def pokeConfig(self):
""" config poke """
LOG.info('config poke agent with body: %s', request.body)
configs = None
result = {}
try:
if request.body:
body = json.loads(request.body)
if 'configs' in body:
configs = body['configs']
result = manifestutil.updateServiceMetaFile('agent', {'configs': configs})
return doneResult(request, response, result = result, controller = self)
raise AgentException(Errors.INVALID_REQUEST, 'Invalid request, expect configs in request body')
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when update agent configs %s - %s' %
(str(excep), traceback.format_exc(2)), controller = self)
finally:
# reload config overrides
configutil.loadConfigOverrides()
def uploadPackage(self):
""" take an upload file """
try:
utils.checkDiskFull()
agt_root = pylons.config['agent_root']
pkg_root = pylons.config['repo_root']
md5 = str(request.POST['md5']) if 'md5' in request.POST else None
dest = str(request.POST['dest']) if 'dest' in request.POST else None
myfile = request.POST['file']
filename = myfile.filename.lstrip(os.sep)
permanent_file = open(os.path.join(pkg_root, filename), 'w')
shutil.copyfileobj(myfile.file, permanent_file)
myfile.file.close()
permanent_file.close()
if md5:
md5Sum = utils.md5sum(permanent_file)
if (md5Sum != md5):
msg = 'package md5 = %s : %s' % (md5Sum, md5)
LOG.warning(msg)
raise AgentException(Errors.DC_FAILED_VALIDATE, msg)
if filename and dest:
utils.copyFile(pkg_root, os.path.join(agt_root, 'service_nodes', dest), filename)
return doneResult(request, response, controller=self)
except AgentException as excp:
return errorResult(request, response, excp.getCode(), excp.getMsg(), controller = self)
except Exception as excp:
errorMsg = 'Exception downloading %s - traceback %s' % (str(excp), traceback.format_exc(2))
return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = errorMsg, controller = self)
def updateMetadata(self, service):
""" create or update .metadata file for a service """
# now connect to state server to store metadata for the service in .metadata
metadata = None
if request.body:
# two flavor of updatemetadata call, one with body, one without
body = json.loads(request.body)
if 'metadata' in body:
metadata = body['metadata']
try:
result = {}
appGlobal = pylons.config['pylons.app_globals']
if metadata is not None and 'monitoring.metric.tags' in metadata:
appGlobal.agentMonitor.reloadMonitors()
if service == 'agent' and metadata is not None and 'hwPath' in metadata:
appGlobal.hwPath = metadata['hwPath']
result = manifestutil.updateServiceMetaFile(service, metadata)
return doneResult(request,
response,
result=result,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg=
'Unknown error when update service metadata (%s) %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller=self)
def post(self, service, manifest):
""" Create a new service object """
from agent.lib.agent_thread.manifest_create import ManifestCreate
try:
LOG.info('Post for service (%s) and manifest (%s) with body: %s',
service, manifest, request.body)
# check to see if the manifest already exists
path = ManifestController.manifestPath(service, manifest)
if (os.path.isdir(path)):
return doneResult(request,
response,
httpStatus=201,
controller=self)
# parse the body
if (request.body == ""):
return errorResult(request,
response,
Errors.MANIFEST_PACKAGE_PARSING_ERROR,
'No body found in post command',
controller=self)
body = json.loads(request.body)
packages = body['package']
forcedPackages = body[
'forcePackageName'] if 'forcePackageName' in body else None
skipProp = asbool(
body['skipProp']
) if 'skipProp' in body else configutil.getConfigAsBool(
'download_skip_prop')
LOG.debug('pkgs = %s, %s', packages, forcedPackages)
# parse the package list
for idx, package in enumerate(packages):
# to support reuse of an package from an existing manifest (active if possible)
# without sending the complete package location in request body
if package.startswith('/'):
packageRef = package
tokens = package.split('/')
pkgnamePrefix = tokens[-1].rstrip()
fullPkgLoc = manifestutil.getPackageByName(
service, manifest=None, pkgnamePrefix=pkgnamePrefix)
if fullPkgLoc is None:
return errorResult(
request,
response,
Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'manifest (%s/%s) package (%s) does not exist' %
(service, manifest, packages),
controller=self)
else:
LOG.info(
'expanding package reuse ref %s with full package location %s'
% (packageRef, fullPkgLoc))
packages[idx] = fullPkgLoc
appGlobal = config['pylons.app_globals']
# start a thread to create the package
manThread = ManifestCreate(appGlobal.threadMgr,
service,
manifest,
packages,
forcePackages=forcedPackages,
skipProp=skipProp)
self.injectJobCtx(manThread)
manThread.start()
manThread.threadMgrEvent.wait()
return statusResult(request, response, manThread, controller=self)
except AgentException as excep:
return errorResult(request,
response,
error=excep.getCode(),
errorMsg=excep.getMsg(),
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error for activateManifest(%s/%s) - %s - %s'
% (service, manifest, str(excep), traceback.format_exc(2)),
controller=self)
def validateToken(self):
""" validate token """
return doneResult(request, response, result="success", controller=self)