def _parse_user_data(self, data):
'''
Take a string in form version|[value|][value|][value|]...
parses according to version and populate the respective self var.
Conductor puts the UUID into the oauth_key field.
At minimum this function expects to find a | in the string
this is in effort not to log oauth secrets.
'''
LOGGER.debug('Parsing User Data')
user_data = data.split('|')
if len(user_data) > 1:
if user_data[0] == '1':
# version 1
# format version|endpoint|oauth_key|oauth_secret
ud_version, endpoint, \
oauth_key, oauth_secret = user_data
self.ud_version = ud_version
self.endpoint = endpoint
self.oauth_key = oauth_key
self.oauth_secret = oauth_secret
return {'endpoint': self.endpoint,
'oauth_key': self.oauth_key,
'oauth_secret': self.oauth_secret, }
else:
raise AAError('Invalid User Data Version: %s' % user_data[0])
else:
raise AAError('Could not get user data version, parse failed')
def unpack_tooling(self):
'''
Description:
Methods used to untar the user provided tarball
Perform validation of the text message sent from the
Config Server. Validate, open and write out the contents
of the user provided tarball.
'''
LOGGER.info('Invoked unpack_tooling()')
# Validate the specified tarfile.
if not os.path.exists(self.tarball):
raise AAError('File does not exist: %s ' % self.tarball)
if not tarfile.is_tarfile(self.tarball):
raise AAErrorInvalidTar('Not a valid tar file: %s' % self.tarball)
# Attempt to extract the contents from the specified tarfile.
# If tarfile access or content is bad report to the user to aid
# problem resolution.
try:
tarf = tarfile.open(self.tarball)
tarf.extractall(path=self.user_dir)
tarf.close()
# Capture and report errors with the tarfile
except (tarfile.TarError, tarfile.ReadError, \
tarfile.CompressionError, tarfile.StreamError, \
tarfile.ExtractError, IOError), (strerror):
raise AAError(('Failed to access tar file %s. Error: %s') % \
(self.tarball, strerror))
def read(self):
#
# If on RHEV-M the user data will be contained on the
# floppy device in file deltacloud-user-data.txt.
# To access it:
# modprobe floppy
# mount /dev/fd0 /media
# read /media/deltacloud-user-data.txt
#
# Note:
# On RHEVm the deltacloud drive had been delivering the user
# data base64 decoded at one point that changed such that the
# deltacloud drive leaves the date base64 encoded. This
# Code segment will handle both base64 encoded and decoded
# user data.
#
# Since ':' is used as a field delimiter in the user data
# and is not a valid base64 char, if ':' is found assume
# the data is already base64 decoded.
#
# modprobe floppy
cmd = ['/sbin/modprobe', 'floppy']
ret = run_cmd(cmd)
if ret['subproc'].returncode != 0:
raise AAError(('Failed command: \n%s \nError: \n%s') % \
(' '.join(cmd), str(ret['err'])))
cmd = ['/bin/mkdir', '/media']
ret = run_cmd(cmd)
# If /media is already there (1) or any other error (0)
if (ret['subproc'].returncode != 1) and \
(ret['subproc'].returncode != 0):
raise AAError(('Failed command: \n%s \nError: \n%s') % \
(' '.join(cmd), str(ret['err'])))
cmd = ['/bin/mount', '/dev/fd0', '/media']
ret = run_cmd(cmd)
# If /media is already mounted (32) or any other error (0)
if (ret['subproc'].returncode != 32) and \
(ret['subproc'].returncode != 0):
raise AAError(('Failed command: \n%s \nError: \n%s') % \
(' '.join(cmd), str(ret['err'])))
# Condfig Server (CS) address:port.
# This could be done using "with open()" but that's not available
# in Python 2.4 as used on RHEL5
try:
user_data_file = open(DELTA_CLOUD_USER_DATA, 'r')
user_data = user_data_file.read().strip()
user_data_file.close()
except Exception, e:
raise AAError('Failed accessing RHEVm user data: %s' % e)
def get_tooling(self):
'''
Description:
get any optional user supplied tooling which is
provided as a tarball
'''
LOGGER.info('Invoked CSClient.get_tooling()')
url = self._cs_url(TOOLING_URL)
headers = {'Accept': 'content-disposition'}
tarball = ''
response, body = self._get(url, headers=headers)
self._validate_http_status(response)
# Parse the file name burried in the response header
# at: response['content-disposition']
# as: 'attachment; tarball="tarball.tgz"'
if (response.status == 200) or (response.status == 202):
tarball = response['content-disposition']. \
lstrip('attachment; filename=').replace('"', '')
# Create the temporary tarfile
try:
self.tmpdir = tempfile.mkdtemp()
self.tarball = self.tmpdir + '/' + tarball
temptar = open(self.tarball, 'w')
temptar.write(body)
temptar.close()
except IOError, err:
raise AAError(('File not found or not a tar file: %s ' + \
'Error: %s') % (self.tarball, err))
def find_tooling(self, service_name):
'''
Description:
Given a service name return the path to the configuration
tooling.
Search for the service start executable in the user
tooling directory.
self.tool_dir + '/user/<service name>/start'
If not found there search for the it in the documented directory
here built in tooling should be placed.
self.tool_dir + '/AUDREY_TOOLING/<service name>/start'
If not found there search for the it in the Red Hat tooling
directory.
self.tool_dir + '/REDHAT/<service name>/start'
If not found there raise an error.
Returns:
return 1 - True if top level tooling found, False otherwise.
return 2 - path to tooling
'''
if not service_name:
raise AAError('Empty service name passed')
# common join
service_start = os.path.join(service_name, 'start')
# returns, check the paths and return the tuple
tooling_paths = [
(True, os.path.join(self.user_dir, 'start')),
(False, os.path.join(self.user_dir, service_start)),
(False, os.path.join(self.audrey_dir, service_start)),
(False, os.path.join(self.redhat_dir, service_start)),
]
for path in tooling_paths:
if os.access(path[1], os.X_OK):
return path
# No tooling found. Raise an error.
raise AAError('No configuration tooling found for service: %s' % \
service_name)
def read(self):
try:
max_attempts = 5
headers = {'Accept': 'text/plain'}
while max_attempts:
response, body = httplib2.Http().request(EC2_USER_DATA_URL,
headers=headers)
if response.status == 200:
break
max_attempts -= 1
if response.status != 200:
raise AAError('Max attempts to get EC2 user data \
exceeded.')
if '|' not in body:
body = base64.b64decode(body)
return self._parse_user_data(body)
except Exception, err:
raise AAError('Failed accessing EC2 user data: %s' % err)
def _validate_http_status(response):
'''
Description:
Confirm the http status is one of:
200 HTTP OK - Success and no more data of this type
202 HTTP Accepted - Success and more data of this type
404 HTTP Not Found - This may be temporary so try again
'''
if isinstance(response, Exception):
raise response
if response.status not in [200, 202, 404]:
raise AAError('Invalid HTTP status code: %s' % response.status)
def read(self):
#
# If on vSphere the user data will be contained on the
# floppy device in file deltacloud-user-data.txt.
# To access it:
# mount /dev/fd0 /media
# read /media/deltacloud-user-data.txt
#
# Note:
# On vSphere the deltacloud drive had been delivering the user
# data base64 decoded at one point that changed such that the
# deltacloud drive leaves the date base64 encoded. This
# Code segment will handle both base64 encoded and decoded
# user data.
cmd = ['/bin/mkdir', '/media']
ret = run_cmd(cmd)
# If /media is already there (1) or any other error (0)
if (ret['subproc'].returncode != 1) and \
(ret['subproc'].returncode != 0):
raise AAError(('Failed command: \n%s \nError: \n%s') % \
(' '.join(cmd), str(ret['err'])))
cmd = ['/bin/mount', '/dev/cdrom', '/media']
ret = run_cmd(cmd)
# If /media is already mounted (32) or any other error (0)
if (ret['subproc'].returncode != 32) and \
(ret['subproc'].returncode != 0):
raise AAError(('Failed command: \n%s \nError: \n%s') % \
(' '.join(cmd), str(ret['err'])))
try:
user_data_file = open(DELTA_CLOUD_USER_DATA, 'r')
user_data = user_data_file.read().strip()
user_data_file.close()
except Exception, e:
raise AAError('Failed accessing vSphere user data file. %s' % e)
def __init__(self, tarball, tool_dir=TOOLING_DIR):
'''
Description:
Set initial state so it can be tracked.
'''
self.tool_dir = tool_dir
self.user_dir = os.path.join(tool_dir, 'user')
self.audrey_dir = os.path.join(tool_dir, 'AUDREY_TOOLING')
self.redhat_dir = os.path.join(tool_dir, 'REDHAT')
self.tarball = tarball
# Create the extraction destination
if not os.path.exists(self.user_dir):
try:
os.makedirs(self.user_dir)
except OSError, err:
raise AAError(('Failed to create directory %s. ' + \
'Error: %s') % (self.user_dir, err))
def discover():
'''
Description:
User Data is passed to the launching instance which
provides the Config Server contact information.
Cloud providers expose the user data differently.
It is necessary to determine which cloud provider
the current instance is running on to determine
how to access the user data. Images built with
image factory will contain a CLOUD_INFO_FILE which
contains a string identifying the cloud provider.
Images not built with Imagefactory will try to
determine what the cloud provider is based on system
information.
'''
LOGGER.debug('Invoked discover')
if os.path.exists(CLOUD_INFO_FILE):
cloud_info = open(CLOUD_INFO_FILE)
cloud_type = cloud_info.read().strip().upper()
cloud_info.close()
else:
cloud_type = _get_cloud_type()
LOGGER.debug('cloud_type: ' + str(cloud_type))
if 'EC2' in cloud_type:
import audrey.user_data_ec2
return audrey.user_data_ec2.UserData()
elif 'RHEV' in cloud_type:
import audrey.user_data_rhev
return audrey.user_data_rhev.UserData()
elif 'VSPHERE' in cloud_type:
import audrey.user_data_vsphere
return audrey.user_data_vsphere.UserData()
else:
raise AAError('Cloud type "%s" is invalid.' % cloud_type)
def test_connection(self, max_retry=5):
'''
call configserver's version endpoint and pass my compat api versions
then parse the response to retireve the api version
we should operate on
'''
# try and wait for connectivity if it's not there
url = self._cs_url(VERSION_URL)
response, body = self._get(url, {'Accept': 'text/xml'})
while isinstance(response, Exception):
if max_retry:
max_retry -= 1
LOGGER.info('Failed attempt to contact config server')
sleep(SLEEP_SECS)
else:
raise AAError("Cannot establish connection to %s" % url)
response, body = self._get(url)
try:
api_v = ElementTree.fromstring(body)
api_v = api_v.find('api-version')
self.api_version = int(api_v.text)
LOGGER.info('Negotiated API V%s' % self.api_version)
except Exception, err:
raise AAErrorApiNegotiation(err)
def get_system_info(facts=None):
'''
Description:
Get the system info to be used for generating this instances
provides back to the Config Server.
Currently utilizes Puppet's facter via a Python subprocess call.
Input:
optional list of fact names
Returns:
A dictionary of system info name/value pairs.
'''
cmd = ['/usr/bin/facter']
if facts:
fact = facts[0]
cmd.extend(facts)
ret = run_cmd(cmd)
if ret['subproc'].returncode != 0:
raise AAError(('Failed command: \n%s \nError: \n%s') % \
(' '.join(cmd), str(ret['err'])))
facts = {}
facts = ret['out'].split('\n')[:-1]
if not facts:
return {}
if len(facts) == 1:
return {fact: facts[0]}
facts = [x.split(' => ') for x in facts]
facts = dict(facts)
for key in facts.keys():
if not facts[key]:
del facts[key]
return facts
def read(self):
'''
Dummy function, indended to be overridden
should return (endpoint, oauth_jey, oauth_secret)
'''
raise AAError("%s read() not overridden. Execution Aborted" % self)
def validate_message(src):
'''
Perform validation of the text message sent from the Config Server.
'''
if not src.startswith('|') or not src.endswith('|'):
raise AAError(('Invalid start and end characters: %s') % (src))
def run(self):
'''
Main loop called by main() in /usr/bin/audrey
'''
provides_len = 0
services_len = 0
retry_ct = 0
status, provides_str = self.client.get_provides()
if status == 200:
services, provides = Provides().parse_cs_str(
provides_str, self.tooling)
else:
raise AAError('HTTP %s from provides & services list' % status)
# process services and provides, removing them from the ques
# as they have been processed.
while services or provides:
# Put the requested provides with values to the Config Server
provides_str = provides.generate_cs_str()
LOGGER.debug('Put Provides: %s' % provides_str)
status = self.client.put_provides(provides_str)[0]
# report non 200 status
if status != 200:
raise AAErrorPutProvides('Put provides returned %s' % status)
# clean regardless of status, otherwise we'll get in
# an infinite loop.
provides.clean()
# check for required configs per service
for service in services.keys():
svc = services[service]
# Get the Required Configs from Config Server for the service
status, configs = self.client.get_configs(svc.name)
svc.parse_configs(configs)
# Configure the system with the provided Required Configs
if status == 202:
# couldn't be given all the configs yet.
continue
else:
if status == 200:
# got all the configs, so invoke and report status
status = svc.invoke_tooling()
LOGGER.info('Service %s returns %s' % (service, status))
# report service status
status = self.client.put_provides(
svc.generate_cs_str(status))[0]
# report non 200 status on service status put
if status != 200:
raise AAErrorPutProvides('Put service status %s' %
status)
# the service has been processed
del services[service]
if services_len == len(services) and provides_len == len(provides):
if retry_ct == MAX_RETRY:
LOGGER.error("Max retry count exceeded. Exiting.")
exit(1)
retry_ct += 1
else:
services_len = len(services)
provides_len = len(provides)
retry_ct = 0
sleep(SLEEP_SECS)
def parse_require_config(src, tooling):
'''
Description:
Parse the required config text message sent from the Config Server.
Input:
The required config string obtained from the Config Server,
delimited by an | and an &
Two tags will mark the sections of the data,
'|service|' and '|parameters|'
To ensure all the data was received the entire string will be
terminated with an "|".
The string "|service|" will precede a service names.
The string "|parameters|" will precede the parameters for
the preceeding service, in the form: names&<b64 encoded values>.
This will be a continuous text string (no CR or New Line).
Format (repeating for each service):
|service|<s1>|parameters|name1&<b64val>|name2&<b64val>|nameN&<b64v>|
e.g.:
|service|ssh::server|parameters|ssh_port&<b64('22')>
|service|apache2::common|apache_port&<b64('8081')>|
Returns:
- A list of ServiceParams objects.
'''
services = []
new = None
CSClient.validate_message(src)
# Message specific validation
if src == '||':
# special case indicating no required config needed.
return []
# split on pipe and chop of first and last, they will always be empty
src = src.split('|')[1:-1]
# get the indexes of the service identifiers
srvs = deque([i for i, x in enumerate(src) if x == 'service'])
srvs.append(len(src))
if srvs[0] != 0:
raise AAError(('|service| is not the first tag found. %s') % (src))
while len(srvs) > 1:
# rebuild a single service's cs string
svc_str = "|%s|" % "|".join(src[srvs[0]:srvs[1]])
name = src[srvs[0] + 1]
if name in ['service', 'parameters'] or '&' in name:
raise AAError('invalid service name: %s' % name)
# instanciate the service with it's name
svc = Service(name, tooling)
svc.parse_configs(svc_str)
services.append(svc)
srvs.popleft()
return services
def run(self):
'''
Main agent loop, called by main() in /usr/bin/audrey
'''
# 0 means don't run again
# -1 is non zero so initial runs will happen
config_status = -1
provides_status = -1
max_retry = MAX_RETRY
loop_count = 60
services = []
# Process the Requires and Provides parameters until the HTTP status
# from the get_configs and the get_params both return 200
while config_status or provides_status:
LOGGER.debug('Config Parameter status: ' + str(config_status))
LOGGER.debug('Return Parameter status: ' + str(provides_status))
# Get the Required Configs from the Config Server
if config_status:
config_status, configs = self.client.get_configs()
# Configure the system with the provided Required Configs
if config_status == 200:
services = Service.parse_require_config(
configs, self.tooling)
self.tooling.invoke_tooling(services)
# don't do any more config status work
# now that the tooling has run
config_status = 0
else:
LOGGER.info(
'No configuration parameters provided. status: ' + \
str(config_status))
# Get the requested provides from the Config Server
if provides_status:
get_status = self.client.get_provides()[0]
# Gather the values from the system for the requested provides
if get_status == 200:
params_values = Provides().generate_cs_str()
else:
params_values = '|&|'
# Put the requested provides with values to the Config Server
provides_status = self.client.put_provides(params_values)[0]
if provides_status == 200:
# don't operate on params anymore, all have been provided.
provides_status = 0
# Retry a number of times if 404 HTTP Not Found is returned.
if config_status == 404 or provides_status == 404:
LOGGER.error('404 from Config Server.')
LOGGER.error('Required Config status: %s' % config_status)
LOGGER.info('Return Parameter status: %s' % provides_status)
max_retry -= 1
if max_retry < 0:
raise AAError('Too many 404 Config Server responses.')
if loop_count:
loop_count -= 1
sleep(SLEEP_SECS)
else:
break