def launchSupervisord(instance_root, logger,
supervisord_additional_argument_list=None):
configuration_file = _getSupervisordConfigurationFilePath(instance_root)
socket = _getSupervisordSocketPath(instance_root)
if os.path.exists(socket):
trynum = 1
while trynum < 6:
try:
supervisor = getSupervisorRPC(socket)
status = supervisor.getState()
except xmlrpclib.Fault as e:
if e.faultCode == 6 and e.faultString == 'SHUTDOWN_STATE':
logger.info('Supervisor in shutdown procedure, will check again later.')
trynum += 1
time.sleep(2 * trynum)
except Exception:
# In case if there is problem with connection, assume that supervisord
# is not running and try to run it
break
else:
if status['statename'] == 'RUNNING' and status['statecode'] == 1:
logger.debug('Supervisord already running.')
_updateWatchdog(socket)
return
elif status['statename'] == 'SHUTDOWN_STATE' and status['statecode'] == 6:
logger.info('Supervisor in shutdown procedure, will check again later.')
trynum += 1
time.sleep(2 * trynum)
else:
log_message = 'Unknown supervisord state %r. Will try to start.' % status
logger.warning(log_message)
break
supervisord_argument_list = ['-c', configuration_file]
if supervisord_additional_argument_list is not None:
supervisord_argument_list.extend(supervisord_additional_argument_list)
logger.info("Launching supervisord with clean environment.")
# Extract python binary to prevent shebang size limit
invocation_list = ["supervisord", '-c']
invocation_list.append(
"import sys ; sys.path=" + str(sys.path) + " ; " +
"import supervisor.supervisord ; " +
"sys.argv[1:1]=" + str(supervisord_argument_list) + " ; " +
"supervisor.supervisord.main()")
supervisord_popen = SlapPopen(invocation_list,
env={},
executable=sys.executable,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
logger=logger)
result = supervisord_popen.communicate()[0]
if supervisord_popen.returncode:
logger.warning('Supervisord unknown problem: %s' % result)
return
try:
default_timeout = socketlib.getdefaulttimeout()
current_timeout = 1
trynum = 1
while trynum < 6:
try:
socketlib.setdefaulttimeout(current_timeout)
supervisor = getSupervisorRPC(socket)
status = supervisor.getState()
if status['statename'] == 'RUNNING' and status['statecode'] == 1:
return
logger.warning('Wrong status name %(statename)r and code '
'%(statecode)r, trying again' % status)
trynum += 1
except Exception:
current_timeout = 5 * trynum
trynum += 1
else:
logger.info('Supervisord started correctly in try %s.' % trynum)
return
logger.warning('Issue while checking supervisord.')
finally:
socketlib.setdefaulttimeout(default_timeout)
def agregateAndSendUsage(self):
"""Will agregate usage from each Computer Partition.
"""
# Prepares environment
self.checkEnvironmentAndCreateStructure()
self._launchSupervisord()
slap_computer_usage = self.slap.registerComputer(self.computer_id)
computer_partition_usage_list = []
self.logger.info('Aggregating and sending usage reports...')
#We retrieve XSD models
try:
computer_consumption_model = \
pkg_resources.resource_string(
'slapos.slap',
'doc/computer_consumption.xsd')
except IOError:
computer_consumption_model = \
pkg_resources.resource_string(
__name__,
'../../../../slapos/slap/doc/computer_consumption.xsd')
try:
partition_consumption_model = \
pkg_resources.resource_string(
'slapos.slap',
'doc/partition_consumption.xsd')
except IOError:
partition_consumption_model = \
pkg_resources.resource_string(
__name__,
'../../../../slapos/slap/doc/partition_consumption.xsd')
clean_run = True
# Loop over the different computer partitions
computer_partition_list = self.FilterComputerPartitionList(
slap_computer_usage.getComputerPartitionList())
for computer_partition in computer_partition_list:
try:
computer_partition_id = computer_partition.getId()
#We want to execute all the script in the report folder
instance_path = os.path.join(self.instance_root,
computer_partition.getId())
report_path = os.path.join(instance_path, 'etc', 'report')
if os.path.isdir(report_path):
script_list_to_run = os.listdir(report_path)
else:
script_list_to_run = []
#We now generate the pseudorandom name for the xml file
# and we add it in the invocation_list
f = tempfile.NamedTemporaryFile()
name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name))
path_to_slapreport = os.path.join(instance_path, 'var', 'xml_report',
name_xml)
failed_script_list = []
for script in script_list_to_run:
invocation_list = []
invocation_list.append(os.path.join(instance_path, 'etc', 'report',
script))
#We add the xml_file name to the invocation_list
#f = tempfile.NamedTemporaryFile()
#name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name))
#path_to_slapreport = os.path.join(instance_path, 'var', name_xml)
invocation_list.append(path_to_slapreport)
#Dropping privileges
uid, gid = None, None
stat_info = os.stat(instance_path)
#stat sys call to get statistics informations
uid = stat_info.st_uid
gid = stat_info.st_gid
process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid, logger=self.logger),
cwd=os.path.join(instance_path, 'etc', 'report'),
env=None,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
logger=self.logger)
if process_handler.returncode is None:
process_handler.kill()
if process_handler.returncode != 0:
clean_run = False
failed_script_list.append("Script %r failed." % script)
self.logger.warning('Failed to run %r' % invocation_list)
if len(failed_script_list):
computer_partition.error('\n'.join(failed_script_list), logger=self.logger)
# Whatever happens, don't stop processing other instances
except Exception:
self.logger.exception('Cannot run usage script(s) for %r:' %
computer_partition.getId())
#Now we loop through the different computer partitions to report
report_usage_issue_cp_list = []
for computer_partition in computer_partition_list:
try:
filename_delete_list = []
computer_partition_id = computer_partition.getId()
instance_path = os.path.join(self.instance_root, computer_partition_id)
dir_reports = os.path.join(instance_path, 'var', 'xml_report')
#The directory xml_report contain a number of files equal
#to the number of software instance running inside the same partition
if os.path.isdir(dir_reports):
filename_list = os.listdir(dir_reports)
else:
filename_list = []
#self.logger.debug('name List %s' % filename_list)
for filename in filename_list:
file_path = os.path.join(dir_reports, filename)
if os.path.exists(file_path):
usage = open(file_path, 'r').read()
#We check the validity of xml content of each reports
if not self.validateXML(usage, partition_consumption_model):
self.logger.info('WARNING: The XML file %s generated by slapreport is '
'not valid - This report is left as is at %s where you can '
'inspect what went wrong ' % (filename, dir_reports))
# Warn the SlapOS Master that a partition generates corrupted xml
# report
else:
computer_partition_usage = self.slap.registerComputerPartition(
self.computer_id, computer_partition_id)
computer_partition_usage.setUsage(usage)
computer_partition_usage_list.append(computer_partition_usage)
filename_delete_list.append(filename)
else:
self.logger.debug('Usage report %r not found, ignored' % file_path)
#After sending the aggregated file we remove all the valid xml reports
for filename in filename_delete_list:
os.remove(os.path.join(dir_reports, filename))
# Whatever happens, don't stop processing other instances
except Exception:
self.logger.exception('Cannot run usage script(s) for %r:' %
computer_partition.getId())
for computer_partition_usage in computer_partition_usage_list:
self.logger.info('computer_partition_usage_list: %s - %s' %
(computer_partition_usage.usage, computer_partition_usage.getId()))
#If there is, at least, one report
if computer_partition_usage_list != []:
try:
#We generate the final XML report with asXML method
computer_consumption = self.asXML(computer_partition_usage_list)
self.logger.info('Final xml report: %s' % computer_consumption)
#We test the XML report before sending it
if self.validateXML(computer_consumption, computer_consumption_model):
self.logger.info('XML file generated by asXML is valid')
slap_computer_usage.reportUsage(computer_consumption)
else:
self.logger.info('XML file generated by asXML is not valid !')
raise ValueError('XML file generated by asXML is not valid !')
except Exception:
issue = "Cannot report usage for %r: %s" % (
computer_partition.getId(),
traceback.format_exc())
self.logger.info(issue)
computer_partition.error(issue, logger=self.logger)
report_usage_issue_cp_list.append(computer_partition_id)
for computer_partition in computer_partition_list:
if computer_partition.getState() == COMPUTER_PARTITION_DESTROYED_STATE:
try:
computer_partition_id = computer_partition.getId()
try:
software_url = computer_partition.getSoftwareRelease().getURI()
software_path = os.path.join(self.software_root, md5digest(software_url))
except (NotFoundError, TypeError):
software_url = None
software_path = None
local_partition = Partition(
software_path=software_path,
instance_path=os.path.join(self.instance_root,
computer_partition.getId()),
supervisord_partition_configuration_path=os.path.join(
self.supervisord_configuration_directory, '%s.conf' %
computer_partition_id),
supervisord_socket=self.supervisord_socket,
computer_partition=computer_partition,
computer_id=self.computer_id,
partition_id=computer_partition_id,
server_url=self.master_url,
software_release_url=software_url,
certificate_repository_path=self.certificate_repository_path,
buildout=self.buildout,
logger=self.logger)
local_partition.stop()
try:
computer_partition.stopped()
except (SystemExit, KeyboardInterrupt):
computer_partition.error(traceback.format_exc(), logger=self.logger)
raise
except Exception:
pass
if computer_partition.getId() in report_usage_issue_cp_list:
self.logger.info('Ignoring destruction of %r, as no report usage was sent' %
computer_partition.getId())
continue
local_partition.destroy()
except (SystemExit, KeyboardInterrupt):
computer_partition.error(traceback.format_exc(), logger=self.logger)
raise
except Exception:
clean_run = False
self.logger.exception('')
exc = traceback.format_exc()
computer_partition.error(exc, logger=self.logger)
try:
computer_partition.destroyed()
except NotFoundError:
self.logger.debug('Ignored slap error while trying to inform about '
'destroying not fully configured Computer Partition %r' %
computer_partition.getId())
except ServerError as server_error:
self.logger.debug('Ignored server error while trying to inform about '
'destroying Computer Partition %r. Error is:\n%r' %
(computer_partition.getId(), server_error.args[0]))
self.logger.info('Finished usage reports.')
# Return success value
if not clean_run:
return SLAPGRID_FAIL
return SLAPGRID_SUCCESS