def runTestSuite(self, node_test_suite, portal_url):
config = self.testnode.config
run_test_suite_path_list = glob.glob(
self.getInstanceRoot(node_test_suite) + "/*/bin/runTestSuite")
try:
run_test_suite_path = min(run_test_suite_path_list)
except ValueError:
raise ValueError(
'No runTestSuite provided in installed partitions.')
# Deal with Shebang size limitation
invocation_list = dealShebang(run_test_suite_path)
invocation_list += (run_test_suite_path, '--master_url', portal_url,
'--revision', node_test_suite.revision,
'--test_suite', node_test_suite.test_suite,
'--test_suite_title',
node_test_suite.test_suite_title)
soft = config['slapos_directory'] + '/soft/'
software_list = [soft + md5digest(x) for x in config['software_list']]
PATH = os.getenv('PATH', '')
PATH = ':'.join(x + '/bin'
for x in software_list) + (PATH and ':' + PATH)
supported_parameter_set = set(
self.testnode.process_manager.getSupportedParameterList(
run_test_suite_path))
def path(name, compat): # BBB
path, = filter(
os.path.exists,
(base + relative
for relative in ('/bin/' + name, '/parts/' + compat)
for base in software_list))
return path
for option, value in (
('--firefox_bin',
lambda: path('firefox', 'firefox/firefox-slapos')),
('--frontend_url', lambda: config['frontend_url']),
('--node_quantity', lambda: config['node_quantity']),
('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')),
('--project_title', lambda: node_test_suite.project_title),
):
if option in supported_parameter_set:
invocation_list += option, value()
# TODO : include testnode correction ( b111682f14890bf )
if hasattr(node_test_suite, 'additional_bt5_repository_id'):
additional_bt5_path = os.path.join(
node_test_suite.working_directory,
node_test_suite.additional_bt5_repository_id)
invocation_list.extend(["--bt5_path", additional_bt5_path])
# From this point, test runner becomes responsible for updating test
# result. We only do cleanup if the test runner itself is not able
# to run.
createFolder(node_test_suite.test_suite_directory, clean=True)
self.testnode.process_manager.spawn(
*invocation_list,
PATH=PATH,
cwd=node_test_suite.test_suite_directory,
log_prefix='runTestSuite',
get_output=False)
def createPartition(
self,
software_release_url,
partition_id=None,
slap_computer_partition=None,
retention_delay=None,
):
"""
Create a partition, and return a Partition object created
from dummy parameters.
"""
# XXX dirty, should disappear when Partition is cleaned up
software_path = os.path.join(
self.software_root,
utils.md5digest(software_release_url)
)
if partition_id is None:
partition_id = 'mypartition'
if slap_computer_partition is None:
slap_computer_partition = SlapComputerPartition(
computer_id='bidon',
partition_id=partition_id)
instance_path = os.path.join(self.instance_root, partition_id)
os.mkdir(instance_path)
os.chmod(instance_path, 0o750)
supervisor_configuration_path = os.path.join(
self.instance_root, 'supervisor')
os.mkdir(supervisor_configuration_path)
partition = Partition(
software_path=software_path,
instance_path=instance_path,
supervisord_partition_configuration_path=os.path.join(
supervisor_configuration_path, partition_id),
supervisord_socket=os.path.join(
supervisor_configuration_path, 'supervisor.sock'),
computer_partition=slap_computer_partition,
computer_id='bidon',
partition_id=partition_id,
server_url='bidon',
software_release_url=software_release_url,
buildout=self.buildout,
logger=logging.getLogger(),
)
partition.updateSupervisor = FakeCallAndNoop
if retention_delay:
partition.retention_delay = retention_delay
return partition
def config_SR_folder(config):
"""Create a symbolik link for each folder in software folder. That allows
the user to customize software release folder"""
config_name = 'slaprunner.config'
def link_to_folder(name, folder):
destination = os.path.join(config['software_link'], name)
source = os.path.join(config['software_root'], folder)
cfg = os.path.join(destination, config_name)
#create symlink
if os.path.lexists(destination):
os.remove(destination)
os.symlink(source, destination)
#write config file
if os.path.exists(source):
with open(cfg, 'w') as cf:
cf.write(current_project + '#' + folder)
# First create the link for current project
current_project = open(os.path.join(config['etc_dir'], ".project")).read()
profile = getCurrentSoftwareReleaseProfile(config)
if current_project[-1] == '/':
current_project = current_project[:-1]
name = current_project.split('/')[-1]
md5sum = md5digest(profile)
link_to_folder(name, md5sum)
# check other links
# XXX-Marco do not shadow 'list'
list = []
for path in os.listdir(config['software_link']):
cfg_path = os.path.join(config['software_link'], path, config_name)
if os.path.exists(cfg_path):
cfg = open(cfg_path).read().split("#")
if len(cfg) != 2:
continue # there is a broken config file
list.append(cfg[1])
if os.path.exists(config['software_root']):
folder_list = os.listdir(config['software_root'])
else:
return
if not folder_list:
return
for folder in folder_list:
if folder in list:
continue # this folder is already registered
else:
link_to_folder(folder, folder)
def __init__(self, url, software_root, buildout, logger,
signature_private_key_file=None, signature_certificate_list=None,
upload_cache_url=None, upload_dir_url=None,
shacache_ca_file=None, shacache_cert_file=None, shacache_key_file=None,
shadir_ca_file=None, shadir_cert_file=None, shadir_key_file=None,
download_binary_cache_url=None, upload_binary_cache_url=None,
download_binary_dir_url=None, upload_binary_dir_url=None,
download_from_binary_cache_url_blacklist=None,
upload_to_binary_cache_url_blacklist=None,
software_min_free_space=None):
"""Initialisation of class parameters
"""
if download_from_binary_cache_url_blacklist is None:
download_from_binary_cache_url_blacklist = []
if upload_to_binary_cache_url_blacklist is None:
upload_to_binary_cache_url_blacklist = []
self.url = url
self.software_root = software_root
self.software_url_hash = md5digest(self.url)
self.software_path = os.path.join(self.software_root,
self.software_url_hash)
self.buildout = buildout
self.logger = logger
self.signature_private_key_file = signature_private_key_file
self.signature_certificate_list = signature_certificate_list
self.upload_cache_url = upload_cache_url
self.upload_dir_url = upload_dir_url
self.shacache_ca_file = shacache_ca_file
self.shacache_cert_file = shacache_cert_file
self.shacache_key_file = shacache_key_file
self.shadir_ca_file = shadir_ca_file
self.shadir_cert_file = shadir_cert_file
self.shadir_key_file = shadir_key_file
self.download_binary_cache_url = download_binary_cache_url
self.upload_binary_cache_url = upload_binary_cache_url
self.download_binary_dir_url = download_binary_dir_url
self.upload_binary_dir_url = upload_binary_dir_url
self.download_from_binary_cache_url_blacklist = \
download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist
self.software_min_free_space = software_min_free_space
def createSoftware(self, url=None, empty=False):
"""
Create an empty software, and return a Software object from
dummy parameters.
"""
if url is None:
url = 'mysoftware'
software_path = os.path.join(self.software_root, utils.md5digest(url))
os.mkdir(software_path)
if not empty:
# Populate the Software Release directory so that it is "complete" and
# "working" from a slapos point of view.
open(os.path.join(software_path, 'instance.cfg'), 'w').close()
return Software(
url=url,
software_root=self.software_root,
buildout=self.buildout,
logger=logging.getLogger(),
)
def agregateAndSendUsage(self):
"""Will agregate usage from each Computer Partition.
"""
# Prepares environment
self.checkEnvironmentAndCreateStructure()
self._launchSupervisord()
slap_computer_usage = self.slap.registerComputer(self.computer_id)
computer_partition_usage_list = []
self.logger.info('Aggregating and sending usage reports...')
#We retrieve XSD models
try:
computer_consumption_model = \
pkg_resources.resource_string(
'slapos.slap',
'doc/computer_consumption.xsd')
except IOError:
computer_consumption_model = \
pkg_resources.resource_string(
__name__,
'../../../../slapos/slap/doc/computer_consumption.xsd')
try:
partition_consumption_model = \
pkg_resources.resource_string(
'slapos.slap',
'doc/partition_consumption.xsd')
except IOError:
partition_consumption_model = \
pkg_resources.resource_string(
__name__,
'../../../../slapos/slap/doc/partition_consumption.xsd')
clean_run = True
# Loop over the different computer partitions
computer_partition_list = self.FilterComputerPartitionList(
slap_computer_usage.getComputerPartitionList())
for computer_partition in computer_partition_list:
try:
computer_partition_id = computer_partition.getId()
#We want to execute all the script in the report folder
instance_path = os.path.join(self.instance_root,
computer_partition.getId())
report_path = os.path.join(instance_path, 'etc', 'report')
if os.path.isdir(report_path):
script_list_to_run = os.listdir(report_path)
else:
script_list_to_run = []
#We now generate the pseudorandom name for the xml file
# and we add it in the invocation_list
f = tempfile.NamedTemporaryFile()
name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name))
path_to_slapreport = os.path.join(instance_path, 'var', 'xml_report',
name_xml)
failed_script_list = []
for script in script_list_to_run:
invocation_list = []
invocation_list.append(os.path.join(instance_path, 'etc', 'report',
script))
#We add the xml_file name to the invocation_list
#f = tempfile.NamedTemporaryFile()
#name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name))
#path_to_slapreport = os.path.join(instance_path, 'var', name_xml)
invocation_list.append(path_to_slapreport)
#Dropping privileges
uid, gid = None, None
stat_info = os.stat(instance_path)
#stat sys call to get statistics informations
uid = stat_info.st_uid
gid = stat_info.st_gid
process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid, logger=self.logger),
cwd=os.path.join(instance_path, 'etc', 'report'),
env=None,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
logger=self.logger)
if process_handler.returncode is None:
process_handler.kill()
if process_handler.returncode != 0:
clean_run = False
failed_script_list.append("Script %r failed." % script)
self.logger.warning('Failed to run %r' % invocation_list)
if len(failed_script_list):
computer_partition.error('\n'.join(failed_script_list), logger=self.logger)
# Whatever happens, don't stop processing other instances
except Exception:
self.logger.exception('Cannot run usage script(s) for %r:' %
computer_partition.getId())
#Now we loop through the different computer partitions to report
report_usage_issue_cp_list = []
for computer_partition in computer_partition_list:
try:
filename_delete_list = []
computer_partition_id = computer_partition.getId()
instance_path = os.path.join(self.instance_root, computer_partition_id)
dir_reports = os.path.join(instance_path, 'var', 'xml_report')
#The directory xml_report contain a number of files equal
#to the number of software instance running inside the same partition
if os.path.isdir(dir_reports):
filename_list = os.listdir(dir_reports)
else:
filename_list = []
#self.logger.debug('name List %s' % filename_list)
for filename in filename_list:
file_path = os.path.join(dir_reports, filename)
if os.path.exists(file_path):
usage = open(file_path, 'r').read()
#We check the validity of xml content of each reports
if not self.validateXML(usage, partition_consumption_model):
self.logger.info('WARNING: The XML file %s generated by slapreport is '
'not valid - This report is left as is at %s where you can '
'inspect what went wrong ' % (filename, dir_reports))
# Warn the SlapOS Master that a partition generates corrupted xml
# report
else:
computer_partition_usage = self.slap.registerComputerPartition(
self.computer_id, computer_partition_id)
computer_partition_usage.setUsage(usage)
computer_partition_usage_list.append(computer_partition_usage)
filename_delete_list.append(filename)
else:
self.logger.debug('Usage report %r not found, ignored' % file_path)
#After sending the aggregated file we remove all the valid xml reports
for filename in filename_delete_list:
os.remove(os.path.join(dir_reports, filename))
# Whatever happens, don't stop processing other instances
except Exception:
self.logger.exception('Cannot run usage script(s) for %r:' %
computer_partition.getId())
for computer_partition_usage in computer_partition_usage_list:
self.logger.info('computer_partition_usage_list: %s - %s' %
(computer_partition_usage.usage, computer_partition_usage.getId()))
#If there is, at least, one report
if computer_partition_usage_list != []:
try:
#We generate the final XML report with asXML method
computer_consumption = self.asXML(computer_partition_usage_list)
self.logger.info('Final xml report: %s' % computer_consumption)
#We test the XML report before sending it
if self.validateXML(computer_consumption, computer_consumption_model):
self.logger.info('XML file generated by asXML is valid')
slap_computer_usage.reportUsage(computer_consumption)
else:
self.logger.info('XML file generated by asXML is not valid !')
raise ValueError('XML file generated by asXML is not valid !')
except Exception:
issue = "Cannot report usage for %r: %s" % (
computer_partition.getId(),
traceback.format_exc())
self.logger.info(issue)
computer_partition.error(issue, logger=self.logger)
report_usage_issue_cp_list.append(computer_partition_id)
for computer_partition in computer_partition_list:
if computer_partition.getState() == COMPUTER_PARTITION_DESTROYED_STATE:
try:
computer_partition_id = computer_partition.getId()
try:
software_url = computer_partition.getSoftwareRelease().getURI()
software_path = os.path.join(self.software_root, md5digest(software_url))
except (NotFoundError, TypeError):
software_url = None
software_path = None
local_partition = Partition(
software_path=software_path,
instance_path=os.path.join(self.instance_root,
computer_partition.getId()),
supervisord_partition_configuration_path=os.path.join(
self.supervisord_configuration_directory, '%s.conf' %
computer_partition_id),
supervisord_socket=self.supervisord_socket,
computer_partition=computer_partition,
computer_id=self.computer_id,
partition_id=computer_partition_id,
server_url=self.master_url,
software_release_url=software_url,
certificate_repository_path=self.certificate_repository_path,
buildout=self.buildout,
logger=self.logger)
local_partition.stop()
try:
computer_partition.stopped()
except (SystemExit, KeyboardInterrupt):
computer_partition.error(traceback.format_exc(), logger=self.logger)
raise
except Exception:
pass
if computer_partition.getId() in report_usage_issue_cp_list:
self.logger.info('Ignoring destruction of %r, as no report usage was sent' %
computer_partition.getId())
continue
local_partition.destroy()
except (SystemExit, KeyboardInterrupt):
computer_partition.error(traceback.format_exc(), logger=self.logger)
raise
except Exception:
clean_run = False
self.logger.exception('')
exc = traceback.format_exc()
computer_partition.error(exc, logger=self.logger)
try:
computer_partition.destroyed()
except NotFoundError:
self.logger.debug('Ignored slap error while trying to inform about '
'destroying not fully configured Computer Partition %r' %
computer_partition.getId())
except ServerError as server_error:
self.logger.debug('Ignored server error while trying to inform about '
'destroying Computer Partition %r. Error is:\n%r' %
(computer_partition.getId(), server_error.args[0]))
self.logger.info('Finished usage reports.')
# Return success value
if not clean_run:
return SLAPGRID_FAIL
return SLAPGRID_SUCCESS
def processComputerPartition(self, computer_partition):
"""
Process a Computer Partition, depending on its state
"""
computer_partition_id = computer_partition.getId()
# Sanity checks before processing
# Those values should not be None or empty string or any falsy value
if not computer_partition_id:
raise ValueError('Computer Partition id is empty.')
# Check if we defined explicit list of partitions to process.
# If so, if current partition not in this list, skip.
if len(self.computer_partition_filter_list) > 0 and \
(computer_partition_id not in self.computer_partition_filter_list):
return
self.logger.info('Processing Computer Partition %s.' % computer_partition_id)
instance_path = os.path.join(self.instance_root, computer_partition_id)
# Try to get partition timestamp (last modification date)
timestamp_path = os.path.join(
instance_path,
COMPUTER_PARTITION_TIMESTAMP_FILENAME
)
parameter_dict = computer_partition.getInstanceParameterDict()
if 'timestamp' in parameter_dict:
timestamp = parameter_dict['timestamp']
else:
timestamp = None
try:
software_url = computer_partition.getSoftwareRelease().getURI()
except NotFoundError:
# Problem with instance: SR URI not set.
# Try to process it anyway, it may need to be deleted.
software_url = None
try:
software_path = os.path.join(self.software_root, md5digest(software_url))
except TypeError:
# Problem with instance: SR URI not set.
# Try to process it anyway, it may need to be deleted.
software_path = None
periodicity = self.maximum_periodicity
if software_path:
periodicity_path = os.path.join(software_path, 'periodicity')
if os.path.exists(periodicity_path):
try:
periodicity = int(open(periodicity_path).read())
except ValueError:
os.remove(periodicity_path)
self.logger.exception('')
# Check if timestamp from server is more recent than local one.
# If not: it's not worth processing this partition (nothing has
# changed).
if (computer_partition_id not in self.computer_partition_filter_list and
not self.develop and os.path.exists(timestamp_path)):
old_timestamp = open(timestamp_path).read()
last_runtime = int(os.path.getmtime(timestamp_path))
if timestamp:
try:
if periodicity == 0:
os.remove(timestamp_path)
elif int(timestamp) <= int(old_timestamp):
if computer_partition.getState() != COMPUTER_PARTITION_STARTED_STATE:
return
# Check periodicity, i.e if periodicity is one day, partition
# should be processed at least every day.
# Only do it for "started" instances
if int(time.time()) <= (last_runtime + periodicity) or periodicity < 0:
self.logger.info('Partition already up-to-date, skipping.')
return
else:
# Periodicity forced processing this partition. Removing
# the timestamp file in case it fails.
os.remove(timestamp_path)
except ValueError:
os.remove(timestamp_path)
self.logger.exception('')
self.logger.info(' Software URL: %s' % software_url)
self.logger.info(' Software path: %s' % software_path)
self.logger.info(' Instance path: %s' % instance_path)
local_partition = Partition(
software_path=software_path,
instance_path=instance_path,
supervisord_partition_configuration_path=os.path.join(
self.supervisord_configuration_directory, '%s.conf' %
computer_partition_id),
supervisord_socket=self.supervisord_socket,
computer_partition=computer_partition,
computer_id=self.computer_id,
partition_id=computer_partition_id,
server_url=self.master_url,
software_release_url=software_url,
certificate_repository_path=self.certificate_repository_path,
buildout=self.buildout,
logger=self.logger)
computer_partition_state = computer_partition.getState()
if computer_partition_state == COMPUTER_PARTITION_STARTED_STATE:
local_partition.install()
computer_partition.available()
local_partition.start()
self._checkPromises(computer_partition)
computer_partition.started()
elif computer_partition_state == COMPUTER_PARTITION_STOPPED_STATE:
try:
local_partition.install()
computer_partition.available()
finally:
# Instance has to be stopped even if buildout/reporting is wrong.
local_partition.stop()
computer_partition.stopped()
elif computer_partition_state == COMPUTER_PARTITION_DESTROYED_STATE:
local_partition.stop()
try:
computer_partition.stopped()
except (SystemExit, KeyboardInterrupt):
computer_partition.error(traceback.format_exc(), logger=self.logger)
raise
except Exception:
pass
else:
error_string = "Computer Partition %r has unsupported state: %s" % \
(computer_partition_id, computer_partition_state)
computer_partition.error(error_string, logger=self.logger)
raise NotImplementedError(error_string)
# If partition has been successfully processed, write timestamp
if timestamp:
timestamp_path = os.path.join(
instance_path,
COMPUTER_PARTITION_TIMESTAMP_FILENAME
)
open(timestamp_path, 'w').write(timestamp)
def processSoftwareReleaseList(self):
"""Will process each Software Release.
"""
self.checkEnvironmentAndCreateStructure()
self.logger.info('Processing software releases...')
# Boolean to know if every instance has correctly been deployed
clean_run = True
for software_release in self.computer.getSoftwareReleaseList():
state = software_release.getState()
try:
software_release_uri = software_release.getURI()
url_hash = md5digest(software_release_uri)
software_path = os.path.join(self.software_root, url_hash)
software = Software(url=software_release_uri,
software_root=self.software_root,
buildout=self.buildout,
logger=self.logger,
signature_private_key_file=self.signature_private_key_file,
signature_certificate_list=self.signature_certificate_list,
download_binary_cache_url=self.download_binary_cache_url,
upload_binary_cache_url=self.upload_binary_cache_url,
download_from_binary_cache_url_blacklist=
self.download_from_binary_cache_url_blacklist,
upload_to_binary_cache_url_blacklist=
self.upload_to_binary_cache_url_blacklist,
upload_cache_url=self.upload_cache_url,
download_binary_dir_url=self.download_binary_dir_url,
upload_binary_dir_url=self.upload_binary_dir_url,
upload_dir_url=self.upload_dir_url,
shacache_cert_file=self.shacache_cert_file,
shacache_key_file=self.shacache_key_file,
shadir_cert_file=self.shadir_cert_file,
shadir_key_file=self.shadir_key_file)
if state == 'available':
completed_tag = os.path.join(software_path, '.completed')
if (self.develop or (not os.path.exists(completed_tag) and
len(self.software_release_filter_list) == 0) or
url_hash in self.software_release_filter_list or
url_hash in (md5digest(uri) for uri in self.software_release_filter_list)):
try:
software_release.building()
except NotFoundError:
pass
software.install()
with open(completed_tag, 'w') as fout:
fout.write(time.asctime())
elif state == 'destroyed':
if os.path.exists(software_path):
self.logger.info('Destroying %r...' % software_release_uri)
software.destroy()
self.logger.info('Destroyed %r.' % software_release_uri)
# Send log before exiting
except (SystemExit, KeyboardInterrupt):
software_release.error(traceback.format_exc(), logger=self.logger)
raise
# Buildout failed: send log but don't print it to output (already done)
except BuildoutFailedError as exc:
clean_run = False
try:
software_release.error(exc, logger=self.logger)
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
self.logger.exception('Problem while reporting error, continuing:')
# For everything else: log it, send it, continue.
except Exception:
self.logger.exception('')
software_release.error(traceback.format_exc(), logger=self.logger)
clean_run = False
else:
if state == 'available':
try:
software_release.available()
except NotFoundError:
pass
elif state == 'destroyed':
try:
software_release.destroyed()
except (NotFoundError, ServerError):
self.logger.exception('')
self.logger.info('Finished software releases.')
# Return success value
if not clean_run:
return SLAPGRID_FAIL
return SLAPGRID_SUCCESS
def processComputerPartition(self, computer_partition):
"""
Process a Computer Partition, depending on its state
"""
computer_partition_id = computer_partition.getId()
# Sanity checks before processing
# Those values should not be None or empty string or any falsy value
if not computer_partition_id:
raise ValueError('Computer Partition id is empty.')
# Check if we defined explicit list of partitions to process.
# If so, if current partition not in this list, skip.
if len(self.computer_partition_filter_list) > 0 and \
(computer_partition_id not in self.computer_partition_filter_list):
return
self.logger.debug('Check if %s requires processing...' % computer_partition_id)
instance_path = os.path.join(self.instance_root, computer_partition_id)
# Try to get partition timestamp (last modification date)
timestamp_path = os.path.join(
instance_path,
COMPUTER_PARTITION_TIMESTAMP_FILENAME
)
parameter_dict = computer_partition.getInstanceParameterDict()
if 'timestamp' in parameter_dict:
timestamp = parameter_dict['timestamp']
else:
timestamp = None
try:
software_url = computer_partition.getSoftwareRelease().getURI()
except NotFoundError:
# Problem with instance: SR URI not set.
# Try to process it anyway, it may need to be deleted.
software_url = None
try:
software_path = os.path.join(self.software_root, md5digest(software_url))
except TypeError:
# Problem with instance: SR URI not set.
# Try to process it anyway, it may need to be deleted.
software_path = None
periodicity = self.maximum_periodicity
if software_path:
periodicity_path = os.path.join(software_path, 'periodicity')
if os.path.exists(periodicity_path):
try:
periodicity = int(open(periodicity_path).read())
except ValueError:
os.remove(periodicity_path)
self.logger.exception('')
# Check if timestamp from server is more recent than local one.
# If not: it's not worth processing this partition (nothing has
# changed).
if (computer_partition_id not in self.computer_partition_filter_list and
not self.develop and os.path.exists(timestamp_path)):
old_timestamp = open(timestamp_path).read()
last_runtime = int(os.path.getmtime(timestamp_path))
if timestamp:
try:
if periodicity == 0:
os.remove(timestamp_path)
elif int(timestamp) <= int(old_timestamp):
# Check periodicity, i.e if periodicity is one day, partition
# should be processed at least every day.
if int(time.time()) <= (last_runtime + periodicity) or periodicity < 0:
self.logger.debug('Partition already up-to-date, skipping.')
return
else:
# Periodicity forced processing this partition. Removing
# the timestamp file in case it fails.
os.remove(timestamp_path)
except ValueError:
os.remove(timestamp_path)
self.logger.exception('')
# Include Partition Logging
log_folder_path = "%s/.slapgrid/log" % instance_path
mkdir_p(log_folder_path)
partition_file_handler = logging.FileHandler(
filename="%s/instance.log" % (log_folder_path)
)
stat_info = os.stat(instance_path)
chownDirectory("%s/.slapgrid" % instance_path,
uid=stat_info.st_uid,
gid=stat_info.st_gid)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)-8s %(name)s %(message)s')
partition_file_handler.setFormatter(formatter)
self.logger.addHandler(partition_file_handler)
try:
self.logger.info('Processing Computer Partition %s.' % computer_partition_id)
self.logger.info(' Software URL: %s' % software_url)
self.logger.info(' Software path: %s' % software_path)
self.logger.info(' Instance path: %s' % instance_path)
filter_dict = getattr(computer_partition, '_filter_dict', None)
if filter_dict:
retention_delay = filter_dict.get('retention_delay', '0')
else:
retention_delay = '0'
local_partition = Partition(
software_path=software_path,
instance_path=instance_path,
supervisord_partition_configuration_path=os.path.join(
_getSupervisordConfigurationDirectory(self.instance_root), '%s.conf' %
computer_partition_id),
supervisord_socket=self.supervisord_socket,
computer_partition=computer_partition,
computer_id=self.computer_id,
partition_id=computer_partition_id,
server_url=self.master_url,
software_release_url=software_url,
certificate_repository_path=self.certificate_repository_path,
buildout=self.buildout,
logger=self.logger,
retention_delay=retention_delay,
instance_min_free_space=self.instance_min_free_space,
instance_storage_home=self.instance_storage_home,
ipv4_global_network=self.ipv4_global_network,
)
computer_partition_state = computer_partition.getState()
# XXX this line breaks 37 tests
# self.logger.info(' Instance type: %s' % computer_partition.getType())
self.logger.info(' Instance status: %s' % computer_partition_state)
if computer_partition_state == COMPUTER_PARTITION_STARTED_STATE:
local_partition.install()
computer_partition.available()
local_partition.start()
self._checkPromises(computer_partition)
computer_partition.started()
elif computer_partition_state == COMPUTER_PARTITION_STOPPED_STATE:
try:
# We want to process the partition, even if stopped, because it should
# propagate the state to children if any.
local_partition.install()
computer_partition.available()
finally:
# Instance has to be stopped even if buildout/reporting is wrong.
local_partition.stop()
computer_partition.stopped()
elif computer_partition_state == COMPUTER_PARTITION_DESTROYED_STATE:
local_partition.stop()
try:
computer_partition.stopped()
except (SystemExit, KeyboardInterrupt):
computer_partition.error(traceback.format_exc(), logger=self.logger)
raise
except Exception:
pass
else:
error_string = "Computer Partition %r has unsupported state: %s" % \
(computer_partition_id, computer_partition_state)
computer_partition.error(error_string, logger=self.logger)
raise NotImplementedError(error_string)
finally:
self.logger.removeHandler(partition_file_handler)
# If partition has been successfully processed, write timestamp
if timestamp:
open(timestamp_path, 'w').write(timestamp)
def runTestSuite(self, node_test_suite, portal_url):
config = self.testnode.config
run_test_suite_path_list = glob.glob(
self.getInstanceRoot(node_test_suite) + "/*/bin/runTestSuite")
try:
run_test_suite_path = min(run_test_suite_path_list)
except ValueError:
raise ValueError(
'No runTestSuite provided in installed partitions.')
# Deal with Shebang size limitation
invocation_list = dealShebang(run_test_suite_path)
invocation_list += (run_test_suite_path, '--master_url', portal_url,
'--revision', node_test_suite.revision,
'--test_node_title', config['test_node_title'],
'--test_suite', node_test_suite.test_suite,
'--test_suite_title',
node_test_suite.test_suite_title)
soft = config['slapos_directory'] + '/soft/'
software_list = [soft + md5digest(x) for x in config['software_list']]
PATH = os.getenv('PATH', '')
PATH = ':'.join(x + '/bin'
for x in software_list) + (PATH and ':' + PATH)
SLAPOS_TEST_SHARED_PART_LIST = os.pathsep.join(
self._getSlapOSControler(node_test_suite.working_directory,
True).shared_part_list)
SLAPOS_TEST_LOG_DIRECTORY = node_test_suite.log_folder_path
supported_parameter_set = set(
self.testnode.process_manager.getSupportedParameterList(
run_test_suite_path))
def path(name, compat): # BBB
path, = filter(
os.path.exists,
(base + relative
for relative in ('/bin/' + name, '/parts/' + compat)
for base in software_list))
return path
for option, value in (
('--firefox_bin',
lambda: path('firefox', 'firefox/firefox-slapos')),
('--frontend_url', lambda: config['frontend_url']),
('--node_quantity', lambda: config['node_quantity']),
('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')),
('--project_title', lambda: node_test_suite.project_title),
('--shared_part_list', lambda: SLAPOS_TEST_SHARED_PART_LIST),
('--log_directory', lambda: SLAPOS_TEST_LOG_DIRECTORY),
):
if option in supported_parameter_set:
invocation_list += option, value()
# TODO : include testnode correction ( b111682f14890bf )
if hasattr(node_test_suite, 'additional_bt5_repository_id'):
additional_bt5_path = os.path.join(
node_test_suite.working_directory,
node_test_suite.additional_bt5_repository_id)
invocation_list.extend(["--bt5_path", additional_bt5_path])
# From this point, test runner becomes responsible for updating test
# result. We only do cleanup if the test runner itself is not able
# to run.
createFolder(node_test_suite.test_suite_directory, clean=True)
# Log the actual command with root logger
root_logger = logging.getLogger()
root_logger.info("Running test suite with: %s",
format_command(*invocation_list, PATH=PATH))
def hide_distributor_url(s):
# type: (bytes) -> bytes
return s.replace(portal_url.encode('utf-8'), b'$DISTRIBUTOR_URL')
self.testnode.process_manager.spawn(
*invocation_list,
PATH=PATH,
SLAPOS_TEST_SHARED_PART_LIST=SLAPOS_TEST_SHARED_PART_LIST,
SLAPOS_TEST_LOG_DIRECTORY=SLAPOS_TEST_LOG_DIRECTORY,
cwd=node_test_suite.test_suite_directory,
log_prefix='runTestSuite',
output_replacers=(hide_distributor_url, ),
get_output=False)
