def updateRevisionList(self, node_test_suite):
config = self.config
revision_list = []
try:
for vcs_repository in node_test_suite.vcs_repository_list:
repository_path = vcs_repository['repository_path']
repository_id = vcs_repository['repository_id']
branch = vcs_repository.get('branch')
# Make sure we have local repository
updater = Updater(
repository_path,
git_binary=config['git_binary'],
branch=branch,
process_manager=self.process_manager,
working_directory=node_test_suite.working_directory,
url=vcs_repository["url"])
updater.checkout()
revision_list.append((repository_id, updater.getRevision()))
except SubprocessError as error:
# only limit to particular error, if we run that code for all errors,
# then if server having most repositories is down for some time, we would
# erase all repositories and facing later hours of downloads
if b'index' in getattr(error, 'stderr', b''):
rmtree(repository_path)
logger.warning(
"Error while getting repository, ignoring this test suite",
exc_info=1)
return False
node_test_suite.revision_list = revision_list
return True
def _cleanupLog(self):
log_directory = self.config['log_directory']
prune_time = time.time() - 86400 * self.max_log_time
for log_folder in os.listdir(log_directory):
folder_path = os.path.join(log_directory, log_folder)
if os.path.isdir(folder_path):
if os.stat(folder_path).st_mtime < prune_time:
logger.debug("deleting log directory %r", folder_path)
rmtree(folder_path)
def purgeOldTestSuite(self, test_suite_data):
reference_set = set(os.listdir(self.working_directory))
reference_set.difference_update(x['test_suite_reference']
for x in test_suite_data)
for reference in reference_set:
fpath = os.path.join(self.working_directory, reference)
self.node_test_suite_dict.pop(reference, None)
logger.info("testnode.purgeOldTestSuite, DELETING : %r", fpath)
if os.path.isdir(fpath):
rmtree(fpath)
else:
os.remove(fpath)
def _cleanupTemporaryFiles(self):
"""
buildout seems letting files under /tmp. To avoid regular error of
missing disk space, remove old logs
"""
temp_directory = self.config["system_temp_folder"]
user_id = os.geteuid()
prune_time = time.time() - 86400 * self.max_temp_time
for temp_folder in os.listdir(temp_directory):
folder_path = os.path.join(temp_directory, temp_folder)
if (temp_folder.startswith("tmp")
or temp_folder.startswith("buildout")):
try:
stat = os.stat(folder_path)
if stat.st_uid == user_id and stat.st_mtime < prune_time:
logger.debug("deleting temp directory %r", folder_path)
if os.path.isdir(folder_path):
rmtree(folder_path)
else:
os.remove(folder_path)
except OSError:
logger.warning("_cleanupTemporaryFiles exception",
exc_info=1)
def createFolder(folder, clean=False):
if os.path.exists(folder):
if not clean:
return
rmtree(folder)
os.mkdir(folder)
def initializeSlapOSControler(self,
slapproxy_log=None,
process_manager=None,
reset_software=False,
software_path_list=None):
self.process_manager = process_manager
self.software_path_list = software_path_list
logger.debug('SlapOSControler, initialize, reset_software: %r',
reset_software)
config = self.config
slapos_config_dict = config.copy()
slapos_config_dict.update(software_root=self.software_root,
instance_root=self.instance_root,
proxy_database=self.proxy_database,
shared_part_list='\n '.join(
self.shared_part_list))
with open(self.slapos_config, 'w') as f:
f.write(
pkg_resources.resource_string(
'erp5.util.testnode', 'template/slapos.cfg.in').decode() %
slapos_config_dict)
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(self.proxy_database):
os.unlink(self.proxy_database)
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
proxy = subprocess.Popen([
config['slapos_binary'], 'proxy', 'start', '--cfg',
self.slapos_config
], **kwargs)
process_manager.process_pid_set.add(proxy.pid)
slap = self.slap = slapos.slap.slap()
# Wait for proxy to accept connections
retries = 0
while True:
time.sleep(1)
try:
slap.initializeConnection(config['master_url'])
computer = slap.registerComputer(config['computer_id'])
# Call a method to ensure connection to master can be established
computer.getComputerPartitionList()
except slapos.slap.ConnectionError as e:
retries += 1
if retries >= 60:
raise
logger.debug("Proxy still not started %s, retrying", e)
else:
break
try:
# register software profile
for path in self.software_path_list:
slap.registerSupply().supply(
path, computer_guid=config['computer_id'])
except Exception:
logger.exception("SlapOSControler.initializeSlapOSControler")
raise ValueError("Unable to registerSupply")
# Reset all previously generated software if needed
if reset_software:
self._resetSoftware()
else:
createFolder(self.software_root)
if os.path.exists(self.old_instance_root): # BBB
rmtree(self.old_instance_root)
instance_root = self.instance_root
# Delete any existing partition in order to not get its data (ex.
# MySQL DB content) from previous runs. To support changes of partition
# naming scheme (which already happened), do this at instance_root level.
createFolder(instance_root, True)
partition_list = []
for i in range(MAX_PARTITIONS):
# create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports
partition_reference = '%s%s' % (config['partition_reference'], i)
partition_path = os.path.join(instance_root, partition_reference)
if not (os.path.exists(partition_path)):
os.mkdir(partition_path)
os.chmod(partition_path, 0o750)
partition_list.append({
'address_list': [
{
'addr': config['ipv4_address'],
'netmask': '255.255.255.255'
},
{
'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'
},
],
'path':
partition_path,
'reference':
partition_reference,
'tap': {
'name': partition_reference
},
})
computer.updateConfiguration(
xml_marshaller.xml_marshaller.dumps({
'address':
config['ipv4_address'],
'instance_root':
instance_root,
'netmask':
'255.255.255.255',
'partition_list':
partition_list,
'reference':
config['computer_id'],
'software_root':
self.software_root
}))