def initializeSlapOSControler(self,
slapproxy_log=None,
process_manager=None,
reset_software=False,
software_path_list=None):
self.process_manager = process_manager
self.software_path_list = software_path_list
logger.debug('SlapOSControler, initialize, reset_software: %r',
reset_software)
config = self.config
slapos_config_dict = config.copy()
slapos_config_dict.update(software_root=self.software_root,
instance_root=self.instance_root,
proxy_database=self.proxy_database)
with open(self.slapos_config, 'w') as f:
f.write(
pkg_resources.resource_string('erp5.util.testnode',
'template/slapos.cfg.in') %
slapos_config_dict)
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(self.proxy_database):
os.unlink(self.proxy_database)
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
# Make sure there is no slapos alive from previous run
process_manager.killall('slapos')
proxy = subprocess.Popen([
config['slapos_binary'], 'proxy', 'start', '--cfg',
self.slapos_config
], **kwargs)
process_manager.process_pid_set.add(proxy.pid)
slap = self.slap = slapos.slap.slap()
# Wait for proxy to accept connections
retries = 0
while True:
time.sleep(.5)
try:
slap.initializeConnection(config['master_url'])
computer = slap.registerComputer(config['computer_id'])
# Call a method to ensure connection to master can be established
computer.getComputerPartitionList()
except slapos.slap.ConnectionError, e:
retries += 1
if retries >= 20:
raise
logger.debug("Proxy still not started %s, retrying", e)
else:
break
def __init__(self, config, log, process_group_pid_set=None,
slapproxy_log=None):
self.log = log
self.config = config
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], **kwargs)
process_group_pid_set.add(proxy.pid)
# XXX: dirty, giving some time for proxy to being able to accept
# connections
time.sleep(10)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
# register software profile
self.software_profile = config['custom_profile_path']
slap.registerSupply().supply(
self.software_profile,
computer_guid=config['computer_id'])
computer = slap.registerComputer(config['computer_id'])
# create partition and configure computer
partition_reference = config['partition_reference']
partition_path = os.path.join(config['instance_root'], partition_reference)
if not os.path.exists(partition_path):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': config['instance_root'],
'netmask': '255.255.255.255',
'partition_list': [{'address_list': [{'addr': config['ipv4_address'],
'netmask': '255.255.255.255'},
{'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'},
],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference},
}
],
'reference': config['computer_id'],
'software_root': config['software_root']}))
def updateProxy(config):
"""
Configure Slapos Node computer and partitions.
Send current Software Release to Slapproxy for compilation and deployment.
"""
startProxy(config)
if not os.path.exists(config['instance_root']):
os.mkdir(config['instance_root'])
slap = slapos.slap.slap()
profile = getCurrentSoftwareReleaseProfile(config)
slap.initializeConnection(config['master_url'])
slap.registerSupply().supply(profile, computer_guid=config['computer_id'])
computer = slap.registerComputer(config['computer_id'])
prefix = 'slappart'
slap_config = {
'address': config['ipv4_address'],
'instance_root': config['instance_root'],
'netmask': '255.255.255.255',
'partition_list': [],
'reference': config['computer_id'],
'software_root': config['software_root']
}
for i in xrange(0, int(config['partition_amount'])):
partition_reference = '%s%s' % (prefix, i)
partition_path = os.path.join(config['instance_root'], partition_reference)
if not os.path.exists(partition_path):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
slap_config['partition_list'].append({
'address_list': [
{
'addr': config['ipv4_address'],
'netmask': '255.255.255.255'
}, {
'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'
},
],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference}})
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps(slap_config))
return True
def getSlapStatus(config):
"""Return all Slapos Partitions with associate information"""
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
partition_list = []
computer = slap.registerComputer(config['computer_id'])
try:
for partition in computer.getComputerPartitionList():
# Note: Internal use of API, as there is no reflexion interface in SLAP
partition_list.append((partition.getId(), partition._connection_dict.copy()))
except Exception:
pass
if partition_list:
for i in xrange(0, int(config['partition_amount'])):
slappart_id = '%s%s' % ("slappart", i)
if not [x[0] for x in partition_list if slappart_id == x[0]]:
partition_list.append((slappart_id, []))
return partition_list
def test_updateInstanceParameter(self):
"""Scenarion 5: Update parameters of current sofware profile"""
self.setAccount()
self.setupTestSoftware()
#Set current projet and run Slapgrid-cp
software = os.path.join(self.software, 'slaprunner-test')
response = loadJson(self.app.post('/setCurrentProject',
data=dict(path=software),
follow_redirects=True))
self.assertEqual(response['result'], "")
self.proxyStatus(True)
#Send paramters for the instance
parameterDict = dict(appname='slaprunnerTest', cacountry='France')
parameterXml = '<?xml version="1.0" encoding="utf-8"?>\n<instance>'
parameterXml += '<parameter id="appname">slaprunnerTest</parameter>\n'
parameterXml += '<parameter id="cacountry">France</parameter>\n</instance>'
software_type = 'production'
response = loadJson(self.app.post('/saveParameterXml',
data=dict(parameter=parameterXml,
software_type=software_type),
follow_redirects=True))
self.assertEqual(response['result'], "")
slap = slapos.slap.slap()
slap.initializeConnection(self.app.config['master_url'])
computer = slap.registerComputer(self.app.config['computer_id'])
partitionList = computer.getComputerPartitionList()
self.assertNotEqual(partitionList, [])
#Assume that the requested partition is partition 0
slapParameterDict = partitionList[0].getInstanceParameterDict()
self.assertTrue('appname' in slapParameterDict)
self.assertTrue('cacountry' in slapParameterDict)
self.assertEqual(slapParameterDict['appname'], 'slaprunnerTest')
self.assertEqual(slapParameterDict['cacountry'], 'France')
self.assertEqual(slapParameterDict['slap_software_type'], 'production')
#test getParameterXml for webrunner UI
response = loadJson(self.app.get('/getParameterXml/xml'))
self.assertEqual(parameterXml, response['result'])
response = loadJson(self.app.get('/getParameterXml/dict'))
self.assertEqual(parameterDict, response['result']['instance'])
self.stopSlapproxy()
def initializeSlapOSControler(self,
slapproxy_log=None,
process_manager=None,
reset_software=False,
software_path_list=None):
self.process_manager = process_manager
self.software_path_list = software_path_list
self.log('SlapOSControler, initialize, reset_software: %r' %
reset_software)
config = self.config
slapos_config_dict = self.config.copy()
slapos_config_dict.update(software_root=self.software_root,
instance_root=self.instance_root,
proxy_database=self.proxy_database)
open(self.slapos_config, 'w').write(
pkg_resources.resource_string('erp5.util.testnode',
'template/slapos.cfg.in') %
slapos_config_dict)
createFolder(self.software_root)
createFolder(self.instance_root)
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(self.proxy_database):
os.unlink(self.proxy_database)
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
proxy = subprocess.Popen(
[config['slapproxy_binary'], self.slapos_config], **kwargs)
process_manager.process_pid_set.add(proxy.pid)
# XXX: dirty, giving some time for proxy to being able to accept
# connections
time.sleep(10)
try:
slap = slapos.slap.slap()
self.slap = slap
self.slap.initializeConnection(config['master_url'])
# register software profile
for path in self.software_path_list:
slap.registerSupply().supply(
path, computer_guid=config['computer_id'])
computer = slap.registerComputer(config['computer_id'])
except:
self.log("SlapOSControler.initializeSlapOSControler, \
exception in registerSupply",
exc_info=sys.exc_info())
raise ValueError("Unable to initializeSlapOSControler")
# Reset all previously generated software if needed
if reset_software:
self._resetSoftware()
instance_root = self.instance_root
if os.path.exists(instance_root):
# delete old paritions which may exists in order to not get its data
# (ex. MySQL db content) from previous testnode's runs
# In order to be able to change partition naming scheme, do this at
# instance_root level (such change happened already, causing problems).
shutil.rmtree(instance_root)
if not (os.path.exists(instance_root)):
os.mkdir(instance_root)
for i in range(0, MAX_PARTIONS):
# create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports
partition_reference = '%s-%s' % (config['partition_reference'], i)
partition_path = os.path.join(instance_root, partition_reference)
if not (os.path.exists(partition_path)):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
computer.updateConfiguration(
xml_marshaller.xml_marshaller.dumps({
'address':
config['ipv4_address'],
'instance_root':
instance_root,
'netmask':
'255.255.255.255',
'partition_list': [{
'address_list': [
{
'addr': config['ipv4_address'],
'netmask': '255.255.255.255'
},
{
'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'
},
],
'path':
partition_path,
'reference':
partition_reference,
'tap': {
'name': partition_reference
},
}],
'reference':
config['computer_id'],
'software_root':
self.software_root
}))
def initializeSlapOSControler(self, slapproxy_log=None, process_manager=None,
reset_software=False, software_path_list=None):
self.process_manager = process_manager
self.software_path_list = software_path_list
self.log('SlapOSControler, initialize, reset_software: %r' % reset_software)
config = self.config
slapos_config_dict = self.config.copy()
slapos_config_dict.update(software_root=self.software_root,
instance_root=self.instance_root,
proxy_database=self.proxy_database)
open(self.slapos_config, 'w').write(pkg_resources.resource_string(
'erp5.util.testnode', 'template/slapos.cfg.in') %
slapos_config_dict)
createFolder(self.software_root)
createFolder(self.instance_root)
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(self.proxy_database):
os.unlink(self.proxy_database)
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
proxy = subprocess.Popen([config['slapproxy_binary'],
self.slapos_config], **kwargs)
process_manager.process_pid_set.add(proxy.pid)
# XXX: dirty, giving some time for proxy to being able to accept
# connections
time.sleep(10)
try:
slap = slapos.slap.slap()
self.slap = slap
self.slap.initializeConnection(config['master_url'])
# register software profile
for path in self.software_path_list:
slap.registerSupply().supply(
path,
computer_guid=config['computer_id'])
computer = slap.registerComputer(config['computer_id'])
except:
self.log("SlapOSControler.initializeSlapOSControler, \
exception in registerSupply", exc_info=sys.exc_info())
raise ValueError("Unable to initializeSlapOSControler")
# Reset all previously generated software if needed
if reset_software:
self._resetSoftware()
instance_root = self.instance_root
if os.path.exists(instance_root):
# delete old paritions which may exists in order to not get its data
# (ex. MySQL db content) from previous testnode's runs
# In order to be able to change partition naming scheme, do this at
# instance_root level (such change happened already, causing problems).
shutil.rmtree(instance_root)
if not(os.path.exists(instance_root)):
os.mkdir(instance_root)
for i in range(0, MAX_PARTIONS):
# create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports
partition_reference = '%s-%s' %(config['partition_reference'], i)
partition_path = os.path.join(instance_root, partition_reference)
if not(os.path.exists(partition_path)):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': instance_root,
'netmask': '255.255.255.255',
'partition_list': [
{'address_list': [{'addr': config['ipv4_address'],
'netmask': '255.255.255.255'},
{'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'},],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference},}],
'reference': config['computer_id'],
'software_root': self.software_root}))
def initializeSlapOSControler(self,
slapproxy_log=None,
process_manager=None,
reset_software=False,
software_path_list=None):
self.process_manager = process_manager
self.software_path_list = software_path_list
logger.debug('SlapOSControler, initialize, reset_software: %r',
reset_software)
config = self.config
slapos_config_dict = config.copy()
slapos_config_dict.update(software_root=self.software_root,
instance_root=self.instance_root,
proxy_database=self.proxy_database,
shared_part_list='\n '.join(
self.shared_part_list))
with open(self.slapos_config, 'w') as f:
f.write(
pkg_resources.resource_string(
'erp5.util.testnode', 'template/slapos.cfg.in').decode() %
slapos_config_dict)
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(self.proxy_database):
os.unlink(self.proxy_database)
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
proxy = subprocess.Popen([
config['slapos_binary'], 'proxy', 'start', '--cfg',
self.slapos_config
], **kwargs)
process_manager.process_pid_set.add(proxy.pid)
slap = self.slap = slapos.slap.slap()
# Wait for proxy to accept connections
retries = 0
while True:
time.sleep(1)
try:
slap.initializeConnection(config['master_url'])
computer = slap.registerComputer(config['computer_id'])
# Call a method to ensure connection to master can be established
computer.getComputerPartitionList()
except slapos.slap.ConnectionError as e:
retries += 1
if retries >= 60:
raise
logger.debug("Proxy still not started %s, retrying", e)
else:
break
try:
# register software profile
for path in self.software_path_list:
slap.registerSupply().supply(
path, computer_guid=config['computer_id'])
except Exception:
logger.exception("SlapOSControler.initializeSlapOSControler")
raise ValueError("Unable to registerSupply")
# Reset all previously generated software if needed
if reset_software:
self._resetSoftware()
else:
createFolder(self.software_root)
instance_root = self.instance_root
# Delete any existing partition in order to not get its data (ex.
# MySQL DB content) from previous runs. To support changes of partition
# naming scheme (which already happened), do this at instance_root level.
createFolder(instance_root, True)
partition_list = []
for i in range(MAX_PARTITIONS):
# create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports
partition_reference = '%s-%s' % (config['partition_reference'], i)
partition_path = os.path.join(instance_root, partition_reference)
if not (os.path.exists(partition_path)):
os.mkdir(partition_path)
os.chmod(partition_path, 0o750)
partition_list.append({
'address_list': [
{
'addr': config['ipv4_address'],
'netmask': '255.255.255.255'
},
{
'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'
},
],
'path':
partition_path,
'reference':
partition_reference,
'tap': {
'name': partition_reference
},
})
computer.updateConfiguration(
xml_marshaller.xml_marshaller.dumps({
'address':
config['ipv4_address'],
'instance_root':
instance_root,
'netmask':
'255.255.255.255',
'partition_list':
partition_list,
'reference':
config['computer_id'],
'software_root':
self.software_root
}))
def __init__(self, config, log,
slapproxy_log=None, process_manager=None, reset_software=False):
log('SlapOSControler, initialize, reset_software: %r' % reset_software)
self.log = log
self.config = config
self.process_manager = process_manager
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], **kwargs)
process_manager.process_pid_set.add(proxy.pid)
# XXX: dirty, giving some time for proxy to being able to accept
# connections
time.sleep(10)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
# register software profile
self.software_profile = config['custom_profile_path']
slap.registerSupply().supply(
self.software_profile,
computer_guid=config['computer_id'])
computer = slap.registerComputer(config['computer_id'])
# Reset all previously generated software if needed
if reset_software:
software_root = config['software_root']
log('SlapOSControler : GOING TO RESET ALL SOFTWARE')
if os.path.exists(software_root):
shutil.rmtree(software_root)
os.mkdir(software_root)
os.chmod(software_root, 0750)
instance_root = config['instance_root']
if os.path.exists(instance_root):
# delete old paritions which may exists in order to not get its data
# (ex. MySQL db content) from previous testnode's runs
# In order to be able to change partition naming scheme, do this at
# instance_root level (such change happened already, causing problems).
shutil.rmtree(instance_root)
os.mkdir(instance_root)
for i in range(0, MAX_PARTIONS):
# create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports
partition_reference = '%s-%s' %(config['partition_reference'], i)
partition_path = os.path.join(instance_root, partition_reference)
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': instance_root,
'netmask': '255.255.255.255',
'partition_list': [{'address_list': [{'addr': config['ipv4_address'],
'netmask': '255.255.255.255'},
{'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'},],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference},
}
],
'reference': config['computer_id'],
'software_root': config['software_root']}))
def run(args):
config = args[0]
for k,v in config['environment'].iteritems():
os.environ[k] = v
proxy = None
slapgrid = None
supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run',
'supervisord.pid')
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
try:
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(proxy.pid)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
while True:
try:
slap.registerSupply().supply(config['profile_url'],
computer_guid=config['computer_id'])
except socket.error:
time.sleep(1)
pass
else:
break
while True:
slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc',
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(slapgrid.pid)
slapgrid.wait()
if slapgrid.returncode == 0:
print 'Software installed properly'
break
print 'Problem with software installation, trying again'
time.sleep(600)
computer = slap.registerComputer(config['computer_id'])
partition_reference = config['partition_reference']
partition_path = os.path.join(config['instance_root'], partition_reference)
if not os.path.exists(partition_path):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
computer.updateConfiguration(xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': config['instance_root'],
'netmask': '255.255.255.255',
'partition_list': [{'address_list': [{'addr': config['ipv4_address'],
'netmask': '255.255.255.255'},
{'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'},
],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference},
}
],
'reference': config['computer_id'],
'software_root': config['software_root']}))
slap.registerOpenOrder().request(config['profile_url'],
partition_reference='testing partition',
partition_parameter_kw=config['instance_dict'])
slapgrid = subprocess.Popen([config['slapgrid_partition_binary'], '-vc',
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
slapgrid.wait()
if slapgrid.returncode != 0:
raise ValueError('Slapgrid instance failed')
runUnitTest = os.path.join(partition_path, 'bin', 'runUnitTest')
if not os.path.exists(runUnitTest):
raise ValueError('No %r provided' % runUnitTest)
except:
try:
if os.path.exists(supervisord_pid_file):
os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM)
except:
pass
raise
finally:
# Nice way to kill *everything* generated by run process -- process
# groups working only in POSIX compilant systems
# Exceptions are swallowed during cleanup phase
if proxy is not None:
os.killpg(proxy.pid, signal.SIGTERM)
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
if slapgrid is not None and slapgrid.returncode is None:
os.killpg(slapgrid.pid, signal.SIGTERM)
try:
bot_env = os.environ.copy()
bot_env['PATH'] = ':'.join([config['bin_directory']] +
bot_env['PATH'].split(':'))
for l in config['bot_environment'].split():
k, v = l.split('=')
bot_env[k] = v
if subprocess.call([config['buildbot_binary'], 'create-slave', '-f',
config['working_directory'], config['buildbot_host'],
config['slave_name'], config['slave_password']]) != 0:
raise ValueError('Buildbot call failed')
process_command_list.append([config['buildbot_binary'], 'stop',
config['working_directory']])
if os.path.exists(os.path.join(config['working_directory'],
'buildbot.tac.new')):
tac = os.path.join(config['working_directory'], 'buildbot.tac')
if os.path.exists(tac):
os.unlink(tac)
os.rename(os.path.join(config['working_directory'],
'buildbot.tac.new'), tac)
if subprocess.call([config['buildbot_binary'], 'start',
config['working_directory']], env=bot_env) != 0:
raise ValueError('Issue during starting buildbot')
while True:
time.sleep(3600)
finally:
try:
subprocess.call([config['buildbot_binary'], 'stop',
config['working_directory']])
except:
pass
try:
if os.path.exists(supervisord_pid_file):
os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM)
except:
pass
