def setUpClass(cls, **kw):
if len(kw) == 0:
return
cls.server_url = kw['server_url']
cls.key_file = kw['key_file']
cls.cert_file = kw['cert_file']
cls.computer_id = kw['computer_id']
cls.partition_id = kw['partition_id']
# Get parameters returned by slapos master
slap = slapos.slap.slap()
slap.initializeConnection(cls.server_url, cls.key_file, cls.cert_file)
cls.partition = slap.registerComputerPartition(
computer_guid=cls.computer_id,
partition_id=cls.partition_id
)
cls.parameter_dict = cls.partition.getConnectionParameterDict()
for attribute, value in cls.parameter_dict.iteritems():
setattr(cls, attribute.replace('-', '_'), value)
#create slaprunner configuration
views.app.config['TESTING'] = True
config = Config()
config.setConfig()
views.app.config.update(**config.__dict__)
cls.app = views.app.test_client()
cls.app.config = views.app.config
# Set up path (needed to find git binary)
os.environ['PATH'] = config.path
def test_error_new_SoftwareRelease_works(self):
"""
Asserts that calling SoftwareRelease.error on software release works
"""
computer_guid = self._getTestComputerId()
software_release_uri = 'http://server/' + self._getTestComputerId()
slap = self.slap
slap.initializeConnection(self.server_url)
def handler(url, req):
qs = urlparse.parse_qs(req.body)
if (url.path == '/softwareReleaseError' and
qs['computer_id'][0] == computer_guid and
qs['url'][0] == software_release_uri and
qs['error_log'][0] == 'some error'):
return {
'status_code': 200
}
return {'status_code': 404}
with httmock.HTTMock(handler):
software_release = self.slap.registerSoftwareRelease(software_release_uri)
software_release._computer_guid = computer_guid
software_release.error('some error')
def test_error_new_ComputerPartition_works(self):
"""
Asserts that calling ComputerPartition.error on new partition works
"""
computer_guid = self._getTestComputerId()
partition_id = 'PARTITION_01'
slap = self.slap
slap.initializeConnection(self.server_url)
def server_response(self, path, method, body, header):
parsed_url = urlparse.urlparse(path.lstrip('/'))
parsed_qs = urlparse.parse_qs(parsed_url.query)
if (parsed_url.path == 'registerComputerPartition' and
parsed_qs['computer_reference'][0] == computer_guid and
parsed_qs['computer_partition_reference'][0] == partition_id):
partition = slapos.slap.ComputerPartition(
computer_guid, partition_id)
return (200, {}, xml_marshaller.xml_marshaller.dumps(partition))
elif parsed_url.path == 'softwareInstanceError':
parsed_qs_body = urlparse.parse_qs(body)
# XXX: why do we have computer_id and not computer_reference?
# XXX: why do we have computer_partition_id and not
# computer_partition_reference?
if (parsed_qs_body['computer_id'][0] == computer_guid and
parsed_qs_body['computer_partition_id'][0] == partition_id and
parsed_qs_body['error_log'][0] == 'some error'):
return (200, {}, '')
return (404, {}, '')
httplib.HTTPConnection._callback = server_response
computer_partition = slap.registerComputerPartition(
computer_guid, partition_id)
# XXX: Interface does not define return value
computer_partition.error('some error')
def requestInstance(config, software_type=None):
"""
Request the main instance of our environment
"""
software_type_path = os.path.join(config['etc_dir'], ".software_type.xml")
if software_type:
# Write it to conf file for later use
open(software_type_path, 'w').write(software_type)
elif os.path.exists(software_type_path):
software_type = open(software_type_path).read().rstrip()
else:
software_type = 'default'
slap = slapos.slap.slap()
profile = getCurrentSoftwareReleaseProfile(config)
slap.initializeConnection(config['master_url'])
param_path = os.path.join(config['etc_dir'], ".parameter.xml")
xml_result = readParameters(param_path)
partition_parameter_kw = None
if type(xml_result) != type('') and 'instance' in xml_result:
partition_parameter_kw = xml_result['instance']
return slap.registerOpenOrder().request(
profile,
partition_reference=getSoftwareReleaseName(config),
partition_parameter_kw=partition_parameter_kw,
software_type=software_type,
filter_kw=None,
state=None,
shared=False)
def _test_new_computer_partition_state(self, state):
"""
Helper method to automate assertions of failing states on new Computer
Partition
"""
computer_guid = self._getTestComputerId()
partition_id = 'PARTITION_01'
slap = self.slap
slap.initializeConnection(self.server_url)
def handler(url, req):
qs = urlparse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' and
qs['computer_reference'][0] == computer_guid and
qs['computer_partition_reference'][0] == partition_id):
partition = slapos.slap.ComputerPartition(
computer_guid, partition_id)
return {
'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(partition)
}
else:
return {'status_code': 404}
with httmock.HTTMock(handler):
computer_partition = self.slap.registerComputerPartition(
computer_guid, partition_id)
self.assertRaises(slapos.slap.NotFoundError,
getattr(computer_partition, state))
def _initializeSlapOSConnection(self):
"""
Initialize communication with slapos
"""
slap = slapos.slap.slap()
retry = 0
while True:
# wait until _hateoas_navigator is loaded.
if retry > 100:
break
slap.initializeConnection(
self.slapos_url,
self.key_path,
self.cert_path,
timeout=120,
slapgrid_rest_uri=self.slapos_api_rest_url)
if getattr(slap, '_hateoas_navigator', None) is None:
retry += 1
logger.info(
"Fail to load _hateoas_navigator waiting a bit and retry.")
time.sleep(30)
else:
break
if getattr(slap, '_hateoas_navigator', None) is None:
raise ValueError("Fail to load _hateoas_navigator")
supply = slap.registerSupply()
order = slap.registerOpenOrder()
return slap, supply, order
def test_computer_getComputerPartitionList_no_partition(self):
"""
Asserts that calling Computer.getComputerPartitionList without Computer
Partitions returns empty list
"""
computer_guid = self._getTestComputerId()
slap = self.slap
slap.initializeConnection(self.server_url)
def server_response(self, path, method, body, header):
parsed_url = urlparse.urlparse(path.lstrip('/'))
parsed_qs = urlparse.parse_qs(parsed_url.query)
if (parsed_url.path == 'registerComputerPartition'
and 'computer_reference' in parsed_qs
and 'computer_partition_reference' in parsed_qs):
slap_partition = slapos.slap.ComputerPartition(
parsed_qs['computer_reference'][0],
parsed_qs['computer_partition_reference'][0])
return (200, {}, xml_marshaller.xml_marshaller.dumps(slap_partition))
elif (parsed_url.path == 'getFullComputerInformation'
and 'computer_id' in parsed_qs):
slap_computer = slapos.slap.Computer(parsed_qs['computer_id'][0])
slap_computer._software_release_list = []
slap_computer._computer_partition_list = []
return (200, {}, xml_marshaller.xml_marshaller.dumps(slap_computer))
elif parsed_url.path == 'requestComputerPartition':
return (408, {}, '')
else:
return (404, {}, '')
httplib.HTTPConnection._callback = server_response
computer = self.slap.registerComputer(computer_guid)
self.assertEqual(computer.getComputerPartitionList(), [])
def install(self):
self.path_list = []
crond = self.installCrond()
slap = slapos.slap.slap()
slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
parameter_dict = slap.registerComputerPartition(
self.computer_id,
self.computer_partition_id,
).getInstanceParameterDict()
# XXX: should probably expect one more (SR-originating) parameter instead
# of using self.work_directory .
configuration_path = os.path.join(self.work_directory, "agent.cfg")
with open(configuration_path, "w") as configuration:
configuration.write(parameter_dict["configuration"])
agent_crond_path = os.path.join(crond, "agent")
with open(agent_crond_path, "w") as agent_crond:
agent_crond.write("*/5 * * * * %s -S %s --pidfile=%s --log=%s "
"%s 2>&1 > /dev/null\n" % (
self.options["python_binary"],
self.options["agent_binary"],
self.options["pidfile"],
self.options["log"],
configuration_path,
))
return self.path_list + [configuration_path, agent_crond_path]
def __init__(
self,
server_url,
key_file,
cert_file,
computer_id,
partition_id,
software,
namebase,
root_instance_name,
sleep_time_between_test=900,
total_instance_count="2",
type=None,
):
self.server_url = server_url
self.key_file = key_file
self.cert_file = cert_file
self.computer_id = computer_id
self.partition_id = partition_id
self.software = software
self.namebase = namebase
self.total_instance_count = total_instance_count
self.root_instance_name = root_instance_name
self.sleep_time_between_test = sleep_time_between_test
self.test_type = type
slap = slapos.slap.slap()
slap.initializeConnection(server_url, key_file, cert_file)
self.partition = slap.registerComputerPartition(computer_guid=computer_id, partition_id=partition_id)
self.logger = logging.getLogger("SlaprunnerResiliencyTest")
self.logger.setLevel(logging.DEBUG)
def _test_new_computer_partition_state(self, state):
"""
Helper method to automate assertions of failing states on new Computer
Partition
"""
computer_guid = self._getTestComputerId()
partition_id = 'PARTITION_01'
slap = self.slap
slap.initializeConnection(self.server_url)
def server_response(self, path, method, body, header):
parsed_url = urlparse.urlparse(path.lstrip('/'))
parsed_qs = urlparse.parse_qs(parsed_url.query)
if (parsed_url.path == 'registerComputerPartition' and
parsed_qs['computer_reference'][0] == computer_guid and
parsed_qs['computer_partition_reference'][0] == partition_id):
partition = slapos.slap.ComputerPartition(
computer_guid, partition_id)
return (200, {}, xml_marshaller.xml_marshaller.dumps(partition))
else:
return (404, {}, '')
httplib.HTTPConnection._callback = server_response
computer_partition = self.slap.registerComputerPartition(
computer_guid, partition_id)
self.assertRaises(slapos.slap.NotFoundError,
getattr(computer_partition, state))
def __init__(self, buildout, name, options):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
options['tap'] = tap_set
parameter_dict = self._expandParameterDict(options, parameter_dict)
match = self.OPTCRE_match
for key, value in parameter_dict.iteritems():
if match(key) is not None:
continue
options['configuration.' + key] = value
def __init__(self, buildout, name, options):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0]
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0]
options['tap'] = tap_set
parameter_dict = self._expandParameterDict(options, parameter_dict)
match = self.OPTCRE_match
for key, value in parameter_dict.iteritems():
if match(key) is not None:
continue
options['configuration.' + key] = value
def initializeSlapOSControler(self,
slapproxy_log=None,
process_manager=None,
reset_software=False,
software_path_list=None):
self.process_manager = process_manager
self.software_path_list = software_path_list
logger.debug('SlapOSControler, initialize, reset_software: %r',
reset_software)
config = self.config
slapos_config_dict = config.copy()
slapos_config_dict.update(software_root=self.software_root,
instance_root=self.instance_root,
proxy_database=self.proxy_database)
with open(self.slapos_config, 'w') as f:
f.write(
pkg_resources.resource_string('erp5.util.testnode',
'template/slapos.cfg.in') %
slapos_config_dict)
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(self.proxy_database):
os.unlink(self.proxy_database)
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
# Make sure there is no slapos alive from previous run
process_manager.killall('slapos')
proxy = subprocess.Popen([
config['slapos_binary'], 'proxy', 'start', '--cfg',
self.slapos_config
], **kwargs)
process_manager.process_pid_set.add(proxy.pid)
slap = self.slap = slapos.slap.slap()
# Wait for proxy to accept connections
retries = 0
while True:
time.sleep(.5)
try:
slap.initializeConnection(config['master_url'])
computer = slap.registerComputer(config['computer_id'])
# Call a method to ensure connection to master can be established
computer.getComputerPartitionList()
except slapos.slap.ConnectionError, e:
retries += 1
if retries >= 20:
raise
logger.debug("Proxy still not started %s, retrying", e)
else:
break
def __init__(self, config, log, process_group_pid_set=None,
slapproxy_log=None):
self.log = log
self.config = config
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], **kwargs)
process_group_pid_set.add(proxy.pid)
# XXX: dirty, giving some time for proxy to being able to accept
# connections
time.sleep(10)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
# register software profile
self.software_profile = config['custom_profile_path']
slap.registerSupply().supply(
self.software_profile,
computer_guid=config['computer_id'])
computer = slap.registerComputer(config['computer_id'])
# create partition and configure computer
partition_reference = config['partition_reference']
partition_path = os.path.join(config['instance_root'], partition_reference)
if not os.path.exists(partition_path):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': config['instance_root'],
'netmask': '255.255.255.255',
'partition_list': [{'address_list': [{'addr': config['ipv4_address'],
'netmask': '255.255.255.255'},
{'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'},
],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference},
}
],
'reference': config['computer_id'],
'software_root': config['software_root']}))
def updateProxy(config):
"""
Configure Slapos Node computer and partitions.
Send current Software Release to Slapproxy for compilation and deployment.
"""
startProxy(config)
if not os.path.exists(config['instance_root']):
os.mkdir(config['instance_root'])
slap = slapos.slap.slap()
profile = getCurrentSoftwareReleaseProfile(config)
slap.initializeConnection(config['master_url'])
slap.registerSupply().supply(profile, computer_guid=config['computer_id'])
computer = slap.registerComputer(config['computer_id'])
prefix = 'slappart'
slap_config = {
'address': config['ipv4_address'],
'instance_root': config['instance_root'],
'netmask': '255.255.255.255',
'partition_list': [],
'reference': config['computer_id'],
'software_root': config['software_root']
}
for i in xrange(0, int(config['partition_amount'])):
partition_reference = '%s%s' % (prefix, i)
partition_path = os.path.join(config['instance_root'], partition_reference)
if not os.path.exists(partition_path):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
slap_config['partition_list'].append({
'address_list': [
{
'addr': config['ipv4_address'],
'netmask': '255.255.255.255'
}, {
'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'
},
],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference}})
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps(slap_config))
return True
def getSlapStatus(config):
"""Return all Slapos Partitions with associate information"""
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
partition_list = []
computer = slap.registerComputer(config['computer_id'])
try:
for partition in computer.getComputerPartitionList():
# Note: Internal use of API, as there is no reflexion interface in SLAP
partition_list.append((partition.getId(), partition._connection_dict.copy()))
except Exception:
pass
if partition_list:
for i in xrange(0, int(config['partition_amount'])):
slappart_id = '%s%s' % ("slappart", i)
if not [x[0] for x in partition_list if slappart_id == x[0]]:
partition_list.append((slappart_id, []))
return partition_list
def checkIfMasterIsCurrentMaster(master_url):
"""
Because there are several ways to contact this server, we can't easily check
in a request() if master_url is ourself or not. So we contact master_url,
and if it returns an ID we know: it is ourself
"""
# Dumb way: compare with listening host/port
host = request.host
port = request.environ['SERVER_PORT']
if master_url == 'http://%s:%s/' % (host, port):
return True
# Hack way: call ourself
slap = slapos.slap.slap()
slap.initializeConnection(master_url)
try:
return run_id == bytes2str(slap._connection_helper.GET('/getRunId'))
except:
return False
def __bang(self, message):
"""
Call bang if requested
"""
if 'master-url' in self.__config and \
'partition-in' in self.__config and \
'computer-id' in self.__config:
slap = slapos.slap.slap()
slap.initializeConnection(
self.__config['master-url'],
self.__config.get('partition-key'),
self.__config.get('partition-cert'),
)
computer_partition = slap.registerComputerPartition(
self.__config['computer-id'],
self.__config['partition-id'],
)
computer_partition.bang(message)
self.logger.info("Bang with message %r." % message)
def test_updateInstanceParameter(self):
"""Scenarion 5: Update parameters of current sofware profile"""
self.setAccount()
self.setupTestSoftware()
#Set current projet and run Slapgrid-cp
software = os.path.join(self.software, 'slaprunner-test')
response = loadJson(self.app.post('/setCurrentProject',
data=dict(path=software),
follow_redirects=True))
self.assertEqual(response['result'], "")
self.proxyStatus(True)
#Send paramters for the instance
parameterDict = dict(appname='slaprunnerTest', cacountry='France')
parameterXml = '<?xml version="1.0" encoding="utf-8"?>\n<instance>'
parameterXml += '<parameter id="appname">slaprunnerTest</parameter>\n'
parameterXml += '<parameter id="cacountry">France</parameter>\n</instance>'
software_type = 'production'
response = loadJson(self.app.post('/saveParameterXml',
data=dict(parameter=parameterXml,
software_type=software_type),
follow_redirects=True))
self.assertEqual(response['result'], "")
slap = slapos.slap.slap()
slap.initializeConnection(self.app.config['master_url'])
computer = slap.registerComputer(self.app.config['computer_id'])
partitionList = computer.getComputerPartitionList()
self.assertNotEqual(partitionList, [])
#Assume that the requested partition is partition 0
slapParameterDict = partitionList[0].getInstanceParameterDict()
self.assertTrue('appname' in slapParameterDict)
self.assertTrue('cacountry' in slapParameterDict)
self.assertEqual(slapParameterDict['appname'], 'slaprunnerTest')
self.assertEqual(slapParameterDict['cacountry'], 'France')
self.assertEqual(slapParameterDict['slap_software_type'], 'production')
#test getParameterXml for webrunner UI
response = loadJson(self.app.get('/getParameterXml/xml'))
self.assertEqual(parameterXml, response['result'])
response = loadJson(self.app.get('/getParameterXml/dict'))
self.assertEqual(parameterDict, response['result']['instance'])
self.stopSlapproxy()
def forwardRequestToExternalMaster(master_url, request_form):
"""
Forward instance request to external SlapOS Master.
"""
master_entry = app.config.get('multimaster').get(master_url, {})
key_file = master_entry.get('key')
cert_file = master_entry.get('cert')
if master_url.startswith('https') and (not key_file or not cert_file):
app.logger.warning('External master %s configuration did not specify key or certificate.' % master_url)
abort(404)
if master_url.startswith('https') and not master_url.startswith('https') and (key_file or cert_file):
app.logger.warning('External master %s configurqtion specifies key or certificate but is using plain http.' % master_url)
abort(404)
slap = slapos.slap.slap()
if key_file:
slap.initializeConnection(master_url, key_file=key_file, cert_file=cert_file)
else:
slap.initializeConnection(master_url)
partition_reference = request_form['partition_reference'].encode()
# Store in database
execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)',
{'partition_reference':partition_reference, 'master_url': master_url})
new_request_form = request_form.copy()
filter_kw = loads(new_request_form['filter_xml'].encode())
filter_kw['source_instance_id'] = partition_reference
new_request_form['filter_xml'] = dumps(filter_kw)
xml = slap._connection_helper.POST('/requestComputerPartition', data=new_request_form)
if type(xml) is unicode:
xml = str(xml)
xml.encode('utf-8')
partition = loads(xml)
# XXX move to other end
partition._master_url = master_url
return dumps(partition)
def test_error_new_ComputerPartition_works(self):
"""
Asserts that calling ComputerPartition.error on new partition works
"""
computer_guid = self._getTestComputerId()
partition_id = 'PARTITION_01'
slap = self.slap
slap.initializeConnection(self.server_url)
def handler(url, req):
qs = urlparse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' and
qs['computer_reference'][0] == computer_guid and
qs['computer_partition_reference'][0] == partition_id):
partition = slapos.slap.ComputerPartition(
computer_guid, partition_id)
return {
'statu_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(partition)
}
elif url.path == '/softwareInstanceError':
parsed_qs_body = urlparse.parse_qs(req.body)
# XXX: why do we have computer_id and not computer_reference?
# XXX: why do we have computer_partition_id and not
# computer_partition_reference?
if (parsed_qs_body['computer_id'][0] == computer_guid and
parsed_qs_body['computer_partition_id'][0] == partition_id and
parsed_qs_body['error_log'][0] == 'some error'):
return {'status_code': 200}
return {'status_code': 404}
with httmock.HTTMock(handler):
computer_partition = slap.registerComputerPartition(
computer_guid, partition_id)
# XXX: Interface does not define return value
computer_partition.error('some error')
def test_computer_getComputerPartitionList_no_partition(self):
"""
Asserts that calling Computer.getComputerPartitionList without Computer
Partitions returns empty list
"""
computer_guid = self._getTestComputerId()
slap = self.slap
slap.initializeConnection(self.server_url)
def handler(url, req):
qs = urlparse.parse_qs(url.query)
if (url.path == '/registerComputerPartition'
and 'computer_reference' in qs
and 'computer_partition_reference' in qs):
slap_partition = slapos.slap.ComputerPartition(
qs['computer_reference'][0],
qs['computer_partition_reference'][0])
return {
'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
}
elif (url.path == '/getFullComputerInformation'
and 'computer_id' in qs):
slap_computer = slapos.slap.Computer(qs['computer_id'][0])
slap_computer._software_release_list = []
slap_computer._computer_partition_list = []
return {
'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
}
elif url.path == '/requestComputerPartition':
return {'status_code': 408}
else:
return {'status_code': 404}
with httmock.HTTMock(handler):
computer = self.slap.registerComputer(computer_guid)
self.assertEqual(computer.getComputerPartitionList(), [])
def test_error_new_SoftwareRelease_works(self):
"""
Asserts that calling SoftwareRelease.error on software release works
"""
computer_guid = self._getTestComputerId()
software_release_uri = 'http://server/' + self._getTestComputerId()
slap = self.slap
slap.initializeConnection(self.server_url)
def server_response(self, path, method, body, header):
parsed_url = urlparse.urlparse(path.lstrip('/'))
parsed_qs = urlparse.parse_qs(body)
if parsed_url.path == 'softwareReleaseError' and \
parsed_qs['computer_id'][0] == computer_guid and \
parsed_qs['url'][0] == software_release_uri and \
parsed_qs['error_log'][0] == 'some error':
return (200, {}, '')
return (404, {}, '')
httplib.HTTPConnection._callback = server_response
software_release = self.slap.registerSoftwareRelease(software_release_uri)
software_release._computer_guid = computer_guid
software_release.error('some error')
def __init__(self, buildout, name, options):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
parameter_dict = slap.registerComputerPartition(
options['computer'],
options['partition'],
).getInstanceParameterDict()
# XXX: those are not partition parameters, strictly speaking.
# Discard them, and make them available as separate section keys.
options['slap-software-type'] = parameter_dict.pop(
'slap_software_type')
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
options['tap'] = tap_set
options['configuration'] = parameter_dict
match = self.OPTCRE_match
for key, value in parameter_dict.iteritems():
if match(key) is not None:
continue
options['configuration.' + key] = value
def main():
"""
Note: This code does not test as much as it monitors.
The goal is to regularily try to build & instantiate a software release
on several machines, to monitor vifib stability and SR stability as time
passes (and things once available online become unavailable).
Part of this function could be reused to make an actual test bot, testing
only when actual changes are committed to a software release, to look for
regressions.
Note: This code does not connect to any instantiated service, it relies on
the presence of a promise section to make instantiation fail until promise
is happy.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--pidfile', '-p', help='pidfile preventing parallel '
'execution.')
parser.add_argument('--log', '-l', help='Log file path.')
parser.add_argument('--verbose', '-v', help='Be verbose.',
action='store_true')
parser.add_argument('configuration_file', type=argparse.FileType(),
help='Slap Test Agent configuration file.')
key_file_dict = {}
args = parser.parse_args()
log = args.log
logger, log_file = getLogger(log, args.verbose)
configuration = ConfigParser.SafeConfigParser()
configuration.readfp(args.configuration_file)
pidfile = args.pidfile
if pidfile:
setRunning(logger=logger, pidfile=pidfile)
try:
while True:
section_dict = loadConfiguration(configuration, logger)
agent_parameter_dict = dict(configuration.items('agent'))
task_distributor = TaskDistributor(agent_parameter_dict['report_url'],
logger=logger)
task_distributor.subscribeNode(
node_title=agent_parameter_dict['node_title'],
computer_guid="None")
test_suite_data = task_distributor.startTestSuite(
node_title=agent_parameter_dict['node_title'],
computer_guid="None")
if type(test_suite_data) == str:
# Backward compatiblity
test_suite_data = json.loads(test_suite_data)
slap_account_key = task_distributor.getSlaposAccountKey()
slap_certificate = task_distributor.getSlaposAccountCertificate()
master_url = task_distributor.getSlaposUrl()
key_file_dict = {}
def asFilenamePair(key, cert):
# Note: python's ssl support only supports fetching key & cert data
# from on-disk files. This is why we need to "convert" direct data
# into file paths, using temporary files.
cert = cert.strip()
try:
temp_key, temp_cert = key_file_dict[cert]
except KeyError:
temp_key = AutoSTemp(key.strip())
temp_cert = AutoSTemp(cert)
key_file_dict[cert] = (temp_key, temp_cert)
return temp_key.name, temp_cert.name
key_file, cert_file = asFilenamePair(slap_account_key,
slap_certificate)
process_manager = ProcessManager(logger.info)
for test_suite in test_suite_data:
full_revision_list = getAndUpdateFullRevisionList(test_suite,
agent_parameter_dict["working_directory"], logger, process_manager)
unit_test_dict = task_distributor.generateConfiguration(
test_suite['test_suite_title'])
if not len(full_revision_list):
# We don't watch git revision but we periodically
# run the test, once a day.
full_revision_list = ["day=%s" % time.strftime('%Y/%m/%d', time.gmtime())]
if type(unit_test_dict) == str:
# Backward compatiblity
unit_test_dict = json.loads(unit_test_dict)
test_result = task_distributor.createTestResult(
revision=','.join(full_revision_list),
test_name_list=unit_test_dict.keys(),
node_title=agent_parameter_dict['node_title'],
allow_restart=False,
test_title=test_suite['test_suite_title'],
project_title=agent_parameter_dict['project_title'],
)
if test_result is None:
# We already have a test result
logger.info('Skiping test for %s, result already available (%s)' %
(test_suite['test_suite_title'], ','.join(full_revision_list)))
continue
test_result.watcher_period = 120
assert test_result is not None
if log_file is not None:
test_result.addWatch(log, log_file, max_history_bytes=10000)
logger.info("Starting to run for %s" % test_result )
test_mapping = TestMap(unit_test_dict)
logger.info("Running %s tests in parallel." % \
len(test_mapping.getGroupList()))
assert master_url.startswith('https:')
slap = slapos.slap.slap()
retry = 0
while True:
if retry > 100:
break
# wait until _hateoas_navigator is loaded.
slap.initializeConnection(
master_url, key_file, cert_file, timeout=120)
if getattr(slap, '_hateoas_navigator', None) is None:
logger.info("Fail to load _hateoas_navigator waiting a bit and retry.")
time.sleep(30)
else:
break
if getattr(slap, '_hateoas_navigator', None) is None:
raise ValueError("Fail to load _hateoas_navigator")
supply = slap.registerSupply()
order = slap.registerOpenOrder()
running_test_dict = {}
logger.info('Starting Test Agent run %s ' % agent_parameter_dict['node_title'])
while True:
# Get up to parallel_task_count tasks to execute
while len(running_test_dict) < len(test_mapping.getGroupList())\
and (len(test_mapping.getGroupList()) > 0):
test_mapping.cleanEmptyGroup()
# Select an unused computer to run the test.
group = test_mapping.getNextGroup(
ignore_list = [group for _, _, group in \
running_test_dict.itervalues()])
# Select a test
test_line = test_result.start(
exclude_list=list(test_mapping.getExcludeList(group)))
logger.info("Test Line: %s " % test_line)
logger.info("Ran Test Set: %s " % test_mapping.ran_test_set)
logger.info("Running test dict: %s " % running_test_dict)
logger.info("Group: %s " % group)
if test_line is None:
logger.info("Removing Group (empty test line): %s " % group)
test_mapping.dropGroup(group)
continue
test_name = test_line.name
try:
section_entry_dict = unit_test_dict[test_name]
except KeyError:
# We don't know how to execute this test. Assume it doesn't
# exist anymore, and fail it in result.
test_line.stop(stderr='This test does not exist on test '
'node %s' % (agent_parameter_dict['node_title'], ))
continue
general_timeout = agent_parameter_dict.get('timeout', 3600)
tester = SoftwareReleaseTester(
test_name + time.strftime('_%Y/%m/%d_%H:%M:%S_+0000', time.gmtime()),
logger,
slap,
order,
supply,
section_entry_dict['url'],
section_entry_dict.get('supply_computer'),
section_entry_dict.get('request_kw'),
agent_parameter_dict.get('software_timeout', general_timeout),
agent_parameter_dict.get('instance_timeout', general_timeout)
)
test_mapping.addRanTest(test_name)
running_test_dict[test_name] = (test_line, tester, group)
if not running_test_dict:
logger.info('No more tests to run...')
break
now = time.time()
# Synchronise refreshes on watcher period, so it doesn't report a
# stalled test node where we are actually still sleeping.
# Change test_result.watcher_period outside this loop if you wish
# to change sleep duration.
next_deadline = now + test_result.watcher_period
for section, (test_line, tester, group) in running_test_dict.items():
logger.info('Checking %s: %r...', section, tester)
try:
deadline = tester.tic(now)
except ConnectionError:
logger.exception('Test execution ConnectionError for %s' % (section))
deadline = next_deadline
except Exception:
logger.exception('Test execution fail for %s' % (section))
test_line.stop(test_count=1, error_count=1, failure_count=0,
skip_count=0, command=tester.getInfo(),
stdout=tester.getFormatedLastMessage(),
stderr=traceback.format_exc())
del running_test_dict[section]
try:
tester.teardown()
except slapos.slap.NotFoundError:
# This exception is ignored because we cannot
# Teardown if SR URL do not exist.
logger.exception('Fail and not found')
pass
except Exception:
logger.exception('teardown failed, human assistance needed for cleanup')
raise
else:
logger.info('%r' % tester)
if deadline is None:
# TODO: report how long each step took.
logger.info('Test execution finished for %s' % (section))
test_line.stop(test_count=1, error_count=0, failure_count=0,
skip_count=0, command=tester.getInfo(), stdout=tester.getFormatedLastMessage())
del running_test_dict[section]
try:
pass #tester.teardown()
except slapos.slap.NotFoundError:
# This exception is ignored because we cannot
# Teardown if SR URL do not exist.
logger.exception('Fail and not found')
pass
except Exception:
logger.exception('teardown failed, human assistance needed for cleanup')
raise
else:
next_deadline = min(deadline, next_deadline)
if running_test_dict:
to_sleep = next_deadline - time.time()
if to_sleep > 0:
logger.info('Sleeping %is...', to_sleep)
time.sleep(to_sleep)
if not test_result.isAlive():
for _, tester, computer_id in running_test_dict.itervalues():
tester.teardown()
time.sleep(300)
finally:
if pidfile:
setFinished(pidfile)
key_file_dict.clear()
def __init__(self, config, log,
slapproxy_log=None, process_manager=None, reset_software=False):
log('SlapOSControler, initialize, reset_software: %r' % reset_software)
self.log = log
self.config = config
self.process_manager = process_manager
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], **kwargs)
process_manager.process_pid_set.add(proxy.pid)
# XXX: dirty, giving some time for proxy to being able to accept
# connections
time.sleep(10)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
# register software profile
self.software_profile = config['custom_profile_path']
slap.registerSupply().supply(
self.software_profile,
computer_guid=config['computer_id'])
computer = slap.registerComputer(config['computer_id'])
# Reset all previously generated software if needed
if reset_software:
software_root = config['software_root']
log('SlapOSControler : GOING TO RESET ALL SOFTWARE')
if os.path.exists(software_root):
shutil.rmtree(software_root)
os.mkdir(software_root)
os.chmod(software_root, 0750)
instance_root = config['instance_root']
if os.path.exists(instance_root):
# delete old paritions which may exists in order to not get its data
# (ex. MySQL db content) from previous testnode's runs
# In order to be able to change partition naming scheme, do this at
# instance_root level (such change happened already, causing problems).
shutil.rmtree(instance_root)
os.mkdir(instance_root)
for i in range(0, MAX_PARTIONS):
# create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports
partition_reference = '%s-%s' %(config['partition_reference'], i)
partition_path = os.path.join(instance_root, partition_reference)
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': instance_root,
'netmask': '255.255.255.255',
'partition_list': [{'address_list': [{'addr': config['ipv4_address'],
'netmask': '255.255.255.255'},
{'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'},],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference},
}
],
'reference': config['computer_id'],
'software_root': config['software_root']}))
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout['slap_connection']
computer_id = slap_connection['computer_id']
computer_partition_id = slap_connection['partition_id']
server_url = slap_connection['server_url']
key_file = slap_connection.get('key_file')
cert_file = slap_connection.get('cert_file')
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id, computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict(
)
software_type = self.parameter_dict['slap_software_type']
self.logger.info('Deploying instance with software type %s' % \
software_type)
# Raise if request software_type does not exist ...
if software_type not in self.options:
# ... Except for backward compatibility. Then use "default".
if software_type in ['RootSoftwareInstance']:
software_type = 'default'
else:
raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
software_type)
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError(
"The specified buildout config file %r does "
"not exist." % instance_file_path)
buildout = ConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name)
if not buildout.has_section('slap-parameter'):
buildout.add_section('slap-parameter')
for parameter, value in self.parameter_dict.items():
# All parameters evaluating to False are... False, and shouldn't
# convey any information.
# Here, all those parameters are simply ignored.
if value:
if isinstance(value, str):
buildout.set('slap-parameter', parameter, value)
else:
buildout.set('slap-parameter', parameter,
json.dumps(value))
buildout.add_section('slap-network-information')
buildout.set('slap-network-information', 'local-ipv4',
self.getLocalIPv4Address())
buildout.set('slap-network-information', 'global-ipv6',
self.getGlobalIPv6Address())
buildout.set('slap-network-information', 'network-interface',
self.getNetworkInterface())
# Copy/paste slap_connection
buildout.add_section('slap-connection')
for key, value in self.buildout['slap_connection'].iteritems():
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set('slap-connection', key.replace('_', '-'), value)
# XXX: Needed for lxc. Use non standard API
buildout.set('slap-connection', 'requested',
self.computer_partition._requested_state)
work_directory = os.path.abspath(
self.buildout['buildout']['directory'])
buildout_filename = os.path.join(work_directory,
'buildout-%s.cfg' % self.name)
with open(buildout_filename, 'w') as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
self.logger.info("Invoking commandline : '%s'",
' '.join(command_line_args))
subprocess.check_call(command_line_args,
cwd=work_directory,
env=os.environ.copy())
return []
def run(args):
config = args[0]
for k,v in config['environment'].iteritems():
os.environ[k] = v
proxy = None
slapgrid = None
last_revision_file = os.path.join(config['working_directory'],
'revision.txt')
if os.path.exists(last_revision_file):
os.unlink(last_revision_file)
# fetch repository from git
repository_clone = os.path.join(config['working_directory'], 'repository')
profile_path = os.path.join(repository_clone, config['profile_path'])
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(proxy.pid)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
while True:
try:
slap.registerSupply().supply(profile_path,
computer_guid=config['computer_id'])
except socket.error:
time.sleep(1)
pass
else:
break
while True:
info_list = []
a = info_list.append
while True:
try:
if os.path.exists(repository_clone):
if getCurrentFetchRemote(config, repository_clone) != config['repository']:
shutil.rmtree(repository_clone)
if not os.path.exists(repository_clone):
subprocess.check_call([config['git_binary'], 'clone',
config['repository'], repository_clone])
# switch to branch
branch = getCurrentBranchName(config, repository_clone)
if branch != config['branch']:
subprocess.check_call([config['git_binary'], 'checkout', '--force',
'--track', '-b', config['branch'], 'origin/'+config['branch']],
cwd=repository_clone)
subprocess.check_call([config['git_binary'], 'reset', '--hard',
'@{upstream}'], cwd=repository_clone)
except Exception:
print 'Retrying git in 60s'
time.sleep(60)
else:
break
a('Tested repository: %s' % config['repository'])
a('Machine identification: %s' % getMachineIdString())
erp5_report = ERP5TestReportHandler(config['test_suite_master_url'],
'@'.join([config['suite_name'], branch]))
last_revision = ''
if os.path.exists(last_revision_file):
last_revision = open(last_revision_file).read().strip()
revision = getRevision(config, repository_clone)
open(last_revision_file, 'w').write(revision)
if revision != last_revision:
print 'Running for revision %r' % revision
while True:
try:
erp5_report.reportStart()
except Exception:
print 'Retrying in 5s'
time.sleep(5)
else:
break
if os.path.exists(config['software_root']):
shutil.rmtree(config['software_root'])
os.mkdir(config['software_root'])
out_file = os.path.join(config['working_directory'], 'slapgrid.out')
if os.path.exists(out_file):
os.unlink(out_file)
out = open(out_file, 'w')
begin = time.time()
slapgrid_environment = os.environ.copy()
for k, v in config['slapgrid_environment'].iteritems():
slapgrid_environment[k] = v
a('Slapgrid environment: %r'% config['slapgrid_environment'])
slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc',
config['slapos_config']], close_fds=True, preexec_fn=os.setsid,
stdout=out, stderr=subprocess.STDOUT, env=slapgrid_environment)
process_group_pid_list.append(slapgrid.pid)
slapgrid.communicate()
out.close()
while True:
try:
erp5_report.reportFinished(out_file,revision,
slapgrid.returncode == 0, time.time() - begin,
'\n'.join(info_list))
except Exception:
print 'Retrying in 5s'
time.sleep(5)
else:
break
print 'Sleeping for 600s'
time.sleep(600)
def main():
"""
Note: This code does not test as much as it monitors.
The goal is to regularily try to build & instantiate a software release
on several machines, to monitor vifib stability and SR stability as time
passes (and things once available online become unavailable).
Part of this function could be reused to make an actual test bot, testing
only when actual changes are committed to a software release, to look for
regressions.
Note: This code does not connect to any instantiated service, it relies on
the presence of a promise section to make instantiation fail until promise
is happy.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--pidfile', '-p', help='pidfile preventing parallel '
'execution.')
parser.add_argument('--log', '-l', help='Log file path.')
parser.add_argument('--verbose', '-v', help='Be verbose.',
action='store_true')
parser.add_argument('configuration_file', type=argparse.FileType(),
help='Slap Test Agent configuration file.')
# Just to keep strong references to AutoSTemp instances
key_file_dict = {}
def asFilenamePair(key, cert):
# Note: python's ssl support only supports fetching key & cert data
# from on-disk files. This is why we need to "convert" direct data
# into file paths, using temporary files.
cert = cert.strip()
try:
temp_key, temp_cert = key_file_dict[cert]
except KeyError:
temp_key = AutoSTemp(key.strip())
temp_cert = AutoSTemp(cert)
key_file_dict[cert] = (temp_key, temp_cert)
return temp_key.name, temp_cert.name
args = parser.parse_args()
log = args.log
formatter = logging.Formatter('%(asctime)s %(message)s')
logger = logging.getLogger()
if args.verbose:
log_level = logging.DEBUG
else:
log_level = logging.INFO
logger.setLevel(log_level)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
logger.addHandler(handler)
if log:
handler = logging.FileHandler(log)
handler.setFormatter(formatter)
logger.addHandler(handler)
log_file = open(log)
log_file.seek(0, 2)
pidfile = args.pidfile
if pidfile:
setRunning(pidfile)
try:
section_dict = collections.OrderedDict()
configuration = ConfigParser.SafeConfigParser()
configuration.readfp(args.configuration_file)
for section in configuration.sections():
if section == 'agent':
continue
section_dict[section] = section_entry_dict = dict(
configuration.items(section))
for key in ('request_kw', 'max_install_duration',
'max_destroy_duration', 'max_request_duration',
'max_uninstall_duration', 'computer_list'
):
if key in section_entry_dict:
try:
if isinstance(section_entry_dict[key], str) or \
isinstance(section_entry_dict[key], unicode):
section_entry_dict[key] = json.loads(
section_entry_dict[key])
except Exception as exc:
logger.error("Fail to load %s on %s" % (key, section_entry_dict))
raise
if 'key' in section_entry_dict:
key_file, cert_file = asFilenamePair(section_entry_dict['key'],
section_entry_dict['cert'])
section_entry_dict['key'] = key_file
section_entry_dict['cert'] = cert_file
if "computer_list" in section_entry_dict:
section_entry_dict["target_computer"] = \
random.choice(section_entry_dict["computer_list"])
agent_parameter_dict = dict(configuration.items('agent'))
# XXX: should node title be auto-generated by installation recipe ?
# For example, using computer guid.
node_title = agent_parameter_dict['node_title']
test_title = agent_parameter_dict['test_title']
project_title = agent_parameter_dict['project_title']
task_distribution_tool = TaskDistributionTool(agent_parameter_dict[
'report_url'])
master_slap_connection_dict = {}
test_result = task_distribution_tool.createTestResult(
revision='',
test_name_list=section_dict.keys(),
node_title=node_title,
allow_restart=True,
test_title=test_title,
project_title=project_title,
)
test_result.watcher_period = 300
if log:
test_result.addWatch(log, log_file, max_history_bytes=10000)
assert test_result is not None
test_mapping = TestMap(section_dict)
logger.info("Running %s tests in parallel." % \
len(test_mapping.getComputerList()))
ran_test_set = set()
running_test_dict = {}
more_tests = True
logger.info('Starting Test Agent run %s ' % node_title)
while True:
# Get up to parallel_task_count tasks to execute
while len(running_test_dict) < len(test_mapping.getComputerList())\
and more_tests:
test_mapping.cleanUp()
target_computer = test_mapping.getNextComputer([computer \
for _, _, computer in running_test_dict.itervalues()])
test_line = test_result.start(
exclude_list= list(ran_test_set) + \
list(test_mapping.getExcludeList(target_computer)))
logger.info("Test Line: %s " % test_line)
logger.info("Ran Test Set: %s " % ran_test_set)
logger.info("Running test dict: %s " % running_test_dict)
logger.info("Target Computer: %s " % target_computer)
if test_line is None:
test_mapping.dropComputer(target_computer)
if len(test_mapping.getComputerList()) == 0:
more_tests = False
continue
test_name = test_line.name
try:
section_entry_dict = section_dict[test_name]
except KeyError:
# We don't know how to execute this test. Assume it doesn't
# exist anymore, and fail it in result.
test_line.stop(stderr='This test does not exist on test '
'node %s' % (node_title, ))
continue
master_url = section_entry_dict['master_url']
master_slap_connection_key = (master_url,
section_entry_dict.get('key'))
try:
supply, order, rpc = master_slap_connection_dict[
master_slap_connection_key]
except KeyError:
key = section_entry_dict.get('key')
cert = section_entry_dict.get('cert')
slap = slapos.slap.slap()
slap.initializeConnection(master_url, key, cert)
supply = slap.registerSupply()
order = slap.registerOpenOrder()
assert master_url.startswith('https:')
rpc = xmlrpclib.ServerProxy(master_url, allow_none=True,
transport=x509Transport(
{'key_file': key, 'cert_file': cert}))
master_slap_connection_dict[
master_slap_connection_key] = (supply, order, rpc)
tester = SoftwareReleaseTester(
test_name + '_' + node_title + time.strftime(
'_%Y/%m/%d_%H:%M:%S_+0000', time.gmtime()),
logger,
rpc,
supply,
order,
section_entry_dict['url'],
section_entry_dict['target_computer'],
section_entry_dict['max_install_duration'],
section_entry_dict['max_uninstall_duration'],
section_entry_dict.get('request_kw'),
section_entry_dict.get('max_request_duration'),
section_entry_dict.get('max_destroy_duration'),
)
ran_test_set.add(test_name)
running_test_dict[test_name] = (test_line, tester, target_computer)
if not running_test_dict:
break
now = time.time()
# Synchronise refreshes on watcher period, so it doesn't report a
# stalled test node where we are actually still sleeping.
# Change test_result.watcher_period outside this loop if you wish
# to change sleep duration.
next_deadline = now + test_result.watcher_period
for section, (test_line, tester, target_computer) in running_test_dict.items():
logger.info('Checking %s: %r...', section, tester)
try:
deadline = tester.tic(now)
except Exception:
logger.exception('Test execution fail for %s' % (section))
test_line.stop(
test_count=1,
error_count=1,
failure_count=0,
skip_count=0,
stderr=traceback.format_exc(),
)
del running_test_dict[section]
try:
tester.teardown()
except slapos.slap.NotFoundError:
# This exception is ignored because we cannot
# Teardown if SR URL do not exist.
logger.exception('Fail and not found')
pass
except Exception:
logger.exception('teardown failed, human '
'assistance needed for cleanup')
raise
else:
logger.info('%r', tester)
if deadline is None:
# TODO: report how long each step took.
logger.info('Test execution finished for %s' % (section))
test_line.stop(
test_count=1,
error_count=0,
failure_count=0,
skip_count=0,
)
del running_test_dict[section]
try:
tester.teardown()
except slapos.slap.NotFoundError:
# This exception is ignored because we cannot
# Teardown if SR URL do not exist.
logger.exception('Fail and not found')
pass
except Exception:
logger.exception('teardown failed, human '
'assistance needed for cleanup')
raise
else:
next_deadline = min(deadline, next_deadline)
if running_test_dict:
to_sleep = next_deadline - time.time()
if to_sleep > 0:
logger.info('Sleeping %is...', to_sleep)
time.sleep(to_sleep)
if not test_result.isAlive():
for _, tester, computer_id in running_test_dict.itervalues():
tester.teardown()
finally:
if pidfile:
setFinished(pidfile)
# Help interpreter get rid of AutoSTemp instances.
key_file_dict.clear()
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout['slap_connection']
computer_id = slap_connection['computer_id']
computer_partition_id = slap_connection['partition_id']
server_url = slap_connection['server_url']
key_file = slap_connection.get('key_file')
cert_file = slap_connection.get('cert_file')
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id,
computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict['slap_software_type']
self.logger.info('Deploying instance with software type %s' % \
software_type)
# Raise if request software_type does not exist ...
if software_type not in self.options:
# ... Except for backward compatibility. Then use "default".
if software_type in ['RootSoftwareInstance']:
software_type = 'default'
else:
raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
software_type)
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError("The specified buildout config file %r does "
"not exist." % instance_file_path)
buildout = ConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name)
if not buildout.has_section('slap-parameter'):
buildout.add_section('slap-parameter')
for parameter, value in self.parameter_dict.items():
# All parameters evaluating to False are... False, and shouldn't
# convey any information.
# Here, all those parameters are simply ignored.
if value:
if isinstance(value, str):
buildout.set('slap-parameter', parameter, value)
else:
buildout.set('slap-parameter', parameter, json.dumps(value))
buildout.add_section('slap-network-information')
buildout.set('slap-network-information', 'local-ipv4',
self.getLocalIPv4Address())
buildout.set('slap-network-information', 'global-ipv6',
self.getGlobalIPv6Address())
buildout.set('slap-network-information', 'network-interface',
self.getNetworkInterface())
# Copy/paste slap_connection
buildout.add_section('slap-connection')
for key, value in self.buildout['slap_connection'].iteritems():
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set('slap-connection', key.replace('_', '-'), value)
# XXX: Needed for lxc. Use non standard API
buildout.set('slap-connection', 'requested', self.computer_partition._requested_state)
work_directory = os.path.abspath(self.buildout['buildout'][
'directory'])
buildout_filename = os.path.join(work_directory,
'buildout-%s.cfg' % self.name)
with open(buildout_filename, 'w') as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
self.logger.info("Invoking commandline : '%s'",
' '.join(command_line_args))
subprocess.check_call(command_line_args, cwd=work_directory,
env=os.environ.copy())
return []
def fetch_parameter_dict(self, options, instance_root):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
if route_ipv4_set:
options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8')
options['tap-network-information-dict'] = dict(ipv4=route_ipv4_set,
netmask=route_mask_set,
gateway=route_gw_set,
network=route_network_set)
else:
options['tap-network-information-dict'] = {}
if route_gw_set:
options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8')
if route_mask_set:
options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8')
if route_network_set:
options['tap-network'] = list(route_network_set)[0].encode('UTF-8')
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(storage_path):
storage_link = os.path.join(instance_root, 'DATA', filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
options['tap'] = tap_set
return self._expandParameterDict(options, parameter_dict)
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout['slap_connection']
computer_id = slap_connection['computer_id']
computer_partition_id = slap_connection['partition_id']
server_url = slap_connection['server_url']
key_file = slap_connection.get('key_file')
cert_file = slap_connection.get('cert_file')
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id,
computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict['slap_software_type']
if software_type not in self.options:
if 'default' in self.options:
software_type = 'default'
else:
raise zc.buildout.UserError("This software type isn't mapped. And"
"there's no default software type.")
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError("The specified buildout config file does not"
"exist.")
buildout = ConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set('buildout', 'installed',
'.installed-%s.cfg' % software_type)
buildout.add_section('slap-parameter')
for parameter, value in self.parameter_dict.items():
buildout.set('slap-parameter', parameter, value)
buildout.add_section('slap-network-information')
buildout.set('slap-network-information', 'local-ipv4',
self.getLocalIPv4Address())
buildout.set('slap-network-information', 'global-ipv6',
self.getGlobalIPv6Address())
# Copy/paste slap_connection
buildout.add_section('slap-connection')
for key, value in self.buildout['slap_connection'].iteritems():
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set('slap-connection', key.replace('_', '-'), value)
work_directory = os.path.abspath(self.buildout['buildout'][
'directory'])
buildout_filename = os.path.join(work_directory,
'buildout-%s.cfg' % software_type)
with open(buildout_filename, 'w') as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
self.logger.info("Invoking commandline : '%s'",
' '.join(command_line_args))
subprocess.check_call(command_line_args, cwd=work_directory,
env=os.environ.copy())
return []
def initializeSlapOSControler(self,
slapproxy_log=None,
process_manager=None,
reset_software=False,
software_path_list=None):
self.process_manager = process_manager
self.software_path_list = software_path_list
logger.debug('SlapOSControler, initialize, reset_software: %r',
reset_software)
config = self.config
slapos_config_dict = config.copy()
slapos_config_dict.update(software_root=self.software_root,
instance_root=self.instance_root,
proxy_database=self.proxy_database,
shared_part_list='\n '.join(
self.shared_part_list))
with open(self.slapos_config, 'w') as f:
f.write(
pkg_resources.resource_string(
'erp5.util.testnode', 'template/slapos.cfg.in').decode() %
slapos_config_dict)
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(self.proxy_database):
os.unlink(self.proxy_database)
kwargs = dict(close_fds=True, preexec_fn=os.setsid)
if slapproxy_log is not None:
slapproxy_log_fp = open(slapproxy_log, 'w')
kwargs['stdout'] = slapproxy_log_fp
kwargs['stderr'] = slapproxy_log_fp
proxy = subprocess.Popen([
config['slapos_binary'], 'proxy', 'start', '--cfg',
self.slapos_config
], **kwargs)
process_manager.process_pid_set.add(proxy.pid)
slap = self.slap = slapos.slap.slap()
# Wait for proxy to accept connections
retries = 0
while True:
time.sleep(1)
try:
slap.initializeConnection(config['master_url'])
computer = slap.registerComputer(config['computer_id'])
# Call a method to ensure connection to master can be established
computer.getComputerPartitionList()
except slapos.slap.ConnectionError as e:
retries += 1
if retries >= 60:
raise
logger.debug("Proxy still not started %s, retrying", e)
else:
break
try:
# register software profile
for path in self.software_path_list:
slap.registerSupply().supply(
path, computer_guid=config['computer_id'])
except Exception:
logger.exception("SlapOSControler.initializeSlapOSControler")
raise ValueError("Unable to registerSupply")
# Reset all previously generated software if needed
if reset_software:
self._resetSoftware()
else:
createFolder(self.software_root)
instance_root = self.instance_root
# Delete any existing partition in order to not get its data (ex.
# MySQL DB content) from previous runs. To support changes of partition
# naming scheme (which already happened), do this at instance_root level.
createFolder(instance_root, True)
partition_list = []
for i in range(MAX_PARTITIONS):
# create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address
# this is not a problem as usually all services are on different ports
partition_reference = '%s-%s' % (config['partition_reference'], i)
partition_path = os.path.join(instance_root, partition_reference)
if not (os.path.exists(partition_path)):
os.mkdir(partition_path)
os.chmod(partition_path, 0o750)
partition_list.append({
'address_list': [
{
'addr': config['ipv4_address'],
'netmask': '255.255.255.255'
},
{
'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'
},
],
'path':
partition_path,
'reference':
partition_reference,
'tap': {
'name': partition_reference
},
})
computer.updateConfiguration(
xml_marshaller.xml_marshaller.dumps({
'address':
config['ipv4_address'],
'instance_root':
instance_root,
'netmask':
'255.255.255.255',
'partition_list':
partition_list,
'reference':
config['computer_id'],
'software_root':
self.software_root
}))
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout['slap_connection']
computer_id = slap_connection['computer_id']
computer_partition_id = slap_connection['partition_id']
server_url = slap_connection['server_url']
key_file = slap_connection.get('key_file')
cert_file = slap_connection.get('cert_file')
instance_root = self.buildout['buildout']['directory']
storage_configuration_dict = self.buildout.get('storage-configuration')
network_dict = self.buildout.get('network-information')
storage_home = ''
global_ipv4_network = ''
if storage_configuration_dict:
storage_home = storage_configuration_dict.get('storage-home')
if network_dict:
global_ipv4_network = network_dict.get('global-ipv4-network')
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id,
computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict['slap_software_type']
# Raise if request software_type does not exist ...
if software_type not in self.options:
# ... Except for backward compatibility. Then use "default".
if software_type in ['RootSoftwareInstance']:
software_type = 'default'
else:
raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
software_type)
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError("The specified buildout config file %r does "
"not exist." % instance_file_path)
buildout = SlapConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name)
if not buildout.has_section('slap-parameter'):
buildout.add_section('slap-parameter')
for parameter, value in self.parameter_dict.items():
# All parameters evaluating to False are... False, and shouldn't
# convey any information.
# Here, all those parameters are simply ignored.
if value:
if isinstance(value, str):
buildout.set('slap-parameter', parameter, value)
else:
buildout.set('slap-parameter', parameter, json.dumps(value))
buildout.add_section('slap-network-information')
buildout.set('slap-network-information', 'local-ipv4',
self.getLocalIPv4Address())
buildout.set('slap-network-information', 'global-ipv6',
self.getGlobalIPv6Address())
buildout.set('slap-network-information', 'network-interface',
self.getNetworkInterface())
tap_ip_list = self.getLocalTapIPv4AddressList()
tap_ipv4 = tap_gateway = tap_netmask = tap_network = ''
if tap_ip_list:
tap_ipv4, tap_gateway, tap_netmask, tap_network= tap_ip_list
buildout.set('slap-network-information', 'tap-ipv4', tap_ipv4)
buildout.set('slap-network-information', 'tap-gateway', tap_gateway)
buildout.set('slap-network-information', 'tap-netmask', tap_netmask)
buildout.set('slap-network-information', 'tap-network', tap_network)
buildout.set('slap-network-information', 'global-ipv4-network',
global_ipv4_network)
# Copy/paste slap_connection
buildout.add_section('slap-connection')
for key, value in self.buildout['slap_connection'].iteritems():
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set('slap-connection', key.replace('_', '-'), value)
# XXX: Needed for lxc. Use non standard API
buildout.set('slap-connection', 'requested', self.computer_partition._requested_state)
# setup storage directory
buildout.add_section('storage-configuration')
buildout.set('storage-configuration', 'storage-home', storage_home)
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
# Create folder instance_root/DATA/ if not exist
data_home = os.path.join(instance_root, 'DATA')
self.mkdir_p(data_home)
for filename in os.listdir(storage_home):
storage_path = os.path.join(storage_home, filename, computer_partition_id)
if os.path.exists(storage_path) and os.path.isdir(storage_path):
storage_link = os.path.join(data_home, filename)
if os.path.lexists(storage_link):
if not os.path.islink(storage_link):
raise zc.buildout.UserError(
'Target %r already exists but is not a link' % storage_link)
#os.unlink(storage_link)
else:
os.symlink(storage_path, storage_link)
buildout.set('storage-configuration', filename, storage_link)
work_directory = os.path.abspath(self.buildout['buildout'][
'directory'])
buildout_filename = os.path.join(work_directory,
'buildout-%s.cfg' % self.name)
with open(buildout_filename, 'w') as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
self.logger.info("Invoking commandline : '%s'",
' '.join(command_line_args))
subprocess.check_call(command_line_args, cwd=work_directory,
env=os.environ.copy())
return []
def fetch_parameter_dict(self, options, instance_root):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
if route_ipv4_set:
options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8')
options['tap-network-information-dict'] = dict(
ipv4=route_ipv4_set,
netmask=route_mask_set,
gateway=route_gw_set,
network=route_network_set)
else:
options['tap-network-information-dict'] = {}
if route_gw_set:
options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8')
if route_mask_set:
options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8')
if route_network_set:
options['tap-network'] = list(route_network_set)[0].encode('UTF-8')
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(
storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(
storage_path):
storage_link = os.path.join(instance_root, 'DATA',
filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
options['tap'] = tap_set
return self._expandParameterDict(options, parameter_dict)
def fetch_parameter_dict(self, options, instance_root):
"""Gather parameters about current computer and partition.
Use two sources of truth
1. SlapOS Master - for external computer/partition information
2. format.Partition.resource_file - for partition specific details
"""
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
# Get Instance and root instance title or return UNKNOWN if not set
options['instance-title'] = parameter_dict.pop('instance_title',
'UNKNOWN Instance')
options['root-instance-title'] = parameter_dict.pop(
'root_instance_title', 'UNKNOWN')
options['instance-guid'] = computer_partition.getInstanceGuid()
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = min(ipv4_set)
if ipv6_set:
options['ipv6-random'] = min(ipv6_set)
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(
storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(
storage_path):
storage_link = os.path.join(instance_root, 'DATA',
filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
# The external information transfered from Slap Master has been processed
# so we extend with information gathered from partition resource file
if hasattr(slapformat.Partition, "resource_file"):
resource_home = instance_root
while not os.path.exists(
os.path.join(resource_home,
slapformat.Partition.resource_file)):
resource_home = os.path.normpath(
os.path.join(resource_home, '..'))
if resource_home == "/":
break
else:
# no break happened - let's add partition resources into options
logger.debug("Using partition resource file {}".format(
os.path.join(resource_home,
slapformat.Partition.resource_file)))
with open(
os.path.join(
resource_home,
slapformat.Partition.resource_file)) as fi:
partition_params = json.load(fi)
# be very careful with overriding master's information
for key, value in flatten_dict(partition_params).items():
if key not in options:
if six.PY2 and isinstance(value, unicode):
value = value.encode('UTF-8')
options[key] = value
# print out augmented options to see what we are passing
logger.debug(str(options))
return self._expandParameterDict(options, parameter_dict)
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout['slap_connection']
computer_id = slap_connection['computer_id']
computer_partition_id = slap_connection['partition_id']
server_url = slap_connection['server_url']
key_file = slap_connection.get('key_file')
cert_file = slap_connection.get('cert_file')
instance_root = self.buildout['buildout']['directory']
storage_configuration_dict = self.buildout.get('storage-configuration')
network_dict = self.buildout.get('network-information')
storage_home = ''
global_ipv4_network = ''
if storage_configuration_dict:
storage_home = storage_configuration_dict.get('storage-home')
if network_dict:
global_ipv4_network = network_dict.get('global-ipv4-network')
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id, computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict(
)
software_type = self.parameter_dict['slap_software_type']
# Raise if request software_type does not exist ...
if software_type not in self.options:
# ... Except for backward compatibility. Then use "default".
if software_type in ['RootSoftwareInstance']:
software_type = 'default'
else:
raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
software_type)
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError(
"The specified buildout config file %r does "
"not exist." % instance_file_path)
if six.PY3:
buildout = SlapConfigParser(strict=False)
else:
buildout = SlapConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name)
if not buildout.has_section('slap-parameter'):
buildout.add_section('slap-parameter')
for parameter, value in self.parameter_dict.items():
# All parameters evaluating to False are... False, and shouldn't
# convey any information.
# Here, all those parameters are simply ignored.
if value:
if isinstance(value, str):
buildout.set('slap-parameter', parameter, value)
else:
buildout.set('slap-parameter', parameter,
json.dumps(value))
buildout.add_section('slap-network-information')
buildout.set('slap-network-information', 'local-ipv4',
self.getLocalIPv4Address())
buildout.set('slap-network-information', 'global-ipv6',
self.getGlobalIPv6Address())
buildout.set('slap-network-information', 'network-interface',
self.getNetworkInterface())
tap_ip_list = self.getLocalTapIPv4AddressList()
tap_ipv4 = tap_gateway = tap_netmask = tap_network = ''
if tap_ip_list:
tap_ipv4, tap_gateway, tap_netmask, tap_network = tap_ip_list
buildout.set('slap-network-information', 'tap-ipv4', tap_ipv4)
buildout.set('slap-network-information', 'tap-gateway', tap_gateway)
buildout.set('slap-network-information', 'tap-netmask', tap_netmask)
buildout.set('slap-network-information', 'tap-network', tap_network)
buildout.set('slap-network-information', 'global-ipv4-network',
global_ipv4_network)
# Copy/paste slap_connection
buildout.add_section('slap-connection')
for key, value in six.iteritems(self.buildout['slap_connection']):
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set('slap-connection', key.replace('_', '-'), value)
# XXX: Needed for lxc. Use non standard API
buildout.set('slap-connection', 'requested',
self.computer_partition._requested_state)
# setup storage directory
buildout.add_section('storage-configuration')
buildout.set('storage-configuration', 'storage-home', storage_home)
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
# Create folder instance_root/DATA/ if not exist
data_home = os.path.join(instance_root, 'DATA')
self.mkdir_p(data_home)
for filename in os.listdir(storage_home):
storage_path = os.path.join(storage_home, filename,
computer_partition_id)
if os.path.exists(storage_path) and os.path.isdir(
storage_path):
storage_link = os.path.join(data_home, filename)
if os.path.lexists(storage_link):
if not os.path.islink(storage_link):
raise zc.buildout.UserError(
'Target %r already exists but is not a link' %
storage_link)
#os.unlink(storage_link)
else:
os.symlink(storage_path, storage_link)
buildout.set('storage-configuration', filename,
storage_link)
work_directory = os.path.abspath(
self.buildout['buildout']['directory'])
buildout_filename = os.path.join(work_directory,
'buildout-%s.cfg' % self.name)
with open(buildout_filename, 'w') as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
self.logger.info("Invoking commandline : '%s'",
' '.join(command_line_args))
subprocess.check_call(command_line_args,
cwd=work_directory,
env=os.environ.copy())
return []
def fetch_parameter_dict(self, options, instance_root):
"""Gather parameters about current computer and partition.
Use two sources of truth
1. SlapOS Master - for external computer/partition information
2. format.Partition.resource_file - for partition specific details
"""
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
# Get Instance and root instance title or return UNKNOWN if not set
options['instance-title'] = parameter_dict.pop('instance_title',
'UNKNOWN Instance').encode('UTF-8')
options['root-instance-title'] = parameter_dict.pop('root_instance_title',
'UNKNOWN').encode('UTF-8')
options['instance-guid'] = computer_partition.getInstanceGuid() \
.encode('UTF-8')
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(storage_path):
storage_link = os.path.join(instance_root, 'DATA', filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
# The external information transfered from Slap Master has been processed
# so we extend with information gathered from partition resource file
if hasattr(slapformat.Partition, "resource_file"):
resource_home = instance_root
while not os.path.exists(os.path.join(resource_home, slapformat.Partition.resource_file)):
resource_home = os.path.normpath(os.path.join(resource_home, '..'))
if resource_home == "/":
break
else:
# no break happened - let's add partition resources into options
logger.debug("Using partition resource file {}".format(
os.path.join(resource_home, slapformat.Partition.resource_file)))
with open(os.path.join(resource_home, slapformat.Partition.resource_file)) as fi:
partition_params = json.load(fi)
# be very careful with overriding master's information
for key, value in flatten_dict(partition_params).items():
if key not in options:
if isinstance(value, unicode):
value = value.encode('UTF-8')
options[key] = value
# print out augmented options to see what we are passing
logger.debug(str(options))
return self._expandParameterDict(options, parameter_dict)
