def __init__(
self,
server_url,
key_file,
cert_file,
computer_id,
partition_id,
software,
namebase,
root_instance_name,
sleep_time_between_test=900,
total_instance_count="2",
type=None,
):
self.server_url = server_url
self.key_file = key_file
self.cert_file = cert_file
self.computer_id = computer_id
self.partition_id = partition_id
self.software = software
self.namebase = namebase
self.total_instance_count = total_instance_count
self.root_instance_name = root_instance_name
self.sleep_time_between_test = sleep_time_between_test
self.test_type = type
slap = slapos.slap.slap()
slap.initializeConnection(server_url, key_file, cert_file)
self.partition = slap.registerComputerPartition(computer_guid=computer_id, partition_id=partition_id)
self.logger = logging.getLogger("SlaprunnerResiliencyTest")
self.logger.setLevel(logging.DEBUG)
def install(self):
self.path_list = []
crond = self.installCrond()
slap = slapos.slap.slap()
slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
parameter_dict = slap.registerComputerPartition(
self.computer_id,
self.computer_partition_id,
).getInstanceParameterDict()
# XXX: should probably expect one more (SR-originating) parameter instead
# of using self.work_directory .
configuration_path = os.path.join(self.work_directory, "agent.cfg")
with open(configuration_path, "w") as configuration:
configuration.write(parameter_dict["configuration"])
agent_crond_path = os.path.join(crond, "agent")
with open(agent_crond_path, "w") as agent_crond:
agent_crond.write("*/5 * * * * %s -S %s --pidfile=%s --log=%s "
"%s 2>&1 > /dev/null\n" % (
self.options["python_binary"],
self.options["agent_binary"],
self.options["pidfile"],
self.options["log"],
configuration_path,
))
return self.path_list + [configuration_path, agent_crond_path]
def setUpClass(cls, **kw):
if len(kw) == 0:
return
cls.server_url = kw['server_url']
cls.key_file = kw['key_file']
cls.cert_file = kw['cert_file']
cls.computer_id = kw['computer_id']
cls.partition_id = kw['partition_id']
# Get parameters returned by slapos master
slap = slapos.slap.slap()
slap.initializeConnection(cls.server_url, cls.key_file, cls.cert_file)
cls.partition = slap.registerComputerPartition(
computer_guid=cls.computer_id,
partition_id=cls.partition_id
)
cls.parameter_dict = cls.partition.getConnectionParameterDict()
for attribute, value in cls.parameter_dict.iteritems():
setattr(cls, attribute.replace('-', '_'), value)
#create slaprunner configuration
views.app.config['TESTING'] = True
config = Config()
config.setConfig()
views.app.config.update(**config.__dict__)
cls.app = views.app.test_client()
cls.app.config = views.app.config
# Set up path (needed to find git binary)
os.environ['PATH'] = config.path
def test_error_new_ComputerPartition_works(self):
"""
Asserts that calling ComputerPartition.error on new partition works
"""
computer_guid = self._getTestComputerId()
partition_id = 'PARTITION_01'
slap = self.slap
slap.initializeConnection(self.server_url)
def server_response(self, path, method, body, header):
parsed_url = urlparse.urlparse(path.lstrip('/'))
parsed_qs = urlparse.parse_qs(parsed_url.query)
if (parsed_url.path == 'registerComputerPartition' and
parsed_qs['computer_reference'][0] == computer_guid and
parsed_qs['computer_partition_reference'][0] == partition_id):
partition = slapos.slap.ComputerPartition(
computer_guid, partition_id)
return (200, {}, xml_marshaller.xml_marshaller.dumps(partition))
elif parsed_url.path == 'softwareInstanceError':
parsed_qs_body = urlparse.parse_qs(body)
# XXX: why do we have computer_id and not computer_reference?
# XXX: why do we have computer_partition_id and not
# computer_partition_reference?
if (parsed_qs_body['computer_id'][0] == computer_guid and
parsed_qs_body['computer_partition_id'][0] == partition_id and
parsed_qs_body['error_log'][0] == 'some error'):
return (200, {}, '')
return (404, {}, '')
httplib.HTTPConnection._callback = server_response
computer_partition = slap.registerComputerPartition(
computer_guid, partition_id)
# XXX: Interface does not define return value
computer_partition.error('some error')
def __init__(self, buildout, name, options):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
options['tap'] = tap_set
parameter_dict = self._expandParameterDict(options, parameter_dict)
match = self.OPTCRE_match
for key, value in parameter_dict.iteritems():
if match(key) is not None:
continue
options['configuration.' + key] = value
def __init__(self, buildout, name, options):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0]
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0]
options['tap'] = tap_set
parameter_dict = self._expandParameterDict(options, parameter_dict)
match = self.OPTCRE_match
for key, value in parameter_dict.iteritems():
if match(key) is not None:
continue
options['configuration.' + key] = value
def __bang(self, message):
"""
Call bang if requested
"""
if 'master-url' in self.__config and \
'partition-in' in self.__config and \
'computer-id' in self.__config:
slap = slapos.slap.slap()
slap.initializeConnection(
self.__config['master-url'],
self.__config.get('partition-key'),
self.__config.get('partition-cert'),
)
computer_partition = slap.registerComputerPartition(
self.__config['computer-id'],
self.__config['partition-id'],
)
computer_partition.bang(message)
self.logger.info("Bang with message %r." % message)
def test_error_new_ComputerPartition_works(self):
"""
Asserts that calling ComputerPartition.error on new partition works
"""
computer_guid = self._getTestComputerId()
partition_id = 'PARTITION_01'
slap = self.slap
slap.initializeConnection(self.server_url)
def handler(url, req):
qs = urlparse.parse_qs(url.query)
if (url.path == '/registerComputerPartition' and
qs['computer_reference'][0] == computer_guid and
qs['computer_partition_reference'][0] == partition_id):
partition = slapos.slap.ComputerPartition(
computer_guid, partition_id)
return {
'statu_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(partition)
}
elif url.path == '/softwareInstanceError':
parsed_qs_body = urlparse.parse_qs(req.body)
# XXX: why do we have computer_id and not computer_reference?
# XXX: why do we have computer_partition_id and not
# computer_partition_reference?
if (parsed_qs_body['computer_id'][0] == computer_guid and
parsed_qs_body['computer_partition_id'][0] == partition_id and
parsed_qs_body['error_log'][0] == 'some error'):
return {'status_code': 200}
return {'status_code': 404}
with httmock.HTTMock(handler):
computer_partition = slap.registerComputerPartition(
computer_guid, partition_id)
# XXX: Interface does not define return value
computer_partition.error('some error')
def __init__(self, buildout, name, options):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
parameter_dict = slap.registerComputerPartition(
options['computer'],
options['partition'],
).getInstanceParameterDict()
# XXX: those are not partition parameters, strictly speaking.
# Discard them, and make them available as separate section keys.
options['slap-software-type'] = parameter_dict.pop(
'slap_software_type')
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
options['tap'] = tap_set
options['configuration'] = parameter_dict
match = self.OPTCRE_match
for key, value in parameter_dict.iteritems():
if match(key) is not None:
continue
options['configuration.' + key] = value
def fetch_parameter_dict(self, options, instance_root):
"""Gather parameters about current computer and partition.
Use two sources of truth
1. SlapOS Master - for external computer/partition information
2. format.Partition.resource_file - for partition specific details
"""
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
# Get Instance and root instance title or return UNKNOWN if not set
options['instance-title'] = parameter_dict.pop('instance_title',
'UNKNOWN Instance').encode('UTF-8')
options['root-instance-title'] = parameter_dict.pop('root_instance_title',
'UNKNOWN').encode('UTF-8')
options['instance-guid'] = computer_partition.getInstanceGuid() \
.encode('UTF-8')
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(storage_path):
storage_link = os.path.join(instance_root, 'DATA', filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
# The external information transfered from Slap Master has been processed
# so we extend with information gathered from partition resource file
if hasattr(slapformat.Partition, "resource_file"):
resource_home = instance_root
while not os.path.exists(os.path.join(resource_home, slapformat.Partition.resource_file)):
resource_home = os.path.normpath(os.path.join(resource_home, '..'))
if resource_home == "/":
break
else:
# no break happened - let's add partition resources into options
logger.debug("Using partition resource file {}".format(
os.path.join(resource_home, slapformat.Partition.resource_file)))
with open(os.path.join(resource_home, slapformat.Partition.resource_file)) as fi:
partition_params = json.load(fi)
# be very careful with overriding master's information
for key, value in flatten_dict(partition_params).items():
if key not in options:
if isinstance(value, unicode):
value = value.encode('UTF-8')
options[key] = value
# print out augmented options to see what we are passing
logger.debug(str(options))
return self._expandParameterDict(options, parameter_dict)
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout['slap_connection']
computer_id = slap_connection['computer_id']
computer_partition_id = slap_connection['partition_id']
server_url = slap_connection['server_url']
key_file = slap_connection.get('key_file')
cert_file = slap_connection.get('cert_file')
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id, computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict(
)
software_type = self.parameter_dict['slap_software_type']
self.logger.info('Deploying instance with software type %s' % \
software_type)
# Raise if request software_type does not exist ...
if software_type not in self.options:
# ... Except for backward compatibility. Then use "default".
if software_type in ['RootSoftwareInstance']:
software_type = 'default'
else:
raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
software_type)
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError(
"The specified buildout config file %r does "
"not exist." % instance_file_path)
buildout = ConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name)
if not buildout.has_section('slap-parameter'):
buildout.add_section('slap-parameter')
for parameter, value in self.parameter_dict.items():
# All parameters evaluating to False are... False, and shouldn't
# convey any information.
# Here, all those parameters are simply ignored.
if value:
if isinstance(value, str):
buildout.set('slap-parameter', parameter, value)
else:
buildout.set('slap-parameter', parameter,
json.dumps(value))
buildout.add_section('slap-network-information')
buildout.set('slap-network-information', 'local-ipv4',
self.getLocalIPv4Address())
buildout.set('slap-network-information', 'global-ipv6',
self.getGlobalIPv6Address())
buildout.set('slap-network-information', 'network-interface',
self.getNetworkInterface())
# Copy/paste slap_connection
buildout.add_section('slap-connection')
for key, value in self.buildout['slap_connection'].iteritems():
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set('slap-connection', key.replace('_', '-'), value)
# XXX: Needed for lxc. Use non standard API
buildout.set('slap-connection', 'requested',
self.computer_partition._requested_state)
work_directory = os.path.abspath(
self.buildout['buildout']['directory'])
buildout_filename = os.path.join(work_directory,
'buildout-%s.cfg' % self.name)
with open(buildout_filename, 'w') as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
self.logger.info("Invoking commandline : '%s'",
' '.join(command_line_args))
subprocess.check_call(command_line_args,
cwd=work_directory,
env=os.environ.copy())
return []
def fetch_parameter_dict(self, options, instance_root):
"""Gather parameters about current computer and partition.
Use two sources of truth
1. SlapOS Master - for external computer/partition information
2. format.Partition.resource_file - for partition specific details
"""
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
# Get Instance and root instance title or return UNKNOWN if not set
options['instance-title'] = parameter_dict.pop('instance_title',
'UNKNOWN Instance')
options['root-instance-title'] = parameter_dict.pop(
'root_instance_title', 'UNKNOWN')
options['instance-guid'] = computer_partition.getInstanceGuid()
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = min(ipv4_set)
if ipv6_set:
options['ipv6-random'] = min(ipv6_set)
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(
storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(
storage_path):
storage_link = os.path.join(instance_root, 'DATA',
filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
# The external information transfered from Slap Master has been processed
# so we extend with information gathered from partition resource file
if hasattr(slapformat.Partition, "resource_file"):
resource_home = instance_root
while not os.path.exists(
os.path.join(resource_home,
slapformat.Partition.resource_file)):
resource_home = os.path.normpath(
os.path.join(resource_home, '..'))
if resource_home == "/":
break
else:
# no break happened - let's add partition resources into options
logger.debug("Using partition resource file {}".format(
os.path.join(resource_home,
slapformat.Partition.resource_file)))
with open(
os.path.join(
resource_home,
slapformat.Partition.resource_file)) as fi:
partition_params = json.load(fi)
# be very careful with overriding master's information
for key, value in flatten_dict(partition_params).items():
if key not in options:
if six.PY2 and isinstance(value, unicode):
value = value.encode('UTF-8')
options[key] = value
# print out augmented options to see what we are passing
logger.debug(str(options))
return self._expandParameterDict(options, parameter_dict)
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout['slap_connection']
computer_id = slap_connection['computer_id']
computer_partition_id = slap_connection['partition_id']
server_url = slap_connection['server_url']
key_file = slap_connection.get('key_file')
cert_file = slap_connection.get('cert_file')
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id,
computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict['slap_software_type']
if software_type not in self.options:
if 'default' in self.options:
software_type = 'default'
else:
raise zc.buildout.UserError("This software type isn't mapped. And"
"there's no default software type.")
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError("The specified buildout config file does not"
"exist.")
buildout = ConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set('buildout', 'installed',
'.installed-%s.cfg' % software_type)
buildout.add_section('slap-parameter')
for parameter, value in self.parameter_dict.items():
buildout.set('slap-parameter', parameter, value)
buildout.add_section('slap-network-information')
buildout.set('slap-network-information', 'local-ipv4',
self.getLocalIPv4Address())
buildout.set('slap-network-information', 'global-ipv6',
self.getGlobalIPv6Address())
# Copy/paste slap_connection
buildout.add_section('slap-connection')
for key, value in self.buildout['slap_connection'].iteritems():
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set('slap-connection', key.replace('_', '-'), value)
work_directory = os.path.abspath(self.buildout['buildout'][
'directory'])
buildout_filename = os.path.join(work_directory,
'buildout-%s.cfg' % software_type)
with open(buildout_filename, 'w') as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
self.logger.info("Invoking commandline : '%s'",
' '.join(command_line_args))
subprocess.check_call(command_line_args, cwd=work_directory,
env=os.environ.copy())
return []
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout['slap_connection']
computer_id = slap_connection['computer_id']
computer_partition_id = slap_connection['partition_id']
server_url = slap_connection['server_url']
key_file = slap_connection.get('key_file')
cert_file = slap_connection.get('cert_file')
instance_root = self.buildout['buildout']['directory']
storage_configuration_dict = self.buildout.get('storage-configuration')
network_dict = self.buildout.get('network-information')
storage_home = ''
global_ipv4_network = ''
if storage_configuration_dict:
storage_home = storage_configuration_dict.get('storage-home')
if network_dict:
global_ipv4_network = network_dict.get('global-ipv4-network')
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id,
computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict['slap_software_type']
# Raise if request software_type does not exist ...
if software_type not in self.options:
# ... Except for backward compatibility. Then use "default".
if software_type in ['RootSoftwareInstance']:
software_type = 'default'
else:
raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
software_type)
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError("The specified buildout config file %r does "
"not exist." % instance_file_path)
buildout = SlapConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name)
if not buildout.has_section('slap-parameter'):
buildout.add_section('slap-parameter')
for parameter, value in self.parameter_dict.items():
# All parameters evaluating to False are... False, and shouldn't
# convey any information.
# Here, all those parameters are simply ignored.
if value:
if isinstance(value, str):
buildout.set('slap-parameter', parameter, value)
else:
buildout.set('slap-parameter', parameter, json.dumps(value))
buildout.add_section('slap-network-information')
buildout.set('slap-network-information', 'local-ipv4',
self.getLocalIPv4Address())
buildout.set('slap-network-information', 'global-ipv6',
self.getGlobalIPv6Address())
buildout.set('slap-network-information', 'network-interface',
self.getNetworkInterface())
tap_ip_list = self.getLocalTapIPv4AddressList()
tap_ipv4 = tap_gateway = tap_netmask = tap_network = ''
if tap_ip_list:
tap_ipv4, tap_gateway, tap_netmask, tap_network= tap_ip_list
buildout.set('slap-network-information', 'tap-ipv4', tap_ipv4)
buildout.set('slap-network-information', 'tap-gateway', tap_gateway)
buildout.set('slap-network-information', 'tap-netmask', tap_netmask)
buildout.set('slap-network-information', 'tap-network', tap_network)
buildout.set('slap-network-information', 'global-ipv4-network',
global_ipv4_network)
# Copy/paste slap_connection
buildout.add_section('slap-connection')
for key, value in self.buildout['slap_connection'].iteritems():
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set('slap-connection', key.replace('_', '-'), value)
# XXX: Needed for lxc. Use non standard API
buildout.set('slap-connection', 'requested', self.computer_partition._requested_state)
# setup storage directory
buildout.add_section('storage-configuration')
buildout.set('storage-configuration', 'storage-home', storage_home)
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
# Create folder instance_root/DATA/ if not exist
data_home = os.path.join(instance_root, 'DATA')
self.mkdir_p(data_home)
for filename in os.listdir(storage_home):
storage_path = os.path.join(storage_home, filename, computer_partition_id)
if os.path.exists(storage_path) and os.path.isdir(storage_path):
storage_link = os.path.join(data_home, filename)
if os.path.lexists(storage_link):
if not os.path.islink(storage_link):
raise zc.buildout.UserError(
'Target %r already exists but is not a link' % storage_link)
#os.unlink(storage_link)
else:
os.symlink(storage_path, storage_link)
buildout.set('storage-configuration', filename, storage_link)
work_directory = os.path.abspath(self.buildout['buildout'][
'directory'])
buildout_filename = os.path.join(work_directory,
'buildout-%s.cfg' % self.name)
with open(buildout_filename, 'w') as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
self.logger.info("Invoking commandline : '%s'",
' '.join(command_line_args))
subprocess.check_call(command_line_args, cwd=work_directory,
env=os.environ.copy())
return []
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout['slap_connection']
computer_id = slap_connection['computer_id']
computer_partition_id = slap_connection['partition_id']
server_url = slap_connection['server_url']
key_file = slap_connection.get('key_file')
cert_file = slap_connection.get('cert_file')
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id,
computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict['slap_software_type']
self.logger.info('Deploying instance with software type %s' % \
software_type)
# Raise if request software_type does not exist ...
if software_type not in self.options:
# ... Except for backward compatibility. Then use "default".
if software_type in ['RootSoftwareInstance']:
software_type = 'default'
else:
raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
software_type)
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError("The specified buildout config file %r does "
"not exist." % instance_file_path)
buildout = ConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name)
if not buildout.has_section('slap-parameter'):
buildout.add_section('slap-parameter')
for parameter, value in self.parameter_dict.items():
# All parameters evaluating to False are... False, and shouldn't
# convey any information.
# Here, all those parameters are simply ignored.
if value:
if isinstance(value, str):
buildout.set('slap-parameter', parameter, value)
else:
buildout.set('slap-parameter', parameter, json.dumps(value))
buildout.add_section('slap-network-information')
buildout.set('slap-network-information', 'local-ipv4',
self.getLocalIPv4Address())
buildout.set('slap-network-information', 'global-ipv6',
self.getGlobalIPv6Address())
buildout.set('slap-network-information', 'network-interface',
self.getNetworkInterface())
# Copy/paste slap_connection
buildout.add_section('slap-connection')
for key, value in self.buildout['slap_connection'].iteritems():
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set('slap-connection', key.replace('_', '-'), value)
# XXX: Needed for lxc. Use non standard API
buildout.set('slap-connection', 'requested', self.computer_partition._requested_state)
work_directory = os.path.abspath(self.buildout['buildout'][
'directory'])
buildout_filename = os.path.join(work_directory,
'buildout-%s.cfg' % self.name)
with open(buildout_filename, 'w') as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
self.logger.info("Invoking commandline : '%s'",
' '.join(command_line_args))
subprocess.check_call(command_line_args, cwd=work_directory,
env=os.environ.copy())
return []
def fetch_parameter_dict(self, options, instance_root):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
if route_ipv4_set:
options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8')
options['tap-network-information-dict'] = dict(ipv4=route_ipv4_set,
netmask=route_mask_set,
gateway=route_gw_set,
network=route_network_set)
else:
options['tap-network-information-dict'] = {}
if route_gw_set:
options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8')
if route_mask_set:
options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8')
if route_network_set:
options['tap-network'] = list(route_network_set)[0].encode('UTF-8')
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(storage_path):
storage_link = os.path.join(instance_root, 'DATA', filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
options['tap'] = tap_set
return self._expandParameterDict(options, parameter_dict)
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout["slap_connection"]
computer_id = slap_connection["computer_id"]
computer_partition_id = slap_connection["partition_id"]
server_url = slap_connection["server_url"]
key_file = slap_connection.get("key_file")
cert_file = slap_connection.get("cert_file")
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(computer_id, computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict["slap_software_type"]
self.logger.info("Deploying instance with software type %s" % software_type)
# Raise if request software_type does not exist ...
if software_type not in self.options:
# ... Except for backward compatibility. Then use "default".
if software_type in ["RootSoftwareInstance"]:
software_type = "default"
else:
raise zc.buildout.UserError("This software type (%s) isn't mapped." % software_type)
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError("The specified buildout config file %r does " "not exist." % instance_file_path)
buildout = ConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set("buildout", "installed", ".installed-%s.cfg" % self.name)
if not buildout.has_section("slap-parameter"):
buildout.add_section("slap-parameter")
for parameter, value in self.parameter_dict.items():
if isinstance(value, str):
buildout.set("slap-parameter", parameter, value)
else:
buildout.set("slap-parameter", parameter, json.dumps(value))
buildout.add_section("slap-network-information")
buildout.set("slap-network-information", "local-ipv4", self.getLocalIPv4Address())
buildout.set("slap-network-information", "global-ipv6", self.getGlobalIPv6Address())
buildout.set("slap-network-information", "network-interface", self.getNetworkInterface())
# Copy/paste slap_connection
buildout.add_section("slap-connection")
for key, value in self.buildout["slap_connection"].iteritems():
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set("slap-connection", key.replace("_", "-"), value)
# XXX: Needed for lxc. Use non standard API
buildout.set("slap-connection", "requested", self.computer_partition._requested_state)
work_directory = os.path.abspath(self.buildout["buildout"]["directory"])
buildout_filename = os.path.join(work_directory, "buildout-%s.cfg" % self.name)
with open(buildout_filename, "w") as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ["-c", buildout_filename]
self.logger.info("Invoking commandline : '%s'", " ".join(command_line_args))
subprocess.check_call(command_line_args, cwd=work_directory, env=os.environ.copy())
return []
def fetch_parameter_dict(self, options, instance_root):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
options.get('key'),
options.get('cert'),
)
computer_partition = slap.registerComputerPartition(
options['computer'],
options['partition'],
)
parameter_dict = computer_partition.getInstanceParameterDict()
options['instance-state'] = computer_partition.getState()
# XXX: those are not partition parameters, strictly speaking.
# Make them available as individual section keys.
for his_key in (
'slap_software_type',
'slap_computer_partition_id',
'slap_computer_id',
'slap_software_release_url',
'slave_instance_list',
'timestamp',
):
try:
value = parameter_dict.pop(his_key)
except KeyError:
pass
else:
options[his_key.replace('_', '-')] = value
ipv4_set = set()
v4_add = ipv4_set.add
ipv6_set = set()
v6_add = ipv6_set.add
tap_set = set()
tap_add = tap_set.add
route_gw_set = set()
route_gw_add = route_gw_set.add
route_mask_set = set()
route_mask_add = route_mask_set.add
route_ipv4_set = set()
route_v4_add = route_ipv4_set.add
route_network_set = set()
route_net_add = route_network_set.add
for tap, ip in parameter_dict.pop('ip_list'):
tap_add(tap)
if valid_ipv4(ip):
v4_add(ip)
elif valid_ipv6(ip):
v6_add(ip)
# XXX: emit warning on unknown address type ?
if 'full_ip_list' in parameter_dict:
for item in parameter_dict.pop('full_ip_list'):
if len(item) == 5:
tap, ip, gw, netmask, network = item
if tap.startswith('route_'):
if valid_ipv4(gw):
route_gw_add(gw)
if valid_ipv4(netmask):
route_mask_add(netmask)
if valid_ipv4(ip):
route_v4_add(ip)
if valid_ipv4(network):
route_net_add(network)
options['ipv4'] = ipv4_set
options['ipv6'] = ipv6_set
# also export single ip values for those recipes that don't support sets.
if ipv4_set:
options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
if ipv6_set:
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
if route_ipv4_set:
options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8')
options['tap-network-information-dict'] = dict(
ipv4=route_ipv4_set,
netmask=route_mask_set,
gateway=route_gw_set,
network=route_network_set)
else:
options['tap-network-information-dict'] = {}
if route_gw_set:
options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8')
if route_mask_set:
options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8')
if route_network_set:
options['tap-network'] = list(route_network_set)[0].encode('UTF-8')
storage_home = options.get('storage-home')
storage_dict = {}
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
for filename in os.listdir(storage_home):
storage_path = os.path.join(
storage_home, filename,
options['slap-computer-partition-id'])
if os.path.exists(storage_path) and os.path.isdir(
storage_path):
storage_link = os.path.join(instance_root, 'DATA',
filename)
mkdir_p(os.path.join(instance_root, 'DATA'))
if not os.path.lexists(storage_link):
os.symlink(storage_path, storage_link)
storage_dict[filename] = storage_link
options['storage-dict'] = storage_dict
options['tap'] = tap_set
return self._expandParameterDict(options, parameter_dict)
def install(self):
slap = slapos.slap.slap()
slap_connection = self.buildout['slap_connection']
computer_id = slap_connection['computer_id']
computer_partition_id = slap_connection['partition_id']
server_url = slap_connection['server_url']
key_file = slap_connection.get('key_file')
cert_file = slap_connection.get('cert_file')
instance_root = self.buildout['buildout']['directory']
storage_configuration_dict = self.buildout.get('storage-configuration')
network_dict = self.buildout.get('network-information')
storage_home = ''
global_ipv4_network = ''
if storage_configuration_dict:
storage_home = storage_configuration_dict.get('storage-home')
if network_dict:
global_ipv4_network = network_dict.get('global-ipv4-network')
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id, computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict(
)
software_type = self.parameter_dict['slap_software_type']
# Raise if request software_type does not exist ...
if software_type not in self.options:
# ... Except for backward compatibility. Then use "default".
if software_type in ['RootSoftwareInstance']:
software_type = 'default'
else:
raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
software_type)
instance_file_path = self.options[software_type]
if not os.path.exists(instance_file_path):
raise zc.buildout.UserError(
"The specified buildout config file %r does "
"not exist." % instance_file_path)
if six.PY3:
buildout = SlapConfigParser(strict=False)
else:
buildout = SlapConfigParser()
with open(instance_file_path) as instance_path:
buildout.readfp(instance_path)
buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name)
if not buildout.has_section('slap-parameter'):
buildout.add_section('slap-parameter')
for parameter, value in self.parameter_dict.items():
# All parameters evaluating to False are... False, and shouldn't
# convey any information.
# Here, all those parameters are simply ignored.
if value:
if isinstance(value, str):
buildout.set('slap-parameter', parameter, value)
else:
buildout.set('slap-parameter', parameter,
json.dumps(value))
buildout.add_section('slap-network-information')
buildout.set('slap-network-information', 'local-ipv4',
self.getLocalIPv4Address())
buildout.set('slap-network-information', 'global-ipv6',
self.getGlobalIPv6Address())
buildout.set('slap-network-information', 'network-interface',
self.getNetworkInterface())
tap_ip_list = self.getLocalTapIPv4AddressList()
tap_ipv4 = tap_gateway = tap_netmask = tap_network = ''
if tap_ip_list:
tap_ipv4, tap_gateway, tap_netmask, tap_network = tap_ip_list
buildout.set('slap-network-information', 'tap-ipv4', tap_ipv4)
buildout.set('slap-network-information', 'tap-gateway', tap_gateway)
buildout.set('slap-network-information', 'tap-netmask', tap_netmask)
buildout.set('slap-network-information', 'tap-network', tap_network)
buildout.set('slap-network-information', 'global-ipv4-network',
global_ipv4_network)
# Copy/paste slap_connection
buildout.add_section('slap-connection')
for key, value in six.iteritems(self.buildout['slap_connection']):
# XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
buildout.set('slap-connection', key.replace('_', '-'), value)
# XXX: Needed for lxc. Use non standard API
buildout.set('slap-connection', 'requested',
self.computer_partition._requested_state)
# setup storage directory
buildout.add_section('storage-configuration')
buildout.set('storage-configuration', 'storage-home', storage_home)
if storage_home and os.path.exists(storage_home) and \
os.path.isdir(storage_home):
# Create folder instance_root/DATA/ if not exist
data_home = os.path.join(instance_root, 'DATA')
self.mkdir_p(data_home)
for filename in os.listdir(storage_home):
storage_path = os.path.join(storage_home, filename,
computer_partition_id)
if os.path.exists(storage_path) and os.path.isdir(
storage_path):
storage_link = os.path.join(data_home, filename)
if os.path.lexists(storage_link):
if not os.path.islink(storage_link):
raise zc.buildout.UserError(
'Target %r already exists but is not a link' %
storage_link)
#os.unlink(storage_link)
else:
os.symlink(storage_path, storage_link)
buildout.set('storage-configuration', filename,
storage_link)
work_directory = os.path.abspath(
self.buildout['buildout']['directory'])
buildout_filename = os.path.join(work_directory,
'buildout-%s.cfg' % self.name)
with open(buildout_filename, 'w') as buildout_file:
buildout.write(buildout_file)
# XXX-Antoine: We gotta find a better way to do this. I tried to check
# out how slapgrid-cp was running buildout. But it is worse than that.
command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
self.logger.info("Invoking commandline : '%s'",
' '.join(command_line_args))
subprocess.check_call(command_line_args,
cwd=work_directory,
env=os.environ.copy())
return []
def install(self):
self.path_list = []
self.crond = self.installCrond()
slap = slapos.slap.slap()
computer_id = self.computer_id
computer_partition_id = self.computer_partition_id
server_url = self.server_url
key_file = self.key_file
cert_file = self.cert_file
slap.initializeConnection(server_url, key_file, cert_file)
self.computer_partition = slap.registerComputerPartition(
computer_id, computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
configuration_path = os.path.join(self.work_directory, "agent.cfg")
configuration = ConfigParser.SafeConfigParser()
configuration.add_section("agent")
configuration.set("agent", "portal_url", self.parameter_dict["portal_url"])
configuration.set("agent", "master_url", self.parameter_dict["master_url"])
configuration.set("agent", "report_url", self.parameter_dict["report_url"])
key_filepath = os.path.join(self.work_directory, "key")
key_file = open(key_filepath, "w")
key_file.write(self.parameter_dict["key"])
key_file.close()
configuration.set("agent", "key_file", key_filepath)
cert_filepath = os.path.join(self.work_directory, "cert")
cert_file = open(cert_filepath, "w")
cert_file.write(self.parameter_dict["cert"])
cert_file.close()
configuration.set("agent", "cert_file", cert_filepath)
configuration.set("agent", "maximum_software_installation_duration",
self.parameter_dict["maximum_software_installation_duration"])
configuration.set("agent", "software_live_duration",
self.parameter_dict["software_live_duration"])
configuration.set("agent", "computer_list",
self.parameter_dict["computer_list"])
configuration.set("agent", "software_list",
self.parameter_dict["software_list"])
configuration.set("agent", "log_directory", self.options["log_directory"])
configuration.set("agent", "state_file", self.options["state_file"])
state = open(self.options["state_file"], "w")
state.write('')
state.close()
configuration.add_section("software_uri")
software_list = json.loads(self.parameter_dict["software_list"])
for software in software_list:
configuration.set("software_uri", software, self.parameter_dict[software])
configuration.write(open(configuration_path, "w"))
agent_crond_path = os.path.join(self.crond, "agent")
agent_crond = open(agent_crond_path, "w")
agent_crond.write("*/3 * * * * %s -S %s" % \
(self.options["python_binary"],
"%s --pidfile=%s %s" % \
(self.options["agent_binary"], self.options["pidfile"],
configuration_path)))
agent_crond.close()
return self.path_list + [configuration_path, key_filepath, cert_filepath, agent_crond_path]