def request(self, software_release, software_type, partition_reference,
shared=False, partition_parameter_kw=None, filter_kw=None,
state=None):
if partition_parameter_kw is None:
partition_parameter_kw = {}
elif not isinstance(partition_parameter_kw, dict):
raise ValueError("Unexpected type of partition_parameter_kw '%s'" %
partition_parameter_kw)
if filter_kw is None:
filter_kw = {}
elif not isinstance(filter_kw, dict):
raise ValueError("Unexpected type of filter_kw '%s'" %
filter_kw)
# Let enforce a default software type
if software_type is None:
software_type = DEFAULT_SOFTWARE_TYPE
request_dict = {
'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
'software_release': software_release,
'software_type': software_type,
'partition_reference': partition_reference,
'shared_xml': xml_marshaller.dumps(shared),
'partition_parameter_xml': xml_marshaller.dumps(
partition_parameter_kw),
'filter_xml': xml_marshaller.dumps(filter_kw),
'state': xml_marshaller.dumps(state),
}
return self._requestComputerPartition(request_dict)
def request(self, software_release, partition_reference,
partition_parameter_kw=None, software_type=None, filter_kw=None,
state=None, shared=False):
if partition_parameter_kw is None:
partition_parameter_kw = {}
if filter_kw is None:
filter_kw = {}
request_dict = {
'software_release': software_release,
'partition_reference': partition_reference,
'partition_parameter_xml': xml_marshaller.dumps(partition_parameter_kw),
'filter_xml': xml_marshaller.dumps(filter_kw),
'state': xml_marshaller.dumps(state),
'shared_xml': xml_marshaller.dumps(shared),
}
if software_type is not None:
request_dict['software_type'] = software_type
try:
self._connection_helper.POST('/requestComputerPartition', request_dict)
except ResourceNotReady:
return ComputerPartition(request_dict=request_dict)
else:
xml = self._connection_helper.response.read()
software_instance = xml_marshaller.loads(xml)
return ComputerPartition(
software_instance.slap_computer_id.encode('UTF-8'),
software_instance.slap_computer_partition_id.encode('UTF-8'))
def request(self, software_release, software_type, partition_reference,
shared=False, partition_parameter_kw=None, filter_kw=None,
state=None):
if partition_parameter_kw is None:
partition_parameter_kw = {}
elif not isinstance(partition_parameter_kw, dict):
raise ValueError("Unexpected type of partition_parameter_kw '%s'" % \
partition_parameter_kw)
if filter_kw is None:
filter_kw = {}
elif not isinstance(filter_kw, dict):
raise ValueError("Unexpected type of filter_kw '%s'" % \
filter_kw)
# Let enforce a default software type
if software_type is None:
software_type = DEFAULT_SOFTWARE_TYPE
request_dict = { 'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
'software_release': software_release,
'software_type': software_type,
'partition_reference': partition_reference,
'shared_xml': xml_marshaller.dumps(shared),
'partition_parameter_xml': xml_marshaller.dumps(
partition_parameter_kw),
'filter_xml': xml_marshaller.dumps(filter_kw),
'state': xml_marshaller.dumps(state),
}
try:
self._connection_helper.POST('/requestComputerPartition', request_dict)
except ResourceNotReady:
return ComputerPartition(
request_dict=request_dict,
connection_helper=self._connection_helper,
)
else:
xml = self._connection_helper.response.read()
software_instance = xml_marshaller.loads(xml)
computer_partition = ComputerPartition(
software_instance.slap_computer_id.encode('UTF-8'),
software_instance.slap_computer_partition_id.encode('UTF-8'),
connection_helper=self._connection_helper,
)
if shared:
computer_partition._synced = True
computer_partition._connection_dict = getattr(software_instance,
'_connection_dict', None)
computer_partition._parameter_dict = getattr(software_instance,
'_parameter_dict', None)
return computer_partition
def setConnectionDict(self, connection_dict, slave_reference=None):
if self.getConnectionParameterDict() != connection_dict:
self._connection_helper.POST('/setComputerPartitionConnectionXml', {
'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
'connection_xml': xml_marshaller.dumps(connection_dict),
'slave_reference': slave_reference})
def setConnectionDict(self, connection_dict, slave_reference=None):
if self.getConnectionParameterDict() == connection_dict:
return
if slave_reference is not None:
# check the connection parameters from the slave
# Should we check existence?
slave_parameter_list = self.getInstanceParameter("slave_instance_list")
slave_connection_dict = {}
for slave_parameter_dict in slave_parameter_list:
if slave_parameter_dict.get("slave_reference") == slave_reference:
connection_parameter_hash = slave_parameter_dict.get("connection-parameter-hash", None)
break
# Skip as nothing changed for the slave
if connection_parameter_hash is not None and \
connection_parameter_hash == hashlib.sha256(str(connection_dict)).hexdigest():
return
self._connection_helper.POST('setComputerPartitionConnectionXml', data={
'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
'connection_xml': xml_marshaller.dumps(connection_dict),
'slave_reference': slave_reference})
def setComputerPartitionRelatedInstanceList(self, instance_reference_list):
self._connection_helper.POST('updateComputerPartitionRelatedInstanceList',
data={
'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
'instance_reference_xml': xml_marshaller.dumps(instance_reference_list)
}
)
def request(self, software_release, partition_reference,
partition_parameter_kw=None, software_type=None, filter_kw=None,
state=None, shared=False):
if partition_parameter_kw is None:
partition_parameter_kw = {}
if filter_kw is None:
filter_kw = {}
request_dict = {
'software_release': software_release,
'partition_reference': partition_reference,
'partition_parameter_xml': xml_marshaller.dumps(partition_parameter_kw),
'filter_xml': xml_marshaller.dumps(filter_kw),
# XXX Cedric: Why state and shared are marshalled? First is a string
# And second is a boolean.
'state': xml_marshaller.dumps(state),
'shared_xml': xml_marshaller.dumps(shared),
}
if software_type is not None:
request_dict['software_type'] = software_type
else:
# Let's enforce a default software type
request_dict['software_type'] = DEFAULT_SOFTWARE_TYPE
try:
self._connection_helper.POST('/requestComputerPartition', request_dict)
except ResourceNotReady:
return ComputerPartition(
request_dict=request_dict,
connection_helper=self._connection_helper,
)
else:
xml = self._connection_helper.response.read()
software_instance = xml_marshaller.loads(xml)
computer_partition = ComputerPartition(
software_instance.slap_computer_id.encode('UTF-8'),
software_instance.slap_computer_partition_id.encode('UTF-8'),
connection_helper=self._connection_helper,
)
if shared:
computer_partition._synced = True
computer_partition._connection_dict = software_instance._connection_dict
computer_partition._parameter_dict = software_instance._parameter_dict
return computer_partition
def stepDirectRequestComputerPartitionHttpRequestTimeoutResponseWithoutState(self,
sequence, **kw):
request_dict = { 'computer_id': sequence['computer_reference'] ,
'computer_partition_id': sequence['computer_partition_reference'],
'software_release': sequence['software_release_uri'],
'software_type': sequence.get('requested_software_type',
'requested_software_type'),
'partition_reference': sequence.get('requested_reference',
'requested_reference'),
'shared_xml': xml_marshaller.dumps(kw.get("shared", False)),
'partition_parameter_xml': xml_marshaller.dumps({}),
'filter_xml': xml_marshaller.dumps({}),
#'state': Note: State is omitted
}
scheme, netloc, path, query, fragment = urlparse.urlsplit(self.server_url)
connection = httplib.HTTPConnection(host=netloc)
connection.request("POST", path + '/requestComputerPartition',
urllib.urlencode(request_dict),
{'Content-type': "application/x-www-form-urlencoded"})
response = connection.getresponse()
self.assertEqual(httplib.REQUEST_TIMEOUT, response.status)
def forwardRequestToExternalMaster(master_url, request_form):
"""
Forward instance request to external SlapOS Master.
"""
master_entry = app.config.get('multimaster').get(master_url, {})
key_file = master_entry.get('key')
cert_file = master_entry.get('cert')
if master_url.startswith('https') and (not key_file or not cert_file):
app.logger.warning('External master %s configuration did not specify key or certificate.' % master_url)
abort(404)
if master_url.startswith('https') and not master_url.startswith('https') and (key_file or cert_file):
app.logger.warning('External master %s configurqtion specifies key or certificate but is using plain http.' % master_url)
abort(404)
slap = slapos.slap.slap()
if key_file:
slap.initializeConnection(master_url, key_file=key_file, cert_file=cert_file)
else:
slap.initializeConnection(master_url)
partition_reference = request_form['partition_reference'].encode()
# Store in database
execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)',
{'partition_reference':partition_reference, 'master_url': master_url})
new_request_form = request_form.copy()
filter_kw = loads(new_request_form['filter_xml'].encode())
filter_kw['source_instance_id'] = partition_reference
new_request_form['filter_xml'] = dumps(filter_kw)
xml = slap._connection_helper.POST('/requestComputerPartition', data=new_request_form)
if type(xml) is unicode:
xml = str(xml)
xml.encode('utf-8')
partition = loads(xml)
# XXX move to other end
partition._master_url = master_url
return dumps(partition)
def dump(self, path_to_xml):
"""
Dump the computer object to an xml file via xml_marshaller.
Args:
path_to_xml: String, path to the file to load.
users: List of User, list of user needed to be add to the dump
(even if they are not related to any tap interface).
"""
computer_dict = _getDict(self)
output_file = open(path_to_xml,'w')
output_file.write(xml_marshaller.dumps(computer_dict))
output_file.close()
def request(self, software_release, partition_reference,
partition_parameter_kw=None, software_type=None,
filter_kw=None, state=None, shared=False):
if partition_parameter_kw is None:
partition_parameter_kw = {}
if filter_kw is None:
filter_kw = {}
request_dict = {
'software_release': software_release,
'partition_reference': partition_reference,
'partition_parameter_xml': xml_marshaller.dumps(partition_parameter_kw),
'filter_xml': xml_marshaller.dumps(filter_kw),
# XXX Cedric: Why state and shared are marshalled? First is a string
# And second is a boolean.
'state': xml_marshaller.dumps(state),
'shared_xml': xml_marshaller.dumps(shared),
}
if software_type is not None:
request_dict['software_type'] = software_type
else:
# Let's enforce a default software type
request_dict['software_type'] = DEFAULT_SOFTWARE_TYPE
return self._requestComputerPartition(request_dict)
def request(self, software_release, software_type, partition_reference,
shared=False, partition_parameter_kw=None, filter_kw=None,
state=None):
if partition_parameter_kw is None:
partition_parameter_kw = {}
elif not isinstance(partition_parameter_kw, dict):
raise ValueError("Unexpected type of partition_parameter_kw '%s'" % \
partition_parameter_kw)
if filter_kw is None:
filter_kw = {}
elif not isinstance(filter_kw, dict):
raise ValueError("Unexpected type of filter_kw '%s'" % \
filter_kw)
request_dict = { 'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
'software_release': software_release,
'software_type': software_type,
'partition_reference': partition_reference,
'shared_xml': xml_marshaller.dumps(shared),
'partition_parameter_xml': xml_marshaller.dumps(
partition_parameter_kw),
'filter_xml': xml_marshaller.dumps(filter_kw),
'state': xml_marshaller.dumps(state),
}
try:
self._connection_helper.POST('/requestComputerPartition', request_dict)
except ResourceNotReady:
return ComputerPartition(request_dict=request_dict)
else:
xml = self._connection_helper.response.read()
software_instance = xml_marshaller.loads(xml)
return ComputerPartition(
software_instance.slap_computer_id.encode('UTF-8'),
software_instance.slap_computer_partition_id.encode('UTF-8'))
def send(self, config):
"""
Send a marshalled dictionary of the computer object serialized via_getDict.
"""
slap_instance = slap.slap()
connection_dict = {}
if config.key_file and config.cert_file:
connection_dict.update(
key_file=config.key_file,
cert_file=config.cert_file)
slap_instance.initializeConnection(config.master_url,
**connection_dict)
slap_computer = slap_instance.registerComputer(self.reference)
if config.dry_run:
return
return slap_computer.updateConfiguration(
xml_marshaller.dumps(_getDict(self)))
def __init__(self, config, process_group_pid_set=None):
self.config = config
# By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates...
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_set.add(proxy.pid)
# XXX: dirty, giving some time for proxy to being able to accept
# connections
time.sleep(10)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
# register software profile
self.software_profile = config['custom_profile_path']
slap.registerSupply().supply(
self.software_profile,
computer_guid=config['computer_id'])
computer = slap.registerComputer(config['computer_id'])
# create partition and configure computer
partition_reference = config['partition_reference']
partition_path = os.path.join(config['instance_root'], partition_reference)
if not os.path.exists(partition_path):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
computer.updateConfiguration(xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': config['instance_root'],
'netmask': '255.255.255.255',
'partition_list': [{'address_list': [{'addr': config['ipv4_address'],
'netmask': '255.255.255.255'},
{'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'},
],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference},
}
],
'reference': config['computer_id'],
'software_root': config['software_root']}))
def requestComputerPartition():
parsed_request_dict = parseRequestComputerPartitionForm(request.form)
# Is it a slave instance?
slave = loads(request.form.get('shared_xml', EMPTY_DICT_XML).encode())
# Check first if instance is already allocated
if slave:
# XXX: change schema to include a simple "partition_reference" which
# is name of the instance. Then, no need to do complex search here.
slave_reference = parsed_request_dict['partition_id'] + '_' + parsed_request_dict['partition_reference']
requested_computer_id = parsed_request_dict['filter_kw'].get('computer_guid', app.config['computer_id'])
matching_partition = getAllocatedSlaveInstance(slave_reference, requested_computer_id)
else:
matching_partition = getAllocatedInstance(parsed_request_dict['partition_reference'])
if matching_partition:
# Then the instance is already allocated, just update it
# XXX: split request and request slave into different update/allocate functions and simplify.
# By default, ALWAYS request instance on default computer
parsed_request_dict['filter_kw'].setdefault('computer_guid', app.config['computer_id'])
if slave:
software_instance = requestSlave(**parsed_request_dict)
else:
software_instance = requestNotSlave(**parsed_request_dict)
else:
# Instance is not yet allocated: try to do it.
external_master_url = isRequestToBeForwardedToExternalMaster(parsed_request_dict)
if external_master_url:
return forwardRequestToExternalMaster(external_master_url, request.form)
# XXX add support for automatic deployment on specific node depending on available SR and partitions on each Node.
# Note: It only deploys on default node if SLA not specified
# XXX: split request and request slave into different update/allocate functions and simplify.
# By default, ALWAYS request instance on default computer
parsed_request_dict['filter_kw'].setdefault('computer_guid', app.config['computer_id'])
if slave:
software_instance = requestSlave(**parsed_request_dict)
else:
software_instance = requestNotSlave(**parsed_request_dict)
return dumps(software_instance)
def send(self, config):
"""
Send a marshalled dictionary of the computer object serialized via_getDict.
"""
slap_instance = slap.slap()
connection_dict = {}
if config.key_file and config.cert_file:
connection_dict.update(
key_file=config.key_file,
cert_file=config.cert_file)
slap_instance.initializeConnection(config.master_url,
**connection_dict)
slap_computer = slap_instance.registerComputer(self.reference)
if config.dry_run:
return
try:
slap_computer.updateConfiguration(xml_marshaller.dumps(_getDict(self)))
except slap.NotFoundError as error:
raise slap.NotFoundError("%s\nERROR : This SlapOS node is not recognised by "
"SlapOS Master. Please make sure computer_id of slapos.cfg looks "
"like 'COMP-123' and is correct.\nError is : 404 Not Found." % error)
return
def run(args):
config = args[0]
for k,v in config['environment'].iteritems():
os.environ[k] = v
proxy = None
slapgrid = None
supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run',
'supervisord.pid')
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
try:
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(proxy.pid)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
while True:
try:
slap.registerSupply().supply(config['profile_url'],
computer_guid=config['computer_id'])
except socket.error:
time.sleep(1)
pass
else:
break
while True:
slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc',
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(slapgrid.pid)
slapgrid.wait()
if slapgrid.returncode == 0:
print 'Software installed properly'
break
print 'Problem with software installation, trying again'
time.sleep(600)
computer = slap.registerComputer(config['computer_id'])
partition_reference = config['partition_reference']
partition_path = os.path.join(config['instance_root'], partition_reference)
if not os.path.exists(partition_path):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
computer.updateConfiguration(xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': config['instance_root'],
'netmask': '255.255.255.255',
'partition_list': [{'address_list': [{'addr': config['ipv4_address'],
'netmask': '255.255.255.255'},
{'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'},
],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference},
}
],
'reference': config['computer_id'],
'software_root': config['software_root']}))
slap.registerOpenOrder().request(config['profile_url'],
partition_reference='testing partition',
partition_parameter_kw=config['instance_dict'])
slapgrid = subprocess.Popen([config['slapgrid_partition_binary'], '-vc',
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
slapgrid.wait()
if slapgrid.returncode != 0:
raise ValueError('Slapgrid instance failed')
runUnitTest = os.path.join(partition_path, 'bin', 'runUnitTest')
if not os.path.exists(runUnitTest):
raise ValueError('No %r provided' % runUnitTest)
except:
try:
if os.path.exists(supervisord_pid_file):
os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM)
except:
pass
raise
finally:
# Nice way to kill *everything* generated by run process -- process
# groups working only in POSIX compilant systems
# Exceptions are swallowed during cleanup phase
if proxy is not None:
os.killpg(proxy.pid, signal.SIGTERM)
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
if slapgrid is not None and slapgrid.returncode is None:
os.killpg(slapgrid.pid, signal.SIGTERM)
try:
bot_env = os.environ.copy()
bot_env['PATH'] = ':'.join([config['bin_directory']] +
bot_env['PATH'].split(':'))
for l in config['bot_environment'].split():
k, v = l.split('=')
bot_env[k] = v
if subprocess.call([config['buildbot_binary'], 'create-slave', '-f',
config['working_directory'], config['buildbot_host'],
config['slave_name'], config['slave_password']]) != 0:
raise ValueError('Buildbot call failed')
process_command_list.append([config['buildbot_binary'], 'stop',
config['working_directory']])
if os.path.exists(os.path.join(config['working_directory'],
'buildbot.tac.new')):
tac = os.path.join(config['working_directory'], 'buildbot.tac')
if os.path.exists(tac):
os.unlink(tac)
os.rename(os.path.join(config['working_directory'],
'buildbot.tac.new'), tac)
if subprocess.call([config['buildbot_binary'], 'start',
config['working_directory']], env=bot_env) != 0:
raise ValueError('Issue during starting buildbot')
while True:
time.sleep(3600)
finally:
try:
subprocess.call([config['buildbot_binary'], 'stop',
config['working_directory']])
except:
pass
try:
if os.path.exists(supervisord_pid_file):
os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM)
except:
pass
import random
import string
from slapos.slap.slap import Computer, ComputerPartition, \
SoftwareRelease, SoftwareInstance, NotFoundError
from slapos.proxy.db_version import DB_VERSION
import slapos.slap
from slapos.util import sqlite_connect
from flask import g, Flask, request, abort
import xml_marshaller
from xml_marshaller.xml_marshaller import loads
from xml_marshaller.xml_marshaller import dumps
app = Flask(__name__)
EMPTY_DICT_XML = dumps({})
class UnauthorizedError(Exception):
pass
# cast everything to string, utf-8 encoded
def to_str(v):
if isinstance(v, str):
return v
if not isinstance(v, unicode):
v = unicode(v)
return v.encode('utf-8')
def xml2dict(xml):