def setUpClass(cls, **kw): if len(kw) == 0: return cls.server_url = kw['server_url'] cls.key_file = kw['key_file'] cls.cert_file = kw['cert_file'] cls.computer_id = kw['computer_id'] cls.partition_id = kw['partition_id'] # Get parameters returned by slapos master slap = slapos.slap.slap() slap.initializeConnection(cls.server_url, cls.key_file, cls.cert_file) cls.partition = slap.registerComputerPartition( computer_guid=cls.computer_id, partition_id=cls.partition_id ) cls.parameter_dict = cls.partition.getConnectionParameterDict() for attribute, value in cls.parameter_dict.iteritems(): setattr(cls, attribute.replace('-', '_'), value) #create slaprunner configuration views.app.config['TESTING'] = True config = Config() config.setConfig() views.app.config.update(**config.__dict__) cls.app = views.app.test_client() cls.app.config = views.app.config # Set up path (needed to find git binary) os.environ['PATH'] = config.path
def test_error_new_SoftwareRelease_works(self): """ Asserts that calling SoftwareRelease.error on software release works """ computer_guid = self._getTestComputerId() software_release_uri = 'http://server/' + self._getTestComputerId() slap = self.slap slap.initializeConnection(self.server_url) def handler(url, req): qs = urlparse.parse_qs(req.body) if (url.path == '/softwareReleaseError' and qs['computer_id'][0] == computer_guid and qs['url'][0] == software_release_uri and qs['error_log'][0] == 'some error'): return { 'status_code': 200 } return {'status_code': 404} with httmock.HTTMock(handler): software_release = self.slap.registerSoftwareRelease(software_release_uri) software_release._computer_guid = computer_guid software_release.error('some error')
def test_error_new_ComputerPartition_works(self): """ Asserts that calling ComputerPartition.error on new partition works """ computer_guid = self._getTestComputerId() partition_id = 'PARTITION_01' slap = self.slap slap.initializeConnection(self.server_url) def server_response(self, path, method, body, header): parsed_url = urlparse.urlparse(path.lstrip('/')) parsed_qs = urlparse.parse_qs(parsed_url.query) if (parsed_url.path == 'registerComputerPartition' and parsed_qs['computer_reference'][0] == computer_guid and parsed_qs['computer_partition_reference'][0] == partition_id): partition = slapos.slap.ComputerPartition( computer_guid, partition_id) return (200, {}, xml_marshaller.xml_marshaller.dumps(partition)) elif parsed_url.path == 'softwareInstanceError': parsed_qs_body = urlparse.parse_qs(body) # XXX: why do we have computer_id and not computer_reference? # XXX: why do we have computer_partition_id and not # computer_partition_reference? if (parsed_qs_body['computer_id'][0] == computer_guid and parsed_qs_body['computer_partition_id'][0] == partition_id and parsed_qs_body['error_log'][0] == 'some error'): return (200, {}, '') return (404, {}, '') httplib.HTTPConnection._callback = server_response computer_partition = slap.registerComputerPartition( computer_guid, partition_id) # XXX: Interface does not define return value computer_partition.error('some error')
def requestInstance(config, software_type=None): """ Request the main instance of our environment """ software_type_path = os.path.join(config['etc_dir'], ".software_type.xml") if software_type: # Write it to conf file for later use open(software_type_path, 'w').write(software_type) elif os.path.exists(software_type_path): software_type = open(software_type_path).read().rstrip() else: software_type = 'default' slap = slapos.slap.slap() profile = getCurrentSoftwareReleaseProfile(config) slap.initializeConnection(config['master_url']) param_path = os.path.join(config['etc_dir'], ".parameter.xml") xml_result = readParameters(param_path) partition_parameter_kw = None if type(xml_result) != type('') and 'instance' in xml_result: partition_parameter_kw = xml_result['instance'] return slap.registerOpenOrder().request( profile, partition_reference=getSoftwareReleaseName(config), partition_parameter_kw=partition_parameter_kw, software_type=software_type, filter_kw=None, state=None, shared=False)
def _test_new_computer_partition_state(self, state): """ Helper method to automate assertions of failing states on new Computer Partition """ computer_guid = self._getTestComputerId() partition_id = 'PARTITION_01' slap = self.slap slap.initializeConnection(self.server_url) def handler(url, req): qs = urlparse.parse_qs(url.query) if (url.path == '/registerComputerPartition' and qs['computer_reference'][0] == computer_guid and qs['computer_partition_reference'][0] == partition_id): partition = slapos.slap.ComputerPartition( computer_guid, partition_id) return { 'status_code': 200, 'content': xml_marshaller.xml_marshaller.dumps(partition) } else: return {'status_code': 404} with httmock.HTTMock(handler): computer_partition = self.slap.registerComputerPartition( computer_guid, partition_id) self.assertRaises(slapos.slap.NotFoundError, getattr(computer_partition, state))
def _initializeSlapOSConnection(self): """ Initialize communication with slapos """ slap = slapos.slap.slap() retry = 0 while True: # wait until _hateoas_navigator is loaded. if retry > 100: break slap.initializeConnection( self.slapos_url, self.key_path, self.cert_path, timeout=120, slapgrid_rest_uri=self.slapos_api_rest_url) if getattr(slap, '_hateoas_navigator', None) is None: retry += 1 logger.info( "Fail to load _hateoas_navigator waiting a bit and retry.") time.sleep(30) else: break if getattr(slap, '_hateoas_navigator', None) is None: raise ValueError("Fail to load _hateoas_navigator") supply = slap.registerSupply() order = slap.registerOpenOrder() return slap, supply, order
def test_computer_getComputerPartitionList_no_partition(self): """ Asserts that calling Computer.getComputerPartitionList without Computer Partitions returns empty list """ computer_guid = self._getTestComputerId() slap = self.slap slap.initializeConnection(self.server_url) def server_response(self, path, method, body, header): parsed_url = urlparse.urlparse(path.lstrip('/')) parsed_qs = urlparse.parse_qs(parsed_url.query) if (parsed_url.path == 'registerComputerPartition' and 'computer_reference' in parsed_qs and 'computer_partition_reference' in parsed_qs): slap_partition = slapos.slap.ComputerPartition( parsed_qs['computer_reference'][0], parsed_qs['computer_partition_reference'][0]) return (200, {}, xml_marshaller.xml_marshaller.dumps(slap_partition)) elif (parsed_url.path == 'getFullComputerInformation' and 'computer_id' in parsed_qs): slap_computer = slapos.slap.Computer(parsed_qs['computer_id'][0]) slap_computer._software_release_list = [] slap_computer._computer_partition_list = [] return (200, {}, xml_marshaller.xml_marshaller.dumps(slap_computer)) elif parsed_url.path == 'requestComputerPartition': return (408, {}, '') else: return (404, {}, '') httplib.HTTPConnection._callback = server_response computer = self.slap.registerComputer(computer_guid) self.assertEqual(computer.getComputerPartitionList(), [])
def install(self): self.path_list = [] crond = self.installCrond() slap = slapos.slap.slap() slap.initializeConnection(self.server_url, self.key_file, self.cert_file) parameter_dict = slap.registerComputerPartition( self.computer_id, self.computer_partition_id, ).getInstanceParameterDict() # XXX: should probably expect one more (SR-originating) parameter instead # of using self.work_directory . configuration_path = os.path.join(self.work_directory, "agent.cfg") with open(configuration_path, "w") as configuration: configuration.write(parameter_dict["configuration"]) agent_crond_path = os.path.join(crond, "agent") with open(agent_crond_path, "w") as agent_crond: agent_crond.write("*/5 * * * * %s -S %s --pidfile=%s --log=%s " "%s 2>&1 > /dev/null\n" % ( self.options["python_binary"], self.options["agent_binary"], self.options["pidfile"], self.options["log"], configuration_path, )) return self.path_list + [configuration_path, agent_crond_path]
def __init__( self, server_url, key_file, cert_file, computer_id, partition_id, software, namebase, root_instance_name, sleep_time_between_test=900, total_instance_count="2", type=None, ): self.server_url = server_url self.key_file = key_file self.cert_file = cert_file self.computer_id = computer_id self.partition_id = partition_id self.software = software self.namebase = namebase self.total_instance_count = total_instance_count self.root_instance_name = root_instance_name self.sleep_time_between_test = sleep_time_between_test self.test_type = type slap = slapos.slap.slap() slap.initializeConnection(server_url, key_file, cert_file) self.partition = slap.registerComputerPartition(computer_guid=computer_id, partition_id=partition_id) self.logger = logging.getLogger("SlaprunnerResiliencyTest") self.logger.setLevel(logging.DEBUG)
def _test_new_computer_partition_state(self, state): """ Helper method to automate assertions of failing states on new Computer Partition """ computer_guid = self._getTestComputerId() partition_id = 'PARTITION_01' slap = self.slap slap.initializeConnection(self.server_url) def server_response(self, path, method, body, header): parsed_url = urlparse.urlparse(path.lstrip('/')) parsed_qs = urlparse.parse_qs(parsed_url.query) if (parsed_url.path == 'registerComputerPartition' and parsed_qs['computer_reference'][0] == computer_guid and parsed_qs['computer_partition_reference'][0] == partition_id): partition = slapos.slap.ComputerPartition( computer_guid, partition_id) return (200, {}, xml_marshaller.xml_marshaller.dumps(partition)) else: return (404, {}, '') httplib.HTTPConnection._callback = server_response computer_partition = self.slap.registerComputerPartition( computer_guid, partition_id) self.assertRaises(slapos.slap.NotFoundError, getattr(computer_partition, state))
def __init__(self, buildout, name, options): slap = slapos.slap.slap() slap.initializeConnection( options['url'], options.get('key'), options.get('cert'), ) computer_partition = slap.registerComputerPartition( options['computer'], options['partition'], ) parameter_dict = computer_partition.getInstanceParameterDict() options['instance-state'] = computer_partition.getState() # XXX: those are not partition parameters, strictly speaking. # Make them available as individual section keys. for his_key in ( 'slap_software_type', 'slap_computer_partition_id', 'slap_computer_id', 'slap_software_release_url', 'slave_instance_list', 'timestamp', ): try: value = parameter_dict.pop(his_key) except KeyError: pass else: options[his_key.replace('_', '-')] = value ipv4_set = set() v4_add = ipv4_set.add ipv6_set = set() v6_add = ipv6_set.add tap_set = set() tap_add = tap_set.add for tap, ip in parameter_dict.pop('ip_list'): tap_add(tap) if valid_ipv4(ip): v4_add(ip) elif valid_ipv6(ip): v6_add(ip) # XXX: emit warning on unknown address type ? options['ipv4'] = ipv4_set options['ipv6'] = ipv6_set # also export single ip values for those recipes that don't support sets. if ipv4_set: options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8') if ipv6_set: options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8') options['tap'] = tap_set parameter_dict = self._expandParameterDict(options, parameter_dict) match = self.OPTCRE_match for key, value in parameter_dict.iteritems(): if match(key) is not None: continue options['configuration.' + key] = value
def __init__(self, buildout, name, options): slap = slapos.slap.slap() slap.initializeConnection( options['url'], options.get('key'), options.get('cert'), ) computer_partition = slap.registerComputerPartition( options['computer'], options['partition'], ) parameter_dict = computer_partition.getInstanceParameterDict() options['instance-state'] = computer_partition.getState() # XXX: those are not partition parameters, strictly speaking. # Make them available as individual section keys. for his_key in ( 'slap_software_type', 'slap_computer_partition_id', 'slap_computer_id', 'slap_software_release_url', 'slave_instance_list', 'timestamp', ): try: value = parameter_dict.pop(his_key) except KeyError: pass else: options[his_key.replace('_', '-')] = value ipv4_set = set() v4_add = ipv4_set.add ipv6_set = set() v6_add = ipv6_set.add tap_set = set() tap_add = tap_set.add for tap, ip in parameter_dict.pop('ip_list'): tap_add(tap) if valid_ipv4(ip): v4_add(ip) elif valid_ipv6(ip): v6_add(ip) # XXX: emit warning on unknown address type ? options['ipv4'] = ipv4_set options['ipv6'] = ipv6_set # also export single ip values for those recipes that don't support sets. if ipv4_set: options['ipv4-random'] = list(ipv4_set)[0] if ipv6_set: options['ipv6-random'] = list(ipv6_set)[0] options['tap'] = tap_set parameter_dict = self._expandParameterDict(options, parameter_dict) match = self.OPTCRE_match for key, value in parameter_dict.iteritems(): if match(key) is not None: continue options['configuration.' + key] = value
def initializeSlapOSControler(self, slapproxy_log=None, process_manager=None, reset_software=False, software_path_list=None): self.process_manager = process_manager self.software_path_list = software_path_list logger.debug('SlapOSControler, initialize, reset_software: %r', reset_software) config = self.config slapos_config_dict = config.copy() slapos_config_dict.update(software_root=self.software_root, instance_root=self.instance_root, proxy_database=self.proxy_database) with open(self.slapos_config, 'w') as f: f.write( pkg_resources.resource_string('erp5.util.testnode', 'template/slapos.cfg.in') % slapos_config_dict) # By erasing everything, we make sure that we are able to "update" # existing profiles. This is quite dirty way to do updates... if os.path.exists(self.proxy_database): os.unlink(self.proxy_database) kwargs = dict(close_fds=True, preexec_fn=os.setsid) if slapproxy_log is not None: slapproxy_log_fp = open(slapproxy_log, 'w') kwargs['stdout'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp # Make sure there is no slapos alive from previous run process_manager.killall('slapos') proxy = subprocess.Popen([ config['slapos_binary'], 'proxy', 'start', '--cfg', self.slapos_config ], **kwargs) process_manager.process_pid_set.add(proxy.pid) slap = self.slap = slapos.slap.slap() # Wait for proxy to accept connections retries = 0 while True: time.sleep(.5) try: slap.initializeConnection(config['master_url']) computer = slap.registerComputer(config['computer_id']) # Call a method to ensure connection to master can be established computer.getComputerPartitionList() except slapos.slap.ConnectionError, e: retries += 1 if retries >= 20: raise logger.debug("Proxy still not started %s, retrying", e) else: break
def __init__(self, config, log, process_group_pid_set=None, slapproxy_log=None): self.log = log self.config = config # By erasing everything, we make sure that we are able to "update" # existing profiles. This is quite dirty way to do updates... if os.path.exists(config['proxy_database']): os.unlink(config['proxy_database']) kwargs = dict(close_fds=True, preexec_fn=os.setsid) if slapproxy_log is not None: slapproxy_log_fp = open(slapproxy_log, 'w') kwargs['stdout'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp proxy = subprocess.Popen([config['slapproxy_binary'], config['slapos_config']], **kwargs) process_group_pid_set.add(proxy.pid) # XXX: dirty, giving some time for proxy to being able to accept # connections time.sleep(10) slap = slapos.slap.slap() slap.initializeConnection(config['master_url']) # register software profile self.software_profile = config['custom_profile_path'] slap.registerSupply().supply( self.software_profile, computer_guid=config['computer_id']) computer = slap.registerComputer(config['computer_id']) # create partition and configure computer partition_reference = config['partition_reference'] partition_path = os.path.join(config['instance_root'], partition_reference) if not os.path.exists(partition_path): os.mkdir(partition_path) os.chmod(partition_path, 0750) computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({ 'address': config['ipv4_address'], 'instance_root': config['instance_root'], 'netmask': '255.255.255.255', 'partition_list': [{'address_list': [{'addr': config['ipv4_address'], 'netmask': '255.255.255.255'}, {'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::'}, ], 'path': partition_path, 'reference': partition_reference, 'tap': {'name': partition_reference}, } ], 'reference': config['computer_id'], 'software_root': config['software_root']}))
def updateProxy(config): """ Configure Slapos Node computer and partitions. Send current Software Release to Slapproxy for compilation and deployment. """ startProxy(config) if not os.path.exists(config['instance_root']): os.mkdir(config['instance_root']) slap = slapos.slap.slap() profile = getCurrentSoftwareReleaseProfile(config) slap.initializeConnection(config['master_url']) slap.registerSupply().supply(profile, computer_guid=config['computer_id']) computer = slap.registerComputer(config['computer_id']) prefix = 'slappart' slap_config = { 'address': config['ipv4_address'], 'instance_root': config['instance_root'], 'netmask': '255.255.255.255', 'partition_list': [], 'reference': config['computer_id'], 'software_root': config['software_root'] } for i in xrange(0, int(config['partition_amount'])): partition_reference = '%s%s' % (prefix, i) partition_path = os.path.join(config['instance_root'], partition_reference) if not os.path.exists(partition_path): os.mkdir(partition_path) os.chmod(partition_path, 0750) slap_config['partition_list'].append({ 'address_list': [ { 'addr': config['ipv4_address'], 'netmask': '255.255.255.255' }, { 'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::' }, ], 'path': partition_path, 'reference': partition_reference, 'tap': {'name': partition_reference}}) computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps(slap_config)) return True
def getSlapStatus(config): """Return all Slapos Partitions with associate information""" slap = slapos.slap.slap() slap.initializeConnection(config['master_url']) partition_list = [] computer = slap.registerComputer(config['computer_id']) try: for partition in computer.getComputerPartitionList(): # Note: Internal use of API, as there is no reflexion interface in SLAP partition_list.append((partition.getId(), partition._connection_dict.copy())) except Exception: pass if partition_list: for i in xrange(0, int(config['partition_amount'])): slappart_id = '%s%s' % ("slappart", i) if not [x[0] for x in partition_list if slappart_id == x[0]]: partition_list.append((slappart_id, [])) return partition_list
def checkIfMasterIsCurrentMaster(master_url): """ Because there are several ways to contact this server, we can't easily check in a request() if master_url is ourself or not. So we contact master_url, and if it returns an ID we know: it is ourself """ # Dumb way: compare with listening host/port host = request.host port = request.environ['SERVER_PORT'] if master_url == 'http://%s:%s/' % (host, port): return True # Hack way: call ourself slap = slapos.slap.slap() slap.initializeConnection(master_url) try: return run_id == bytes2str(slap._connection_helper.GET('/getRunId')) except: return False
def __bang(self, message): """ Call bang if requested """ if 'master-url' in self.__config and \ 'partition-in' in self.__config and \ 'computer-id' in self.__config: slap = slapos.slap.slap() slap.initializeConnection( self.__config['master-url'], self.__config.get('partition-key'), self.__config.get('partition-cert'), ) computer_partition = slap.registerComputerPartition( self.__config['computer-id'], self.__config['partition-id'], ) computer_partition.bang(message) self.logger.info("Bang with message %r." % message)
def test_updateInstanceParameter(self): """Scenarion 5: Update parameters of current sofware profile""" self.setAccount() self.setupTestSoftware() #Set current projet and run Slapgrid-cp software = os.path.join(self.software, 'slaprunner-test') response = loadJson(self.app.post('/setCurrentProject', data=dict(path=software), follow_redirects=True)) self.assertEqual(response['result'], "") self.proxyStatus(True) #Send paramters for the instance parameterDict = dict(appname='slaprunnerTest', cacountry='France') parameterXml = '<?xml version="1.0" encoding="utf-8"?>\n<instance>' parameterXml += '<parameter id="appname">slaprunnerTest</parameter>\n' parameterXml += '<parameter id="cacountry">France</parameter>\n</instance>' software_type = 'production' response = loadJson(self.app.post('/saveParameterXml', data=dict(parameter=parameterXml, software_type=software_type), follow_redirects=True)) self.assertEqual(response['result'], "") slap = slapos.slap.slap() slap.initializeConnection(self.app.config['master_url']) computer = slap.registerComputer(self.app.config['computer_id']) partitionList = computer.getComputerPartitionList() self.assertNotEqual(partitionList, []) #Assume that the requested partition is partition 0 slapParameterDict = partitionList[0].getInstanceParameterDict() self.assertTrue('appname' in slapParameterDict) self.assertTrue('cacountry' in slapParameterDict) self.assertEqual(slapParameterDict['appname'], 'slaprunnerTest') self.assertEqual(slapParameterDict['cacountry'], 'France') self.assertEqual(slapParameterDict['slap_software_type'], 'production') #test getParameterXml for webrunner UI response = loadJson(self.app.get('/getParameterXml/xml')) self.assertEqual(parameterXml, response['result']) response = loadJson(self.app.get('/getParameterXml/dict')) self.assertEqual(parameterDict, response['result']['instance']) self.stopSlapproxy()
def forwardRequestToExternalMaster(master_url, request_form): """ Forward instance request to external SlapOS Master. """ master_entry = app.config.get('multimaster').get(master_url, {}) key_file = master_entry.get('key') cert_file = master_entry.get('cert') if master_url.startswith('https') and (not key_file or not cert_file): app.logger.warning('External master %s configuration did not specify key or certificate.' % master_url) abort(404) if master_url.startswith('https') and not master_url.startswith('https') and (key_file or cert_file): app.logger.warning('External master %s configurqtion specifies key or certificate but is using plain http.' % master_url) abort(404) slap = slapos.slap.slap() if key_file: slap.initializeConnection(master_url, key_file=key_file, cert_file=cert_file) else: slap.initializeConnection(master_url) partition_reference = request_form['partition_reference'].encode() # Store in database execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)', {'partition_reference':partition_reference, 'master_url': master_url}) new_request_form = request_form.copy() filter_kw = loads(new_request_form['filter_xml'].encode()) filter_kw['source_instance_id'] = partition_reference new_request_form['filter_xml'] = dumps(filter_kw) xml = slap._connection_helper.POST('/requestComputerPartition', data=new_request_form) if type(xml) is unicode: xml = str(xml) xml.encode('utf-8') partition = loads(xml) # XXX move to other end partition._master_url = master_url return dumps(partition)
def test_error_new_ComputerPartition_works(self): """ Asserts that calling ComputerPartition.error on new partition works """ computer_guid = self._getTestComputerId() partition_id = 'PARTITION_01' slap = self.slap slap.initializeConnection(self.server_url) def handler(url, req): qs = urlparse.parse_qs(url.query) if (url.path == '/registerComputerPartition' and qs['computer_reference'][0] == computer_guid and qs['computer_partition_reference'][0] == partition_id): partition = slapos.slap.ComputerPartition( computer_guid, partition_id) return { 'statu_code': 200, 'content': xml_marshaller.xml_marshaller.dumps(partition) } elif url.path == '/softwareInstanceError': parsed_qs_body = urlparse.parse_qs(req.body) # XXX: why do we have computer_id and not computer_reference? # XXX: why do we have computer_partition_id and not # computer_partition_reference? if (parsed_qs_body['computer_id'][0] == computer_guid and parsed_qs_body['computer_partition_id'][0] == partition_id and parsed_qs_body['error_log'][0] == 'some error'): return {'status_code': 200} return {'status_code': 404} with httmock.HTTMock(handler): computer_partition = slap.registerComputerPartition( computer_guid, partition_id) # XXX: Interface does not define return value computer_partition.error('some error')
def test_computer_getComputerPartitionList_no_partition(self): """ Asserts that calling Computer.getComputerPartitionList without Computer Partitions returns empty list """ computer_guid = self._getTestComputerId() slap = self.slap slap.initializeConnection(self.server_url) def handler(url, req): qs = urlparse.parse_qs(url.query) if (url.path == '/registerComputerPartition' and 'computer_reference' in qs and 'computer_partition_reference' in qs): slap_partition = slapos.slap.ComputerPartition( qs['computer_reference'][0], qs['computer_partition_reference'][0]) return { 'status_code': 200, 'content': xml_marshaller.xml_marshaller.dumps(slap_partition) } elif (url.path == '/getFullComputerInformation' and 'computer_id' in qs): slap_computer = slapos.slap.Computer(qs['computer_id'][0]) slap_computer._software_release_list = [] slap_computer._computer_partition_list = [] return { 'status_code': 200, 'content': xml_marshaller.xml_marshaller.dumps(slap_computer) } elif url.path == '/requestComputerPartition': return {'status_code': 408} else: return {'status_code': 404} with httmock.HTTMock(handler): computer = self.slap.registerComputer(computer_guid) self.assertEqual(computer.getComputerPartitionList(), [])
def test_error_new_SoftwareRelease_works(self): """ Asserts that calling SoftwareRelease.error on software release works """ computer_guid = self._getTestComputerId() software_release_uri = 'http://server/' + self._getTestComputerId() slap = self.slap slap.initializeConnection(self.server_url) def server_response(self, path, method, body, header): parsed_url = urlparse.urlparse(path.lstrip('/')) parsed_qs = urlparse.parse_qs(body) if parsed_url.path == 'softwareReleaseError' and \ parsed_qs['computer_id'][0] == computer_guid and \ parsed_qs['url'][0] == software_release_uri and \ parsed_qs['error_log'][0] == 'some error': return (200, {}, '') return (404, {}, '') httplib.HTTPConnection._callback = server_response software_release = self.slap.registerSoftwareRelease(software_release_uri) software_release._computer_guid = computer_guid software_release.error('some error')
def __init__(self, buildout, name, options): slap = slapos.slap.slap() slap.initializeConnection( options['url'], options.get('key'), options.get('cert'), ) parameter_dict = slap.registerComputerPartition( options['computer'], options['partition'], ).getInstanceParameterDict() # XXX: those are not partition parameters, strictly speaking. # Discard them, and make them available as separate section keys. options['slap-software-type'] = parameter_dict.pop( 'slap_software_type') ipv4_set = set() v4_add = ipv4_set.add ipv6_set = set() v6_add = ipv6_set.add tap_set = set() tap_add = tap_set.add for tap, ip in parameter_dict.pop('ip_list'): tap_add(tap) if valid_ipv4(ip): v4_add(ip) elif valid_ipv6(ip): v6_add(ip) # XXX: emit warning on unknown address type ? options['ipv4'] = ipv4_set options['ipv6'] = ipv6_set options['tap'] = tap_set options['configuration'] = parameter_dict match = self.OPTCRE_match for key, value in parameter_dict.iteritems(): if match(key) is not None: continue options['configuration.' + key] = value
def main(): """ Note: This code does not test as much as it monitors. The goal is to regularily try to build & instantiate a software release on several machines, to monitor vifib stability and SR stability as time passes (and things once available online become unavailable). Part of this function could be reused to make an actual test bot, testing only when actual changes are committed to a software release, to look for regressions. Note: This code does not connect to any instantiated service, it relies on the presence of a promise section to make instantiation fail until promise is happy. """ parser = argparse.ArgumentParser() parser.add_argument('--pidfile', '-p', help='pidfile preventing parallel ' 'execution.') parser.add_argument('--log', '-l', help='Log file path.') parser.add_argument('--verbose', '-v', help='Be verbose.', action='store_true') parser.add_argument('configuration_file', type=argparse.FileType(), help='Slap Test Agent configuration file.') key_file_dict = {} args = parser.parse_args() log = args.log logger, log_file = getLogger(log, args.verbose) configuration = ConfigParser.SafeConfigParser() configuration.readfp(args.configuration_file) pidfile = args.pidfile if pidfile: setRunning(logger=logger, pidfile=pidfile) try: while True: section_dict = loadConfiguration(configuration, logger) agent_parameter_dict = dict(configuration.items('agent')) task_distributor = TaskDistributor(agent_parameter_dict['report_url'], logger=logger) task_distributor.subscribeNode( node_title=agent_parameter_dict['node_title'], computer_guid="None") test_suite_data = task_distributor.startTestSuite( node_title=agent_parameter_dict['node_title'], computer_guid="None") if type(test_suite_data) == str: # Backward compatiblity test_suite_data = json.loads(test_suite_data) slap_account_key = task_distributor.getSlaposAccountKey() slap_certificate = task_distributor.getSlaposAccountCertificate() master_url = task_distributor.getSlaposUrl() key_file_dict = {} def asFilenamePair(key, cert): # Note: python's ssl support only supports fetching key & cert data # from on-disk files. This is why we need to "convert" direct data # into file paths, using temporary files. cert = cert.strip() try: temp_key, temp_cert = key_file_dict[cert] except KeyError: temp_key = AutoSTemp(key.strip()) temp_cert = AutoSTemp(cert) key_file_dict[cert] = (temp_key, temp_cert) return temp_key.name, temp_cert.name key_file, cert_file = asFilenamePair(slap_account_key, slap_certificate) process_manager = ProcessManager(logger.info) for test_suite in test_suite_data: full_revision_list = getAndUpdateFullRevisionList(test_suite, agent_parameter_dict["working_directory"], logger, process_manager) unit_test_dict = task_distributor.generateConfiguration( test_suite['test_suite_title']) if not len(full_revision_list): # We don't watch git revision but we periodically # run the test, once a day. full_revision_list = ["day=%s" % time.strftime('%Y/%m/%d', time.gmtime())] if type(unit_test_dict) == str: # Backward compatiblity unit_test_dict = json.loads(unit_test_dict) test_result = task_distributor.createTestResult( revision=','.join(full_revision_list), test_name_list=unit_test_dict.keys(), node_title=agent_parameter_dict['node_title'], allow_restart=False, test_title=test_suite['test_suite_title'], project_title=agent_parameter_dict['project_title'], ) if test_result is None: # We already have a test result logger.info('Skiping test for %s, result already available (%s)' % (test_suite['test_suite_title'], ','.join(full_revision_list))) continue test_result.watcher_period = 120 assert test_result is not None if log_file is not None: test_result.addWatch(log, log_file, max_history_bytes=10000) logger.info("Starting to run for %s" % test_result ) test_mapping = TestMap(unit_test_dict) logger.info("Running %s tests in parallel." % \ len(test_mapping.getGroupList())) assert master_url.startswith('https:') slap = slapos.slap.slap() retry = 0 while True: if retry > 100: break # wait until _hateoas_navigator is loaded. slap.initializeConnection( master_url, key_file, cert_file, timeout=120) if getattr(slap, '_hateoas_navigator', None) is None: logger.info("Fail to load _hateoas_navigator waiting a bit and retry.") time.sleep(30) else: break if getattr(slap, '_hateoas_navigator', None) is None: raise ValueError("Fail to load _hateoas_navigator") supply = slap.registerSupply() order = slap.registerOpenOrder() running_test_dict = {} logger.info('Starting Test Agent run %s ' % agent_parameter_dict['node_title']) while True: # Get up to parallel_task_count tasks to execute while len(running_test_dict) < len(test_mapping.getGroupList())\ and (len(test_mapping.getGroupList()) > 0): test_mapping.cleanEmptyGroup() # Select an unused computer to run the test. group = test_mapping.getNextGroup( ignore_list = [group for _, _, group in \ running_test_dict.itervalues()]) # Select a test test_line = test_result.start( exclude_list=list(test_mapping.getExcludeList(group))) logger.info("Test Line: %s " % test_line) logger.info("Ran Test Set: %s " % test_mapping.ran_test_set) logger.info("Running test dict: %s " % running_test_dict) logger.info("Group: %s " % group) if test_line is None: logger.info("Removing Group (empty test line): %s " % group) test_mapping.dropGroup(group) continue test_name = test_line.name try: section_entry_dict = unit_test_dict[test_name] except KeyError: # We don't know how to execute this test. Assume it doesn't # exist anymore, and fail it in result. test_line.stop(stderr='This test does not exist on test ' 'node %s' % (agent_parameter_dict['node_title'], )) continue general_timeout = agent_parameter_dict.get('timeout', 3600) tester = SoftwareReleaseTester( test_name + time.strftime('_%Y/%m/%d_%H:%M:%S_+0000', time.gmtime()), logger, slap, order, supply, section_entry_dict['url'], section_entry_dict.get('supply_computer'), section_entry_dict.get('request_kw'), agent_parameter_dict.get('software_timeout', general_timeout), agent_parameter_dict.get('instance_timeout', general_timeout) ) test_mapping.addRanTest(test_name) running_test_dict[test_name] = (test_line, tester, group) if not running_test_dict: logger.info('No more tests to run...') break now = time.time() # Synchronise refreshes on watcher period, so it doesn't report a # stalled test node where we are actually still sleeping. # Change test_result.watcher_period outside this loop if you wish # to change sleep duration. next_deadline = now + test_result.watcher_period for section, (test_line, tester, group) in running_test_dict.items(): logger.info('Checking %s: %r...', section, tester) try: deadline = tester.tic(now) except ConnectionError: logger.exception('Test execution ConnectionError for %s' % (section)) deadline = next_deadline except Exception: logger.exception('Test execution fail for %s' % (section)) test_line.stop(test_count=1, error_count=1, failure_count=0, skip_count=0, command=tester.getInfo(), stdout=tester.getFormatedLastMessage(), stderr=traceback.format_exc()) del running_test_dict[section] try: tester.teardown() except slapos.slap.NotFoundError: # This exception is ignored because we cannot # Teardown if SR URL do not exist. logger.exception('Fail and not found') pass except Exception: logger.exception('teardown failed, human assistance needed for cleanup') raise else: logger.info('%r' % tester) if deadline is None: # TODO: report how long each step took. logger.info('Test execution finished for %s' % (section)) test_line.stop(test_count=1, error_count=0, failure_count=0, skip_count=0, command=tester.getInfo(), stdout=tester.getFormatedLastMessage()) del running_test_dict[section] try: pass #tester.teardown() except slapos.slap.NotFoundError: # This exception is ignored because we cannot # Teardown if SR URL do not exist. logger.exception('Fail and not found') pass except Exception: logger.exception('teardown failed, human assistance needed for cleanup') raise else: next_deadline = min(deadline, next_deadline) if running_test_dict: to_sleep = next_deadline - time.time() if to_sleep > 0: logger.info('Sleeping %is...', to_sleep) time.sleep(to_sleep) if not test_result.isAlive(): for _, tester, computer_id in running_test_dict.itervalues(): tester.teardown() time.sleep(300) finally: if pidfile: setFinished(pidfile) key_file_dict.clear()
def __init__(self, config, log, slapproxy_log=None, process_manager=None, reset_software=False): log('SlapOSControler, initialize, reset_software: %r' % reset_software) self.log = log self.config = config self.process_manager = process_manager # By erasing everything, we make sure that we are able to "update" # existing profiles. This is quite dirty way to do updates... if os.path.exists(config['proxy_database']): os.unlink(config['proxy_database']) kwargs = dict(close_fds=True, preexec_fn=os.setsid) if slapproxy_log is not None: slapproxy_log_fp = open(slapproxy_log, 'w') kwargs['stdout'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp proxy = subprocess.Popen([config['slapproxy_binary'], config['slapos_config']], **kwargs) process_manager.process_pid_set.add(proxy.pid) # XXX: dirty, giving some time for proxy to being able to accept # connections time.sleep(10) slap = slapos.slap.slap() slap.initializeConnection(config['master_url']) # register software profile self.software_profile = config['custom_profile_path'] slap.registerSupply().supply( self.software_profile, computer_guid=config['computer_id']) computer = slap.registerComputer(config['computer_id']) # Reset all previously generated software if needed if reset_software: software_root = config['software_root'] log('SlapOSControler : GOING TO RESET ALL SOFTWARE') if os.path.exists(software_root): shutil.rmtree(software_root) os.mkdir(software_root) os.chmod(software_root, 0750) instance_root = config['instance_root'] if os.path.exists(instance_root): # delete old paritions which may exists in order to not get its data # (ex. MySQL db content) from previous testnode's runs # In order to be able to change partition naming scheme, do this at # instance_root level (such change happened already, causing problems). shutil.rmtree(instance_root) os.mkdir(instance_root) for i in range(0, MAX_PARTIONS): # create partition and configure computer # XXX: at the moment all partitions do share same virtual interface address # this is not a problem as usually all services are on different ports partition_reference = '%s-%s' %(config['partition_reference'], i) partition_path = os.path.join(instance_root, partition_reference) os.mkdir(partition_path) os.chmod(partition_path, 0750) computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({ 'address': config['ipv4_address'], 'instance_root': instance_root, 'netmask': '255.255.255.255', 'partition_list': [{'address_list': [{'addr': config['ipv4_address'], 'netmask': '255.255.255.255'}, {'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::'},], 'path': partition_path, 'reference': partition_reference, 'tap': {'name': partition_reference}, } ], 'reference': config['computer_id'], 'software_root': config['software_root']}))
def install(self): slap = slapos.slap.slap() slap_connection = self.buildout['slap_connection'] computer_id = slap_connection['computer_id'] computer_partition_id = slap_connection['partition_id'] server_url = slap_connection['server_url'] key_file = slap_connection.get('key_file') cert_file = slap_connection.get('cert_file') slap.initializeConnection(server_url, key_file, cert_file) self.computer_partition = slap.registerComputerPartition( computer_id, computer_partition_id) self.parameter_dict = self.computer_partition.getInstanceParameterDict( ) software_type = self.parameter_dict['slap_software_type'] self.logger.info('Deploying instance with software type %s' % \ software_type) # Raise if request software_type does not exist ... if software_type not in self.options: # ... Except for backward compatibility. Then use "default". if software_type in ['RootSoftwareInstance']: software_type = 'default' else: raise zc.buildout.UserError("This software type (%s) isn't mapped." % \ software_type) instance_file_path = self.options[software_type] if not os.path.exists(instance_file_path): raise zc.buildout.UserError( "The specified buildout config file %r does " "not exist." % instance_file_path) buildout = ConfigParser() with open(instance_file_path) as instance_path: buildout.readfp(instance_path) buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name) if not buildout.has_section('slap-parameter'): buildout.add_section('slap-parameter') for parameter, value in self.parameter_dict.items(): # All parameters evaluating to False are... False, and shouldn't # convey any information. # Here, all those parameters are simply ignored. if value: if isinstance(value, str): buildout.set('slap-parameter', parameter, value) else: buildout.set('slap-parameter', parameter, json.dumps(value)) buildout.add_section('slap-network-information') buildout.set('slap-network-information', 'local-ipv4', self.getLocalIPv4Address()) buildout.set('slap-network-information', 'global-ipv6', self.getGlobalIPv6Address()) buildout.set('slap-network-information', 'network-interface', self.getNetworkInterface()) # Copy/paste slap_connection buildout.add_section('slap-connection') for key, value in self.buildout['slap_connection'].iteritems(): # XXX: Waiting for SlapBaseRecipe to use dash instead of underscores buildout.set('slap-connection', key.replace('_', '-'), value) # XXX: Needed for lxc. Use non standard API buildout.set('slap-connection', 'requested', self.computer_partition._requested_state) work_directory = os.path.abspath( self.buildout['buildout']['directory']) buildout_filename = os.path.join(work_directory, 'buildout-%s.cfg' % self.name) with open(buildout_filename, 'w') as buildout_file: buildout.write(buildout_file) # XXX-Antoine: We gotta find a better way to do this. I tried to check # out how slapgrid-cp was running buildout. But it is worse than that. command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename] self.logger.info("Invoking commandline : '%s'", ' '.join(command_line_args)) subprocess.check_call(command_line_args, cwd=work_directory, env=os.environ.copy()) return []
def run(args): config = args[0] for k,v in config['environment'].iteritems(): os.environ[k] = v proxy = None slapgrid = None last_revision_file = os.path.join(config['working_directory'], 'revision.txt') if os.path.exists(last_revision_file): os.unlink(last_revision_file) # fetch repository from git repository_clone = os.path.join(config['working_directory'], 'repository') profile_path = os.path.join(repository_clone, config['profile_path']) if os.path.exists(config['proxy_database']): os.unlink(config['proxy_database']) proxy = subprocess.Popen([config['slapproxy_binary'], config['slapos_config']], close_fds=True, preexec_fn=os.setsid) process_group_pid_list.append(proxy.pid) slap = slapos.slap.slap() slap.initializeConnection(config['master_url']) while True: try: slap.registerSupply().supply(profile_path, computer_guid=config['computer_id']) except socket.error: time.sleep(1) pass else: break while True: info_list = [] a = info_list.append while True: try: if os.path.exists(repository_clone): if getCurrentFetchRemote(config, repository_clone) != config['repository']: shutil.rmtree(repository_clone) if not os.path.exists(repository_clone): subprocess.check_call([config['git_binary'], 'clone', config['repository'], repository_clone]) # switch to branch branch = getCurrentBranchName(config, repository_clone) if branch != config['branch']: subprocess.check_call([config['git_binary'], 'checkout', '--force', '--track', '-b', config['branch'], 'origin/'+config['branch']], cwd=repository_clone) subprocess.check_call([config['git_binary'], 'reset', '--hard', '@{upstream}'], cwd=repository_clone) except Exception: print 'Retrying git in 60s' time.sleep(60) else: break a('Tested repository: %s' % config['repository']) a('Machine identification: %s' % getMachineIdString()) erp5_report = ERP5TestReportHandler(config['test_suite_master_url'], '@'.join([config['suite_name'], branch])) last_revision = '' if os.path.exists(last_revision_file): last_revision = open(last_revision_file).read().strip() revision = getRevision(config, repository_clone) open(last_revision_file, 'w').write(revision) if revision != last_revision: print 'Running for revision %r' % revision while True: try: erp5_report.reportStart() except Exception: print 'Retrying in 5s' time.sleep(5) else: break if os.path.exists(config['software_root']): shutil.rmtree(config['software_root']) os.mkdir(config['software_root']) out_file = os.path.join(config['working_directory'], 'slapgrid.out') if os.path.exists(out_file): os.unlink(out_file) out = open(out_file, 'w') begin = time.time() slapgrid_environment = os.environ.copy() for k, v in config['slapgrid_environment'].iteritems(): slapgrid_environment[k] = v a('Slapgrid environment: %r'% config['slapgrid_environment']) slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc', config['slapos_config']], close_fds=True, preexec_fn=os.setsid, stdout=out, stderr=subprocess.STDOUT, env=slapgrid_environment) process_group_pid_list.append(slapgrid.pid) slapgrid.communicate() out.close() while True: try: erp5_report.reportFinished(out_file,revision, slapgrid.returncode == 0, time.time() - begin, '\n'.join(info_list)) except Exception: print 'Retrying in 5s' time.sleep(5) else: break print 'Sleeping for 600s' time.sleep(600)
def main(): """ Note: This code does not test as much as it monitors. The goal is to regularily try to build & instantiate a software release on several machines, to monitor vifib stability and SR stability as time passes (and things once available online become unavailable). Part of this function could be reused to make an actual test bot, testing only when actual changes are committed to a software release, to look for regressions. Note: This code does not connect to any instantiated service, it relies on the presence of a promise section to make instantiation fail until promise is happy. """ parser = argparse.ArgumentParser() parser.add_argument('--pidfile', '-p', help='pidfile preventing parallel ' 'execution.') parser.add_argument('--log', '-l', help='Log file path.') parser.add_argument('--verbose', '-v', help='Be verbose.', action='store_true') parser.add_argument('configuration_file', type=argparse.FileType(), help='Slap Test Agent configuration file.') # Just to keep strong references to AutoSTemp instances key_file_dict = {} def asFilenamePair(key, cert): # Note: python's ssl support only supports fetching key & cert data # from on-disk files. This is why we need to "convert" direct data # into file paths, using temporary files. cert = cert.strip() try: temp_key, temp_cert = key_file_dict[cert] except KeyError: temp_key = AutoSTemp(key.strip()) temp_cert = AutoSTemp(cert) key_file_dict[cert] = (temp_key, temp_cert) return temp_key.name, temp_cert.name args = parser.parse_args() log = args.log formatter = logging.Formatter('%(asctime)s %(message)s') logger = logging.getLogger() if args.verbose: log_level = logging.DEBUG else: log_level = logging.INFO logger.setLevel(log_level) handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) logger.addHandler(handler) if log: handler = logging.FileHandler(log) handler.setFormatter(formatter) logger.addHandler(handler) log_file = open(log) log_file.seek(0, 2) pidfile = args.pidfile if pidfile: setRunning(pidfile) try: section_dict = collections.OrderedDict() configuration = ConfigParser.SafeConfigParser() configuration.readfp(args.configuration_file) for section in configuration.sections(): if section == 'agent': continue section_dict[section] = section_entry_dict = dict( configuration.items(section)) for key in ('request_kw', 'max_install_duration', 'max_destroy_duration', 'max_request_duration', 'max_uninstall_duration', 'computer_list' ): if key in section_entry_dict: try: if isinstance(section_entry_dict[key], str) or \ isinstance(section_entry_dict[key], unicode): section_entry_dict[key] = json.loads( section_entry_dict[key]) except Exception as exc: logger.error("Fail to load %s on %s" % (key, section_entry_dict)) raise if 'key' in section_entry_dict: key_file, cert_file = asFilenamePair(section_entry_dict['key'], section_entry_dict['cert']) section_entry_dict['key'] = key_file section_entry_dict['cert'] = cert_file if "computer_list" in section_entry_dict: section_entry_dict["target_computer"] = \ random.choice(section_entry_dict["computer_list"]) agent_parameter_dict = dict(configuration.items('agent')) # XXX: should node title be auto-generated by installation recipe ? # For example, using computer guid. node_title = agent_parameter_dict['node_title'] test_title = agent_parameter_dict['test_title'] project_title = agent_parameter_dict['project_title'] task_distribution_tool = TaskDistributionTool(agent_parameter_dict[ 'report_url']) master_slap_connection_dict = {} test_result = task_distribution_tool.createTestResult( revision='', test_name_list=section_dict.keys(), node_title=node_title, allow_restart=True, test_title=test_title, project_title=project_title, ) test_result.watcher_period = 300 if log: test_result.addWatch(log, log_file, max_history_bytes=10000) assert test_result is not None test_mapping = TestMap(section_dict) logger.info("Running %s tests in parallel." % \ len(test_mapping.getComputerList())) ran_test_set = set() running_test_dict = {} more_tests = True logger.info('Starting Test Agent run %s ' % node_title) while True: # Get up to parallel_task_count tasks to execute while len(running_test_dict) < len(test_mapping.getComputerList())\ and more_tests: test_mapping.cleanUp() target_computer = test_mapping.getNextComputer([computer \ for _, _, computer in running_test_dict.itervalues()]) test_line = test_result.start( exclude_list= list(ran_test_set) + \ list(test_mapping.getExcludeList(target_computer))) logger.info("Test Line: %s " % test_line) logger.info("Ran Test Set: %s " % ran_test_set) logger.info("Running test dict: %s " % running_test_dict) logger.info("Target Computer: %s " % target_computer) if test_line is None: test_mapping.dropComputer(target_computer) if len(test_mapping.getComputerList()) == 0: more_tests = False continue test_name = test_line.name try: section_entry_dict = section_dict[test_name] except KeyError: # We don't know how to execute this test. Assume it doesn't # exist anymore, and fail it in result. test_line.stop(stderr='This test does not exist on test ' 'node %s' % (node_title, )) continue master_url = section_entry_dict['master_url'] master_slap_connection_key = (master_url, section_entry_dict.get('key')) try: supply, order, rpc = master_slap_connection_dict[ master_slap_connection_key] except KeyError: key = section_entry_dict.get('key') cert = section_entry_dict.get('cert') slap = slapos.slap.slap() slap.initializeConnection(master_url, key, cert) supply = slap.registerSupply() order = slap.registerOpenOrder() assert master_url.startswith('https:') rpc = xmlrpclib.ServerProxy(master_url, allow_none=True, transport=x509Transport( {'key_file': key, 'cert_file': cert})) master_slap_connection_dict[ master_slap_connection_key] = (supply, order, rpc) tester = SoftwareReleaseTester( test_name + '_' + node_title + time.strftime( '_%Y/%m/%d_%H:%M:%S_+0000', time.gmtime()), logger, rpc, supply, order, section_entry_dict['url'], section_entry_dict['target_computer'], section_entry_dict['max_install_duration'], section_entry_dict['max_uninstall_duration'], section_entry_dict.get('request_kw'), section_entry_dict.get('max_request_duration'), section_entry_dict.get('max_destroy_duration'), ) ran_test_set.add(test_name) running_test_dict[test_name] = (test_line, tester, target_computer) if not running_test_dict: break now = time.time() # Synchronise refreshes on watcher period, so it doesn't report a # stalled test node where we are actually still sleeping. # Change test_result.watcher_period outside this loop if you wish # to change sleep duration. next_deadline = now + test_result.watcher_period for section, (test_line, tester, target_computer) in running_test_dict.items(): logger.info('Checking %s: %r...', section, tester) try: deadline = tester.tic(now) except Exception: logger.exception('Test execution fail for %s' % (section)) test_line.stop( test_count=1, error_count=1, failure_count=0, skip_count=0, stderr=traceback.format_exc(), ) del running_test_dict[section] try: tester.teardown() except slapos.slap.NotFoundError: # This exception is ignored because we cannot # Teardown if SR URL do not exist. logger.exception('Fail and not found') pass except Exception: logger.exception('teardown failed, human ' 'assistance needed for cleanup') raise else: logger.info('%r', tester) if deadline is None: # TODO: report how long each step took. logger.info('Test execution finished for %s' % (section)) test_line.stop( test_count=1, error_count=0, failure_count=0, skip_count=0, ) del running_test_dict[section] try: tester.teardown() except slapos.slap.NotFoundError: # This exception is ignored because we cannot # Teardown if SR URL do not exist. logger.exception('Fail and not found') pass except Exception: logger.exception('teardown failed, human ' 'assistance needed for cleanup') raise else: next_deadline = min(deadline, next_deadline) if running_test_dict: to_sleep = next_deadline - time.time() if to_sleep > 0: logger.info('Sleeping %is...', to_sleep) time.sleep(to_sleep) if not test_result.isAlive(): for _, tester, computer_id in running_test_dict.itervalues(): tester.teardown() finally: if pidfile: setFinished(pidfile) # Help interpreter get rid of AutoSTemp instances. key_file_dict.clear()
def install(self): slap = slapos.slap.slap() slap_connection = self.buildout['slap_connection'] computer_id = slap_connection['computer_id'] computer_partition_id = slap_connection['partition_id'] server_url = slap_connection['server_url'] key_file = slap_connection.get('key_file') cert_file = slap_connection.get('cert_file') slap.initializeConnection(server_url, key_file, cert_file) self.computer_partition = slap.registerComputerPartition( computer_id, computer_partition_id) self.parameter_dict = self.computer_partition.getInstanceParameterDict() software_type = self.parameter_dict['slap_software_type'] self.logger.info('Deploying instance with software type %s' % \ software_type) # Raise if request software_type does not exist ... if software_type not in self.options: # ... Except for backward compatibility. Then use "default". if software_type in ['RootSoftwareInstance']: software_type = 'default' else: raise zc.buildout.UserError("This software type (%s) isn't mapped." % \ software_type) instance_file_path = self.options[software_type] if not os.path.exists(instance_file_path): raise zc.buildout.UserError("The specified buildout config file %r does " "not exist." % instance_file_path) buildout = ConfigParser() with open(instance_file_path) as instance_path: buildout.readfp(instance_path) buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name) if not buildout.has_section('slap-parameter'): buildout.add_section('slap-parameter') for parameter, value in self.parameter_dict.items(): # All parameters evaluating to False are... False, and shouldn't # convey any information. # Here, all those parameters are simply ignored. if value: if isinstance(value, str): buildout.set('slap-parameter', parameter, value) else: buildout.set('slap-parameter', parameter, json.dumps(value)) buildout.add_section('slap-network-information') buildout.set('slap-network-information', 'local-ipv4', self.getLocalIPv4Address()) buildout.set('slap-network-information', 'global-ipv6', self.getGlobalIPv6Address()) buildout.set('slap-network-information', 'network-interface', self.getNetworkInterface()) # Copy/paste slap_connection buildout.add_section('slap-connection') for key, value in self.buildout['slap_connection'].iteritems(): # XXX: Waiting for SlapBaseRecipe to use dash instead of underscores buildout.set('slap-connection', key.replace('_', '-'), value) # XXX: Needed for lxc. Use non standard API buildout.set('slap-connection', 'requested', self.computer_partition._requested_state) work_directory = os.path.abspath(self.buildout['buildout'][ 'directory']) buildout_filename = os.path.join(work_directory, 'buildout-%s.cfg' % self.name) with open(buildout_filename, 'w') as buildout_file: buildout.write(buildout_file) # XXX-Antoine: We gotta find a better way to do this. I tried to check # out how slapgrid-cp was running buildout. But it is worse than that. command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename] self.logger.info("Invoking commandline : '%s'", ' '.join(command_line_args)) subprocess.check_call(command_line_args, cwd=work_directory, env=os.environ.copy()) return []
def fetch_parameter_dict(self, options, instance_root): slap = slapos.slap.slap() slap.initializeConnection( options['url'], options.get('key'), options.get('cert'), ) computer_partition = slap.registerComputerPartition( options['computer'], options['partition'], ) parameter_dict = computer_partition.getInstanceParameterDict() options['instance-state'] = computer_partition.getState() # XXX: those are not partition parameters, strictly speaking. # Make them available as individual section keys. for his_key in ( 'slap_software_type', 'slap_computer_partition_id', 'slap_computer_id', 'slap_software_release_url', 'slave_instance_list', 'timestamp', ): try: value = parameter_dict.pop(his_key) except KeyError: pass else: options[his_key.replace('_', '-')] = value ipv4_set = set() v4_add = ipv4_set.add ipv6_set = set() v6_add = ipv6_set.add tap_set = set() tap_add = tap_set.add route_gw_set = set() route_gw_add = route_gw_set.add route_mask_set = set() route_mask_add = route_mask_set.add route_ipv4_set = set() route_v4_add = route_ipv4_set.add route_network_set = set() route_net_add = route_network_set.add for tap, ip in parameter_dict.pop('ip_list'): tap_add(tap) if valid_ipv4(ip): v4_add(ip) elif valid_ipv6(ip): v6_add(ip) # XXX: emit warning on unknown address type ? if 'full_ip_list' in parameter_dict: for item in parameter_dict.pop('full_ip_list'): if len(item) == 5: tap, ip, gw, netmask, network = item if tap.startswith('route_'): if valid_ipv4(gw): route_gw_add(gw) if valid_ipv4(netmask): route_mask_add(netmask) if valid_ipv4(ip): route_v4_add(ip) if valid_ipv4(network): route_net_add(network) options['ipv4'] = ipv4_set options['ipv6'] = ipv6_set # also export single ip values for those recipes that don't support sets. if ipv4_set: options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8') if ipv6_set: options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8') if route_ipv4_set: options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8') options['tap-network-information-dict'] = dict(ipv4=route_ipv4_set, netmask=route_mask_set, gateway=route_gw_set, network=route_network_set) else: options['tap-network-information-dict'] = {} if route_gw_set: options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8') if route_mask_set: options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8') if route_network_set: options['tap-network'] = list(route_network_set)[0].encode('UTF-8') storage_home = options.get('storage-home') storage_dict = {} if storage_home and os.path.exists(storage_home) and \ os.path.isdir(storage_home): for filename in os.listdir(storage_home): storage_path = os.path.join(storage_home, filename, options['slap-computer-partition-id']) if os.path.exists(storage_path) and os.path.isdir(storage_path): storage_link = os.path.join(instance_root, 'DATA', filename) mkdir_p(os.path.join(instance_root, 'DATA')) if not os.path.lexists(storage_link): os.symlink(storage_path, storage_link) storage_dict[filename] = storage_link options['storage-dict'] = storage_dict options['tap'] = tap_set return self._expandParameterDict(options, parameter_dict)
def install(self): slap = slapos.slap.slap() slap_connection = self.buildout['slap_connection'] computer_id = slap_connection['computer_id'] computer_partition_id = slap_connection['partition_id'] server_url = slap_connection['server_url'] key_file = slap_connection.get('key_file') cert_file = slap_connection.get('cert_file') slap.initializeConnection(server_url, key_file, cert_file) self.computer_partition = slap.registerComputerPartition( computer_id, computer_partition_id) self.parameter_dict = self.computer_partition.getInstanceParameterDict() software_type = self.parameter_dict['slap_software_type'] if software_type not in self.options: if 'default' in self.options: software_type = 'default' else: raise zc.buildout.UserError("This software type isn't mapped. And" "there's no default software type.") instance_file_path = self.options[software_type] if not os.path.exists(instance_file_path): raise zc.buildout.UserError("The specified buildout config file does not" "exist.") buildout = ConfigParser() with open(instance_file_path) as instance_path: buildout.readfp(instance_path) buildout.set('buildout', 'installed', '.installed-%s.cfg' % software_type) buildout.add_section('slap-parameter') for parameter, value in self.parameter_dict.items(): buildout.set('slap-parameter', parameter, value) buildout.add_section('slap-network-information') buildout.set('slap-network-information', 'local-ipv4', self.getLocalIPv4Address()) buildout.set('slap-network-information', 'global-ipv6', self.getGlobalIPv6Address()) # Copy/paste slap_connection buildout.add_section('slap-connection') for key, value in self.buildout['slap_connection'].iteritems(): # XXX: Waiting for SlapBaseRecipe to use dash instead of underscores buildout.set('slap-connection', key.replace('_', '-'), value) work_directory = os.path.abspath(self.buildout['buildout'][ 'directory']) buildout_filename = os.path.join(work_directory, 'buildout-%s.cfg' % software_type) with open(buildout_filename, 'w') as buildout_file: buildout.write(buildout_file) # XXX-Antoine: We gotta find a better way to do this. I tried to check # out how slapgrid-cp was running buildout. But it is worse than that. command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename] self.logger.info("Invoking commandline : '%s'", ' '.join(command_line_args)) subprocess.check_call(command_line_args, cwd=work_directory, env=os.environ.copy()) return []
def initializeSlapOSControler(self, slapproxy_log=None, process_manager=None, reset_software=False, software_path_list=None): self.process_manager = process_manager self.software_path_list = software_path_list logger.debug('SlapOSControler, initialize, reset_software: %r', reset_software) config = self.config slapos_config_dict = config.copy() slapos_config_dict.update(software_root=self.software_root, instance_root=self.instance_root, proxy_database=self.proxy_database, shared_part_list='\n '.join( self.shared_part_list)) with open(self.slapos_config, 'w') as f: f.write( pkg_resources.resource_string( 'erp5.util.testnode', 'template/slapos.cfg.in').decode() % slapos_config_dict) # By erasing everything, we make sure that we are able to "update" # existing profiles. This is quite dirty way to do updates... if os.path.exists(self.proxy_database): os.unlink(self.proxy_database) kwargs = dict(close_fds=True, preexec_fn=os.setsid) if slapproxy_log is not None: slapproxy_log_fp = open(slapproxy_log, 'w') kwargs['stdout'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp proxy = subprocess.Popen([ config['slapos_binary'], 'proxy', 'start', '--cfg', self.slapos_config ], **kwargs) process_manager.process_pid_set.add(proxy.pid) slap = self.slap = slapos.slap.slap() # Wait for proxy to accept connections retries = 0 while True: time.sleep(1) try: slap.initializeConnection(config['master_url']) computer = slap.registerComputer(config['computer_id']) # Call a method to ensure connection to master can be established computer.getComputerPartitionList() except slapos.slap.ConnectionError as e: retries += 1 if retries >= 60: raise logger.debug("Proxy still not started %s, retrying", e) else: break try: # register software profile for path in self.software_path_list: slap.registerSupply().supply( path, computer_guid=config['computer_id']) except Exception: logger.exception("SlapOSControler.initializeSlapOSControler") raise ValueError("Unable to registerSupply") # Reset all previously generated software if needed if reset_software: self._resetSoftware() else: createFolder(self.software_root) instance_root = self.instance_root # Delete any existing partition in order to not get its data (ex. # MySQL DB content) from previous runs. To support changes of partition # naming scheme (which already happened), do this at instance_root level. createFolder(instance_root, True) partition_list = [] for i in range(MAX_PARTITIONS): # create partition and configure computer # XXX: at the moment all partitions do share same virtual interface address # this is not a problem as usually all services are on different ports partition_reference = '%s-%s' % (config['partition_reference'], i) partition_path = os.path.join(instance_root, partition_reference) if not (os.path.exists(partition_path)): os.mkdir(partition_path) os.chmod(partition_path, 0o750) partition_list.append({ 'address_list': [ { 'addr': config['ipv4_address'], 'netmask': '255.255.255.255' }, { 'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::' }, ], 'path': partition_path, 'reference': partition_reference, 'tap': { 'name': partition_reference }, }) computer.updateConfiguration( xml_marshaller.xml_marshaller.dumps({ 'address': config['ipv4_address'], 'instance_root': instance_root, 'netmask': '255.255.255.255', 'partition_list': partition_list, 'reference': config['computer_id'], 'software_root': self.software_root }))
def install(self): slap = slapos.slap.slap() slap_connection = self.buildout['slap_connection'] computer_id = slap_connection['computer_id'] computer_partition_id = slap_connection['partition_id'] server_url = slap_connection['server_url'] key_file = slap_connection.get('key_file') cert_file = slap_connection.get('cert_file') instance_root = self.buildout['buildout']['directory'] storage_configuration_dict = self.buildout.get('storage-configuration') network_dict = self.buildout.get('network-information') storage_home = '' global_ipv4_network = '' if storage_configuration_dict: storage_home = storage_configuration_dict.get('storage-home') if network_dict: global_ipv4_network = network_dict.get('global-ipv4-network') slap.initializeConnection(server_url, key_file, cert_file) self.computer_partition = slap.registerComputerPartition( computer_id, computer_partition_id) self.parameter_dict = self.computer_partition.getInstanceParameterDict() software_type = self.parameter_dict['slap_software_type'] # Raise if request software_type does not exist ... if software_type not in self.options: # ... Except for backward compatibility. Then use "default". if software_type in ['RootSoftwareInstance']: software_type = 'default' else: raise zc.buildout.UserError("This software type (%s) isn't mapped." % \ software_type) instance_file_path = self.options[software_type] if not os.path.exists(instance_file_path): raise zc.buildout.UserError("The specified buildout config file %r does " "not exist." % instance_file_path) buildout = SlapConfigParser() with open(instance_file_path) as instance_path: buildout.readfp(instance_path) buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name) if not buildout.has_section('slap-parameter'): buildout.add_section('slap-parameter') for parameter, value in self.parameter_dict.items(): # All parameters evaluating to False are... False, and shouldn't # convey any information. # Here, all those parameters are simply ignored. if value: if isinstance(value, str): buildout.set('slap-parameter', parameter, value) else: buildout.set('slap-parameter', parameter, json.dumps(value)) buildout.add_section('slap-network-information') buildout.set('slap-network-information', 'local-ipv4', self.getLocalIPv4Address()) buildout.set('slap-network-information', 'global-ipv6', self.getGlobalIPv6Address()) buildout.set('slap-network-information', 'network-interface', self.getNetworkInterface()) tap_ip_list = self.getLocalTapIPv4AddressList() tap_ipv4 = tap_gateway = tap_netmask = tap_network = '' if tap_ip_list: tap_ipv4, tap_gateway, tap_netmask, tap_network= tap_ip_list buildout.set('slap-network-information', 'tap-ipv4', tap_ipv4) buildout.set('slap-network-information', 'tap-gateway', tap_gateway) buildout.set('slap-network-information', 'tap-netmask', tap_netmask) buildout.set('slap-network-information', 'tap-network', tap_network) buildout.set('slap-network-information', 'global-ipv4-network', global_ipv4_network) # Copy/paste slap_connection buildout.add_section('slap-connection') for key, value in self.buildout['slap_connection'].iteritems(): # XXX: Waiting for SlapBaseRecipe to use dash instead of underscores buildout.set('slap-connection', key.replace('_', '-'), value) # XXX: Needed for lxc. Use non standard API buildout.set('slap-connection', 'requested', self.computer_partition._requested_state) # setup storage directory buildout.add_section('storage-configuration') buildout.set('storage-configuration', 'storage-home', storage_home) if storage_home and os.path.exists(storage_home) and \ os.path.isdir(storage_home): # Create folder instance_root/DATA/ if not exist data_home = os.path.join(instance_root, 'DATA') self.mkdir_p(data_home) for filename in os.listdir(storage_home): storage_path = os.path.join(storage_home, filename, computer_partition_id) if os.path.exists(storage_path) and os.path.isdir(storage_path): storage_link = os.path.join(data_home, filename) if os.path.lexists(storage_link): if not os.path.islink(storage_link): raise zc.buildout.UserError( 'Target %r already exists but is not a link' % storage_link) #os.unlink(storage_link) else: os.symlink(storage_path, storage_link) buildout.set('storage-configuration', filename, storage_link) work_directory = os.path.abspath(self.buildout['buildout'][ 'directory']) buildout_filename = os.path.join(work_directory, 'buildout-%s.cfg' % self.name) with open(buildout_filename, 'w') as buildout_file: buildout.write(buildout_file) # XXX-Antoine: We gotta find a better way to do this. I tried to check # out how slapgrid-cp was running buildout. But it is worse than that. command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename] self.logger.info("Invoking commandline : '%s'", ' '.join(command_line_args)) subprocess.check_call(command_line_args, cwd=work_directory, env=os.environ.copy()) return []
def fetch_parameter_dict(self, options, instance_root): slap = slapos.slap.slap() slap.initializeConnection( options['url'], options.get('key'), options.get('cert'), ) computer_partition = slap.registerComputerPartition( options['computer'], options['partition'], ) parameter_dict = computer_partition.getInstanceParameterDict() options['instance-state'] = computer_partition.getState() # XXX: those are not partition parameters, strictly speaking. # Make them available as individual section keys. for his_key in ( 'slap_software_type', 'slap_computer_partition_id', 'slap_computer_id', 'slap_software_release_url', 'slave_instance_list', 'timestamp', ): try: value = parameter_dict.pop(his_key) except KeyError: pass else: options[his_key.replace('_', '-')] = value ipv4_set = set() v4_add = ipv4_set.add ipv6_set = set() v6_add = ipv6_set.add tap_set = set() tap_add = tap_set.add route_gw_set = set() route_gw_add = route_gw_set.add route_mask_set = set() route_mask_add = route_mask_set.add route_ipv4_set = set() route_v4_add = route_ipv4_set.add route_network_set = set() route_net_add = route_network_set.add for tap, ip in parameter_dict.pop('ip_list'): tap_add(tap) if valid_ipv4(ip): v4_add(ip) elif valid_ipv6(ip): v6_add(ip) # XXX: emit warning on unknown address type ? if 'full_ip_list' in parameter_dict: for item in parameter_dict.pop('full_ip_list'): if len(item) == 5: tap, ip, gw, netmask, network = item if tap.startswith('route_'): if valid_ipv4(gw): route_gw_add(gw) if valid_ipv4(netmask): route_mask_add(netmask) if valid_ipv4(ip): route_v4_add(ip) if valid_ipv4(network): route_net_add(network) options['ipv4'] = ipv4_set options['ipv6'] = ipv6_set # also export single ip values for those recipes that don't support sets. if ipv4_set: options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8') if ipv6_set: options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8') if route_ipv4_set: options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8') options['tap-network-information-dict'] = dict( ipv4=route_ipv4_set, netmask=route_mask_set, gateway=route_gw_set, network=route_network_set) else: options['tap-network-information-dict'] = {} if route_gw_set: options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8') if route_mask_set: options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8') if route_network_set: options['tap-network'] = list(route_network_set)[0].encode('UTF-8') storage_home = options.get('storage-home') storage_dict = {} if storage_home and os.path.exists(storage_home) and \ os.path.isdir(storage_home): for filename in os.listdir(storage_home): storage_path = os.path.join( storage_home, filename, options['slap-computer-partition-id']) if os.path.exists(storage_path) and os.path.isdir( storage_path): storage_link = os.path.join(instance_root, 'DATA', filename) mkdir_p(os.path.join(instance_root, 'DATA')) if not os.path.lexists(storage_link): os.symlink(storage_path, storage_link) storage_dict[filename] = storage_link options['storage-dict'] = storage_dict options['tap'] = tap_set return self._expandParameterDict(options, parameter_dict)
def fetch_parameter_dict(self, options, instance_root): """Gather parameters about current computer and partition. Use two sources of truth 1. SlapOS Master - for external computer/partition information 2. format.Partition.resource_file - for partition specific details """ slap = slapos.slap.slap() slap.initializeConnection( options['url'], options.get('key'), options.get('cert'), ) computer_partition = slap.registerComputerPartition( options['computer'], options['partition'], ) parameter_dict = computer_partition.getInstanceParameterDict() options['instance-state'] = computer_partition.getState() # XXX: those are not partition parameters, strictly speaking. # Make them available as individual section keys. for his_key in ( 'slap_software_type', 'slap_computer_partition_id', 'slap_computer_id', 'slap_software_release_url', 'slave_instance_list', 'timestamp', ): try: value = parameter_dict.pop(his_key) except KeyError: pass else: options[his_key.replace('_', '-')] = value # Get Instance and root instance title or return UNKNOWN if not set options['instance-title'] = parameter_dict.pop('instance_title', 'UNKNOWN Instance') options['root-instance-title'] = parameter_dict.pop( 'root_instance_title', 'UNKNOWN') options['instance-guid'] = computer_partition.getInstanceGuid() ipv4_set = set() v4_add = ipv4_set.add ipv6_set = set() v6_add = ipv6_set.add tap_set = set() tap_add = tap_set.add route_gw_set = set() route_gw_add = route_gw_set.add route_mask_set = set() route_mask_add = route_mask_set.add route_ipv4_set = set() route_v4_add = route_ipv4_set.add route_network_set = set() route_net_add = route_network_set.add for tap, ip in parameter_dict.pop('ip_list'): tap_add(tap) if valid_ipv4(ip): v4_add(ip) elif valid_ipv6(ip): v6_add(ip) # XXX: emit warning on unknown address type ? if 'full_ip_list' in parameter_dict: for item in parameter_dict.pop('full_ip_list'): if len(item) == 5: tap, ip, gw, netmask, network = item if tap.startswith('route_'): if valid_ipv4(gw): route_gw_add(gw) if valid_ipv4(netmask): route_mask_add(netmask) if valid_ipv4(ip): route_v4_add(ip) if valid_ipv4(network): route_net_add(network) options['ipv4'] = ipv4_set options['ipv6'] = ipv6_set # also export single ip values for those recipes that don't support sets. if ipv4_set: options['ipv4-random'] = min(ipv4_set) if ipv6_set: options['ipv6-random'] = min(ipv6_set) storage_home = options.get('storage-home') storage_dict = {} if storage_home and os.path.exists(storage_home) and \ os.path.isdir(storage_home): for filename in os.listdir(storage_home): storage_path = os.path.join( storage_home, filename, options['slap-computer-partition-id']) if os.path.exists(storage_path) and os.path.isdir( storage_path): storage_link = os.path.join(instance_root, 'DATA', filename) mkdir_p(os.path.join(instance_root, 'DATA')) if not os.path.lexists(storage_link): os.symlink(storage_path, storage_link) storage_dict[filename] = storage_link options['storage-dict'] = storage_dict # The external information transfered from Slap Master has been processed # so we extend with information gathered from partition resource file if hasattr(slapformat.Partition, "resource_file"): resource_home = instance_root while not os.path.exists( os.path.join(resource_home, slapformat.Partition.resource_file)): resource_home = os.path.normpath( os.path.join(resource_home, '..')) if resource_home == "/": break else: # no break happened - let's add partition resources into options logger.debug("Using partition resource file {}".format( os.path.join(resource_home, slapformat.Partition.resource_file))) with open( os.path.join( resource_home, slapformat.Partition.resource_file)) as fi: partition_params = json.load(fi) # be very careful with overriding master's information for key, value in flatten_dict(partition_params).items(): if key not in options: if six.PY2 and isinstance(value, unicode): value = value.encode('UTF-8') options[key] = value # print out augmented options to see what we are passing logger.debug(str(options)) return self._expandParameterDict(options, parameter_dict)
def install(self): slap = slapos.slap.slap() slap_connection = self.buildout['slap_connection'] computer_id = slap_connection['computer_id'] computer_partition_id = slap_connection['partition_id'] server_url = slap_connection['server_url'] key_file = slap_connection.get('key_file') cert_file = slap_connection.get('cert_file') instance_root = self.buildout['buildout']['directory'] storage_configuration_dict = self.buildout.get('storage-configuration') network_dict = self.buildout.get('network-information') storage_home = '' global_ipv4_network = '' if storage_configuration_dict: storage_home = storage_configuration_dict.get('storage-home') if network_dict: global_ipv4_network = network_dict.get('global-ipv4-network') slap.initializeConnection(server_url, key_file, cert_file) self.computer_partition = slap.registerComputerPartition( computer_id, computer_partition_id) self.parameter_dict = self.computer_partition.getInstanceParameterDict( ) software_type = self.parameter_dict['slap_software_type'] # Raise if request software_type does not exist ... if software_type not in self.options: # ... Except for backward compatibility. Then use "default". if software_type in ['RootSoftwareInstance']: software_type = 'default' else: raise zc.buildout.UserError("This software type (%s) isn't mapped." % \ software_type) instance_file_path = self.options[software_type] if not os.path.exists(instance_file_path): raise zc.buildout.UserError( "The specified buildout config file %r does " "not exist." % instance_file_path) if six.PY3: buildout = SlapConfigParser(strict=False) else: buildout = SlapConfigParser() with open(instance_file_path) as instance_path: buildout.readfp(instance_path) buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name) if not buildout.has_section('slap-parameter'): buildout.add_section('slap-parameter') for parameter, value in self.parameter_dict.items(): # All parameters evaluating to False are... False, and shouldn't # convey any information. # Here, all those parameters are simply ignored. if value: if isinstance(value, str): buildout.set('slap-parameter', parameter, value) else: buildout.set('slap-parameter', parameter, json.dumps(value)) buildout.add_section('slap-network-information') buildout.set('slap-network-information', 'local-ipv4', self.getLocalIPv4Address()) buildout.set('slap-network-information', 'global-ipv6', self.getGlobalIPv6Address()) buildout.set('slap-network-information', 'network-interface', self.getNetworkInterface()) tap_ip_list = self.getLocalTapIPv4AddressList() tap_ipv4 = tap_gateway = tap_netmask = tap_network = '' if tap_ip_list: tap_ipv4, tap_gateway, tap_netmask, tap_network = tap_ip_list buildout.set('slap-network-information', 'tap-ipv4', tap_ipv4) buildout.set('slap-network-information', 'tap-gateway', tap_gateway) buildout.set('slap-network-information', 'tap-netmask', tap_netmask) buildout.set('slap-network-information', 'tap-network', tap_network) buildout.set('slap-network-information', 'global-ipv4-network', global_ipv4_network) # Copy/paste slap_connection buildout.add_section('slap-connection') for key, value in six.iteritems(self.buildout['slap_connection']): # XXX: Waiting for SlapBaseRecipe to use dash instead of underscores buildout.set('slap-connection', key.replace('_', '-'), value) # XXX: Needed for lxc. Use non standard API buildout.set('slap-connection', 'requested', self.computer_partition._requested_state) # setup storage directory buildout.add_section('storage-configuration') buildout.set('storage-configuration', 'storage-home', storage_home) if storage_home and os.path.exists(storage_home) and \ os.path.isdir(storage_home): # Create folder instance_root/DATA/ if not exist data_home = os.path.join(instance_root, 'DATA') self.mkdir_p(data_home) for filename in os.listdir(storage_home): storage_path = os.path.join(storage_home, filename, computer_partition_id) if os.path.exists(storage_path) and os.path.isdir( storage_path): storage_link = os.path.join(data_home, filename) if os.path.lexists(storage_link): if not os.path.islink(storage_link): raise zc.buildout.UserError( 'Target %r already exists but is not a link' % storage_link) #os.unlink(storage_link) else: os.symlink(storage_path, storage_link) buildout.set('storage-configuration', filename, storage_link) work_directory = os.path.abspath( self.buildout['buildout']['directory']) buildout_filename = os.path.join(work_directory, 'buildout-%s.cfg' % self.name) with open(buildout_filename, 'w') as buildout_file: buildout.write(buildout_file) # XXX-Antoine: We gotta find a better way to do this. I tried to check # out how slapgrid-cp was running buildout. But it is worse than that. command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename] self.logger.info("Invoking commandline : '%s'", ' '.join(command_line_args)) subprocess.check_call(command_line_args, cwd=work_directory, env=os.environ.copy()) return []
def fetch_parameter_dict(self, options, instance_root): """Gather parameters about current computer and partition. Use two sources of truth 1. SlapOS Master - for external computer/partition information 2. format.Partition.resource_file - for partition specific details """ slap = slapos.slap.slap() slap.initializeConnection( options['url'], options.get('key'), options.get('cert'), ) computer_partition = slap.registerComputerPartition( options['computer'], options['partition'], ) parameter_dict = computer_partition.getInstanceParameterDict() options['instance-state'] = computer_partition.getState() # XXX: those are not partition parameters, strictly speaking. # Make them available as individual section keys. for his_key in ( 'slap_software_type', 'slap_computer_partition_id', 'slap_computer_id', 'slap_software_release_url', 'slave_instance_list', 'timestamp', ): try: value = parameter_dict.pop(his_key) except KeyError: pass else: options[his_key.replace('_', '-')] = value # Get Instance and root instance title or return UNKNOWN if not set options['instance-title'] = parameter_dict.pop('instance_title', 'UNKNOWN Instance').encode('UTF-8') options['root-instance-title'] = parameter_dict.pop('root_instance_title', 'UNKNOWN').encode('UTF-8') options['instance-guid'] = computer_partition.getInstanceGuid() \ .encode('UTF-8') ipv4_set = set() v4_add = ipv4_set.add ipv6_set = set() v6_add = ipv6_set.add tap_set = set() tap_add = tap_set.add route_gw_set = set() route_gw_add = route_gw_set.add route_mask_set = set() route_mask_add = route_mask_set.add route_ipv4_set = set() route_v4_add = route_ipv4_set.add route_network_set = set() route_net_add = route_network_set.add for tap, ip in parameter_dict.pop('ip_list'): tap_add(tap) if valid_ipv4(ip): v4_add(ip) elif valid_ipv6(ip): v6_add(ip) # XXX: emit warning on unknown address type ? if 'full_ip_list' in parameter_dict: for item in parameter_dict.pop('full_ip_list'): if len(item) == 5: tap, ip, gw, netmask, network = item if tap.startswith('route_'): if valid_ipv4(gw): route_gw_add(gw) if valid_ipv4(netmask): route_mask_add(netmask) if valid_ipv4(ip): route_v4_add(ip) if valid_ipv4(network): route_net_add(network) options['ipv4'] = ipv4_set options['ipv6'] = ipv6_set # also export single ip values for those recipes that don't support sets. if ipv4_set: options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8') if ipv6_set: options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8') storage_home = options.get('storage-home') storage_dict = {} if storage_home and os.path.exists(storage_home) and \ os.path.isdir(storage_home): for filename in os.listdir(storage_home): storage_path = os.path.join(storage_home, filename, options['slap-computer-partition-id']) if os.path.exists(storage_path) and os.path.isdir(storage_path): storage_link = os.path.join(instance_root, 'DATA', filename) mkdir_p(os.path.join(instance_root, 'DATA')) if not os.path.lexists(storage_link): os.symlink(storage_path, storage_link) storage_dict[filename] = storage_link options['storage-dict'] = storage_dict # The external information transfered from Slap Master has been processed # so we extend with information gathered from partition resource file if hasattr(slapformat.Partition, "resource_file"): resource_home = instance_root while not os.path.exists(os.path.join(resource_home, slapformat.Partition.resource_file)): resource_home = os.path.normpath(os.path.join(resource_home, '..')) if resource_home == "/": break else: # no break happened - let's add partition resources into options logger.debug("Using partition resource file {}".format( os.path.join(resource_home, slapformat.Partition.resource_file))) with open(os.path.join(resource_home, slapformat.Partition.resource_file)) as fi: partition_params = json.load(fi) # be very careful with overriding master's information for key, value in flatten_dict(partition_params).items(): if key not in options: if isinstance(value, unicode): value = value.encode('UTF-8') options[key] = value # print out augmented options to see what we are passing logger.debug(str(options)) return self._expandParameterDict(options, parameter_dict)