def initializeSlapOSControler(self, slapproxy_log=None, process_manager=None, reset_software=False, software_path_list=None): self.process_manager = process_manager self.software_path_list = software_path_list logger.debug('SlapOSControler, initialize, reset_software: %r', reset_software) config = self.config slapos_config_dict = config.copy() slapos_config_dict.update(software_root=self.software_root, instance_root=self.instance_root, proxy_database=self.proxy_database) with open(self.slapos_config, 'w') as f: f.write( pkg_resources.resource_string('erp5.util.testnode', 'template/slapos.cfg.in') % slapos_config_dict) # By erasing everything, we make sure that we are able to "update" # existing profiles. This is quite dirty way to do updates... if os.path.exists(self.proxy_database): os.unlink(self.proxy_database) kwargs = dict(close_fds=True, preexec_fn=os.setsid) if slapproxy_log is not None: slapproxy_log_fp = open(slapproxy_log, 'w') kwargs['stdout'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp # Make sure there is no slapos alive from previous run process_manager.killall('slapos') proxy = subprocess.Popen([ config['slapos_binary'], 'proxy', 'start', '--cfg', self.slapos_config ], **kwargs) process_manager.process_pid_set.add(proxy.pid) slap = self.slap = slapos.slap.slap() # Wait for proxy to accept connections retries = 0 while True: time.sleep(.5) try: slap.initializeConnection(config['master_url']) computer = slap.registerComputer(config['computer_id']) # Call a method to ensure connection to master can be established computer.getComputerPartitionList() except slapos.slap.ConnectionError, e: retries += 1 if retries >= 20: raise logger.debug("Proxy still not started %s, retrying", e) else: break
def __init__(self, config, log, process_group_pid_set=None, slapproxy_log=None): self.log = log self.config = config # By erasing everything, we make sure that we are able to "update" # existing profiles. This is quite dirty way to do updates... if os.path.exists(config['proxy_database']): os.unlink(config['proxy_database']) kwargs = dict(close_fds=True, preexec_fn=os.setsid) if slapproxy_log is not None: slapproxy_log_fp = open(slapproxy_log, 'w') kwargs['stdout'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp proxy = subprocess.Popen([config['slapproxy_binary'], config['slapos_config']], **kwargs) process_group_pid_set.add(proxy.pid) # XXX: dirty, giving some time for proxy to being able to accept # connections time.sleep(10) slap = slapos.slap.slap() slap.initializeConnection(config['master_url']) # register software profile self.software_profile = config['custom_profile_path'] slap.registerSupply().supply( self.software_profile, computer_guid=config['computer_id']) computer = slap.registerComputer(config['computer_id']) # create partition and configure computer partition_reference = config['partition_reference'] partition_path = os.path.join(config['instance_root'], partition_reference) if not os.path.exists(partition_path): os.mkdir(partition_path) os.chmod(partition_path, 0750) computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({ 'address': config['ipv4_address'], 'instance_root': config['instance_root'], 'netmask': '255.255.255.255', 'partition_list': [{'address_list': [{'addr': config['ipv4_address'], 'netmask': '255.255.255.255'}, {'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::'}, ], 'path': partition_path, 'reference': partition_reference, 'tap': {'name': partition_reference}, } ], 'reference': config['computer_id'], 'software_root': config['software_root']}))
def updateProxy(config): """ Configure Slapos Node computer and partitions. Send current Software Release to Slapproxy for compilation and deployment. """ startProxy(config) if not os.path.exists(config['instance_root']): os.mkdir(config['instance_root']) slap = slapos.slap.slap() profile = getCurrentSoftwareReleaseProfile(config) slap.initializeConnection(config['master_url']) slap.registerSupply().supply(profile, computer_guid=config['computer_id']) computer = slap.registerComputer(config['computer_id']) prefix = 'slappart' slap_config = { 'address': config['ipv4_address'], 'instance_root': config['instance_root'], 'netmask': '255.255.255.255', 'partition_list': [], 'reference': config['computer_id'], 'software_root': config['software_root'] } for i in xrange(0, int(config['partition_amount'])): partition_reference = '%s%s' % (prefix, i) partition_path = os.path.join(config['instance_root'], partition_reference) if not os.path.exists(partition_path): os.mkdir(partition_path) os.chmod(partition_path, 0750) slap_config['partition_list'].append({ 'address_list': [ { 'addr': config['ipv4_address'], 'netmask': '255.255.255.255' }, { 'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::' }, ], 'path': partition_path, 'reference': partition_reference, 'tap': {'name': partition_reference}}) computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps(slap_config)) return True
def getSlapStatus(config): """Return all Slapos Partitions with associate information""" slap = slapos.slap.slap() slap.initializeConnection(config['master_url']) partition_list = [] computer = slap.registerComputer(config['computer_id']) try: for partition in computer.getComputerPartitionList(): # Note: Internal use of API, as there is no reflexion interface in SLAP partition_list.append((partition.getId(), partition._connection_dict.copy())) except Exception: pass if partition_list: for i in xrange(0, int(config['partition_amount'])): slappart_id = '%s%s' % ("slappart", i) if not [x[0] for x in partition_list if slappart_id == x[0]]: partition_list.append((slappart_id, [])) return partition_list
def test_updateInstanceParameter(self): """Scenarion 5: Update parameters of current sofware profile""" self.setAccount() self.setupTestSoftware() #Set current projet and run Slapgrid-cp software = os.path.join(self.software, 'slaprunner-test') response = loadJson(self.app.post('/setCurrentProject', data=dict(path=software), follow_redirects=True)) self.assertEqual(response['result'], "") self.proxyStatus(True) #Send paramters for the instance parameterDict = dict(appname='slaprunnerTest', cacountry='France') parameterXml = '<?xml version="1.0" encoding="utf-8"?>\n<instance>' parameterXml += '<parameter id="appname">slaprunnerTest</parameter>\n' parameterXml += '<parameter id="cacountry">France</parameter>\n</instance>' software_type = 'production' response = loadJson(self.app.post('/saveParameterXml', data=dict(parameter=parameterXml, software_type=software_type), follow_redirects=True)) self.assertEqual(response['result'], "") slap = slapos.slap.slap() slap.initializeConnection(self.app.config['master_url']) computer = slap.registerComputer(self.app.config['computer_id']) partitionList = computer.getComputerPartitionList() self.assertNotEqual(partitionList, []) #Assume that the requested partition is partition 0 slapParameterDict = partitionList[0].getInstanceParameterDict() self.assertTrue('appname' in slapParameterDict) self.assertTrue('cacountry' in slapParameterDict) self.assertEqual(slapParameterDict['appname'], 'slaprunnerTest') self.assertEqual(slapParameterDict['cacountry'], 'France') self.assertEqual(slapParameterDict['slap_software_type'], 'production') #test getParameterXml for webrunner UI response = loadJson(self.app.get('/getParameterXml/xml')) self.assertEqual(parameterXml, response['result']) response = loadJson(self.app.get('/getParameterXml/dict')) self.assertEqual(parameterDict, response['result']['instance']) self.stopSlapproxy()
def initializeSlapOSControler(self, slapproxy_log=None, process_manager=None, reset_software=False, software_path_list=None): self.process_manager = process_manager self.software_path_list = software_path_list self.log('SlapOSControler, initialize, reset_software: %r' % reset_software) config = self.config slapos_config_dict = self.config.copy() slapos_config_dict.update(software_root=self.software_root, instance_root=self.instance_root, proxy_database=self.proxy_database) open(self.slapos_config, 'w').write( pkg_resources.resource_string('erp5.util.testnode', 'template/slapos.cfg.in') % slapos_config_dict) createFolder(self.software_root) createFolder(self.instance_root) # By erasing everything, we make sure that we are able to "update" # existing profiles. This is quite dirty way to do updates... if os.path.exists(self.proxy_database): os.unlink(self.proxy_database) kwargs = dict(close_fds=True, preexec_fn=os.setsid) if slapproxy_log is not None: slapproxy_log_fp = open(slapproxy_log, 'w') kwargs['stdout'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp proxy = subprocess.Popen( [config['slapproxy_binary'], self.slapos_config], **kwargs) process_manager.process_pid_set.add(proxy.pid) # XXX: dirty, giving some time for proxy to being able to accept # connections time.sleep(10) try: slap = slapos.slap.slap() self.slap = slap self.slap.initializeConnection(config['master_url']) # register software profile for path in self.software_path_list: slap.registerSupply().supply( path, computer_guid=config['computer_id']) computer = slap.registerComputer(config['computer_id']) except: self.log("SlapOSControler.initializeSlapOSControler, \ exception in registerSupply", exc_info=sys.exc_info()) raise ValueError("Unable to initializeSlapOSControler") # Reset all previously generated software if needed if reset_software: self._resetSoftware() instance_root = self.instance_root if os.path.exists(instance_root): # delete old paritions which may exists in order to not get its data # (ex. MySQL db content) from previous testnode's runs # In order to be able to change partition naming scheme, do this at # instance_root level (such change happened already, causing problems). shutil.rmtree(instance_root) if not (os.path.exists(instance_root)): os.mkdir(instance_root) for i in range(0, MAX_PARTIONS): # create partition and configure computer # XXX: at the moment all partitions do share same virtual interface address # this is not a problem as usually all services are on different ports partition_reference = '%s-%s' % (config['partition_reference'], i) partition_path = os.path.join(instance_root, partition_reference) if not (os.path.exists(partition_path)): os.mkdir(partition_path) os.chmod(partition_path, 0750) computer.updateConfiguration( xml_marshaller.xml_marshaller.dumps({ 'address': config['ipv4_address'], 'instance_root': instance_root, 'netmask': '255.255.255.255', 'partition_list': [{ 'address_list': [ { 'addr': config['ipv4_address'], 'netmask': '255.255.255.255' }, { 'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::' }, ], 'path': partition_path, 'reference': partition_reference, 'tap': { 'name': partition_reference }, }], 'reference': config['computer_id'], 'software_root': self.software_root }))
def initializeSlapOSControler(self, slapproxy_log=None, process_manager=None, reset_software=False, software_path_list=None): self.process_manager = process_manager self.software_path_list = software_path_list self.log('SlapOSControler, initialize, reset_software: %r' % reset_software) config = self.config slapos_config_dict = self.config.copy() slapos_config_dict.update(software_root=self.software_root, instance_root=self.instance_root, proxy_database=self.proxy_database) open(self.slapos_config, 'w').write(pkg_resources.resource_string( 'erp5.util.testnode', 'template/slapos.cfg.in') % slapos_config_dict) createFolder(self.software_root) createFolder(self.instance_root) # By erasing everything, we make sure that we are able to "update" # existing profiles. This is quite dirty way to do updates... if os.path.exists(self.proxy_database): os.unlink(self.proxy_database) kwargs = dict(close_fds=True, preexec_fn=os.setsid) if slapproxy_log is not None: slapproxy_log_fp = open(slapproxy_log, 'w') kwargs['stdout'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp proxy = subprocess.Popen([config['slapproxy_binary'], self.slapos_config], **kwargs) process_manager.process_pid_set.add(proxy.pid) # XXX: dirty, giving some time for proxy to being able to accept # connections time.sleep(10) try: slap = slapos.slap.slap() self.slap = slap self.slap.initializeConnection(config['master_url']) # register software profile for path in self.software_path_list: slap.registerSupply().supply( path, computer_guid=config['computer_id']) computer = slap.registerComputer(config['computer_id']) except: self.log("SlapOSControler.initializeSlapOSControler, \ exception in registerSupply", exc_info=sys.exc_info()) raise ValueError("Unable to initializeSlapOSControler") # Reset all previously generated software if needed if reset_software: self._resetSoftware() instance_root = self.instance_root if os.path.exists(instance_root): # delete old paritions which may exists in order to not get its data # (ex. MySQL db content) from previous testnode's runs # In order to be able to change partition naming scheme, do this at # instance_root level (such change happened already, causing problems). shutil.rmtree(instance_root) if not(os.path.exists(instance_root)): os.mkdir(instance_root) for i in range(0, MAX_PARTIONS): # create partition and configure computer # XXX: at the moment all partitions do share same virtual interface address # this is not a problem as usually all services are on different ports partition_reference = '%s-%s' %(config['partition_reference'], i) partition_path = os.path.join(instance_root, partition_reference) if not(os.path.exists(partition_path)): os.mkdir(partition_path) os.chmod(partition_path, 0750) computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({ 'address': config['ipv4_address'], 'instance_root': instance_root, 'netmask': '255.255.255.255', 'partition_list': [ {'address_list': [{'addr': config['ipv4_address'], 'netmask': '255.255.255.255'}, {'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::'},], 'path': partition_path, 'reference': partition_reference, 'tap': {'name': partition_reference},}], 'reference': config['computer_id'], 'software_root': self.software_root}))
def initializeSlapOSControler(self, slapproxy_log=None, process_manager=None, reset_software=False, software_path_list=None): self.process_manager = process_manager self.software_path_list = software_path_list logger.debug('SlapOSControler, initialize, reset_software: %r', reset_software) config = self.config slapos_config_dict = config.copy() slapos_config_dict.update(software_root=self.software_root, instance_root=self.instance_root, proxy_database=self.proxy_database, shared_part_list='\n '.join( self.shared_part_list)) with open(self.slapos_config, 'w') as f: f.write( pkg_resources.resource_string( 'erp5.util.testnode', 'template/slapos.cfg.in').decode() % slapos_config_dict) # By erasing everything, we make sure that we are able to "update" # existing profiles. This is quite dirty way to do updates... if os.path.exists(self.proxy_database): os.unlink(self.proxy_database) kwargs = dict(close_fds=True, preexec_fn=os.setsid) if slapproxy_log is not None: slapproxy_log_fp = open(slapproxy_log, 'w') kwargs['stdout'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp proxy = subprocess.Popen([ config['slapos_binary'], 'proxy', 'start', '--cfg', self.slapos_config ], **kwargs) process_manager.process_pid_set.add(proxy.pid) slap = self.slap = slapos.slap.slap() # Wait for proxy to accept connections retries = 0 while True: time.sleep(1) try: slap.initializeConnection(config['master_url']) computer = slap.registerComputer(config['computer_id']) # Call a method to ensure connection to master can be established computer.getComputerPartitionList() except slapos.slap.ConnectionError as e: retries += 1 if retries >= 60: raise logger.debug("Proxy still not started %s, retrying", e) else: break try: # register software profile for path in self.software_path_list: slap.registerSupply().supply( path, computer_guid=config['computer_id']) except Exception: logger.exception("SlapOSControler.initializeSlapOSControler") raise ValueError("Unable to registerSupply") # Reset all previously generated software if needed if reset_software: self._resetSoftware() else: createFolder(self.software_root) instance_root = self.instance_root # Delete any existing partition in order to not get its data (ex. # MySQL DB content) from previous runs. To support changes of partition # naming scheme (which already happened), do this at instance_root level. createFolder(instance_root, True) partition_list = [] for i in range(MAX_PARTITIONS): # create partition and configure computer # XXX: at the moment all partitions do share same virtual interface address # this is not a problem as usually all services are on different ports partition_reference = '%s-%s' % (config['partition_reference'], i) partition_path = os.path.join(instance_root, partition_reference) if not (os.path.exists(partition_path)): os.mkdir(partition_path) os.chmod(partition_path, 0o750) partition_list.append({ 'address_list': [ { 'addr': config['ipv4_address'], 'netmask': '255.255.255.255' }, { 'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::' }, ], 'path': partition_path, 'reference': partition_reference, 'tap': { 'name': partition_reference }, }) computer.updateConfiguration( xml_marshaller.xml_marshaller.dumps({ 'address': config['ipv4_address'], 'instance_root': instance_root, 'netmask': '255.255.255.255', 'partition_list': partition_list, 'reference': config['computer_id'], 'software_root': self.software_root }))
def __init__(self, config, log, slapproxy_log=None, process_manager=None, reset_software=False): log('SlapOSControler, initialize, reset_software: %r' % reset_software) self.log = log self.config = config self.process_manager = process_manager # By erasing everything, we make sure that we are able to "update" # existing profiles. This is quite dirty way to do updates... if os.path.exists(config['proxy_database']): os.unlink(config['proxy_database']) kwargs = dict(close_fds=True, preexec_fn=os.setsid) if slapproxy_log is not None: slapproxy_log_fp = open(slapproxy_log, 'w') kwargs['stdout'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp proxy = subprocess.Popen([config['slapproxy_binary'], config['slapos_config']], **kwargs) process_manager.process_pid_set.add(proxy.pid) # XXX: dirty, giving some time for proxy to being able to accept # connections time.sleep(10) slap = slapos.slap.slap() slap.initializeConnection(config['master_url']) # register software profile self.software_profile = config['custom_profile_path'] slap.registerSupply().supply( self.software_profile, computer_guid=config['computer_id']) computer = slap.registerComputer(config['computer_id']) # Reset all previously generated software if needed if reset_software: software_root = config['software_root'] log('SlapOSControler : GOING TO RESET ALL SOFTWARE') if os.path.exists(software_root): shutil.rmtree(software_root) os.mkdir(software_root) os.chmod(software_root, 0750) instance_root = config['instance_root'] if os.path.exists(instance_root): # delete old paritions which may exists in order to not get its data # (ex. MySQL db content) from previous testnode's runs # In order to be able to change partition naming scheme, do this at # instance_root level (such change happened already, causing problems). shutil.rmtree(instance_root) os.mkdir(instance_root) for i in range(0, MAX_PARTIONS): # create partition and configure computer # XXX: at the moment all partitions do share same virtual interface address # this is not a problem as usually all services are on different ports partition_reference = '%s-%s' %(config['partition_reference'], i) partition_path = os.path.join(instance_root, partition_reference) os.mkdir(partition_path) os.chmod(partition_path, 0750) computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({ 'address': config['ipv4_address'], 'instance_root': instance_root, 'netmask': '255.255.255.255', 'partition_list': [{'address_list': [{'addr': config['ipv4_address'], 'netmask': '255.255.255.255'}, {'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::'},], 'path': partition_path, 'reference': partition_reference, 'tap': {'name': partition_reference}, } ], 'reference': config['computer_id'], 'software_root': config['software_root']}))
def run(args): config = args[0] for k,v in config['environment'].iteritems(): os.environ[k] = v proxy = None slapgrid = None supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run', 'supervisord.pid') if os.path.exists(config['proxy_database']): os.unlink(config['proxy_database']) try: proxy = subprocess.Popen([config['slapproxy_binary'], config['slapos_config']], close_fds=True, preexec_fn=os.setsid) process_group_pid_list.append(proxy.pid) slap = slapos.slap.slap() slap.initializeConnection(config['master_url']) while True: try: slap.registerSupply().supply(config['profile_url'], computer_guid=config['computer_id']) except socket.error: time.sleep(1) pass else: break while True: slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc', config['slapos_config']], close_fds=True, preexec_fn=os.setsid) process_group_pid_list.append(slapgrid.pid) slapgrid.wait() if slapgrid.returncode == 0: print 'Software installed properly' break print 'Problem with software installation, trying again' time.sleep(600) computer = slap.registerComputer(config['computer_id']) partition_reference = config['partition_reference'] partition_path = os.path.join(config['instance_root'], partition_reference) if not os.path.exists(partition_path): os.mkdir(partition_path) os.chmod(partition_path, 0750) computer.updateConfiguration(xml_marshaller.dumps({ 'address': config['ipv4_address'], 'instance_root': config['instance_root'], 'netmask': '255.255.255.255', 'partition_list': [{'address_list': [{'addr': config['ipv4_address'], 'netmask': '255.255.255.255'}, {'addr': config['ipv6_address'], 'netmask': 'ffff:ffff:ffff::'}, ], 'path': partition_path, 'reference': partition_reference, 'tap': {'name': partition_reference}, } ], 'reference': config['computer_id'], 'software_root': config['software_root']})) slap.registerOpenOrder().request(config['profile_url'], partition_reference='testing partition', partition_parameter_kw=config['instance_dict']) slapgrid = subprocess.Popen([config['slapgrid_partition_binary'], '-vc', config['slapos_config']], close_fds=True, preexec_fn=os.setsid) slapgrid.wait() if slapgrid.returncode != 0: raise ValueError('Slapgrid instance failed') runUnitTest = os.path.join(partition_path, 'bin', 'runUnitTest') if not os.path.exists(runUnitTest): raise ValueError('No %r provided' % runUnitTest) except: try: if os.path.exists(supervisord_pid_file): os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM) except: pass raise finally: # Nice way to kill *everything* generated by run process -- process # groups working only in POSIX compilant systems # Exceptions are swallowed during cleanup phase if proxy is not None: os.killpg(proxy.pid, signal.SIGTERM) if os.path.exists(config['proxy_database']): os.unlink(config['proxy_database']) if slapgrid is not None and slapgrid.returncode is None: os.killpg(slapgrid.pid, signal.SIGTERM) try: bot_env = os.environ.copy() bot_env['PATH'] = ':'.join([config['bin_directory']] + bot_env['PATH'].split(':')) for l in config['bot_environment'].split(): k, v = l.split('=') bot_env[k] = v if subprocess.call([config['buildbot_binary'], 'create-slave', '-f', config['working_directory'], config['buildbot_host'], config['slave_name'], config['slave_password']]) != 0: raise ValueError('Buildbot call failed') process_command_list.append([config['buildbot_binary'], 'stop', config['working_directory']]) if os.path.exists(os.path.join(config['working_directory'], 'buildbot.tac.new')): tac = os.path.join(config['working_directory'], 'buildbot.tac') if os.path.exists(tac): os.unlink(tac) os.rename(os.path.join(config['working_directory'], 'buildbot.tac.new'), tac) if subprocess.call([config['buildbot_binary'], 'start', config['working_directory']], env=bot_env) != 0: raise ValueError('Issue during starting buildbot') while True: time.sleep(3600) finally: try: subprocess.call([config['buildbot_binary'], 'stop', config['working_directory']]) except: pass try: if os.path.exists(supervisord_pid_file): os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM) except: pass