def runTestSuite(self, node_test_suite, portal_url): config = self.testnode.config run_test_suite_path_list = glob.glob( self.getInstanceRoot(node_test_suite) + "/*/bin/runTestSuite") try: run_test_suite_path = min(run_test_suite_path_list) except ValueError: raise ValueError( 'No runTestSuite provided in installed partitions.') # Deal with Shebang size limitation invocation_list = dealShebang(run_test_suite_path) invocation_list += (run_test_suite_path, '--master_url', portal_url, '--revision', node_test_suite.revision, '--test_suite', node_test_suite.test_suite, '--test_suite_title', node_test_suite.test_suite_title) soft = config['slapos_directory'] + '/soft/' software_list = [soft + md5digest(x) for x in config['software_list']] PATH = os.getenv('PATH', '') PATH = ':'.join(x + '/bin' for x in software_list) + (PATH and ':' + PATH) supported_parameter_set = set( self.testnode.process_manager.getSupportedParameterList( run_test_suite_path)) def path(name, compat): # BBB path, = filter( os.path.exists, (base + relative for relative in ('/bin/' + name, '/parts/' + compat) for base in software_list)) return path for option, value in ( ('--firefox_bin', lambda: path('firefox', 'firefox/firefox-slapos')), ('--frontend_url', lambda: config['frontend_url']), ('--node_quantity', lambda: config['node_quantity']), ('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')), ('--project_title', lambda: node_test_suite.project_title), ): if option in supported_parameter_set: invocation_list += option, value() # TODO : include testnode correction ( b111682f14890bf ) if hasattr(node_test_suite, 'additional_bt5_repository_id'): additional_bt5_path = os.path.join( node_test_suite.working_directory, node_test_suite.additional_bt5_repository_id) invocation_list.extend(["--bt5_path", additional_bt5_path]) # From this point, test runner becomes responsible for updating test # result. We only do cleanup if the test runner itself is not able # to run. createFolder(node_test_suite.test_suite_directory, clean=True) self.testnode.process_manager.spawn( *invocation_list, PATH=PATH, cwd=node_test_suite.test_suite_directory, log_prefix='runTestSuite', get_output=False)
def createPartition( self, software_release_url, partition_id=None, slap_computer_partition=None, retention_delay=None, ): """ Create a partition, and return a Partition object created from dummy parameters. """ # XXX dirty, should disappear when Partition is cleaned up software_path = os.path.join( self.software_root, utils.md5digest(software_release_url) ) if partition_id is None: partition_id = 'mypartition' if slap_computer_partition is None: slap_computer_partition = SlapComputerPartition( computer_id='bidon', partition_id=partition_id) instance_path = os.path.join(self.instance_root, partition_id) os.mkdir(instance_path) os.chmod(instance_path, 0o750) supervisor_configuration_path = os.path.join( self.instance_root, 'supervisor') os.mkdir(supervisor_configuration_path) partition = Partition( software_path=software_path, instance_path=instance_path, supervisord_partition_configuration_path=os.path.join( supervisor_configuration_path, partition_id), supervisord_socket=os.path.join( supervisor_configuration_path, 'supervisor.sock'), computer_partition=slap_computer_partition, computer_id='bidon', partition_id=partition_id, server_url='bidon', software_release_url=software_release_url, buildout=self.buildout, logger=logging.getLogger(), ) partition.updateSupervisor = FakeCallAndNoop if retention_delay: partition.retention_delay = retention_delay return partition
def config_SR_folder(config): """Create a symbolik link for each folder in software folder. That allows the user to customize software release folder""" config_name = 'slaprunner.config' def link_to_folder(name, folder): destination = os.path.join(config['software_link'], name) source = os.path.join(config['software_root'], folder) cfg = os.path.join(destination, config_name) #create symlink if os.path.lexists(destination): os.remove(destination) os.symlink(source, destination) #write config file if os.path.exists(source): with open(cfg, 'w') as cf: cf.write(current_project + '#' + folder) # First create the link for current project current_project = open(os.path.join(config['etc_dir'], ".project")).read() profile = getCurrentSoftwareReleaseProfile(config) if current_project[-1] == '/': current_project = current_project[:-1] name = current_project.split('/')[-1] md5sum = md5digest(profile) link_to_folder(name, md5sum) # check other links # XXX-Marco do not shadow 'list' list = [] for path in os.listdir(config['software_link']): cfg_path = os.path.join(config['software_link'], path, config_name) if os.path.exists(cfg_path): cfg = open(cfg_path).read().split("#") if len(cfg) != 2: continue # there is a broken config file list.append(cfg[1]) if os.path.exists(config['software_root']): folder_list = os.listdir(config['software_root']) else: return if not folder_list: return for folder in folder_list: if folder in list: continue # this folder is already registered else: link_to_folder(folder, folder)
def __init__(self, url, software_root, buildout, logger, signature_private_key_file=None, signature_certificate_list=None, upload_cache_url=None, upload_dir_url=None, shacache_ca_file=None, shacache_cert_file=None, shacache_key_file=None, shadir_ca_file=None, shadir_cert_file=None, shadir_key_file=None, download_binary_cache_url=None, upload_binary_cache_url=None, download_binary_dir_url=None, upload_binary_dir_url=None, download_from_binary_cache_url_blacklist=None, upload_to_binary_cache_url_blacklist=None, software_min_free_space=None): """Initialisation of class parameters """ if download_from_binary_cache_url_blacklist is None: download_from_binary_cache_url_blacklist = [] if upload_to_binary_cache_url_blacklist is None: upload_to_binary_cache_url_blacklist = [] self.url = url self.software_root = software_root self.software_url_hash = md5digest(self.url) self.software_path = os.path.join(self.software_root, self.software_url_hash) self.buildout = buildout self.logger = logger self.signature_private_key_file = signature_private_key_file self.signature_certificate_list = signature_certificate_list self.upload_cache_url = upload_cache_url self.upload_dir_url = upload_dir_url self.shacache_ca_file = shacache_ca_file self.shacache_cert_file = shacache_cert_file self.shacache_key_file = shacache_key_file self.shadir_ca_file = shadir_ca_file self.shadir_cert_file = shadir_cert_file self.shadir_key_file = shadir_key_file self.download_binary_cache_url = download_binary_cache_url self.upload_binary_cache_url = upload_binary_cache_url self.download_binary_dir_url = download_binary_dir_url self.upload_binary_dir_url = upload_binary_dir_url self.download_from_binary_cache_url_blacklist = \ download_from_binary_cache_url_blacklist self.upload_to_binary_cache_url_blacklist = \ upload_to_binary_cache_url_blacklist self.software_min_free_space = software_min_free_space
def createSoftware(self, url=None, empty=False): """ Create an empty software, and return a Software object from dummy parameters. """ if url is None: url = 'mysoftware' software_path = os.path.join(self.software_root, utils.md5digest(url)) os.mkdir(software_path) if not empty: # Populate the Software Release directory so that it is "complete" and # "working" from a slapos point of view. open(os.path.join(software_path, 'instance.cfg'), 'w').close() return Software( url=url, software_root=self.software_root, buildout=self.buildout, logger=logging.getLogger(), )
def agregateAndSendUsage(self): """Will agregate usage from each Computer Partition. """ # Prepares environment self.checkEnvironmentAndCreateStructure() self._launchSupervisord() slap_computer_usage = self.slap.registerComputer(self.computer_id) computer_partition_usage_list = [] self.logger.info('Aggregating and sending usage reports...') #We retrieve XSD models try: computer_consumption_model = \ pkg_resources.resource_string( 'slapos.slap', 'doc/computer_consumption.xsd') except IOError: computer_consumption_model = \ pkg_resources.resource_string( __name__, '../../../../slapos/slap/doc/computer_consumption.xsd') try: partition_consumption_model = \ pkg_resources.resource_string( 'slapos.slap', 'doc/partition_consumption.xsd') except IOError: partition_consumption_model = \ pkg_resources.resource_string( __name__, '../../../../slapos/slap/doc/partition_consumption.xsd') clean_run = True # Loop over the different computer partitions computer_partition_list = self.FilterComputerPartitionList( slap_computer_usage.getComputerPartitionList()) for computer_partition in computer_partition_list: try: computer_partition_id = computer_partition.getId() #We want to execute all the script in the report folder instance_path = os.path.join(self.instance_root, computer_partition.getId()) report_path = os.path.join(instance_path, 'etc', 'report') if os.path.isdir(report_path): script_list_to_run = os.listdir(report_path) else: script_list_to_run = [] #We now generate the pseudorandom name for the xml file # and we add it in the invocation_list f = tempfile.NamedTemporaryFile() name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name)) path_to_slapreport = os.path.join(instance_path, 'var', 'xml_report', name_xml) failed_script_list = [] for script in script_list_to_run: invocation_list = [] invocation_list.append(os.path.join(instance_path, 'etc', 'report', script)) #We add the xml_file name to the invocation_list #f = tempfile.NamedTemporaryFile() #name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name)) #path_to_slapreport = os.path.join(instance_path, 'var', name_xml) invocation_list.append(path_to_slapreport) #Dropping privileges uid, gid = None, None stat_info = os.stat(instance_path) #stat sys call to get statistics informations uid = stat_info.st_uid gid = stat_info.st_gid process_handler = SlapPopen(invocation_list, preexec_fn=lambda: dropPrivileges(uid, gid, logger=self.logger), cwd=os.path.join(instance_path, 'etc', 'report'), env=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, logger=self.logger) if process_handler.returncode is None: process_handler.kill() if process_handler.returncode != 0: clean_run = False failed_script_list.append("Script %r failed." % script) self.logger.warning('Failed to run %r' % invocation_list) if len(failed_script_list): computer_partition.error('\n'.join(failed_script_list), logger=self.logger) # Whatever happens, don't stop processing other instances except Exception: self.logger.exception('Cannot run usage script(s) for %r:' % computer_partition.getId()) #Now we loop through the different computer partitions to report report_usage_issue_cp_list = [] for computer_partition in computer_partition_list: try: filename_delete_list = [] computer_partition_id = computer_partition.getId() instance_path = os.path.join(self.instance_root, computer_partition_id) dir_reports = os.path.join(instance_path, 'var', 'xml_report') #The directory xml_report contain a number of files equal #to the number of software instance running inside the same partition if os.path.isdir(dir_reports): filename_list = os.listdir(dir_reports) else: filename_list = [] #self.logger.debug('name List %s' % filename_list) for filename in filename_list: file_path = os.path.join(dir_reports, filename) if os.path.exists(file_path): usage = open(file_path, 'r').read() #We check the validity of xml content of each reports if not self.validateXML(usage, partition_consumption_model): self.logger.info('WARNING: The XML file %s generated by slapreport is ' 'not valid - This report is left as is at %s where you can ' 'inspect what went wrong ' % (filename, dir_reports)) # Warn the SlapOS Master that a partition generates corrupted xml # report else: computer_partition_usage = self.slap.registerComputerPartition( self.computer_id, computer_partition_id) computer_partition_usage.setUsage(usage) computer_partition_usage_list.append(computer_partition_usage) filename_delete_list.append(filename) else: self.logger.debug('Usage report %r not found, ignored' % file_path) #After sending the aggregated file we remove all the valid xml reports for filename in filename_delete_list: os.remove(os.path.join(dir_reports, filename)) # Whatever happens, don't stop processing other instances except Exception: self.logger.exception('Cannot run usage script(s) for %r:' % computer_partition.getId()) for computer_partition_usage in computer_partition_usage_list: self.logger.info('computer_partition_usage_list: %s - %s' % (computer_partition_usage.usage, computer_partition_usage.getId())) #If there is, at least, one report if computer_partition_usage_list != []: try: #We generate the final XML report with asXML method computer_consumption = self.asXML(computer_partition_usage_list) self.logger.info('Final xml report: %s' % computer_consumption) #We test the XML report before sending it if self.validateXML(computer_consumption, computer_consumption_model): self.logger.info('XML file generated by asXML is valid') slap_computer_usage.reportUsage(computer_consumption) else: self.logger.info('XML file generated by asXML is not valid !') raise ValueError('XML file generated by asXML is not valid !') except Exception: issue = "Cannot report usage for %r: %s" % ( computer_partition.getId(), traceback.format_exc()) self.logger.info(issue) computer_partition.error(issue, logger=self.logger) report_usage_issue_cp_list.append(computer_partition_id) for computer_partition in computer_partition_list: if computer_partition.getState() == COMPUTER_PARTITION_DESTROYED_STATE: try: computer_partition_id = computer_partition.getId() try: software_url = computer_partition.getSoftwareRelease().getURI() software_path = os.path.join(self.software_root, md5digest(software_url)) except (NotFoundError, TypeError): software_url = None software_path = None local_partition = Partition( software_path=software_path, instance_path=os.path.join(self.instance_root, computer_partition.getId()), supervisord_partition_configuration_path=os.path.join( self.supervisord_configuration_directory, '%s.conf' % computer_partition_id), supervisord_socket=self.supervisord_socket, computer_partition=computer_partition, computer_id=self.computer_id, partition_id=computer_partition_id, server_url=self.master_url, software_release_url=software_url, certificate_repository_path=self.certificate_repository_path, buildout=self.buildout, logger=self.logger) local_partition.stop() try: computer_partition.stopped() except (SystemExit, KeyboardInterrupt): computer_partition.error(traceback.format_exc(), logger=self.logger) raise except Exception: pass if computer_partition.getId() in report_usage_issue_cp_list: self.logger.info('Ignoring destruction of %r, as no report usage was sent' % computer_partition.getId()) continue local_partition.destroy() except (SystemExit, KeyboardInterrupt): computer_partition.error(traceback.format_exc(), logger=self.logger) raise except Exception: clean_run = False self.logger.exception('') exc = traceback.format_exc() computer_partition.error(exc, logger=self.logger) try: computer_partition.destroyed() except NotFoundError: self.logger.debug('Ignored slap error while trying to inform about ' 'destroying not fully configured Computer Partition %r' % computer_partition.getId()) except ServerError as server_error: self.logger.debug('Ignored server error while trying to inform about ' 'destroying Computer Partition %r. Error is:\n%r' % (computer_partition.getId(), server_error.args[0])) self.logger.info('Finished usage reports.') # Return success value if not clean_run: return SLAPGRID_FAIL return SLAPGRID_SUCCESS
def processComputerPartition(self, computer_partition): """ Process a Computer Partition, depending on its state """ computer_partition_id = computer_partition.getId() # Sanity checks before processing # Those values should not be None or empty string or any falsy value if not computer_partition_id: raise ValueError('Computer Partition id is empty.') # Check if we defined explicit list of partitions to process. # If so, if current partition not in this list, skip. if len(self.computer_partition_filter_list) > 0 and \ (computer_partition_id not in self.computer_partition_filter_list): return self.logger.info('Processing Computer Partition %s.' % computer_partition_id) instance_path = os.path.join(self.instance_root, computer_partition_id) # Try to get partition timestamp (last modification date) timestamp_path = os.path.join( instance_path, COMPUTER_PARTITION_TIMESTAMP_FILENAME ) parameter_dict = computer_partition.getInstanceParameterDict() if 'timestamp' in parameter_dict: timestamp = parameter_dict['timestamp'] else: timestamp = None try: software_url = computer_partition.getSoftwareRelease().getURI() except NotFoundError: # Problem with instance: SR URI not set. # Try to process it anyway, it may need to be deleted. software_url = None try: software_path = os.path.join(self.software_root, md5digest(software_url)) except TypeError: # Problem with instance: SR URI not set. # Try to process it anyway, it may need to be deleted. software_path = None periodicity = self.maximum_periodicity if software_path: periodicity_path = os.path.join(software_path, 'periodicity') if os.path.exists(periodicity_path): try: periodicity = int(open(periodicity_path).read()) except ValueError: os.remove(periodicity_path) self.logger.exception('') # Check if timestamp from server is more recent than local one. # If not: it's not worth processing this partition (nothing has # changed). if (computer_partition_id not in self.computer_partition_filter_list and not self.develop and os.path.exists(timestamp_path)): old_timestamp = open(timestamp_path).read() last_runtime = int(os.path.getmtime(timestamp_path)) if timestamp: try: if periodicity == 0: os.remove(timestamp_path) elif int(timestamp) <= int(old_timestamp): if computer_partition.getState() != COMPUTER_PARTITION_STARTED_STATE: return # Check periodicity, i.e if periodicity is one day, partition # should be processed at least every day. # Only do it for "started" instances if int(time.time()) <= (last_runtime + periodicity) or periodicity < 0: self.logger.info('Partition already up-to-date, skipping.') return else: # Periodicity forced processing this partition. Removing # the timestamp file in case it fails. os.remove(timestamp_path) except ValueError: os.remove(timestamp_path) self.logger.exception('') self.logger.info(' Software URL: %s' % software_url) self.logger.info(' Software path: %s' % software_path) self.logger.info(' Instance path: %s' % instance_path) local_partition = Partition( software_path=software_path, instance_path=instance_path, supervisord_partition_configuration_path=os.path.join( self.supervisord_configuration_directory, '%s.conf' % computer_partition_id), supervisord_socket=self.supervisord_socket, computer_partition=computer_partition, computer_id=self.computer_id, partition_id=computer_partition_id, server_url=self.master_url, software_release_url=software_url, certificate_repository_path=self.certificate_repository_path, buildout=self.buildout, logger=self.logger) computer_partition_state = computer_partition.getState() if computer_partition_state == COMPUTER_PARTITION_STARTED_STATE: local_partition.install() computer_partition.available() local_partition.start() self._checkPromises(computer_partition) computer_partition.started() elif computer_partition_state == COMPUTER_PARTITION_STOPPED_STATE: try: local_partition.install() computer_partition.available() finally: # Instance has to be stopped even if buildout/reporting is wrong. local_partition.stop() computer_partition.stopped() elif computer_partition_state == COMPUTER_PARTITION_DESTROYED_STATE: local_partition.stop() try: computer_partition.stopped() except (SystemExit, KeyboardInterrupt): computer_partition.error(traceback.format_exc(), logger=self.logger) raise except Exception: pass else: error_string = "Computer Partition %r has unsupported state: %s" % \ (computer_partition_id, computer_partition_state) computer_partition.error(error_string, logger=self.logger) raise NotImplementedError(error_string) # If partition has been successfully processed, write timestamp if timestamp: timestamp_path = os.path.join( instance_path, COMPUTER_PARTITION_TIMESTAMP_FILENAME ) open(timestamp_path, 'w').write(timestamp)
def processSoftwareReleaseList(self): """Will process each Software Release. """ self.checkEnvironmentAndCreateStructure() self.logger.info('Processing software releases...') # Boolean to know if every instance has correctly been deployed clean_run = True for software_release in self.computer.getSoftwareReleaseList(): state = software_release.getState() try: software_release_uri = software_release.getURI() url_hash = md5digest(software_release_uri) software_path = os.path.join(self.software_root, url_hash) software = Software(url=software_release_uri, software_root=self.software_root, buildout=self.buildout, logger=self.logger, signature_private_key_file=self.signature_private_key_file, signature_certificate_list=self.signature_certificate_list, download_binary_cache_url=self.download_binary_cache_url, upload_binary_cache_url=self.upload_binary_cache_url, download_from_binary_cache_url_blacklist= self.download_from_binary_cache_url_blacklist, upload_to_binary_cache_url_blacklist= self.upload_to_binary_cache_url_blacklist, upload_cache_url=self.upload_cache_url, download_binary_dir_url=self.download_binary_dir_url, upload_binary_dir_url=self.upload_binary_dir_url, upload_dir_url=self.upload_dir_url, shacache_cert_file=self.shacache_cert_file, shacache_key_file=self.shacache_key_file, shadir_cert_file=self.shadir_cert_file, shadir_key_file=self.shadir_key_file) if state == 'available': completed_tag = os.path.join(software_path, '.completed') if (self.develop or (not os.path.exists(completed_tag) and len(self.software_release_filter_list) == 0) or url_hash in self.software_release_filter_list or url_hash in (md5digest(uri) for uri in self.software_release_filter_list)): try: software_release.building() except NotFoundError: pass software.install() with open(completed_tag, 'w') as fout: fout.write(time.asctime()) elif state == 'destroyed': if os.path.exists(software_path): self.logger.info('Destroying %r...' % software_release_uri) software.destroy() self.logger.info('Destroyed %r.' % software_release_uri) # Send log before exiting except (SystemExit, KeyboardInterrupt): software_release.error(traceback.format_exc(), logger=self.logger) raise # Buildout failed: send log but don't print it to output (already done) except BuildoutFailedError as exc: clean_run = False try: software_release.error(exc, logger=self.logger) except (SystemExit, KeyboardInterrupt): raise except Exception: self.logger.exception('Problem while reporting error, continuing:') # For everything else: log it, send it, continue. except Exception: self.logger.exception('') software_release.error(traceback.format_exc(), logger=self.logger) clean_run = False else: if state == 'available': try: software_release.available() except NotFoundError: pass elif state == 'destroyed': try: software_release.destroyed() except (NotFoundError, ServerError): self.logger.exception('') self.logger.info('Finished software releases.') # Return success value if not clean_run: return SLAPGRID_FAIL return SLAPGRID_SUCCESS
def processComputerPartition(self, computer_partition): """ Process a Computer Partition, depending on its state """ computer_partition_id = computer_partition.getId() # Sanity checks before processing # Those values should not be None or empty string or any falsy value if not computer_partition_id: raise ValueError('Computer Partition id is empty.') # Check if we defined explicit list of partitions to process. # If so, if current partition not in this list, skip. if len(self.computer_partition_filter_list) > 0 and \ (computer_partition_id not in self.computer_partition_filter_list): return self.logger.debug('Check if %s requires processing...' % computer_partition_id) instance_path = os.path.join(self.instance_root, computer_partition_id) # Try to get partition timestamp (last modification date) timestamp_path = os.path.join( instance_path, COMPUTER_PARTITION_TIMESTAMP_FILENAME ) parameter_dict = computer_partition.getInstanceParameterDict() if 'timestamp' in parameter_dict: timestamp = parameter_dict['timestamp'] else: timestamp = None try: software_url = computer_partition.getSoftwareRelease().getURI() except NotFoundError: # Problem with instance: SR URI not set. # Try to process it anyway, it may need to be deleted. software_url = None try: software_path = os.path.join(self.software_root, md5digest(software_url)) except TypeError: # Problem with instance: SR URI not set. # Try to process it anyway, it may need to be deleted. software_path = None periodicity = self.maximum_periodicity if software_path: periodicity_path = os.path.join(software_path, 'periodicity') if os.path.exists(periodicity_path): try: periodicity = int(open(periodicity_path).read()) except ValueError: os.remove(periodicity_path) self.logger.exception('') # Check if timestamp from server is more recent than local one. # If not: it's not worth processing this partition (nothing has # changed). if (computer_partition_id not in self.computer_partition_filter_list and not self.develop and os.path.exists(timestamp_path)): old_timestamp = open(timestamp_path).read() last_runtime = int(os.path.getmtime(timestamp_path)) if timestamp: try: if periodicity == 0: os.remove(timestamp_path) elif int(timestamp) <= int(old_timestamp): # Check periodicity, i.e if periodicity is one day, partition # should be processed at least every day. if int(time.time()) <= (last_runtime + periodicity) or periodicity < 0: self.logger.debug('Partition already up-to-date, skipping.') return else: # Periodicity forced processing this partition. Removing # the timestamp file in case it fails. os.remove(timestamp_path) except ValueError: os.remove(timestamp_path) self.logger.exception('') # Include Partition Logging log_folder_path = "%s/.slapgrid/log" % instance_path mkdir_p(log_folder_path) partition_file_handler = logging.FileHandler( filename="%s/instance.log" % (log_folder_path) ) stat_info = os.stat(instance_path) chownDirectory("%s/.slapgrid" % instance_path, uid=stat_info.st_uid, gid=stat_info.st_gid) formatter = logging.Formatter( '[%(asctime)s] %(levelname)-8s %(name)s %(message)s') partition_file_handler.setFormatter(formatter) self.logger.addHandler(partition_file_handler) try: self.logger.info('Processing Computer Partition %s.' % computer_partition_id) self.logger.info(' Software URL: %s' % software_url) self.logger.info(' Software path: %s' % software_path) self.logger.info(' Instance path: %s' % instance_path) filter_dict = getattr(computer_partition, '_filter_dict', None) if filter_dict: retention_delay = filter_dict.get('retention_delay', '0') else: retention_delay = '0' local_partition = Partition( software_path=software_path, instance_path=instance_path, supervisord_partition_configuration_path=os.path.join( _getSupervisordConfigurationDirectory(self.instance_root), '%s.conf' % computer_partition_id), supervisord_socket=self.supervisord_socket, computer_partition=computer_partition, computer_id=self.computer_id, partition_id=computer_partition_id, server_url=self.master_url, software_release_url=software_url, certificate_repository_path=self.certificate_repository_path, buildout=self.buildout, logger=self.logger, retention_delay=retention_delay, instance_min_free_space=self.instance_min_free_space, instance_storage_home=self.instance_storage_home, ipv4_global_network=self.ipv4_global_network, ) computer_partition_state = computer_partition.getState() # XXX this line breaks 37 tests # self.logger.info(' Instance type: %s' % computer_partition.getType()) self.logger.info(' Instance status: %s' % computer_partition_state) if computer_partition_state == COMPUTER_PARTITION_STARTED_STATE: local_partition.install() computer_partition.available() local_partition.start() self._checkPromises(computer_partition) computer_partition.started() elif computer_partition_state == COMPUTER_PARTITION_STOPPED_STATE: try: # We want to process the partition, even if stopped, because it should # propagate the state to children if any. local_partition.install() computer_partition.available() finally: # Instance has to be stopped even if buildout/reporting is wrong. local_partition.stop() computer_partition.stopped() elif computer_partition_state == COMPUTER_PARTITION_DESTROYED_STATE: local_partition.stop() try: computer_partition.stopped() except (SystemExit, KeyboardInterrupt): computer_partition.error(traceback.format_exc(), logger=self.logger) raise except Exception: pass else: error_string = "Computer Partition %r has unsupported state: %s" % \ (computer_partition_id, computer_partition_state) computer_partition.error(error_string, logger=self.logger) raise NotImplementedError(error_string) finally: self.logger.removeHandler(partition_file_handler) # If partition has been successfully processed, write timestamp if timestamp: open(timestamp_path, 'w').write(timestamp)
def runTestSuite(self, node_test_suite, portal_url): config = self.testnode.config run_test_suite_path_list = glob.glob( self.getInstanceRoot(node_test_suite) + "/*/bin/runTestSuite") try: run_test_suite_path = min(run_test_suite_path_list) except ValueError: raise ValueError( 'No runTestSuite provided in installed partitions.') # Deal with Shebang size limitation invocation_list = dealShebang(run_test_suite_path) invocation_list += (run_test_suite_path, '--master_url', portal_url, '--revision', node_test_suite.revision, '--test_node_title', config['test_node_title'], '--test_suite', node_test_suite.test_suite, '--test_suite_title', node_test_suite.test_suite_title) soft = config['slapos_directory'] + '/soft/' software_list = [soft + md5digest(x) for x in config['software_list']] PATH = os.getenv('PATH', '') PATH = ':'.join(x + '/bin' for x in software_list) + (PATH and ':' + PATH) SLAPOS_TEST_SHARED_PART_LIST = os.pathsep.join( self._getSlapOSControler(node_test_suite.working_directory, True).shared_part_list) SLAPOS_TEST_LOG_DIRECTORY = node_test_suite.log_folder_path supported_parameter_set = set( self.testnode.process_manager.getSupportedParameterList( run_test_suite_path)) def path(name, compat): # BBB path, = filter( os.path.exists, (base + relative for relative in ('/bin/' + name, '/parts/' + compat) for base in software_list)) return path for option, value in ( ('--firefox_bin', lambda: path('firefox', 'firefox/firefox-slapos')), ('--frontend_url', lambda: config['frontend_url']), ('--node_quantity', lambda: config['node_quantity']), ('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')), ('--project_title', lambda: node_test_suite.project_title), ('--shared_part_list', lambda: SLAPOS_TEST_SHARED_PART_LIST), ('--log_directory', lambda: SLAPOS_TEST_LOG_DIRECTORY), ): if option in supported_parameter_set: invocation_list += option, value() # TODO : include testnode correction ( b111682f14890bf ) if hasattr(node_test_suite, 'additional_bt5_repository_id'): additional_bt5_path = os.path.join( node_test_suite.working_directory, node_test_suite.additional_bt5_repository_id) invocation_list.extend(["--bt5_path", additional_bt5_path]) # From this point, test runner becomes responsible for updating test # result. We only do cleanup if the test runner itself is not able # to run. createFolder(node_test_suite.test_suite_directory, clean=True) # Log the actual command with root logger root_logger = logging.getLogger() root_logger.info("Running test suite with: %s", format_command(*invocation_list, PATH=PATH)) def hide_distributor_url(s): # type: (bytes) -> bytes return s.replace(portal_url.encode('utf-8'), b'$DISTRIBUTOR_URL') self.testnode.process_manager.spawn( *invocation_list, PATH=PATH, SLAPOS_TEST_SHARED_PART_LIST=SLAPOS_TEST_SHARED_PART_LIST, SLAPOS_TEST_LOG_DIRECTORY=SLAPOS_TEST_LOG_DIRECTORY, cwd=node_test_suite.test_suite_directory, log_prefix='runTestSuite', output_replacers=(hide_distributor_url, ), get_output=False)