def setUp(self): self.sandbox_path = tempfile.mkdtemp( prefix="jens_sandbox_%s-" % self._testMethodName, suffix='-' + str(time.time())) init_sandbox(self.sandbox_path) self.keep_sandbox = bool(os.getenv('JENS_TEST_KEEP_SANDBOX', False)) self.debug_level = JensTestCase.DEFAULT_DEBUG_LEVEL self.config_file_path = "%s/etc/main.conf" % self.sandbox_path config_file = open(self.config_file_path, 'w+') config_file.write(BASE_CONFIG.substitute( sandbox=self.sandbox_path, hashprefix=JensTestCase.COMMIT_PREFIX, debuglevel=self.debug_level, mandatory_branches=','.join(JensTestCase.MANDATORY_BRANCHES))) config_file.close() root = logging.getLogger() for handler in root.handlers[:]: root.removeHandler(handler) for _filter in root.filters[:]: root.removeFilter(_filter) self.settings = Settings("jens-test") self.settings.parse_config(self.config_file_path) self.lock = JensLockFactory.make_lock(self.settings)
def _queue_item(item): settings = Settings() try: queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA) except OSError, error: raise JensMessagingError("Failed to create Queue object (%s)" % error)
class JensTestCase(unittest.TestCase): COMMIT_PREFIX = 'commit/' DEFAULT_DEBUG_LEVEL = 'DEBUG' MANDATORY_BRANCHES = ['master', 'qa'] def setUp(self): self.sandbox_path = tempfile.mkdtemp( prefix="jens_sandbox_%s-" % self._testMethodName, suffix='-' + str(time.time())) init_sandbox(self.sandbox_path) self.keep_sandbox = bool(os.getenv('JENS_TEST_KEEP_SANDBOX', False)) self.debug_level = JensTestCase.DEFAULT_DEBUG_LEVEL self.config_file_path = "%s/etc/main.conf" % self.sandbox_path config_file = open(self.config_file_path, 'w+') config_file.write(BASE_CONFIG.substitute( sandbox=self.sandbox_path, hashprefix=JensTestCase.COMMIT_PREFIX, debuglevel=self.debug_level, mandatory_branches=','.join(JensTestCase.MANDATORY_BRANCHES))) config_file.close() root = logging.getLogger() for handler in root.handlers[:]: root.removeHandler(handler) for _filter in root.filters[:]: root.removeFilter(_filter) self.settings = Settings("jens-test") self.settings.parse_config(self.config_file_path) self.lock = JensLockFactory.make_lock(self.settings) def tearDown(self): if self.keep_sandbox: print("Sandbox kept in", self.sandbox_path) else: destroy_sandbox(self.sandbox_path) def assertLogNoErrors(self): try: self.log except AttributeError: self.log = open("%s/jens-test.log" % self.settings.LOGDIR) for line in self.log.readlines(): if re.match(r'.+ERROR.+', line): raise AssertionError(line) def assertLogErrors(self, errorRegexp=None): try: self.log except AttributeError: self.log = open("%s/jens-test.log" % self.settings.LOGDIR) found = False regexp = r'.+ERROR.+' if errorRegexp is not None: regexp = r'.+ERROR.+%s.+' % errorRegexp for line in self.log.readlines(): if re.match(regexp, line): found = True if not found: raise AssertionError("There should be errors") def assertClone(self, identifier, pointsto=None): partition, element, dirname = identifier.split('/') path = "%s/%s/%s/%s" % (self.settings.CLONEDIR, \ partition, element, dirname) if not os.path.isdir(path): raise AssertionError("Clone '%s' not found" % path) if not os.path.isdir("%s/code" % path): raise AssertionError("Clone '%s' does not have code dir" % path) if not os.path.isdir("%s/data" % path): raise AssertionError("Clone '%s' does not have data dir" % path) inventory = get_inventory() self.assertTrue(partition in inventory) self.assertTrue(element in inventory[partition]) refname = dirname_to_refname(dirname) self.assertTrue(refname in inventory[partition][element]) if pointsto is not None: self.assertEqual(get_repository_head(path), pointsto) def assertCloneFileExists(self, identifier, fname): partition, element, dirname = identifier.split('/') path = "%s/%s/%s/%s/%s" % (self.settings.CLONEDIR, \ partition, element, dirname, fname) if not os.path.isfile(path): raise AssertionError("File '%s' not found" % path) def assertNotClone(self, identifier): try: self.assertClone(identifier) except AssertionError: return raise AssertionError("Clone '%s' seems present" % identifier) def assertBare(self, identifier): partition, element = identifier.split('/') path = "%s/%s/%s" % (self.settings.BAREDIR, \ partition, element) if not os.path.isdir(path): raise AssertionError("Bare '%s' not found" % path) if not os.path.isfile("%s/HEAD" % path): raise AssertionError("Bare '%s' does not have HEAD" % path) def assertNotBare(self, identifier): try: self.assertBare(identifier) except AssertionError: return raise AssertionError("Bare '%s' seems present" % identifier) def assertEnvironmentLinks(self, environment): base_path = "%s/%s" % (self.settings.ENVIRONMENTSDIR, environment) common_hieradata_items = set(self.settings.COMMON_HIERADATA_ITEMS) if not os.path.isdir(base_path): raise AssertionError("Environment '%s' not present" % environment) for path, dirs, files in os.walk(base_path): if path.endswith('hieradata'): self.assertTrue(common_hieradata_items.issubset(set(dirs + files))) for file in files + dirs: file_apath = "%s/%s" % (path, file) if os.path.islink(file_apath): if not self._verify_link(path, file_apath): raise AssertionError("Environment '%s' -- '%s' link broken" % \ (environment, file_apath)) def assertEnvironmentHasAConfigFile(self, environment): # https://docs.puppetlabs.com/puppet/latest/reference/config_file_environment.html conf_file_path = "%s/%s/environment.conf" % \ (self.settings.ENVIRONMENTSDIR, environment) if not os.path.isfile(conf_file_path): raise AssertionError("Environment '%s' doesn't have a config file" % environment) if not os.stat(conf_file_path).st_size > 0: raise AssertionError("Environment '%s''s config file seems empty" % environment) def assertEnvironmentHasAConfigFileAndParserSet(self, environment, parser): conf_file_path = "%s/%s/environment.conf" % \ (self.settings.ENVIRONMENTSDIR, environment) config = ConfigObj(conf_file_path) if config.get('parser', None) != parser: raise AssertionError("Environment '%s''s parser:'s value is not %s" % (environment, parser)) def assertEnvironmentDoesNotHaveAConfigFile(self, environment): try: self.assertEnvironmentHasAConfigFile(environment) except AssertionError: return raise AssertionError("Environment '%s' seems to have a config file" % environment) def assertEnvironmentBrokenLinks(self, environment): try: self.assertEnvironmentLinks(environment) except AssertionError: return raise AssertionError("Environment '%s' seems fine" % environment) def assertEnvironmentDoesntExist(self, environment): base_path = "%s/%s" % (self.settings.ENVIRONMENTSDIR, environment) if os.path.isdir(base_path): raise AssertionError("Environment '%s' present" % environment) def assertEnvironmentOverride(self, environment, identifier, desired): base_path = "%s/%s" % (self.settings.ENVIRONMENTSDIR, environment) partition, element = identifier.split('/') links = [] links.append("%s/%s/%s" % (base_path, partition, element)) if partition == 'modules': links.append("%s/hieradata/module_names/%s" % (base_path, element)) if partition == 'hostgroups': canonical_name = element.replace('hg_', '') links.append("%s/hieradata/hostgroups/%s" % \ (base_path, canonical_name)) links.append("%s/hieradata/fqdns/%s" % \ (base_path, canonical_name)) for link in links: # Override created if not os.path.lexists(link): raise AssertionError("Env '%s' -- '%s' -- '%s' does not exist" % \ (environment, identifier, link)) target = os.readlink(link) dirname = refname_to_dirname(desired) # Link not broken link_parent = os.path.abspath(os.path.join(link, os.pardir)) if not self._verify_link(link_parent, link): raise AssertionError("Env '%s' -- '%s' -- '%s' is broken" % \ (environment, identifier, link)) # And points to the correct clone link_type = 'data' if re.match(r'^%s/hieradata/.+' % \ base_path, link) else 'code' if not re.match(r".+/%s/%s.*" % (dirname, link_type), target): raise AssertionError("Env '%s' '%s' // '%s' -> '%s' not to '%s' (%s)" % \ (environment, identifier, link, target, dirname, desired)) def assertEnvironmentOverrideDoesntExist(self, environment, identifier): base_path = "%s/%s" % (self.settings.ENVIRONMENTSDIR, environment) partition, element = identifier.split('/') links = [] links.append("%s/%s/%s" % (base_path, partition, element)) if partition == 'modules': links.append("%s/hieradata/module_names/%s" % (base_path, element)) if partition == 'hostgroups': canonical_name = element.replace('hg_', '') links.append("%s/hieradata/hostgroups/%s" % \ (base_path, canonical_name)) links.append("%s/hieradata/fqdns/%s" % \ (base_path, canonical_name)) for link in links: if os.path.lexists(link): raise AssertionError("Env '%s' -- '%s' -- '%s' exists" % \ (environment, identifier, link)) def assertEnvironmentOverrideExistsButBroken(self, environment, identifier, desired): base_path = "%s/%s" % (self.settings.ENVIRONMENTSDIR, environment) partition, element = identifier.split('/') links = [] links.append("%s/%s/%s" % (base_path, partition, element)) if partition == 'modules': links.append("%s/hieradata/module_names/%s" % (base_path, element)) if partition == 'hostgroups': canonical_name = element.replace('hg_', '') links.append("%s/hieradata/hostgroups/%s" % \ (base_path, canonical_name)) links.append("%s/hieradata/fqdns/%s" % \ (base_path, canonical_name)) for link in links: # Override created if not os.path.lexists(link): raise AssertionError("Env '%s' -- '%s' -- '%s' does not exist" % \ (environment, identifier, link)) target = os.readlink(link) dirname = refname_to_dirname(desired) link_parent = os.path.abspath(os.path.join(link, os.pardir)) # Link broken if self._verify_link(link_parent, link): raise AssertionError("Env '%s' -- '%s' -- '%s' is not broken" % \ (environment, identifier, link)) def assertEnvironmentNumberOf(self, environment, partition, count): base_path = "%s/%s/%s" % \ (self.settings.ENVIRONMENTSDIR, environment, partition) actual = len(os.listdir(base_path)) if actual != count: raise AssertionError("'%s' has %d %s (expected: %d)" % \ (environment, actual, partition, count)) def _verify_link(self, base, path): cwd = os.getcwd() os.chdir(base) try: os.stat(os.readlink(path)) except OSError: return False finally: os.chdir(cwd) return True
def _read_inventory_from_disk(): settings = Settings() inventory_file = open(settings.CACHEDIR + "/repositories", "r") return pickle.load(inventory_file)
def _compose_bare_repository_path(name, partition): settings = Settings() return settings.BAREDIR + "/%s/%s" % (partition, name)
def __init__(self, tries, waittime): self.settings = Settings() self.tries = tries self.waittime = waittime
class JensTestCase(unittest.TestCase): COMMIT_PREFIX = 'commit/' DEFAULT_DEBUG_LEVEL = 'DEBUG' MANDATORY_BRANCHES = ['master', 'qa'] def setUp(self): self.sandbox_path = tempfile.mkdtemp(prefix="jens_sandbox_%s-" % self._testMethodName, suffix='-' + str(time.time())) init_sandbox(self.sandbox_path) self.keep_sandbox = bool(os.getenv('JENS_TEST_KEEP_SANDBOX', False)) self.debug_level = JensTestCase.DEFAULT_DEBUG_LEVEL self.config_file_path = "%s/etc/main.conf" % self.sandbox_path config_file = open(self.config_file_path, 'w+') config_file.write( BASE_CONFIG.substitute(sandbox=self.sandbox_path, hashprefix=JensTestCase.COMMIT_PREFIX, debuglevel=self.debug_level, mandatory_branches=','.join( JensTestCase.MANDATORY_BRANCHES))) config_file.close() root = logging.getLogger() map(root.removeHandler, root.handlers[:]) map(root.removeFilter, root.filters[:]) self.settings = Settings("jens-test") self.settings.parse_config(self.config_file_path) self.lock = JensLockFactory.make_lock(self.settings) def tearDown(self): if self.keep_sandbox: print "Sandbox kept in %s" % self.sandbox_path else: destroy_sandbox(self.sandbox_path) def assertLogNoErrors(self): try: self.log except AttributeError: self.log = open("%s/jens-test.log" % self.settings.LOGDIR) for line in self.log.readlines(): if re.match(r'.+ERROR.+', line): raise AssertionError(line) def assertLogErrors(self, errorRegexp=None): try: self.log except AttributeError: self.log = open("%s/jens-test.log" % self.settings.LOGDIR) found = False regexp = r'.+ERROR.+' if errorRegexp is not None: regexp = r'.+ERROR.+%s.+' % errorRegexp for line in self.log.readlines(): if re.match(regexp, line): found = True if not found: raise AssertionError("There should be errors") def assertClone(self, identifier, pointsto=None): partition, element, dirname = identifier.split('/') path = "%s/%s/%s/%s" % (self.settings.CLONEDIR, \ partition, element, dirname) if not os.path.isdir(path): raise AssertionError("Clone '%s' not found" % path) if not os.path.isdir("%s/code" % path): raise AssertionError("Clone '%s' does not have code dir" % path) if not os.path.isdir("%s/data" % path): raise AssertionError("Clone '%s' does not have data dir" % path) inventory = get_inventory() self.assertTrue(partition in inventory) self.assertTrue(element in inventory[partition]) refname = dirname_to_refname(dirname) self.assertTrue(refname in inventory[partition][element]) if pointsto is not None: self.assertEquals(get_repository_head(path), pointsto) def assertCloneFileExists(self, identifier, fname): partition, element, dirname = identifier.split('/') path = "%s/%s/%s/%s/%s" % (self.settings.CLONEDIR, \ partition, element, dirname, fname) if not os.path.isfile(path): raise AssertionError("File '%s' not found" % path) def assertNotClone(self, identifier): try: self.assertClone(identifier) except AssertionError: return raise AssertionError("Clone '%s' seems present" % identifier) def assertBare(self, identifier): partition, element = identifier.split('/') path = "%s/%s/%s" % (self.settings.BAREDIR, \ partition, element) if not os.path.isdir(path): raise AssertionError("Bare '%s' not found" % path) if not os.path.isfile("%s/HEAD" % path): raise AssertionError("Bare '%s' does not have HEAD" % path) def assertNotBare(self, identifier): try: self.assertBare(identifier) except AssertionError: return raise AssertionError("Bare '%s' seems present" % identifier) def assertEnvironmentLinks(self, environment): base_path = "%s/%s" % (self.settings.ENVIRONMENTSDIR, environment) common_hieradata_items = set(self.settings.COMMON_HIERADATA_ITEMS) if not os.path.isdir(base_path): raise AssertionError("Environment '%s' not present" % environment) for path, dirs, files in os.walk(base_path): if path.endswith('hieradata'): self.assertTrue( common_hieradata_items.issubset(set(dirs + files))) for file in files + dirs: file_apath = "%s/%s" % (path, file) if os.path.islink(file_apath): if not self._verify_link(path, file_apath): raise AssertionError("Environment '%s' -- '%s' link broken" % \ (environment, file_apath)) def assertEnvironmentHasAConfigFile(self, environment): # https://docs.puppetlabs.com/puppet/latest/reference/config_file_environment.html conf_file_path = "%s/%s/environment.conf" % \ (self.settings.ENVIRONMENTSDIR, environment) if not os.path.isfile(conf_file_path): raise AssertionError( "Environment '%s' doesn't have a config file" % environment) if not os.stat(conf_file_path).st_size > 0: raise AssertionError("Environment '%s''s config file seems empty" % environment) def assertEnvironmentHasAConfigFileAndParserSet(self, environment, parser): conf_file_path = "%s/%s/environment.conf" % \ (self.settings.ENVIRONMENTSDIR, environment) config = ConfigObj(conf_file_path) if config.get('parser', None) != parser: raise AssertionError( "Environment '%s''s parser:'s value is not %s" % (environment, parser)) def assertEnvironmentDoesNotHaveAConfigFile(self, environment): try: self.assertEnvironmentHasAConfigFile(environment) except AssertionError: return raise AssertionError("Environment '%s' seems to have a config file" % environment) def assertEnvironmentBrokenLinks(self, environment): try: self.assertEnvironmentLinks(environment) except AssertionError: return raise AssertionError("Environment '%s' seems fine" % environment) def assertEnvironmentDoesntExist(self, environment): base_path = "%s/%s" % (self.settings.ENVIRONMENTSDIR, environment) if os.path.isdir(base_path): raise AssertionError("Environment '%s' present" % environment) def assertEnvironmentOverride(self, environment, identifier, desired): base_path = "%s/%s" % (self.settings.ENVIRONMENTSDIR, environment) partition, element = identifier.split('/') links = [] links.append("%s/%s/%s" % (base_path, partition, element)) if partition == 'modules': links.append("%s/hieradata/module_names/%s" % (base_path, element)) if partition == 'hostgroups': canonical_name = element.replace('hg_', '') links.append("%s/hieradata/hostgroups/%s" % \ (base_path, canonical_name)) links.append("%s/hieradata/fqdns/%s" % \ (base_path, canonical_name)) for link in links: # Override created if not os.path.lexists(link): raise AssertionError("Env '%s' -- '%s' -- '%s' does not exist" % \ (environment, identifier, link)) target = os.readlink(link) dirname = refname_to_dirname(desired) # Link not broken link_parent = os.path.abspath(os.path.join(link, os.pardir)) if not self._verify_link(link_parent, link): raise AssertionError("Env '%s' -- '%s' -- '%s' is broken" % \ (environment, identifier, link)) # And points to the correct clone link_type = 'data' if re.match(r'^%s/hieradata/.+' % \ base_path, link) else 'code' if not re.match(r".+/%s/%s.*" % (dirname, link_type), target): raise AssertionError("Env '%s' '%s' // '%s' -> '%s' not to '%s' (%s)" % \ (environment, identifier, link, target, dirname, desired)) def assertEnvironmentOverrideDoesntExist(self, environment, identifier): base_path = "%s/%s" % (self.settings.ENVIRONMENTSDIR, environment) partition, element = identifier.split('/') links = [] links.append("%s/%s/%s" % (base_path, partition, element)) if partition == 'modules': links.append("%s/hieradata/module_names/%s" % (base_path, element)) if partition == 'hostgroups': canonical_name = element.replace('hg_', '') links.append("%s/hieradata/hostgroups/%s" % \ (base_path, canonical_name)) links.append("%s/hieradata/fqdns/%s" % \ (base_path, canonical_name)) for link in links: if os.path.lexists(link): raise AssertionError("Env '%s' -- '%s' -- '%s' exists" % \ (environment, identifier, link)) def assertEnvironmentOverrideExistsButBroken(self, environment, identifier, desired): base_path = "%s/%s" % (self.settings.ENVIRONMENTSDIR, environment) partition, element = identifier.split('/') links = [] links.append("%s/%s/%s" % (base_path, partition, element)) if partition == 'modules': links.append("%s/hieradata/module_names/%s" % (base_path, element)) if partition == 'hostgroups': canonical_name = element.replace('hg_', '') links.append("%s/hieradata/hostgroups/%s" % \ (base_path, canonical_name)) links.append("%s/hieradata/fqdns/%s" % \ (base_path, canonical_name)) for link in links: # Override created if not os.path.lexists(link): raise AssertionError("Env '%s' -- '%s' -- '%s' does not exist" % \ (environment, identifier, link)) target = os.readlink(link) dirname = refname_to_dirname(desired) link_parent = os.path.abspath(os.path.join(link, os.pardir)) # Link broken if self._verify_link(link_parent, link): raise AssertionError("Env '%s' -- '%s' -- '%s' is not broken" % \ (environment, identifier, link)) def assertEnvironmentNumberOf(self, environment, partition, count): base_path = "%s/%s/%s" % \ (self.settings.ENVIRONMENTSDIR, environment, partition) actual = len(os.listdir(base_path)) if actual != count: raise AssertionError("'%s' has %d %s (expected: %d)" % \ (environment, actual, partition, count)) def _verify_link(self, base, path): cwd = os.getcwd() os.chdir(base) try: os.stat(os.readlink(path)) except OSError: return False finally: os.chdir(cwd) return True
def ref_is_commit(refname): settings = Settings() return re.match(r'^%s([0-9A-Fa-f]+)' % settings.HASHPREFIX, refname, re.IGNORECASE)
def dirname_to_refname(dirname): settings = Settings() match = re.match(r'^\.([^\.]+)', dirname) if match: return "%s%s" % (settings.HASHPREFIX, match.group(1)) return dirname
def add_msg_to_queue(msg): settings = Settings() queue = Queue(settings.MESSAGING_QUEUEDIR, schema=MSG_SCHEMA) queue.enqueue(msg)
def init_repositories(): settings = Settings() data = {'repositories': {'modules': {}, 'hostgroups': {}, 'common': {}}} repositories_file = open(settings.REPO_METADATA, 'w+') yaml.dump(data, repositories_file, default_flow_style=False) repositories_file.close()
def destroy_environment(envname): settings = Settings() os.remove("%s/%s.yaml" % (settings.ENV_METADATADIR, envname))