def test_is_virtual_repository_path(self): virtual_path_example = "target/virtual/foobarrepo" static_path_example = "target/static/foobarrepo" self.assertTrue(RepoConfigService().is_virtual_repository_path( virtual_path_example)) self.assertFalse(RepoConfigService().is_virtual_repository_path( static_path_example))
def filter(self, request, repos): if 'name' in request.GET: pattern = re.compile(request.GET['name']) repos = filter(lambda d: pattern.match(d), repos) if 'tag' in request.GET: tags = set(request.GET['tag'].split(',')) taggingService = RepoTaggingService() repos = filter(lambda d: len(tags.intersection(taggingService.getTags(d))) > 0, repos) if 'notag' in request.GET: forbiddentags = set(request.GET['notag'].split(',')) taggingService = RepoTaggingService() repos = filter(lambda d: len(forbiddentags.intersection(taggingService.getTags(d))) == 0, repos) if 'older' in request.GET: pastTime = self.get_past_time(int(request.GET['older'])) configService = RepoConfigService() repos = filter(lambda d: os.stat(configService.getStaticRepoDir(d)).st_mtime < pastTime, repos) if 'newer' in request.GET: pastTime = self.get_past_time(int(request.GET['newer'])) configService = RepoConfigService() repos = filter(lambda d: os.stat(configService.getStaticRepoDir(d)).st_mtime > pastTime, repos) return repos
def test_get_no_rpm_group_if_less_than_max_rpm(self): rpm_file_names = ['rss-4-1.1.noarch.rpm'] dict_group_with_obsolete_rpms = RepoConfigService()._get_rpm_group_with_obsolete_files_by_file_name( rpm_file_names, 3) self.assertEqual([], dict_group_with_obsolete_rpms.keys()) self.assertEqual([], dict_group_with_obsolete_rpms.values())
def rpm_info_static(request, rpm): config = RepoConfigService() rpm_path = os.path.join(config.getStaticRepoDir(), rpm) if '../' in rpm: raise Http404('../ not allowed') return rpm_info(rpm_path)
def rpm_info_virtual(request, reponame, rpm): config = RepoConfigService() repoConfig = config.getConfig(reponame) rpm_path = os.path.join(config.getRepoDir(), repoConfig.destination[1:], rpm) if '../' in rpm: raise Http404('../ not allowed') return rpm_info(rpm_path)
def test_filter_by_older_then(self): repo1 = self.createNewRepoAndAssertValid() repo2 = self.createNewRepoAndAssertValid() configService = RepoConfigService() repo2dir = configService.getStaticRepoDir(repo2) self.set_mtime(repo2dir) response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC+".txt?older=5") self.assertEqual(repo2 + "\n", response.read())
def test_get_no_rpm_group_if_rpms_have_different_names(self): rpm_file_names = ['rss-4-1.1.noarch.rpm', 'rss-4-1.2.noarch.rpm', 'rss-5-1.1.noarch.rpm', 'feed-rss-7-1.4.noarch.rpm'] dict_group_with_obsolete_rpms = RepoConfigService()._get_rpm_group_with_obsolete_files_by_file_name( rpm_file_names, 3) self.assertEqual([], dict_group_with_obsolete_rpms.keys()) self.assertEqual([], dict_group_with_obsolete_rpms.values())
def test_filter_by_newer_then(self): repo1 = self.createNewRepoAndAssertValid() repo2 = self.createNewRepoAndAssertValid() configService = RepoConfigService() repo2dir = configService.getStaticRepoDir(repo2) self.set_mtime(repo2dir) response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC + ".txt?newer=5") self.assertEqual(repo1 + "\n", response.read())
def virtual_repo_info(request, reponame): config = RepoConfigService() repoConfig = config.getConfig(reponame) template = loader.select_template(['static/virtual_repo_info.html']) context = Context({ 'reponame': reponame, 'config': repoConfig, 'staticRepos': config.staticRepos, }) return HttpResponse(template.render(context))
def virtual_repo_info(request, reponame): config = RepoConfigService() repoConfig = config.getConfig(reponame) template = loader.select_template(['static/virtual_repo_info.html']) context = Context({ 'reponame' : reponame, 'config' : repoConfig, 'staticRepos' : config.staticRepos, }) return HttpResponse(template.render(context))
def test_should_determine_repository_path(self): service = RepoConfigService() when(service).getStaticRepoDir(any_value()).thenReturn("path/to/repository") when(yum_repo_server.api.services.repoConfigService.os.path).exists(any_value()).thenReturn(True) actual_repository_path = service.determine_static_repository_path("repository-name") self.assertEqual("path/to/repository", actual_repository_path) verify(service).getStaticRepoDir("repository-name") verify(yum_repo_server.api.services.repoConfigService.os.path).exists("path/to/repository") unstub()
class YumMetaDataHandler(BaseHandler): repoConfigService = RepoConfigService() def create(self, request, reponame): full_path_to_repo = self.repoConfigService.getStaticRepoDir(reponame) if not os.path.exists(full_path_to_repo): resp = rc.NOT_FOUND resp.content = 'Repository %s not found' % full_path_to_repo return resp try: self.repoConfigService.doCreateRepo(full_path_to_repo, reponame) except: resp = rc.BAD_REQUEST resp.content = 'createrepo has finished with Error' return resp resp = rc.CREATED resp.content = config.get_repo_dir() return resp def read(self, request, reponame): return serve(request, '/' + reponame + '/repodata/', self.repoConfigService.getStaticRepoDir(), True)
class YumRepoAliasHandler(BaseHandler): POST_PARAM_DESTINATION_NAME = 'destination' POST_PARAM_VIRTUAL_REPO_NAME = 'name' repoConfigService = RepoConfigService() audit = RepoAuditService() def create(self, request, text): try: virtual_repo_name, destination_repo = self.check_request_sanity( request) except RequestFailException, rfe: return rfe.args[0] try: result = self.repoConfigService.createVirtualRepo( virtual_repo_name, destination_repo) except RepoNotFoundException: resp = rc.NOT_HERE #Do NOT disclose the actual path to the client -> return relative destination resp.content = 'The destination repository at %s does not exist.' % destination_repo return resp self.audit.log_action( "created a virtual link from %s to %s" % (virtual_repo_name, destination_repo), request) response = rc.CREATED response.content = result return response
class TestCleanup(BaseIntegrationTestCase): config = RepoConfigService() def test_cleanup_cache_dir(self): reponame1 = self.createStaticRepoWithContent() reponame2 = self.createStaticRepoWithContent() reponame3 = self.createStaticRepoWithContent() # delete repo 1 shutil.rmtree(self.config.getStaticRepoDir(reponame1)) # lock repo 2 open(self.config.getRepoLockFile(reponame2), 'w').close() MetaDataGenerationScheduler().cleanupCacheDir() self.assertFalse(os.path.exists( self.config.getRepoCacheDir(reponame1))) self.assertTrue(os.path.exists(self.config.getRepoCacheDir(reponame2))) self.assertFalse(os.path.exists( self.config.getRepoCacheDir(reponame3))) def createStaticRepoWithContent(self): reponame = self.createNewRepoAndAssertValid() testRPMFilePath = Constants.TEST_RPM_FILE_LOC + Constants.TEST_RPM_FILE_NAME self.upload_testfile(reponame, testRPMFilePath) self.generate_metadata(reponame) repoCacheDir = self.config.getRepoCacheDir(reponame) self.assertTrue(os.path.exists(repoCacheDir)) return reponame
def test_do_not_remove_rpms_when_rpm_max_keep_is_zero(self): os.makedirs(self.testRepo) self.touchRpms(self.testRepo) try: RepoConfigService().doCleanup(self.targetDir, 0) self.assertTrue(os.path.exists(self.testRepo + '/rss-4-1.1.noarch.rpm'), 'rpm-file was deleted.') finally: shutil.rmtree(self.targetDir)
def test_find_rpms_to_delete_returns_empty_list_when_single_group_is_not_big_enough(self): rpm_file_names = ['rss-4-1.1.noarch.rpm', 'rss-4-1.2.noarch.rpm', 'rss-5-1.1.noarch.rpm', 'rss-feed-7-1.4.noarch.rpm'] rpm_to_delete = RepoConfigService()._find_rpms_to_delete(rpm_file_names, 3) self.assertEquals(0, len(rpm_to_delete))
def test_doCleanup(self): os.makedirs(self.testRepo) self.touchRpms(self.testRepo) try: RepoConfigService().doCleanup(self.targetDir, 3) self.assertFalse(os.path.exists(self.testRepo + '/rss-4-1.1.noarch.rpm')) finally: shutil.rmtree(self.targetDir)
def test_find_rpms_to_delete(self): rpm_file_names = ['rss-4-1.1.noarch.rpm', 'rss-4-1.2.noarch.rpm', 'rss-5-1.1.noarch.rpm', 'rss-7-1.4.noarch.rpm'] rpm_to_delete = RepoConfigService()._find_rpms_to_delete(rpm_file_names, 3) self.assertEquals(1, len(rpm_to_delete)) self.assertEquals('rss-4-1.1.noarch.rpm', rpm_to_delete[0].file_name)
def filter(self, request, repos): if 'name' in request.GET: pattern = re.compile(request.GET['name']) repos = filter(lambda d: pattern.match(d), repos) if 'tag' in request.GET: tags = set(request.GET['tag'].split(',')) taggingService = RepoTaggingService() repos = filter( lambda d: len(tags.intersection(taggingService.getTags(d))) > 0, repos) if 'notag' in request.GET: forbiddentags = set(request.GET['notag'].split(',')) taggingService = RepoTaggingService() repos = filter( lambda d: len( forbiddentags.intersection(taggingService.getTags(d))) == 0, repos) if 'older' in request.GET: pastTime = self.get_past_time(int(request.GET['older'])) configService = RepoConfigService() repos = filter( lambda d: os.stat(configService.getStaticRepoDir(d)).st_mtime < pastTime, repos) if 'newer' in request.GET: pastTime = self.get_past_time(int(request.GET['newer'])) configService = RepoConfigService() repos = filter( lambda d: os.stat(configService.getStaticRepoDir(d)).st_mtime > pastTime, repos) return repos
def assertVirtualRepoConfig(self, virtual_reponame, static_reponame): metadatapath = RepoConfigService().getVirtualRepoDir( virtual_reponame) + "/" + RepoConfigService.ALIAS_METADATA_FILENAME self.assertTrue(os.path.exists(metadatapath), "No metadata generated -> alias failed") alias_file = open(metadatapath) alias_config = yaml.load(alias_file) alias_file.close() alias_metadata = dict(alias_config.items()) config_destination = alias_metadata['destination'] expected_destination = "/static/" + static_reponame self.assertEquals(config_destination, expected_destination, "Alias points to wrong repo")
class YumRepoHandler(BaseHandler): NAME_REGEX = '^[a-zA-Z0-9][a-zA-Z0-9_\-\.]*$' repoConfigService = RepoConfigService() audit = RepoAuditService() def create(self, request, text): data = request.POST if not data: resp = rc.BAD_REQUEST resp.content = 'POST data missing' return resp name = data.get('name', None) #set None as default if the key is missing if not name: resp = rc.BAD_REQUEST resp.content = 'The name attribute is missing' return resp if not re.match(self.NAME_REGEX, data['name']): resp = rc.BAD_REQUEST resp.content = 'The provided name is invalid. It must match this regular expression : ' + self.NAME_REGEX return resp path = self.repoConfigService.getStaticRepoDir(name) if os.path.exists(path): resp = rc.DUPLICATE_ENTRY resp.content = 'The repository at ' + path + ' already exists.' return resp os.makedirs(path) resp = rc.CREATED resp.content = dict(name=request.POST['name'], dir=path) self.audit.log_action("created static repository %s" % name, request) return resp def read(self, request, text): static_path = self.repoConfigService.getStaticRepoDir() return serve(request=request, path='/', document_root=static_path, add_virtual=True, show_indexes=True, parent_dir_type=ParentDirType.STATIC)
class MetaDataGenerationScheduler(): def __init__(self, updateIntervalSeconds=30): self.interval = updateIntervalSeconds config = {'apscheduler.daemonic': False} self.sched = Scheduler(config) # initialize these per instance. self.repo_timestamps = {} self.jobs = {} repo_timestamps = {} #dictionary with jobName (=reponame) : last scheduler modification timestamp (float) jobs = {} #dictionary with jobName (=reponame) : jobHandle configService = RepoConfigService() static_root_dir = configService.getStaticRepoDir() sched = None interval = None def start(self): self.update_program_config() #read configs, schedule jobs # schedule an update as a job self.sched.add_interval_job(self.update_program_config, seconds=self.interval) # schedule cleanup cache self.sched.add_cron_job(self.cleanupCacheDir, hour = 23, minute = 17, second = 20) self.sched.start() def createrepo_with_optional_cleanup_job(self, *argList): monitor = JobMonitorer() monitor.job_starts() repoDir = argList[0] reponame = argList[1] rpm_max_keep = argList[2] didCleanUp=False try: if rpm_max_keep != None: didCleanUp=True self.configService.doCleanup(repoDir, rpm_max_keep) logging.info("job RpmCleanup on "+reponame+" took "+str(monitor.get_execution_time_until_now_seconds())+" seconds") self.configService.doCreateRepo(repoDir, reponame) monitor.job_finishes() logging.info(monitor.get_pretty_job_summary("createrepo on "+reponame+" (cleanup included : "+str(didCleanUp)+")")) except Exception, ex: logging.error(traceback.format_exc())
class RepoTaggingHandler(BaseHandler): repoConfigService = RepoConfigService() repoTaggingService = RepoTaggingService() repoAuditService = RepoAuditService() def create(self, request, repodir,tag): try: tag = self.check_request_sanity(request) except RequestFailException, rfe: return rfe.args[0] try: result = self.repoTaggingService.tagRepo(repodir,tag) except CouldNotLockTagsException, lockErr: response = rc.BAD_REQUEST response.content = "Could not lock tags file" return response
class RepoContentService(object): """ Service to retrieve information about the content of a repository. """ METADATA_DIRECTORY = "repodata" repoConfigService = RepoConfigService() def list_architectures(self, repository_name): """ @return: list of architecture paths """ repository_path = self.repoConfigService.getStaticRepoDir( repository_name) directories_in_repository = os.listdir(repository_path) available_architectures = [] for directory in directories_in_repository: architecture_path = os.path.join(repository_path, directory) if directory != self.METADATA_DIRECTORY: if os.path.isdir(architecture_path): if len(os.listdir(architecture_path)) > 0: available_architectures.append(architecture_path) return available_architectures def list_packages(self, repository_name): """ @return: a list of tuples (first element is the architecture name, second element is the package file name) """ available_architectures = self.list_architectures(repository_name) packages_in_repository = [] for architecture_path in available_architectures: packages_in_architecture_dir = os.listdir(architecture_path) for package in packages_in_architecture_dir: package_path = os.path.join(architecture_path, package) packages_in_repository.append(package_path) return packages_in_repository
class VirtualRepoConfigHandler(BaseHandler): repoConfigService = RepoConfigService() def read(self, request, reponame): if '../' in reponame: return rc.BAD_REQUESST repo_dir = self.repoConfigService.getVirtualRepoDir(reponame) if not os.path.exists(repo_dir): return rc.NOT_FOUND repo_config = self.repoConfigService.getConfig(reponame) resp = rc.ALL_OK resp.content = '' json.dump(repo_config.data, resp) return resp
class VirtualRepoHandler(BaseHandler): repoConfigService = RepoConfigService() audit = RepoAuditService() def read(self, request, reponame, rpm='/'): virtual_reponame = reponame path_relative_to_repository = rpm virtual_repo_dir = self.repoConfigService.getVirtualRepoDir( virtual_reponame) if not os.path.exists(virtual_repo_dir): resp = rc.NOT_FOUND resp.content = 'Virtual Repository does not exists!' return resp repoConfig = self.repoConfigService.getConfig(virtual_reponame) if re.match('^https?://.*', repoConfig.destination): return HttpResponseRedirect(repoConfig.destination + rpm) else: absoluteDestinationRepoPath = config.get_repo_dir( ) + repoConfig.destination return serve(request, path_relative_to_repository, absoluteDestinationRepoPath, True) def delete(self, request, reponame, rpm='/'): if not rpm is None and len(rpm) > 1: resp = rc.NOT_IMPLEMENTED resp.content = "You are just allowed to remove virtual repositories" return resp repo_path = self.repoConfigService.getVirtualRepoDir(reponame) if not os.path.exists(repo_path): resp = rc.NOT_FOUND resp.content = 'Virtual Repository does not exists!' return resp self.audit.log_action("deleted virtual repository %s" % (reponame), request) shutil.rmtree(repo_path) return rc.DELETED
def directory_index(path, fullpath, add_virtual = False, parent_dir_type = ParentDirType.NONE): t = loader.select_template(['static/directory_index.html', 'static/directory_index']) unsorted_files = [] for f in os.listdir(fullpath): if not f.startswith('.'): unsorted_files.append(FileInfo(f, fullpath, parent_dir_type)) if add_virtual: virtual_repodir = RepoConfigService().getVirtualRepoDir() repodir=os.path.join(virtual_repodir,os.path.pardir) relative_virtualdir=os.path.basename(os.path.normpath(virtual_repodir)) file_info = FileInfo(relative_virtualdir, repodir) if os.path.exists(file_info.fullpath): unsorted_files.append(file_info) files = sorted(unsorted_files, key = attrgetter('isFile', 'filename')) c = Context({ 'directory' : path, 'file_list' : files, 'file_count': len(files), }) return HttpResponse(t.render(c))
def read(self, request, path): return serve(request, path, RepoConfigService().getStaticRepoDir(), True, False, False)
def _repository_contains(self, architecture, file_name, repository): repository_path = RepoConfigService().getStaticRepoDir(repository) path_to_file = os.path.join(repository_path, architecture, file_name) repository_contains_file = os.path.exists(path_to_file) return repository_contains_file
class RepoPropagationService(object): repoConfigService = RepoConfigService() repoContentService = RepoContentService() rpmService = RpmService() def propagatePackage(self, package_name, source_repository, destination_repository, architecture): source_repo_path = self.determine_repository_path(source_repository) destination_repo_path = self.determine_repository_path(destination_repository) source_architecture_path = os.path.join(source_repo_path, architecture) file_name = self._determine_rpm_file_name(source_architecture_path, package_name) source_rpm_path = os.path.join(source_repo_path, architecture, file_name) destination_rpm_parent_dir = os.path.join(destination_repo_path, architecture) destination_rpm_path = os.path.join(destination_rpm_parent_dir, file_name) if not os.path.exists(source_rpm_path): raise PropagationException("Package '{0}' could not be found.".format(source_rpm_path)) if not os.path.exists(destination_rpm_parent_dir): os.mkdir(destination_rpm_parent_dir) shutil.move(source_rpm_path, destination_rpm_path) return file_name def _determine_rpm_file_name(self, directory, rpm): if create_rpm_file_object(rpm) is None: latest_rpm = self.rpmService.get_latest_rpm(rpm, directory) if latest_rpm is None: raise PropagationException("Package for {0} could not be found in {1}".format(rpm, directory)) return latest_rpm return rpm def propagate_repository(self, source_repository, destination_repository): destination_repository_path = self.determine_repository_path(destination_repository) packages_to_propagate = self.repoContentService.list_packages(source_repository) propagated_packages = [] for package_path in packages_to_propagate: architecture_path = os.path.dirname(package_path) architecture = os.path.basename(architecture_path) destination_path = os.path.join(destination_repository_path, architecture) if not os.path.exists(destination_path): os.makedirs(destination_path) shutil.move(package_path, destination_path) propagated_packages.append(package_path) return propagated_packages def determine_repository_path(self, repository_name): repository_path = self.repoConfigService.getStaticRepoDir(repository_name) if not os.path.exists(repository_path): raise PropagationException("Static repository '{0}' does not exist.".format(repository_name)) return repository_path
def test_do_cleanup_should_raise_valueError_when_rpm_max_keep_is_not_an_integer(self): try: RepoConfigService().doCleanup(self.targetDir, '3') self.fail('do cleanup should check, that rpm_max_keep is an integer.') except ValueError: pass
class CsvListingHandler(BaseHandler): config = RepoConfigService() def read(self, request, repodir=''): if not self._valid(repodir): response = rc.BAD_REQUEST response.write("You are not allowed to look into %s" % repodir) return response is_virtual = repodir == 'virtual' show_destination = is_virtual and 'showDestination' in request.GET and request.GET[ 'showDestination'].lower() == 'true' root_dir = self.config.getStaticRepoDir() if is_virtual: root_dir = self.config.getVirtualRepoDir() repos = os.listdir(root_dir) repos = self.filter(request, repos) response = rc.ALL_OK response.content = "" for repo in repos: response.write(self._write_repo(repo, show_destination)) response.write("\n") return response def _valid(self, repodir): if repodir in ["virtual", ""]: return True return False def _write_repo(self, repo, show_destination=False): if show_destination: destination = self.config.getConfig(repo).destination return '%s:%s' % (repo, destination) return repo def filter(self, request, repos): if 'name' in request.GET: pattern = re.compile(request.GET['name']) repos = filter(lambda d: pattern.match(d), repos) if 'tag' in request.GET: tags = set(request.GET['tag'].split(',')) taggingService = RepoTaggingService() repos = filter( lambda d: len(tags.intersection(taggingService.getTags(d))) > 0, repos) if 'notag' in request.GET: forbiddentags = set(request.GET['notag'].split(',')) taggingService = RepoTaggingService() repos = filter( lambda d: len( forbiddentags.intersection(taggingService.getTags(d))) == 0, repos) if 'older' in request.GET: pastTime = self.get_past_time(int(request.GET['older'])) configService = RepoConfigService() repos = filter( lambda d: os.stat(configService.getStaticRepoDir(d)).st_mtime < pastTime, repos) if 'newer' in request.GET: pastTime = self.get_past_time(int(request.GET['newer'])) configService = RepoConfigService() repos = filter( lambda d: os.stat(configService.getStaticRepoDir(d)).st_mtime > pastTime, repos) return repos def get_past_time(self, days): pastDay = datetime.datetime.now() - datetime.timedelta(days=days) return int(time.mktime(pastDay.timetuple()))
class TestCsvListing(BaseIntegrationTestCase): config = RepoConfigService() def test_static_listing_is_200_ok(self): response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC + ".txt") self.assertEquals(httplib.OK, response.status) def test_virtual_listing_is_200_ok(self): response = self.helper.do_http_get(Constants.HTTP_PATH_VIRTUAL + ".txt") self.assertEquals(httplib.OK, response.status) def test_does_not_list_invalid_repos(self): response = self.helper.do_http_get(Constants.HTTP_PATH_VIRTUAL + "sldknlkdnlnsd.txt") self.assertEquals(httplib.BAD_REQUEST, response.status) def test_static_listing_lists_repos(self): reponame = self.createNewRepoAndAssertValid() response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC + ".txt") self.assertTrue(reponame in response.read()) shutil.rmtree(self.config.getStaticRepoDir(reponame)) def test_static_listing_lists_repos_without_destination_url_when_parameter_is_given( self): reponame = self.createNewRepoAndAssertValid() response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC + ".txt?showDestination=true") response_text = response.read().strip() self.assertEquals(reponame, response_text) def test_virtual_listing_lists_repos(self): reponame = unique_repo_name() virtualRepoPath = self.config.getVirtualRepoDir() + reponame os.makedirs(virtualRepoPath) response = self.helper.do_http_get(Constants.HTTP_PATH_VIRTUAL + ".txt") self.assertTrue(reponame in response.read()) shutil.rmtree(virtualRepoPath) def test_virtual_listing_has_destination_url(self): static_repo, virtual_repo = self.assert_create_virtual_repo() response = self.helper.do_http_get(Constants.HTTP_PATH_VIRTUAL + ".txt?showDestination=true") response_text = response.read().strip() self.assertEquals('%s:/static/%s' % (virtual_repo, static_repo), response_text) def test_filter_by_name(self): self.createNewRepoAndAssertValid() reponame = 'some-app-321.1.1-' + unique_repo_name() self.repoclient().createStaticRepo(reponame) response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC + ".txt?name=some-app-.*") self.assertEqual(reponame + "\n", response.read()) def test_filter_by_name_with_regex(self): self.createNewRepoAndAssertValid() reponame = 'some-app-321.1.12' self.repoclient().createStaticRepo(reponame) encoded_url = urllib.urlencode({'name': 'some-app-[\d\.]+'}) response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC + '.txt?%s' % encoded_url) self.assertEqual(reponame + "\n", response.read()) def test_filter_by_tags_inclusive(self): repo1 = self.createNewRepoAndAssertValid() self.repoclient().tagRepo(repo1, "atag") repo2 = self.createNewRepoAndAssertValid() self.repoclient().tagRepo(repo2, "atag") self.repoclient().tagRepo(repo2, "atag2") response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC + ".txt?tag=atag2,btag") self.assertEqual(repo2 + "\n", response.read()) def test_filter_by_tags_exclusive(self): repo1 = self.createNewRepoAndAssertValid() self.repoclient().tagRepo(repo1, "atag") self.repoclient().tagRepo(repo1, "atag2") repo2 = self.createNewRepoAndAssertValid() self.repoclient().tagRepo(repo2, "atag") response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC + ".txt?notag=atag2") self.assertEqual(repo2 + "\n", response.read()) def test_filter_by_older_then(self): repo1 = self.createNewRepoAndAssertValid() repo2 = self.createNewRepoAndAssertValid() configService = RepoConfigService() repo2dir = configService.getStaticRepoDir(repo2) self.set_mtime(repo2dir) response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC + ".txt?older=5") self.assertEqual(repo2 + "\n", response.read()) def test_filter_by_newer_then(self): repo1 = self.createNewRepoAndAssertValid() repo2 = self.createNewRepoAndAssertValid() configService = RepoConfigService() repo2dir = configService.getStaticRepoDir(repo2) self.set_mtime(repo2dir) response = self.helper.do_http_get(Constants.HTTP_PATH_STATIC + ".txt?newer=5") self.assertEqual(repo1 + "\n", response.read()) def set_mtime(self, path): today = datetime.datetime.now() pastday = today - datetime.timedelta(days=11) atime = int(time.mktime(pastday.timetuple())) os.utime(path, (atime, atime))
class RepoTaggingService(object): config = RepoConfigService() def tagRepo(self, static_reponame, tag): repo = self.config.getStaticRepoDir(static_reponame) tag_encoded = self.utf8Encode(tag) if not os.path.exists(repo): raise IsNotAStaticRepoException() tagpath = self.config.getTagsFileForStaticRepo(static_reponame) lock = lockfile.FileLock(tagpath) while not lock.i_am_locking(): try: lock.acquire(timeout=15) #wait 15sec max except LockTimeout: raise CouldNotLogTagsException() try: fileHandle = open(tagpath, 'a') fileHandle.write(tag_encoded) fileHandle.write('\n') fileHandle.close() finally: lock.release() return "Tagged OK" def utf8Encode(self, string): return string.encode('utf-8') def unTagRepo(self, static_reponame, tag): initialTags = self.getTags(static_reponame) tag_encoded = self.utf8Encode(tag) if not tag_encoded in initialTags: raise NoSuchTagException() initialTags.remove(tag_encoded) repo = self.config.getStaticRepoDir(static_reponame) if not os.path.exists(repo): raise IsNotAStaticRepoException() tagpath = self.config.getTagsFileForStaticRepo(static_reponame) lock = lockfile.FileLock(tagpath) while not lock.i_am_locking(): try: lock.acquire(timeout=15) #wait 15sec max except LockTimeout: raise CouldNotLogTagsException() try: fileHandle = open(tagpath, 'w') #replace instead of appending for tag in initialTags: tag_encoded = self.utf8Encode(tag) fileHandle.write(tag_encoded) fileHandle.write('\n') fileHandle.close() finally: lock.release() return "Untagged OK" def getTags(self, static_reponame): filepath = self.config.getTagsFileForStaticRepo(static_reponame) if not os.path.exists(self.config.getStaticRepoDir(static_reponame)): raise NotFoundException() if not os.path.exists(filepath): return set() f = open(filepath, "r") try: tags = set(f.read().split('\n')) finally: f.close() tags.discard('') return tags def getDefaultTags(self): tagFilePath = os.path.join(self.config.getRepoDir(), 'defaultTags.txt') if os.path.exists(tagFilePath): f = open(tagFilePath) try: content = f.read() tags = content.split('\n') tags = [tag.strip() for tag in tags] tagsSet = set(tags) tagsSet.discard('') return tagsSet finally: f.close() return set()
class TestScheduler(BaseIntegrationTestCase): configService = RepoConfigService() METADATA_SCHEDULED = """ generation_type: scheduled generation_interval: 1 """ METADATA_MANUAL = """ generation_type: manual """ def setUp(self): self.newRepoName = unique_repo_name() self.createdRepoDir = self.configService.getStaticRepoDir(self.newRepoName) self.mockedscheduler = self.get_mocked_scheduler() def tearDown(self): if os.path.exists(self.createdRepoDir): shutil.rmtree(self.createdRepoDir) def test_mockscheduler_schedules_update(self): self.create_dummy_repo_with_metadata_generation(None) #creates only repo, no metadata self.mockedscheduler.start() #should NOT queue a job for the repo jobs = self.mockedscheduler.sched.get_active_jobs() self.assertTrue(jobs["NOPATH"]=="update_program_config") def test_mockscheduler_does_not_care_about_manual_jobs(self): self.create_dummy_repo_with_metadata_generation(self.METADATA_MANUAL) self.mockedscheduler.start() #should NOT queue a job for the repo (since is manual) jobs = self.mockedscheduler.sched.get_active_jobs() self.assertFalse(self.createdRepoDir in jobs) def test_mockscheduler_works_on_scheduled_jobs(self): self.create_dummy_repo_with_metadata_generation(self.METADATA_SCHEDULED) self.mockedscheduler.start() #should NOT queue a job for the repo (since is manual) jobs = self.mockedscheduler.sched.get_active_jobs() self.assertTrue(self.createdRepoDir in jobs) def test_mockscheduler_schedules_update_only_once_after_alternation(self): self.create_dummy_repo_with_metadata_generation(self.METADATA_SCHEDULED) self.mockedscheduler.start() self.unscheduled_counter=0 self.add_job_counter=0 def count_unschedule(reponame): self.unscheduled_counter += 1 MetaDataGenerationScheduler.unschedule_by_reponame(self.mockedscheduler, reponame) def count_add_job(reponame): self.add_job_counter += 1 MetaDataGenerationScheduler.add_job_for_repo(self.mockedscheduler, reponame) self.mockedscheduler.unschedule_by_reponame=count_unschedule self.mockedscheduler.add_job_for_repo=count_add_job # update timestamp of metadataFile metadata_file = self._get_metadata_generation_file_path(self.createdRepoDir) # ensure, that the timestamp will be changed. time.sleep(1) self._touch_file(metadata_file) self.mockedscheduler.update_program_config() self.mockedscheduler.update_program_config() self.mockedscheduler.update_program_config() self.assertEqual(1, self.unscheduled_counter, 'too many jobs were unscheduled.') self.assertEqual(1, self.add_job_counter, 'too many jobs were added.') def test_mockscheduler_works_on_scheduled_jobs_and_stops_them_when_repo_deleted(self): self.create_dummy_repo_with_metadata_generation(self.METADATA_SCHEDULED) self.mockedscheduler.start() #queues a job self.assertTrue(self.createdRepoDir in self.mockedscheduler.sched.get_active_jobs()) shutil.rmtree(self.createdRepoDir) self.mockedscheduler.update_program_config() #should remove the queued job since repo was deleted self.assertFalse(self.createdRepoDir in self.mockedscheduler.sched.get_active_jobs()) def test_mockscheduler_works_on_scheduled_jobs_and_stops_them_when_metadata_file_deleted(self): self.create_dummy_repo_with_metadata_generation(self.METADATA_SCHEDULED) self.mockedscheduler.start() #queues a job self.assertTrue(self.createdRepoDir in self.mockedscheduler.sched.get_active_jobs()) os.remove(self.configService.getMetaDataGenerationFilePathRelativeToRepoDirByRepoName(self.newRepoName)) self.mockedscheduler.update_program_config() #should remove the queued job since file was deleted self.assertFalse(self.createdRepoDir in self.mockedscheduler.sched.get_active_jobs()) def test_mockscheduler_works_on_scheduled_jobs_and_stops_them_when_switching_to_manual(self): self.create_dummy_repo_with_metadata_generation(self.METADATA_SCHEDULED) self.mockedscheduler.start() #queues a job self.assertTrue(self.createdRepoDir in self.mockedscheduler.sched.get_active_jobs()) time.sleep(1) #THIS IS NECESSARY!! if not present, in some cases the timestamp does not change because the test is so fast, causing it to fail (no change = no update..) self.create_dummy_repo_with_metadata_generation(self.METADATA_MANUAL) #overwrite with manual self.mockedscheduler.update_program_config() #should remove the queued job since was set to manual self.assertFalse(self.createdRepoDir in self.mockedscheduler.sched.get_active_jobs()) def test_mockscheduler_works_on_scheduled_jobs_after_switching_from_manual_to_scheduled(self): self.create_dummy_repo_with_metadata_generation(self.METADATA_MANUAL) self.mockedscheduler.start() #queues a job self.assertFalse(self.createdRepoDir in self.mockedscheduler.sched.get_active_jobs()) time.sleep(2) #THIS IS NECESSARY!! if not present, in some cases the timestamp does not change because the test is so fast, causing it to fail (no change = no update..) self.create_dummy_repo_with_metadata_generation(self.METADATA_SCHEDULED) #overwrite with scheduled self.mockedscheduler.update_program_config() #should add a job since was set to scheduled self.assertTrue(self.createdRepoDir in self.mockedscheduler.sched.get_active_jobs()) def get_mocked_scheduler(self): mockedscheduler = MetaDataGenerationScheduler(5) mockedscheduler.sched=MockedScheduler() return mockedscheduler def create_dummy_repo_with_metadata_generation(self,metadataContents): if not os.path.exists(self.createdRepoDir): os.makedirs(self.createdRepoDir) if not metadataContents: return filepath = self._get_metadata_generation_file_path(self.createdRepoDir) schedulingFile = open(filepath,'w') schedulingFile.write(metadataContents) schedulingFile.close() def _get_metadata_generation_file_path(self, repoDir): return repoDir+"/"+self.configService.METADATA_GENERATION_FILENAME def _touch_file(self, filename): handle = file(filename, 'a') try: os.utime(filename, None) finally: handle.close()
class TestTagRepo(BaseIntegrationTestCase): config=RepoConfigService() def test_tag_repo_fails_with_no_tag_in_postdata(self): reponame=self.createNewRepoAndAssertValid() response = self.doHttpPost(Constants.HTTP_PATH_STATIC+"/"+reponame+"/tags/","hello world how are you today") self.assertTrue(response.status, httplib.BAD_REQUEST) def test_tag_repo_post_ok(self): reponame=self.createNewRepoAndAssertValid() tag="tag_"+reponame response = self.repoclient().tagRepo(reponame,tag) self.assertEquals(response.status, httplib.CREATED) def test_tag_repo_read_ok(self): reponame=self.createNewRepoAndAssertValid() tag="tag_"+reponame self.repoclient().tagRepo(reponame,tag) response = self.repoclient().tagList(reponame) read_tags=response.read() self.assertTrue(tag in read_tags) def test_tag_repo_writes_tag(self): reponame=self.createNewRepoAndAssertValid() tag="tag_"+reponame self.repoclient().tagRepo(reponame,tag) targetFile=self.config.getTagsFileForStaticRepo(reponame) self.assertTrue(self.fileContains(targetFile,tag)) def test_simple_tag_untag_repo(self): reponame=self.createNewRepoAndAssertValid() tag="tag_"+reponame self.repoclient().tagRepo(reponame,tag) targetFile=self.config.getTagsFileForStaticRepo(reponame) self.assertTrue(self.fileContains(targetFile,tag)) response=self.repoclient().untagRepo(reponame,tag) self.assertEquals(response.status, httplib.NO_CONTENT) self.assertFalse(self.fileContains(targetFile,tag)) def test_many_tags_untag(self): reponame=self.createNewRepoAndAssertValid() for i in range(10): self.repoclient().tagRepo(reponame,str(i)) self.assertTrue(str(i) in self.repoclient().tagList(reponame).read()) targetFile=self.config.getTagsFileForStaticRepo(reponame) self.assertTrue(self.fileContains(targetFile,str(5))) response=self.repoclient().untagRepo(reponame,str(5)) self.assertEquals(response.status, httplib.NO_CONTENT) self.assertFalse(self.fileContains(targetFile,str(5))) for i in range(10): if not i==5: self.assertTrue(self.fileContains(targetFile,str(i))) def test_untag_tag_fails_if_tag_not_present(self): reponame=self.createNewRepoAndAssertValid() tag="tag_"+reponame self.repoclient().tagRepo(reponame,tag) msg=self.repoclient().doHttpDelete(Constants.HTTP_PATH_STATIC+"/"+reponame+"/tags/asladalkdsla") self.assertEquals(msg.status, httplib.NOT_FOUND) def test_tag_twice_writes_tags(self): reponame=self.createNewRepoAndAssertValid() tag1="tag1_"+reponame tag2="tag2_"+reponame self.repoclient().tagRepo(reponame,tag1) self.repoclient().tagRepo(reponame,tag2) targetFile=self.config.getTagsFileForStaticRepo(reponame) self.assertTrue(self.fileContains(targetFile,tag1)) self.assertTrue(self.fileContains(targetFile,tag2)) def fileContains(self,targetFile,contents): f = open(targetFile, "r") text = f.read() f.close() if contents in text: return True return False
def test_get_rpm_group_with_obsolete_files_by_file_name(self): dict_group_with_obsolete_rpms = RepoConfigService()._get_rpm_group_with_obsolete_files_by_file_name( self.rpm_file_names, 3) self.assertEqual(self.dict_rpm_tuples.keys(), dict_group_with_obsolete_rpms.keys()) self.assertEqual(self.dict_rpm_tuples.values(), dict_group_with_obsolete_rpms.values())