def test_map_discarding_regex_group(self): cmap = CloneMapper( [{'name': 'plugins/(.*)', 'dest': 'project/'}], ['plugins/Plugin_1']) self.assertEqual( {'plugins/Plugin_1': '/basepath/project'}, cmap.expand('/basepath'))
def test_map_using_regex(self): """One can use regex in maps and use \\1 to forge the directory""" cmap = CloneMapper( [{'name': 'plugins/(.*)', 'dest': 'project/plugins/\\1'}], ['plugins/PluginFirst']) self.assertEqual( {'plugins/PluginFirst': '/basepath/project/plugins/PluginFirst'}, cmap.expand('/basepath'))
def test_map_to_a_dot_dir(self): """Verify we normalize path, hence '.' refers to the basepath""" cmap = CloneMapper( [{'name': 'mediawiki/core', 'dest': '.'}], ['mediawiki/core']) self.assertEqual( {'mediawiki/core': '/basepath'}, cmap.expand('/basepath'))
def execute(self): mapper = CloneMapper(self.clone_map, self.projects) dests = mapper.expand(workspace=self.workspace) self.log.info("Preparing %s repositories", len(dests)) for project, dest in six.iteritems(dests): self.prepareRepo(project, dest) self.log.info("Prepared all repositories")
def test_map_with_dot_and_regex(self): """Combining relative path and regex""" cmap = CloneMapper( [{'name': 'plugins/(.*)', 'dest': './\\1'}], ['plugins/PluginInBasePath']) self.assertEqual( {'plugins/PluginInBasePath': '/basepath/PluginInBasePath'}, cmap.expand('/basepath'))
def execute(self): mapper = CloneMapper(self.clone_map, self.projects) dests = mapper.expand(workspace=self.workspace) self.log.info("Preparing %s repositories", len(dests)) for project, dest in dests.iteritems(): self.prepareRepo(project, dest) self.log.info("Prepared all repositories")
def test_empty_mapper(self): """Given an empty map, the slashes in project names are directory separators""" cmap = CloneMapper({}, ['project1', 'plugins/plugin1']) self.assertEqual( { 'project1': '/basepath/project1', 'plugins/plugin1': '/basepath/plugins/plugin1' }, cmap.expand('/basepath'))
def test_empty_mapper(self): """Given an empty map, the slashes in project names are directory separators""" cmap = CloneMapper( {}, [ 'project1', 'plugins/plugin1' ]) self.assertEqual( {'project1': '/basepath/project1', 'plugins/plugin1': '/basepath/plugins/plugin1'}, cmap.expand('/basepath') )
def test_cant_dupe_destinations(self): """We cant clone multiple projects in the same directory""" cmap = CloneMapper( [{'name': 'plugins/(.*)', 'dest': 'catchall/'}], ['plugins/plugin1', 'plugins/plugin2'] ) self.assertRaises(Exception, cmap.expand, '/basepath')
def repo_dir(repo): mapper = CloneMapper(CLONE_MAP, [repo]) return mapper.expand(workspace='./')[repo]
def getProjectMap(self, project): cm = CloneMapper( yaml.safe_load(self.yaml_conf).get('clonemap'), [project]) return cm.expand(self.TEST_ROOT)
def getProjectMap(self, project): cm = CloneMapper(yaml.load(self.yaml_conf).get('clonemap'), [project]) return cm.expand(self.TEST_ROOT)
def clone(branch, cache_dir, project_branch, projects, workers, workspace, zuul_branch, zuul_newrev, zuul_project, zuul_ref, zuul_url): log = logging.getLogger('quibble.zuul.clone') if isinstance(projects, str): projects = [projects] if zuul_ref is not None and zuul_url is None: raise Exception('Zuul ref requires a Zuul url') if zuul_newrev is not None and zuul_project is None: raise Exception('Zuul newrev requires a Zuul project') project_branches = {} if project_branch: for x in project_branch: p, p_branch = x[0].split('=') project_branches[p] = p_branch zuul_cloner = Cloner( git_base_url='https://gerrit.wikimedia.org/r', projects=projects, workspace=workspace, zuul_branch=zuul_branch, zuul_ref=zuul_ref, zuul_url=zuul_url, branch=branch, project_branches=project_branches, cache_dir=cache_dir, zuul_newrev=zuul_newrev, zuul_project=zuul_project, cache_no_hardlinks=False, # False allows hardlink ) # The constructor expects a file, set the value directly zuul_cloner.clone_map = CLONE_MAP # Reimplement Cloner.execute() to make sure mediawiki/core is cloned first # and clone the rest in parallel. mapper = CloneMapper(CLONE_MAP, projects) dests = mapper.expand(workspace=workspace) if workers == 1: return zuul_cloner.execute() # Reimplement the cloner execute method with parallelism and logging # suitable for multiplexed output. log.info("Preparing %s repositories with %s workers" % ( len(dests), workers)) mw_git_dir = os.path.join(dests['mediawiki/core'], '.git') if not os.path.exists(mw_git_dir): log.info("Cloning mediawiki/core first") zuul_cloner.prepareRepo('mediawiki/core', dests['mediawiki/core']) del(dests['mediawiki/core']) with ThreadPoolExecutor(max_workers=workers) as executor: for project, dest in dests.items(): # Copy and hijack the logger project_cloner = copy.copy(zuul_cloner) project_cloner.log = project_cloner.log.getChild(project) executor.submit(project_cloner.prepareRepo, project, dest) log.info("Prepared all repositories")