def test_map_discarding_regex_group(self):
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': 'project/'}],
['plugins/Plugin_1'])
self.assertEqual(
{'plugins/Plugin_1': '/basepath/project'},
cmap.expand('/basepath'))
def test_map_discarding_regex_group(self):
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': 'project/'}],
['plugins/Plugin_1'])
self.assertEqual(
{'plugins/Plugin_1': '/basepath/project'},
cmap.expand('/basepath'))
def test_map_using_regex(self):
"""One can use regex in maps and use \\1 to forge the directory"""
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': 'project/plugins/\\1'}],
['plugins/PluginFirst'])
self.assertEqual(
{'plugins/PluginFirst': '/basepath/project/plugins/PluginFirst'},
cmap.expand('/basepath'))
def test_map_to_a_dot_dir(self):
"""Verify we normalize path, hence '.' refers to the basepath"""
cmap = CloneMapper(
[{'name': 'mediawiki/core', 'dest': '.'}],
['mediawiki/core'])
self.assertEqual(
{'mediawiki/core': '/basepath'},
cmap.expand('/basepath'))
def execute(self):
mapper = CloneMapper(self.clone_map, self.projects)
dests = mapper.expand(workspace=self.workspace)
self.log.info("Preparing %s repositories", len(dests))
for project, dest in six.iteritems(dests):
self.prepareRepo(project, dest)
self.log.info("Prepared all repositories")
def test_map_with_dot_and_regex(self):
"""Combining relative path and regex"""
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': './\\1'}],
['plugins/PluginInBasePath'])
self.assertEqual(
{'plugins/PluginInBasePath': '/basepath/PluginInBasePath'},
cmap.expand('/basepath'))
def test_map_using_regex(self):
"""One can use regex in maps and use \\1 to forge the directory"""
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': 'project/plugins/\\1'}],
['plugins/PluginFirst'])
self.assertEqual(
{'plugins/PluginFirst': '/basepath/project/plugins/PluginFirst'},
cmap.expand('/basepath'))
def test_map_to_a_dot_dir(self):
"""Verify we normalize path, hence '.' refers to the basepath"""
cmap = CloneMapper(
[{'name': 'mediawiki/core', 'dest': '.'}],
['mediawiki/core'])
self.assertEqual(
{'mediawiki/core': '/basepath'},
cmap.expand('/basepath'))
def test_map_with_dot_and_regex(self):
"""Combining relative path and regex"""
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': './\\1'}],
['plugins/PluginInBasePath'])
self.assertEqual(
{'plugins/PluginInBasePath': '/basepath/PluginInBasePath'},
cmap.expand('/basepath'))
def execute(self):
mapper = CloneMapper(self.clone_map, self.projects)
dests = mapper.expand(workspace=self.workspace)
self.log.info("Preparing %s repositories", len(dests))
for project, dest in dests.iteritems():
self.prepareRepo(project, dest)
self.log.info("Prepared all repositories")
def test_empty_mapper(self):
"""Given an empty map, the slashes in project names are directory
separators"""
cmap = CloneMapper({}, ['project1', 'plugins/plugin1'])
self.assertEqual(
{
'project1': '/basepath/project1',
'plugins/plugin1': '/basepath/plugins/plugin1'
}, cmap.expand('/basepath'))
def test_empty_mapper(self):
"""Given an empty map, the slashes in project names are directory
separators"""
cmap = CloneMapper(
{},
[
'project1',
'plugins/plugin1'
])
self.assertEqual(
{'project1': '/basepath/project1',
'plugins/plugin1': '/basepath/plugins/plugin1'},
cmap.expand('/basepath')
)
def test_cant_dupe_destinations(self):
"""We cant clone multiple projects in the same directory"""
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': 'catchall/'}],
['plugins/plugin1', 'plugins/plugin2']
)
self.assertRaises(Exception, cmap.expand, '/basepath')
def repo_dir(repo):
mapper = CloneMapper(CLONE_MAP, [repo])
return mapper.expand(workspace='./')[repo]
def getProjectMap(self, project):
cm = CloneMapper(
yaml.safe_load(self.yaml_conf).get('clonemap'), [project])
return cm.expand(self.TEST_ROOT)
def getProjectMap(self, project):
cm = CloneMapper(yaml.load(self.yaml_conf).get('clonemap'),
[project])
return cm.expand(self.TEST_ROOT)
def clone(branch, cache_dir, project_branch, projects, workers, workspace,
zuul_branch, zuul_newrev, zuul_project, zuul_ref, zuul_url):
log = logging.getLogger('quibble.zuul.clone')
if isinstance(projects, str):
projects = [projects]
if zuul_ref is not None and zuul_url is None:
raise Exception('Zuul ref requires a Zuul url')
if zuul_newrev is not None and zuul_project is None:
raise Exception('Zuul newrev requires a Zuul project')
project_branches = {}
if project_branch:
for x in project_branch:
p, p_branch = x[0].split('=')
project_branches[p] = p_branch
zuul_cloner = Cloner(
git_base_url='https://gerrit.wikimedia.org/r',
projects=projects,
workspace=workspace,
zuul_branch=zuul_branch,
zuul_ref=zuul_ref,
zuul_url=zuul_url,
branch=branch,
project_branches=project_branches,
cache_dir=cache_dir,
zuul_newrev=zuul_newrev,
zuul_project=zuul_project,
cache_no_hardlinks=False, # False allows hardlink
)
# The constructor expects a file, set the value directly
zuul_cloner.clone_map = CLONE_MAP
# Reimplement Cloner.execute() to make sure mediawiki/core is cloned first
# and clone the rest in parallel.
mapper = CloneMapper(CLONE_MAP, projects)
dests = mapper.expand(workspace=workspace)
if workers == 1:
return zuul_cloner.execute()
# Reimplement the cloner execute method with parallelism and logging
# suitable for multiplexed output.
log.info("Preparing %s repositories with %s workers" % (
len(dests), workers))
mw_git_dir = os.path.join(dests['mediawiki/core'], '.git')
if not os.path.exists(mw_git_dir):
log.info("Cloning mediawiki/core first")
zuul_cloner.prepareRepo('mediawiki/core', dests['mediawiki/core'])
del(dests['mediawiki/core'])
with ThreadPoolExecutor(max_workers=workers) as executor:
for project, dest in dests.items():
# Copy and hijack the logger
project_cloner = copy.copy(zuul_cloner)
project_cloner.log = project_cloner.log.getChild(project)
executor.submit(project_cloner.prepareRepo, project, dest)
log.info("Prepared all repositories")
