def test_clone_fetch_if_local_repo_missing(self, sh_mock):
repositories.refreshrepo('url', 'path', branch='branch', local=True)
expected = [mock.call(sh.git.clone, 'url', 'path'),
mock.call(sh.git.fetch, 'origin'),
mock.call(sh.git.checkout, '-f', 'branch'),
mock.call(sh.git.reset, '--hard', 'origin/branch'),
mock.call(sh.git.log, '--pretty=format:%H %ct', '-1', '.')]
self.assertEqual(sh_mock.call_args_list, expected)
def test_dont_clone_if_cloned(self, path_mock, shutil_mock, sh_mock):
repositories.refreshrepo('url', 'path', branch='branch')
expected = [mock.call(sh.git, 'remote', '-v'),
mock.call(sh.git.clone, 'url', 'path'),
mock.call(sh.git.fetch, 'origin'),
mock.call(sh.git.checkout, '-f', 'branch'),
mock.call(sh.git.reset, '--hard', 'origin/branch'),
mock.call(sh.git.log, '--pretty=format:%H %ct', '-1', '.')]
self.assertEqual(sh_mock.call_args_list, expected)
def test_clone_fallback_var(self, sh_mock):
config = configparser.RawConfigParser()
config.read("projects.ini")
config.set('DEFAULT', 'fallback_to_master', '1')
config.set('DEFAULT', 'nonfallback_branches', '^foo-')
self.config = ConfigOptions(config)
with mock.patch.object(sh.Command, '__call__') as new_mock:
new_mock.side_effect = _aux_sh
result = repositories.refreshrepo('url', 'path', branch='bar')
self.assertEqual(result, ['master', 'None'])
expected = [
mock.call('url', 'path'),
mock.call('origin'),
mock.call('-f', 'bar'),
mock.call('master'),
mock.call('--hard', 'origin/master'),
mock.call('--pretty=format:%H %ct', '-1', '.')
]
self.assertEqual(new_mock.call_args_list, expected)
def getinfo(self, **kwargs):
project = kwargs.get('project')
package = kwargs.get('package')
since = kwargs.get('since')
local = kwargs.get('local')
dev_mode = kwargs.get('dev_mode')
datadir = self.config_options.datadir
repo = package['upstream']
distro = package['master-distgit']
tags_only = buildtagsonly(package)
distro_dir = self._distgit_clone_dir(package['name'])
distro_dir_full = self.distgit_dir(package['name'])
distro_branch = getdistrobranch(package)
source_branch = getsourcebranch(package)
if dev_mode is False:
try:
distro_branch, distro_hash, dt_distro = refreshrepo(
distro, distro_dir, distro_branch, local=local,
full_path=distro_dir_full)
except Exception:
# The error was already logged by refreshrepo, and we want
# to avoid halting the whole run because this distgit repo
# failed, so return an empty list
return []
else:
distro_hash = "dev"
dt_distro = 0 # Doesn't get used in dev mode
if not os.path.isdir(distro_dir):
# We should fail in this case, since we are running
# in dev mode, so no try/except
refreshrepo(distro, distro_dir, distro_branch, local=local,
full_path=distro_dir_full)
# repo is usually a string, but if it contains more then one entry we
# git clone into a project subdirectory
repos = [repo]
if isinstance(repo, list):
repos = repo
project_toprocess = []
for repo in repos:
repo_dir = os.path.join(datadir, project)
if len(repos) > 1:
repo_dir = os.path.join(repo_dir, os.path.split(repo)[1])
try:
source_branch, _, _ = refreshrepo(repo, repo_dir, source_branch,
local=local)
except Exception:
# The error was already logged by refreshrepo, and the only
# side-effect is that we are not adding this commit to the
# list of commits to be processed, so we can ignore it and
# move on to the next repo
continue
git = sh.git.bake(_cwd=repo_dir, _tty_out=False)
# Git gives us commits already sorted in the right order
if tags_only is True:
logger.info('Building tags only for %s' % project)
if since == '-1':
# we need 2 entries as HEAD will be listed too
since = '-2'
lines = filter(
lambda x: x.find('tag: ') >= 0,
git.log('--simplify-by-decoration',
"--pretty=format:'%ct %H %d'",
since, "--first-parent",
"--reverse", "%s" % source_branch))
else:
lines = git.log("--pretty=format:'%ct %H'",
since, "--first-parent",
"--reverse")
for line in lines:
dt, commit_hash = str(line).strip().strip("'").split(" ")[:2]
commit = Commit(dt_commit=float(dt), project_name=project,
type="rpm",
commit_hash=commit_hash, repo_dir=repo_dir,
distro_hash=distro_hash, dt_distro=dt_distro,
distgit_dir=self.distgit_dir(package['name']),
commit_branch=source_branch,
dt_extended=0, extended_hash=None)
project_toprocess.append(commit)
return project_toprocess
def getinfo(self, **kwargs):
project = kwargs.get('project')
package = kwargs.get('package')
local = kwargs.get('local')
dev_mode = kwargs.get('dev_mode')
datadir = self.config_options.datadir
repo = package['upstream']
distgit_attr = self.config_options.downstream_distgit_key or 'distgit'
distro = package.get(distgit_attr)
if not distro:
fail_req_attr_missing(distgit_attr, package)
distro_dir = self._distgit_clone_dir(package['name'])
distro_dir_full = self.distgit_dir(package['name'])
distro_branch = self.config_options.downstream_distro_branch
if not distro_branch:
fail_req_config_missing('downstream_distro_branch')
source_branch = getsourcebranch(package)
versions = self._getversions()
ups_distro = package['master-distgit']
ups_distro_dir = self._upstream_distgit_clone_dir(package['name'])
ups_distro_dir_full = self.upstream_distgit_dir(package['name'])
ups_distro_branch = getdistrobranch(package)
# only process packages present in versions.csv
if package['name'] not in versions:
logger.warning('Package %s not present in %s - skipping.' % (
package['name'], self.config_options.versions_url))
return []
version = versions[package['name']]
dt_distro = 0 # In this driver we do not care about dt_distro
if dev_mode is False:
try:
distro_branch, extended_hash, dt_extended = refreshrepo(
distro, distro_dir, distro_branch, local=local,
full_path=distro_dir_full)
# extract distro_hash from versions.csv
distro_hash = version[3]
# Also download upstream distgit
_, _, _ = refreshrepo(
ups_distro, ups_distro_dir, distro_hash,
local=local, full_path=ups_distro_dir_full)
except Exception:
# The error was already logged by refreshrepo, and we want
# to avoid halting the whole run because this distgit repo
# failed, so return an empty list
return []
else:
distro_hash = "dev"
extended_hash = "dev"
dt_extended = 0
if not os.path.isdir(distro_dir):
# We should fail in this case, since we are running
# in dev mode, so no try/except
refreshrepo(distro, distro_dir, distro_branch, local=local,
full_path=distro_dir_full)
if not os.path.isdir(ups_distro_dir):
refreshrepo(ups_distro, ups_distro_dir, ups_distro_branch,
local=local, full_path=ups_distro_dir_full)
# repo is usually a string, but if it contains more then one entry we
# git clone into a project subdirectory
repos = [repo]
if isinstance(repo, list):
repos = repo
project_toprocess = []
for repo in repos:
repo_dir = os.path.join(datadir, project)
if len(repos) > 1:
repo_dir = os.path.join(repo_dir, os.path.split(repo)[1])
try:
source_branch, _, _ = refreshrepo(repo, repo_dir,
source_branch,
local=local)
except Exception:
# The error was already logged by refreshrepo, and the only
# side-effect is that we are not adding this commit to the
# list of commits to be processed, so we can ignore it and
# move on to the next repo
continue
dt = version[5]
commit_hash = version[1]
commit = Commit(dt_commit=float(dt), project_name=project,
type='rpm',
commit_hash=commit_hash, repo_dir=repo_dir,
distro_hash=distro_hash, dt_distro=dt_distro,
extended_hash=extended_hash,
dt_extended=dt_extended,
distgit_dir=self.distgit_dir(package['name']),
commit_branch=source_branch)
project_toprocess.append(commit)
return project_toprocess
def getinfo(self, **kwargs):
project = kwargs.get('project')
package = kwargs.get('package')
since = kwargs.get('since')
local = kwargs.get('local')
dev_mode = kwargs.get('dev_mode')
datadir = self.config_options.datadir
repo = package['upstream']
distro = package['master-distgit']
tags_only = buildtagsonly(package)
distro_dir = self._distgit_clone_dir(package['name'])
distro_dir_full = self.distgit_dir(package['name'])
distro_branch = getdistrobranch(package)
source_branch = getsourcebranch(package)
if dev_mode is False:
try:
distro_branch, distro_hash, dt_distro = refreshrepo(
distro,
distro_dir,
distro_branch,
local=local,
full_path=distro_dir_full)
except Exception:
# The error was already logged by refreshrepo, and we want
# to avoid halting the whole run because this distgit repo
# failed, so return an empty list
return []
else:
distro_hash = "dev"
dt_distro = 0 # Doesn't get used in dev mode
if not os.path.isdir(distro_dir):
# We should fail in this case, since we are running
# in dev mode, so no try/except
refreshrepo(distro,
distro_dir,
distro_branch,
local=local,
full_path=distro_dir_full)
# repo is usually a string, but if it contains more then one entry we
# git clone into a project subdirectory
repos = [repo]
if isinstance(repo, list):
repos = repo
project_toprocess = []
for repo in repos:
repo_dir = os.path.join(datadir, project)
if len(repos) > 1:
repo_dir = os.path.join(repo_dir, os.path.split(repo)[1])
try:
source_branch, _, _ = refreshrepo(repo,
repo_dir,
source_branch,
local=local)
except Exception:
# The error was already logged by refreshrepo, and the only
# side-effect is that we are not adding this commit to the
# list of commits to be processed, so we can ignore it and
# move on to the next repo
continue
git = sh.git.bake(_cwd=repo_dir, _tty_out=False)
# Git gives us commits already sorted in the right order
if tags_only is True:
logger.info('Building tags only for %s' % project)
if since == '-1':
# we need 2 entries as HEAD will be listed too
since = '-2'
lines = filter(
lambda x: x.find('tag: ') >= 0,
git.log('--simplify-by-decoration',
"--pretty=format:'%ct %H %d'", since,
"--first-parent", "--reverse",
"%s" % source_branch))
else:
lines = git.log("--pretty=format:'%ct %H'", since,
"--first-parent", "--reverse")
for line in lines:
dt, commit_hash = str(line).strip().strip("'").split(" ")[:2]
commit = Commit(dt_commit=float(dt),
project_name=project,
commit_hash=commit_hash,
repo_dir=repo_dir,
distro_hash=distro_hash,
dt_distro=dt_distro,
distgit_dir=self.distgit_dir(package['name']),
commit_branch=source_branch)
project_toprocess.append(commit)
return project_toprocess
def test_dont_fetch_if_local_repo_exists(self, path_mock, sh_mock):
repositories.refreshrepo('url', 'path', branch='branch', local=True)
expected = [mock.call(sh.git.log, '--pretty=format:%H %ct', '-1', '.')]
self.assertEqual(sh_mock.call_args_list, expected)
def getinfo(self, **kwargs):
project = kwargs.get('project')
package = kwargs.get('package')
local = kwargs.get('local')
dev_mode = kwargs.get('dev_mode')
datadir = self.config_options.datadir
repo = package['upstream']
distgit_attr = self.config_options.downstream_distgit_key or 'distgit'
distro = package.get(distgit_attr)
if not distro:
fail_req_attr_missing(distgit_attr, package)
distro_dir = self._distgit_clone_dir(package['name'])
distro_dir_full = self.distgit_dir(package['name'])
distro_branch = self.config_options.downstream_distro_branch
if not distro_branch:
fail_req_config_missing('downstream_distro_branch')
source_branch = getsourcebranch(package)
versions = self._getversions()
ups_distro = package['master-distgit']
ups_distro_dir = self._upstream_distgit_clone_dir(package['name'])
ups_distro_dir_full = self.upstream_distgit_dir(package['name'])
ups_distro_branch = getdistrobranch(package)
# Downstream source git
dsgit_attr = self.config_options.downstream_source_git_key
ds_source = package.get(dsgit_attr)
if not distro:
fail_req_attr_missing(dsgit_attr, package)
dsgit_dir = self._downstream_git_clone_dir(package['name'])
dsgit_branch = self.config_options.downstream_source_git_branch
# only process packages present in versions.csv
if package['name'] not in versions:
logger.warning('Package %s not present in %s - skipping.' %
(package['name'], self.config_options.versions_url))
return PkgInfoDriver.Info([], True)
version = versions[package['name']]
dt_distro = 0 # In this driver we do not care about dt_distro
if dev_mode is False:
try:
distro_branch, extended_hash_dsdist, dt_extended = refreshrepo(
distro,
distro_dir,
distro_branch,
local=local,
full_path=distro_dir_full)
_, extended_hash_dssource, _ = refreshrepo(ds_source,
dsgit_dir,
dsgit_branch,
local=local,
full_path=dsgit_dir)
extended_hash = '%s_%s' % (extended_hash_dsdist,
extended_hash_dssource)
# extract distro_hash from versions.csv
distro_hash = version[3]
# Also download upstream distgit
_, _, _ = refreshrepo(ups_distro,
ups_distro_dir,
distro_hash,
local=local,
full_path=ups_distro_dir_full)
except Exception:
# The error was already logged by refreshrepo, and we want
# to avoid halting the whole run because this distgit repo
# failed, so return an empty list
return PkgInfoDriver.Info([], True)
else:
distro_hash = "dev"
extended_hash = "dev"
dt_extended = 0
if not os.path.isdir(distro_dir):
# We should fail in this case, since we are running
# in dev mode, so no try/except
refreshrepo(distro,
distro_dir,
distro_branch,
local=local,
full_path=distro_dir_full)
if not os.path.isdir(ups_distro_dir):
refreshrepo(ups_distro,
ups_distro_dir,
ups_distro_branch,
local=local,
full_path=ups_distro_dir_full)
# repo is usually a string, but if it contains more then one entry we
# git clone into a project subdirectory
repos = [repo]
if isinstance(repo, list):
repos = repo
project_toprocess = []
for repo in repos:
repo_dir = os.path.join(datadir, project)
if len(repos) > 1:
repo_dir = os.path.join(repo_dir, os.path.split(repo)[1])
try:
source_branch, _, _ = refreshrepo(repo,
repo_dir,
source_branch,
local=local)
except Exception:
# The error was already logged by refreshrepo, and the only
# side-effect is that we are not adding this commit to the
# list of commits to be processed, so we can ignore it and
# move on to the next repo
return PkgInfoDriver.Info([], True)
if not local:
# This is the default behavior
dt = version[5]
commit_hash = version[1]
else:
# When running with --local, we really want to use the local
# source git, regardless of the upstream versions.csv info
git = sh.git.bake(_cwd=repo_dir, _tty_out=False)
lines = git.log("--pretty=format:'%ct %H'", "-1",
"--first-parent", "--reverse")
for line in lines:
# There is only one line
dt, commit_hash = str(line).strip().strip("'").\
split(" ")[:2]
if self.config_options.use_components and 'component' in package:
component = package['component']
else:
component = None
commit = Commit(dt_commit=float(dt),
project_name=project,
type='rpm',
commit_hash=commit_hash,
repo_dir=repo_dir,
distro_hash=distro_hash,
dt_distro=dt_distro,
extended_hash=extended_hash,
dt_extended=dt_extended,
distgit_dir=self.distgit_dir(package['name']),
commit_branch=source_branch,
component=component)
project_toprocess.append(commit)
return PkgInfoDriver.Info(project_toprocess, False)
def getinfo(self, **kwargs):
project = kwargs.get('project')
package = kwargs.get('package')
since = kwargs.get('since')
local = kwargs.get('local')
dev_mode = kwargs.get('dev_mode')
datadir = self.config_options.datadir
for url in base_urls:
if check_url("%s/%s" % (url, module2upstream(package['name']))):
package['upstream'] = ("%s/%s" %
(url,
module2upstream(package['name'])))
break
else:
logger.error("Could not find upstream URL for project %s" %
package)
repo = package['upstream']
distro_dir = self.distgit_dir(package['name'])
source_branch = getsourcebranch(package)
if dev_mode is True:
distro_hash = "dev"
dt_distro = 0 # Doesn't get used in dev mode
else:
# Get distro_hash from last commit in distgit directory
git = sh.git.bake(_cwd=distro_dir, _tty_out=False)
repoinfo = str(git.log("--pretty=format:%H %ct", "-1", ".")
).strip().split(" ")
distro_hash = repoinfo[0]
dt_distro = repoinfo[1]
# repo is usually a string, but if it contains more then one entry we
# git clone into a project subdirectory
repos = [repo]
if isinstance(repo, list):
repos = repo
project_toprocess = []
for repo in repos:
repo_dir = os.path.join(datadir, project)
if len(repos) > 1:
repo_dir = os.path.join(repo_dir, os.path.split(repo)[1])
try:
source_branch, _, _ = refreshrepo(repo, repo_dir,
source_branch, local=local)
except Exception:
# The error was already logged by refreshrepo, and the only
# side-effect is that we are not adding this commit to the
# list of commits to be processed, so we can ignore it and
# move on to the next repo
continue
git = sh.git.bake(_cwd=repo_dir, _tty_out=False)
# Git gives us commits already sorted in the right order
lines = git.log("--pretty=format:'%ct %H'", since,
"--first-parent", "--reverse")
for line in lines:
dt, commit_hash = str(line).strip().strip("'").split(" ")
commit = Commit(dt_commit=float(dt), project_name=project,
commit_hash=commit_hash, repo_dir=repo_dir,
distro_hash=distro_hash, dt_distro=dt_distro,
distgit_dir=distro_dir,
commit_branch=source_branch)
project_toprocess.append(commit)
return project_toprocess
def getinfo(self, **kwargs):
project = kwargs.get('project')
package = kwargs.get('package')
since = kwargs.get('since')
local = kwargs.get('local')
dev_mode = kwargs.get('dev_mode')
datadir = self.config_options.datadir
for url in base_urls:
if check_url("%s/%s" % (url, module2upstream(package['name']))):
package['upstream'] = ("%s/%s" %
(url,
module2upstream(package['name'])))
break
else:
logger.error("Could not find upstream URL for project %s" %
package)
repo = package['upstream']
distro_dir = self.distgit_dir(package['name'])
source_branch = getsourcebranch(package)
if dev_mode is True:
distro_hash = "dev"
dt_distro = 0 # Doesn't get used in dev mode
else:
# Get distro_hash from last commit in distgit directory
git = sh.git.bake(_cwd=distro_dir, _tty_out=False)
repoinfo = str(git.log("--pretty=format:%H %ct", "-1", ".")
).strip().split(" ")
distro_hash = repoinfo[0]
dt_distro = repoinfo[1]
# repo is usually a string, but if it contains more then one entry we
# git clone into a project subdirectory
repos = [repo]
if isinstance(repo, list):
repos = repo
project_toprocess = []
for repo in repos:
repo_dir = os.path.join(datadir, project)
if len(repos) > 1:
repo_dir = os.path.join(repo_dir, os.path.split(repo)[1])
try:
source_branch, _, _ = refreshrepo(repo, repo_dir,
source_branch, local=local)
except Exception:
# The error was already logged by refreshrepo, and the only
# side-effect is that we are not adding this commit to the
# list of commits to be processed, so we can ignore it and
# move on to the next repo
continue
git = sh.git.bake(_cwd=repo_dir, _tty_out=False)
# Git gives us commits already sorted in the right order
lines = git.log("--pretty=format:'%ct %H'", since,
"--first-parent", "--reverse")
for line in lines:
dt, commit_hash = str(line).strip().strip("'").split(" ")
commit = Commit(dt_commit=float(dt), project_name=project,
type='rpm',
commit_hash=commit_hash, repo_dir=repo_dir,
distro_hash=distro_hash, dt_distro=dt_distro,
distgit_dir=distro_dir,
commit_branch=source_branch,
dt_extended=0, extended_hash=None)
project_toprocess.append(commit)
return project_toprocess
