def analyzeDevelBuild(self, name):
"""
Expects build in the current directory
"""
# create build directory in the current directory
cwd = os.getcwd()
if os.path.exists("%s/%s" % (cwd, 'build')):
return {}
try:
os.mkdir("build", 0755);
except OSError as error:
return {}
os.chdir('build')
runCommand('rpm2cpio ../%s | cpio -idmv' % name)
# scan builds using native golang parser
package_xml = ProjectToXml("", "%s/%s" % (os.getcwd(), 'usr/share/gocode/src/'), self.latest_build)
os.chdir(cwd)
try:
shutil.rmtree("build")
except OSError as error:
return {}
return {
"xmlobj": package_xml
}
def test(self):
# Parse spec file
self.sp_obj = SpecParser(self.spec)
if not self.sp_obj.parse():
self.err = self.sp_obj.getError()
return False
# Parse sources file
if self.sources != "":
self.src_obj = Sources(self.sources)
if not self.src_obj.parse():
self.err = self.src_obj.getError()
return False
# Inspect tarball
self.prj_obj = ProjectInfo(noGodeps = self.noGodeps)
# create a temp directory
dir = tempfile.mkdtemp()
# extract tarball to the directory
so, se, rc = runCommand("tar -xf %s --directory=%s" % (self.archive, dir))
if rc != 0:
self.err = se
return False
so, se, rc = runCommand("ls %s" % dir)
if rc != 0:
self.err = "Unable to archive's extracted folder"
return False
so = so.split('\n')[0]
if not self.prj_obj.retrieve("%s/%s" % (dir, so), skip_errors = True):
self.err = self.prj_obj.getError()
return False
shutil.rmtree(dir)
tests = []
# test package name
tests.append(self.testPackageName)
# test commit
tests.append(self.testCommit)
# test import path macros
tests.append(self.testImportPathMacros)
# test provider, provider_tld, ...
tests.append(self.testRepositoryMacros)
# test source0 macro
tests.append(self.testSpecFileSource)
# test devel subpackage
tests.append(self.testDevel)
for test in tests:
if not test():
self.printTResult(self.t_result)
elif self.verbose:
self.printTResult(self.t_result)
return True
def parse(self): f = tempfile.NamedTemporaryFile(delete=True) cmd_str = "curl http://pkgs.fedoraproject.org/cgit/%s.git/plain/%s.spec > %s" runCommand(cmd_str % (self.package, self.package, f.name)) self.sp_obj = SpecParser(f.name) if not self.sp_obj.parse(): self.err = self.sp_obj.getError() f.close() return False f.close() return True
def getTarball(self, importpath, commit):
"""
Search for a tarball in tarball directory.
If it does not exist, download the tarball.
TODO: Create a DirectoryStorage on top of TarballStorage?
1) extract the archive to
2) rename its directory to TARBALLSIGNATURE_DIR
3) return TARBALLSIGNATURE_DIR
"""
# tarball directory must exist
if not os.path.exists(self.tarball_directory):
self.err = "Tarball directory %s does not exist" % self.tarball_directory
return ""
ri = RepositoryInfo(importpath, commit)
if not ri.retrieve():
self.err = ri.getError()
return ""
# tarball exists?
tarball_path = "%s/%s" % (self.tarball_directory, ri.getSignature())
if not os.path.exists(tarball_path):
ai = ri.getArchiveInfo()
if self.verbose:
print "Downloading %s ..." % ai.archive_url
# download tarball
so, se, rc = runCommand("wget -nv %s --no-check-certificate -O %s" % (ai.archive_url, tarball_path))
if rc != 0:
print "Unable to download tarball:\n%s" % (se)
return ""
return tarball_path
def packageInPkgdb(pkg):
_, _, rt = runCommand("git ls-remote http://pkgs.fedoraproject.org/cgit/" + pkg + ".git/")
if rt == 0:
return True
return False
def getSpecLines(self, spec):
"""
Get interpreted content of spec file
"""
stdout, stderr, rt = runCommand('rpmspec -P %s' % spec)
if rt != 0:
self.err = stderr
return []
return stdout.split('\n')
def getGoSymbols(self, path, imports_only=False):
script_dir = getScriptDir() + "/.."
options = ""
if imports_only:
options = "-imports"
so, se, rc = runCommand("%s/parseGo %s %s" % (script_dir, options, path))
if rc != 0:
return (1, se)
return (0, so)
def run(self):
so, se, rc = runCommand("koji taskinfo %s" % self.task_id)
if rc != 0:
self.err = "Unable to get taskinfo for %s branch's %s task: %s" % (branch, task_id, se)
return
state_lines = filter(lambda l: l.startswith("State"), so.split("\n"))
state = state_lines[0].split(" ")[1]
if state == "closed":
self.state = True
def compareNVRs(nvr1, nvr2):
so, se, rc = runCommand("rpmdev-vercmp %s %s" % (nvr1, nvr2))
if rc != 0:
# if the command is not working => nvr1 < nvr2
return 1;
so = so.split("\n")[0]
if "<" in so:
return -1
elif ">" in so:
return 1
return 0
def downloadBuilds(self, tag = 'rawhide'):
"""
Download all builds in the currect directory
"""
self.latest_build = self.getLatestBuilds(tag)
if self.latest_build == "":
return False
so, se, rc = runCommand("koji download-build %s" % self.latest_build)
if rc != 0:
return False
return True
def runFedpkgScratchBuild(self, srpm):
"""
Run 'fedpkg scratch-build --nowait --srpm=SRPM'
"""
if self.debug == True:
print "Running 'fedpkg scratch-build --nowait --srpm=SRPM'"
if self.dry == True:
so = "Created task: 1"
se = ""
rc = 0
return so, se, rc
else:
return runCommand("fedpkg scratch-build --nowait --srpm=%s" % srpm)
def runFedpkgBuild(self):
"""
Run 'fedpkg build --nowait'
"""
if self.debug == True:
print "Running 'fedpkg build --nowait'"
if self.dry == True:
so = "Created task: 1"
se = ""
rc = 0
return so, se, rc
else:
return runCommand("fedpkg build --nowait")
def runGitPull(self):
"""
Run 'git pull'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'git pull'"
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("git pull")
def runFedpkgPush(self):
"""
Run 'fedpkg push'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'fedpkg push'"
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("fedpkg push")
def runFedpkgSwitchBranch(self, branch):
"""
Run 'fedpkg switch-branch'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'fedpkg switch-branch'"
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("fedpkg switch-branch %s" % branch)
def runFedpkgCherryPick(self, branch):
"""
Run 'git cherry-pick BRANCH'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'git cherry-pick BRANCH'"
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("git cherry-pick %s" % branch)
def runGitCherryPick(self, commit="master"):
"""
Run 'git cherry-pick COMMIT'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'git cherry-pick %s'" % commit
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("git cherry-pick %s" % commit)
def runGitReset(self, branch):
"""
Run 'git reset --hard remotes/origin/BRANCH'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'git reset --hard remotes/origin/%s'" % branch
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("git reset --hard remotes/origin/%s" % branch)
def runGitMerge(self, branch):
"""
Run 'git merge BRANCH'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'git merge %s'" % branch
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("git merge %s" % branch)
def runFedpkgSrpm(self):
"""
Run 'fedpkg srpm'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'fedpkg srpm'"
if self.dry == True:
so = "Wrote: gofed-test-0.6.2-0.3.git89088df.fc20.src.rpm"
se = ""
rc = 0
return so, se, rc
else:
return runCommand("fedpkg srpm")
def runGitPull(self):
"""
Run 'git pull'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'git pull'"
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("git pull")
def getPackagesFromPkgDb():
so, _, rc = runCommand("koji search --regex package '^golang-'")
if rc != 0:
return []
pkgs = []
for line in so.split('\n'):
line = line.strip()
if line == "":
continue
line = line.strip()
pkgs.append(line)
return pkgs
def getPackagesFromPkgDb():
so, _, rc = runCommand("koji search --regex package '^golang-'")
if rc != 0:
return []
pkgs = []
for line in so.split('\n'):
line = line.strip()
if line == "":
continue
line = line.strip()
pkgs.append(line)
return pkgs
def runKojiWaitOverride(self, branch, name):
"""
Run 'koji wait-repo TAG --build=BUILD'.
It returns so, se, rc triple.
"""
build = "%s.%s" % (name, BranchMapper().branch2tag(branch))
build_tag = BranchMapper().branch2build(branch)
if self.debug == True:
print "Running 'koji wait-repo %s --build=%s'" % (build_tag, build)
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("koji wait-repo %s --build=%s" % (build_tag, build))
def runKojiWaitOverride(self, branch, name):
"""
Run 'koji wait-repo TAG --build=BUILD'.
It returns so, se, rc triple.
"""
build = "%s.%s" % (name, branch2tag[branch])
build_tag = branch2build[branch]
if self.debug == True:
print "Running 'koji wait-repo %s --build=%s'" % (build_tag, build)
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("koji wait-repo %s --build=%s" % (build_tag, build))
def analyzeBuilds(self):
"""
For each build return its import paths and provides,
which are computed from source codes, not from quering a build.
"""
builds = {}
so, _, _ = runCommand("ls")
for line in so.split('\n'):
line = line.strip()
if line == '':
continue
# from N-V-R get name
parts = line.split('-')
if len(parts) < 3:
continue
name = "-".join(parts[:-2])
# all devel packages should end with devel sufix
if name.endswith('devel'):
info = self.analyzeDevelBuild(line)
if info != {}:
builds[name] = info
# but some does not have to
# 1) must be binary free or not? Maybe scripts are allowed?
# 2) must contain at least one package different from
# example*, main
else:
info = self.analyzeDevelBuild(line)
if info == {} or "xmlobj" not in info:
continue
if self.isBuildMinimalDevel(info["xmlobj"]):
builds[name] = info
else:
self.other_builds.append(name)
return builds
def analyzeBuilds(self):
"""
For each build return its import paths and provides,
which are computed from source codes, not from quering a build.
"""
builds = {}
so, _, _ = runCommand("ls")
for line in so.split('\n'):
line = line.strip()
if line == '':
continue
# from N-V-R get name
parts = line.split('-')
if len(parts) < 3:
continue
name = "-".join(parts[:-2])
# all devel packages should end with devel sufix
if name.endswith('devel'):
info = self.analyzeDevelBuild(line)
if info != {}:
builds[name] = info
# but some does not have to
# 1) must be binary free or not? Maybe scripts are allowed?
# 2) must contain at least one package different from
# example*, main
else:
info = self.analyzeDevelBuild(line)
if info == {} or "xmlobj" not in info:
continue
if self.isBuildMinimalDevel(info["xmlobj"]):
builds[name] = info
else:
self.other_builds.append(name)
return builds
def runBodhiOverride(self, branch, name):
"""
Run 'bodhi --buildroot-override=BUILD for TAG --duration=DURATION --notes=NOTES'.
It returns so, se, rc triple.
"""
build = "%s.%s" % (name, BranchMapper().branch2tag(branch))
long_tag = BranchMapper().branch2buildCandidate(branch)
build_tag = BranchMapper().branch2build(branch)
if self.debug == True:
print "Running 'bodhi --buildroot-override=%s for %s --duration=20 --notes='temp non-stable dependecy waiting for stable''" % (build, long_tag)
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("bodhi --buildroot-override=%s for %s --duration=20 --notes='temp non-stable dependecy waiting for stable'" % (build, long_tag))
def getDirectory(self, importpath, commit):
tarball = self.storage.getTarball(importpath, commit)
if tarball == "":
return ""
tarball_dir = "%s_dir" % tarball
# extract tarball if not already
if not os.path.exists(tarball_dir):
# let's suppose all tarballs are tar.gz
# tar xf archive.tar -C /target/directory --strip-components=1
os.mkdir(tarball_dir)
so, se, rc = runCommand("tar -xf %s -C %s --strip-components=1" % (tarball, tarball_dir))
if rc != 0 or se != "":
self.err = "Unable to extract tarball"
print "Unable to extract tarball: %s" % se
shutil.rmtree(tarball_dir)
return ""
return tarball_dir
def runBodhiOverride(self, branch, name):
"""
Run 'bodhi --buildroot-override=BUILD for TAG --duration=DURATION --notes=NOTES'.
It returns so, se, rc triple.
"""
build = "%s.%s" % (name, branch2tag[branch])
long_tag = branch2bc[branch]
build_tag = branch2build[branch]
if self.debug == True:
print "Running 'bodhi --buildroot-override=%s for %s --duration=20 --notes='temp non-stable dependecy waiting for stable''" % (build, long_tag)
if self.dry == True:
so = ""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("bodhi --buildroot-override=%s for %s --duration=20 --notes='temp non-stable dependecy waiting for stable'" % (build, long_tag))
def getDirectory(self, importpath, commit):
tarball = self.storage.getTarball(importpath, commit)
if tarball == "":
return ""
tarball_dir = "%s_dir" % tarball
# extract tarball if not already
if not os.path.exists(tarball_dir):
# let's suppose all tarballs are tar.gz
# tar xf archive.tar -C /target/directory --strip-components=1
os.mkdir(tarball_dir)
so, se, rc = runCommand("tar -xf %s -C %s --strip-components=1" %
(tarball, tarball_dir))
if rc != 0 or se != "":
self.err = "Unable to extract tarball"
print "Unable to extract tarball: %s" % se
shutil.rmtree(tarball_dir)
return ""
return tarball_dir
def fetchLatestBuilds(self, tag="rawhide"):
pkgs = self.loadPackages()
outdated = {}
so, se, rc = runCommand("koji -q latest-build %s %s" % (tag, " ".join(pkgs)))
if rc != 0:
print se
return {}
for line in so.split("\n"):
line = line.strip()
if line == "":
continue
nvr = re.sub(r'[ \t]+', ' ', line).split(' ')[0]
parts = nvr.split("-")
pkg = "-".join(parts[0:-2])
outdated[pkg] = nvr
return outdated
def fetchLatestBuilds(self, tag="rawhide"):
pkgs = self.loadPackages()
outdated = {}
so, se, rc = runCommand("koji -q latest-build %s %s" %
(tag, " ".join(pkgs)))
if rc != 0:
print se
return {}
for line in so.split("\n"):
line = line.strip()
if line == "":
continue
nvr = re.sub(r'[ \t]+', ' ', line).split(' ')[0]
parts = nvr.split("-")
pkg = "-".join(parts[0:-2])
outdated[pkg] = nvr
return outdated
def runGitLog(self, depth):
"""
Run 'git log --pretty=format:"%%H" -n DEPTH'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'git log --pretty=format:\"%%H\" -n %s'" % depth
if self.dry == True:
so = """4e604fecc22b498e0d46854ee4bfccdfc1932b12
c46cdd60710b184f834b54cff80502027b66c5e0
6170e22ecb5923bbd22a311f172fcf59c5f16c08
0fc92e675c90e7b9e1eaba0c4837093b9b365317
21cf1880e696fd7047f8b0f5605ffa72dde6c504
8025678aab1c404aecdd6d7e5b3afaf9942ef6c6
6ed91011294946fd7ca6e6382b9686e12deda9be
ec0ebc48684bccbd4793b83edf14c59076edb1eb
adf728db9355a86332e17436a78f54a769e194be"""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("git log --pretty=format:\"%%H\" -n %s" % depth)
def runGitLog(self, depth):
"""
Run 'git log --pretty=format:"%%H" -n DEPTH'.
It returns so, se, rc triple.
"""
if self.debug == True:
print "Running 'git log --pretty=format:\"%%H\" -n %s'" % depth
if self.dry == True:
so = """4e604fecc22b498e0d46854ee4bfccdfc1932b12
c46cdd60710b184f834b54cff80502027b66c5e0
6170e22ecb5923bbd22a311f172fcf59c5f16c08
0fc92e675c90e7b9e1eaba0c4837093b9b365317
21cf1880e696fd7047f8b0f5605ffa72dde6c504
8025678aab1c404aecdd6d7e5b3afaf9942ef6c6
6ed91011294946fd7ca6e6382b9686e12deda9be
ec0ebc48684bccbd4793b83edf14c59076edb1eb
adf728db9355a86332e17436a78f54a769e194be"""
se = ""
rc = 0
return so, se, rc
else:
return runCommand("git log --pretty=format:\"%%H\" -n %s" % depth)
def getRepoCommits(path, repo, pull=True):
# path does not exists? create one
runCommand("mkdir -p %s" % path)
cwd = os.getcwd()
os.chdir('/'.join(path.split('/')[:-1]))
logs = ""
# git or hg?
if len(repo) < 4 or repo[-4:] != '.git':
runCommand("hg clone %s" % repo)
repo_dir = repo.split('/')[-1]
os.chdir(repo_dir)
if pull:
runCommand("hg pull")
logs, se, rc = runCommand(
'hg log --template "{date|hgdate} {node}\n" | cut -d" " -f1,3 | sed "s/ /:/g"'
)
else:
runCommand("git clone %s" % repo)
repo_dir = repo.split('/')[-1][:-4]
os.chdir(repo_dir)
if pull:
runCommand('git pull')
logs, se, rc = runCommand('git log --pretty=format:"%ct:%H"')
commits = {}
for line in logs.split('\n'):
line = line.strip().split(':')
if len(line) != 2:
continue
# timestamp:commit
commits[line[1]] = line[0]
os.chdir(cwd)
return commits
def getLatestBuilds(self, tag='rawhide'):
so, _, rc = runCommand("koji -q latest-build %s %s" %
(tag, self.pkg_name))
if rc != 0:
return ''
return re.sub(r'[ \t]+', ' ', so.strip()).split(' ')[0]
def run(self):
runCommand("koji watch-task %s --quiet" % self.task_id)
def checkBuild(self, task_id):
so, se, rc = runCommand("koji taskinfo -rv %s" % task_id)
if rc != 0:
self.err = "Unable to get taskinfo for task: %s" % (task_id, se)
return {}
build_data = so
info = {}
keys = {}
items = [info]
for line in build_data.split("\n"):
parts = line.split(":", 1)
if len(parts) > 1:
if parts[1] == "":
del(parts[1])
if len(parts) > 0:
# next item in a list
if parts[0] == "":
del(items[1:])
if "items" not in items[-1]:
items[-1]["items"] = [{}]
else:
items[-1]["items"].append({})
items.append(items[-1]["items"][-1])
continue
# process single item
indent = re.search(r"[^ ]", parts[0]).start()
key_level = indent / 2
if len(items) > key_level + 1:
del(items[key_level + 1:])
key = parts[0].strip()
if len(parts) == 2:
items[-1][key] = parts[1].strip()
else:
items[-1][key] = {}
items.append(items[-1][key])
# Atm, I am interested in status of each child build on a given architecture
status = {}
status["State"] = info["State"]
status["Items"] = []
for item in info["items"]:
if not item:
continue
# skip the SRPM
if "Build Arch" not in item["Request Parameters"]:
continue
status["Items"].append({"State": item["State"], "BuildArch": item["Request Parameters"]["Build Arch"]})
return status
def getLatestBuilds(self, tag = 'rawhide'):
so, _, rc = runCommand("koji -q latest-build %s %s" % (tag, self.pkg_name))
if rc != 0:
return ''
return re.sub(r'[ \t]+', ' ', so.strip()).split(' ')[0]
def run(self):
runCommand("koji watch-task %s --quiet" % self.task_id)
def checkBuild(self, task_id):
so, se, rc = runCommand("koji taskinfo -rv %s" % task_id)
if rc != 0:
self.err = "Unable to get taskinfo for task: %s" % (task_id, se)
return {}
build_data = so
info = {}
keys = {}
items = [info]
for line in build_data.split("\n"):
parts = line.split(":", 1)
if len(parts) > 1:
if parts[1] == "":
del (parts[1])
if len(parts) > 0:
# next item in a list
if parts[0] == "":
del (items[1:])
if "items" not in items[-1]:
items[-1]["items"] = [{}]
else:
items[-1]["items"].append({})
items.append(items[-1]["items"][-1])
continue
# process single item
indent = re.search(r"[^ ]", parts[0]).start()
key_level = indent / 2
if len(items) > key_level + 1:
del (items[key_level + 1:])
key = parts[0].strip()
if len(parts) == 2:
items[-1][key] = parts[1].strip()
else:
items[-1][key] = {}
items.append(items[-1][key])
# Atm, I am interested in status of each child build on a given architecture
status = {}
status["State"] = info["State"]
status["Items"] = []
for item in info["items"]:
if not item:
continue
# skip the SRPM
if "Build Arch" not in item["Request Parameters"]:
continue
status["Items"].append({
"State":
item["State"],
"BuildArch":
item["Request Parameters"]["Build Arch"]
})
return status
def test(self, source_code_directory = ""):
# Parse spec file
self.sp_obj = SpecParser(self.spec)
if not self.sp_obj.parse():
self.err = self.sp_obj.getError()
return False
# Parse sources file
if self.sources != "":
self.src_obj = Sources(self.sources)
if not self.src_obj.parse():
self.err = self.src_obj.getError()
return False
if source_code_directory == "":
# create a temp directory
dir = tempfile.mkdtemp()
# extract tarball to the directory
so, se, rc = runCommand("tar -xf %s --directory=%s" % (self.archive, dir))
if rc != 0:
self.err = se
return False
so, se, rc = runCommand("ls %s" % dir)
if rc != 0:
self.err = "Unable to archive's extracted folder"
return False
sc_directory = "%s/%s" % (dir, so.split('\n')[0])
else:
sc_directory = source_code_directory
data = {
"type": "user_directory",
"resource": path.abspath(sc_directory),
#"directories_to_skip": self.noGodeps,
"ipprefix": "."
}
try:
data = ActFactory().bake("go-code-inspection").call(data)
except Exception as e:
logging.error(e)
exit(1)
# TODO(jchaloup) catch exceptions, at least ValueError
self.prj_info.construct(data[ARTEFACT_GOLANG_PROJECT_PACKAGES])
if source_code_directory == "":
shutil.rmtree(dir)
tests = []
# test package name
tests.append(self.testPackageName)
# test commit
tests.append(self.testCommit)
# test import path macros
tests.append(self.testImportPathMacros)
# test provider, provider_tld, ...
tests.append(self.testRepositoryMacros)
# test source0 macro
tests.append(self.testSpecFileSource)
# test devel subpackage
tests.append(self.testDevel)
for test in tests:
if not test():
self.printTResult(self.t_result)
elif self.verbose:
self.printTResult(self.t_result)
return True