def clean_orig(self):
with pwd(self.pkg_src_dir):
sourceball = tarfile.open(self.sourceball)
with pwd(self.branches_dir):
rm(self.orig_dir(self.name))
sourceball.extractall()
move(self.name, self.orig_dir(self.name))
def __init__(self, name=None, tarball=""):
if tarball:
tmp_dir = mkdtemp()
with pwd(tmp_dir):
sourceball_name = copy(tarball, split(tarball)[1])
log.debug("sourceball_name " + sourceball_name)
sourceball = tarfile.open(sourceball_name)
extract_dir = base_dir(sourceball)
if name and not name == extract_dir:
log.debug("hahahahhaah")
raise ExecutionException("tarball is not target directory")
if not name:
name = extract_dir
super(SourceBall, self).__init__(name)
if tarball:
self.cfg["tarball_source"] = tarball
with pwd(self.parent):
sourceball.extractall()
with pwd(self.branches_dir):
sourceball.extractall()
move(self.name, self.orig_dir(self.name))
with pwd(self.pkg_src_dir):
move(join(tmp_dir, sourceball_name), sourceball_name)
self.cfg["sourceball"] = sourceball_name
self.set_cur_to("head")
def set_cur_to(self, *args):
'''passes arbitrary args to darcs get and makes a branch out of it
this is not really optimal, because it only does things temporary,
we need to look at as systematic way to handle branching.
looking at a potential git and a potential cvs module may help
'''
if len(args) >= 2 and args[0] == 'new':
branch_name = args[1]
args = args[2:]
else:
branch_name = ''
if len(args) == 1 and args[0] == 'head':
log.debug('doing head')
self.cfg['source'] = '.'
elif len(args) == 2 and args[0] == 'branch':
self.cfg['source'] = join(self.branches, args[1])
else:
log.debug('creating new branch')
index = str(hash(args))
index_dir = join(self.branches, index)
with pwd(self.dir):
if not exists(index_dir):
log.debug('index dir should be ' + index_dir)
self.get(self.source(), index_dir, *args)
self.cfg['source'] = index_dir
if branch_name:
with pwd(self.branches_dir):
self.get(index, branch_name)
self.set_current_src()
def rpmbuild(self, param, package, profile=None):
'''The work horse of building rpms
Given a directive param for rpmbuild, the package, and a possible
profile, it runs rpmbuild appropriatly.
package is a directory name to a Package (Directory).
profile is a Profile object, this parameter should only be used internally
returns nothing
'''
pkg = DirFactory(package)
log.debug(package)
log.debug(pkg)
if profile:
defines = join_defines(profile.dist_defines,
dir_defines(self.dir))
else:
defines = dir_defines(self.dir)
log.debug('defines are ' + str(defines))
log.debug('spec file is ' + pkg.spec_file)
cmd = ['rpmbuild'] + defines + ['-v', param, pkg.spec_file]
log.debug('cmd is ' + str(cmd))
with pwd(pkg.dir):
with log_file('rpmbuild.log') as rpm_out:
with pwd(join(self.dir, 'SPECS')):
p = Popen(cmd, stdout=rpm_out, stderr=rpm_out)
log.info('building %s... please wait'
% pkg.spec_file)
p.wait()
log.debug(p.returncode)
def __init__(self, name=None, fedora_name=None):
if fedora_name:
if not name:
name = fedora_name
tmp_dir = mkdtemp()
with pwd(tmp_dir):
env = copy(environ)
env['CVSROOT'] = CVSROOT
cmd = ['cvs', 'co', fedora_name]
with log_file('cvs.log') as cvs_log:
p = Popen(cmd, stdout=cvs_log, stderr=cvs_log,
env=env)
log.info('Fetching full source from CVS... please wait...')
p.communicate()
with pwd(join(fedora_name, 'common')):
with file('branches', 'r') as branches:
branches = (branch for branch in branches if ':' in branch)
branches = (branch.split(':')[0] for branch in branches)
branches = list(branches)
with pwd(fedora_name):
p_branches = os.listdir('.')
v_branches = (branch for branch in branches if branch in p_branches)
v_branches = list(v_branches)
for branch in v_branches:
dir_name = fedora_name + '.' + branch
cvs_branch = join(fedora_name, branch)
cmd = ['git', 'cvsimport', '-o', branch, '-C', dir_name, cvs_branch]
with log_file('cvs.log') as cvs_log:
p = Popen(cmd, stdout=cvs_log, stderr=cvs_log,
env=env)
log.info('Fetching partial source from CVS for %s... please wait...' % branch)
p.communicate()
non_devel_v_branches = (branch for branch in v_branches if branch is not 'devel')
for branch in non_devel_v_branches:
dir_name = fedora_name + '.' + branch
cvs_branch = join(fedora_name, branch)
refspec = branch + ':' + branch
cmd = ['git', 'fetch', join(tmp_dir, dir_name), refspec]
with log_file('cvs.log') as cvs_log:
with pwd(fedora_name + '.devel'):
p = Popen(cmd, stdout=cvs_log, stderr=cvs_log,
env=env)
log.info('Combining CVS sources for %s ... please wait...' % branch)
p.communicate()
move(join(tmp_dir, fedora_name + '.devel'), name)
move(join(tmp_dir, fedora_name), join(name, '.fedora_cvs'))
move(join(tmp_dir, 'cvs.log'), join(name, 'cvs.log'))
super(RpmCvsPackage, self).__init__(name)
if fedora_name:
self.cfg['branches'] = v_branches
self.cfg['pkg_name'] = fedora_name
def generate_patch(self, patch_name):
prefix = self.next_patch_num + "."
patch_name = patch_name + ".patch" if not patch_name.endswith(".patch") else patch_name
patch_name = prefix + patch_name
log.debug("patch_name " + patch_name)
with self.patches_applied("orig"):
with self.do_diff(self.branch("orig"), ".") as diff:
with pwd(self.dir):
with file(join(self.pkg_src_dir, patch_name), "w") as patch_file:
diff(patch_file)
# We have to do this last step because of the context manager
# it assumes that this last patch has also been applied
patch_name = join(self.pkg_src_dir, patch_name)
with pwd(self.branch_dir("orig")):
self.apply_patch(patch_name)
def checkout(self, *args):
with pwd(self.source_dir):
with log_file('git.log') as git_out:
# NB: args is a tuple, must be converted
p = Popen(['git', 'checkout'] + list(args),
stdout=git_out, stderr=git_out)
log.info('git checkout %s, please wait....' % str(args))
p.communicate()
def import_other_patch(self, patch_file, patch_level):
if type(patch_level) is int:
patch_level = str(patch_level)
cmd = ["patch", "-p" + patch_level]
with pwd(self.dir):
self.do_patch(patch_file, cmd)
patch_name = basename(patch_file)
self.generate_patch(patch_name)
def install_tag(self, tag):
'''assuming a package that supports tagging, install a specific tag
tag is the name of the tag in the DVCS
'''
with pwd(self.package.dir):
with self.package.tag(tag):
self.install()
def verify_patches(self, *args):
test_file = join(self.pkg_src_dir, "test.patch.")
with self.patches_applied(*args):
with self.do_diff(self.branch("orig"), ".") as diff:
with pwd(self.dir):
with file(test_file, "w") as patch_file:
diff(patch_file)
clean = False if len(file(test_file).read()) else True
log.info("Verified clean patches: " + str(clean))
return clean
def whatis_sysdir(dir):
''' given a dir, determine it's type'''
with pwd(dir):
cfg = ConfigObj('.devshell')
try:
type = cfg['type']
log.debug('is type ' + type)
return type
except KeyError, e:
return 'directory'
def do_diff(self, src, tgt):
cmd = ["diff", "-r", "-u", "-X", DIFF_EXCLUDES_FILE, src, tgt]
with pwd(self.pkg_src_dir):
with log_file("diff.log") as diff_out:
def command(fd):
p = Popen(cmd, stdout=fd, stderr=diff_out)
log.info("generating patch %s, please wait..." % fd.name)
p.communicate()
yield command
def add_sourceball(self, sourceball):
'''copies a tarball into the package
tarball is a path to some tarball
'''
with pwd(self.pkg.dir):
pkg_src = self.sourceball('', sourceball)
pkg_src.set_buildsystem(self.builder._type)
name = pkg_src.name
self.pkg.add_source(name)
return close_later(pkg_src)
def fetch_sourceballs(self, profile=None, regen=True):
pkg_srcen = self.sources
pkg_srcen = (DirFactory(pkg_src) for pkg_src in pkg_srcen)
self.cfg['sourceballen'] = list()
with pwd(self.dir):
for pkg_src in pkg_srcen:
if regen:
pkg_src.setup_sourceball(self.ver(profile))
symlink(pkg_src.sourceball_loc, pkg_src.sourceball)
self.cfg['sourceballen'].append(pkg_src.sourceball)
pkg_src.close()
def add_vcs(self, url, tgt, *args):
'''creates a darcs variant of a cabal package using darcs source
url is a url to some darcs repo
tgt is the local name of the darcs repo
'''
with pwd(self.pkg.dir):
pkg_src = self.revision_control(tgt, url, *args)
pkg_src.set_buildsystem(self.builder._type)
name = pkg_src.name
self.pkg.add_source(name)
return close_later(pkg_src)
def fetch_build(self, package):
'''fetches the built source tree from the execution of rpmbuild for review
This is usefull to see if rpmbuild did anything funny in execution
of the spec file. Hopefully you'll never need this. The results
end up in the top level directory of the given package
package is a directory name to a Package (Directory).
'''
pkg = DirFactory(package)
with pwd(self.dir):
source = pkg.cfg['source']
move(join('BUILD', source), join(pkg.dir, 'results'))
def configure_from_system(self, branch):
'''sets up a profile based on system profiles
branch is a branch name from the fedora-cvs
'''
self.cfg['branch'] = branch
d = distro[branch]
self.cfg['koji_target'] = d[TARGET]
self.cfg['distval'] = d[DISTVAL]
self.cfg['distvar'] = d[DISTVAR]
self.cfg['dist'] = d[DIST]
with pwd(self.dir):
copytree(MOCK_CFG_DIR, self.mock_cfg_dir)
def setup_sourceball(self, ver, delete_old=False):
'''gets darcs to spit out a sourceball to be used in rpm making by other modules'''
log.debug('someone set us up the bomb')
if delete_old:
with pwd(self.pkg_src_dir):
rm(self.sourceball)
name = self.upstream_name
date = self.date
full_name = '%s-%s.%sdarcs' % (name, ver, date)
log.debug('full name is ' + full_name)
sourceball = full_name + '.tar.gz'
with pwd(self.source_dir):
with log_file('darcs.log') as darcs_out:
log.debug('we get signal')
p = Popen(['darcs', 'dist', '-d', full_name],
stdout=darcs_out, stderr=darcs_out)
log.info('generating tarball %s.tar.gz, please wait...'
% full_name)
p.communicate()
with pwd(self.pkg_src_dir):
move(join(self.source_dir, sourceball), sourceball)
self.cfg['sourceball'] = sourceball
def setup_sourceball(self, ver, delete_old=False):
'''gets darcs to spit out a sourceball to be used in rpm making by other modules'''
log.debug('someone set us up the bomb')
if delete_old:
with pwd(self.pkg_src_dir):
rm(self.sourceball)
name = self.upstream_name
date = self.date
full_name = '%s-%s.%s' % (name, ver, self.alphatag)
log.debug('full name is ' + full_name)
sourceball = full_name + '.tar.gz'
with pwd(self.source_dir):
with log_file('git.log') as git_out:
p1 = Popen(['git', 'archive', '--format=tar', "--prefix=%s/" % full_name, "HEAD"],
stdout=PIPE, stderr=git_out)
log.info('generating tarball %s.tar.gz, please wait...'
% full_name)
with pwd(self.pkg_src_dir):
with file(sourceball, "wb") as sourceball_file:
p2 = Popen(['gzip'], stdin=p1.stdout,
stdout=sourceball_file, stderr=git_out)
p2.communicate()
self.cfg['sourceball'] = sourceball
self.cfg['full_name'] = full_name
def prepare_sourceballs(self, install=False, force=False, *args):
tmp_dir = mkdtemp()
pkg_srcen = self.pkg.sources
pkg_srcen = (DirFactory(pkg_src) for pkg_src in pkg_srcen)
with self.pkg.in_dir():
for pkg_src in pkg_srcen:
with pwd(tmp_dir):
sourceball = SourceBall(pkg_src.full_name, pkg_src.sourceball_loc)
builder = self.builder(sourceball)
builder.prepare(install, force, *args)
rm(sourceball.pkg_src_dir)
tar_file = tarfile.open(pkg_src.sourceball_loc, 'w:gz')
tar_file.add(pkg_src.full_name)
tar_file.close()
rm(tmp_dir)
def setup_source(self, package):
'''Given a package, set's it up in the buildroot for being built with rpmbuild
package is a directory name to a Package (Directory).
returns nothing
'''
pkg = DirFactory(package)
with pwd(pkg.dir):
symlink(pkg.spec_file,
join(self.dir, 'SPECS', pkg.spec_file))
pkg.fetch_sourceballs()
for source in pkg.spec_all_sources():
log.debug(source)
symlink(source, join(self.dir, 'SOURCES', source))
def __init__(self, name=None, url=None, *args):
if url:
#split chokes on URLs that end in a /
url = url[:-1] if url.endswith('/') else url
tgt = basename(url)
if not name:
name = tgt
self.get(url, tgt, *args)
super(Darcs, self).__init__(name)
if url:
self.cfg['vc_url'] = url
self.cfg['upstream_name'] = tgt
self.cfg['source'] = '.'
with pwd(self.branches_dir):
self.get(self.dir, 'orig')
self.set_current_src()
def fetch_rpms(self, target_dir):
'''fetches all rpm files from the buildroot to another target directory
As a module author, it is a good idea to use this to clean up
afterwards. This module pulls all RPMs said module may have made
as well as any other RPMs from earlier incantations of other modules.
Please be courteous to other module authors and run this liberally.
target_dir is a path to the destination for all the rpms
'''
target_dir = abspath(target_dir)
with pwd(self.dir):
for path, dirs, files in walk('.'):
for f in files:
if f.endswith('.rpm'):
move(join(path, f), join(target_dir, f))
def set_current_src(self):
'''sets the current internal state to reflect the current head
chances are, the user should rarely mess with this.
this may change, because rather than using .patch files for rpm
handling, we may ask the user to commit all the changes to darcs,
and then have devshell generate .patch files automatically instead
'''
with pwd(self.source_dir):
p = Popen(['darcs', 'changes', '--xml-output', '--last=1'],
stdout = PIPE, stderr = PIPE)
change = p.communicate()[0]
hash = hash_re.search(change).groups()[0]
date = date_re.search(change).groups()[0]
self.cfg['head'] = (hash, date)
def build(self, package):
pkg = DirFactory(package)
srpm_name = pkg.get_srpm_name(self.profile)
mock_cfg = self.profile.mock_cfg
result_dir = self.profile.result_dir
config_dir = self.profile.mock_cfg_dir
log.debug("mock_cfg is " + mock_cfg)
log.debug("result_dir is " + result_dir)
log.debug("config_dir is " + config_dir)
cmd = ["mock", "-r", mock_cfg, "--configdir=%s" % config_dir, "--resultdir=%s" % result_dir, srpm_name]
log.debug("cmd is " + str(cmd))
with pwd(result_dir):
with log_file("mock.log") as mock_out:
p = Popen(cmd, stdout=mock_out, stderr=mock_out)
log.info("mock compiling %s... please wait" % srpm_name)
p.communicate()
def add_spec(self, spec_file):
'''add's a spec file to the package, and sets the canonical package
name based on the spec file, possibly renaming the spec file to match
within fedora guidelines'''
log.debug('spec_file is %s' % spec_file)
log.debug('spec_file_name is %s' % self.name + '.spec')
#TODO: get the spec file name, copy
# Then get the actual package name and set pkg_name to the right one
spec_file = abspath(spec_file)
spec_fname = basename(spec_file)
with pwd(self.dir):
try:
copy(spec_file, spec_fname)
rpm = RPMSpec(spec_fname)
self.cfg['pkg_name'] = rpm.name()
if not spec_fname == self.spec_file:
move(spec_fname, self.spec_file)
except IOError, e:
log.error(str(e))
raise ExecutionException(e, 'spec-file could not be added')
def set_current_src(self):
'''sets the current internal state to reflect the current head
chances are, the user should rarely mess with this.
this may change, because rather than using .patch files for rpm
handling, we may ask the user to commit all the changes to darcs,
and then have devshell generate .patch files automatically instead
'''
with pwd(self.source_dir):
p = Popen(['git', 'log', '--pretty=format:"%H%n%ad"', 'HEAD^..'],
stdout = PIPE, stderr = PIPE)
change = p.communicate()[0]
change = change.split('\n')
# for some reason, there are extra quotation marks that shouldn't be there
# so we have to strip them away
hash = change[0][1:]
date = dateutil.parser.parse(change[1][:-1]).strftime('%Y%m%d%H%M%S')
print hash, date
self.cfg['head'] = (hash, date)
self.cfg['alphatag'] = date[:8] + 'git.' + hash[:12]
def in_branches_dir(self):
with pwd(self.branches_dir):
yield
def rpm_obj(self, *args):
with pwd(self.dir):
yield RPMSpec(*args)
def del_source(self, source):
self.rem_source(source)
with pwd(self.dir):
rm(source)
