def clean_orig(self): with pwd(self.pkg_src_dir): sourceball = tarfile.open(self.sourceball) with pwd(self.branches_dir): rm(self.orig_dir(self.name)) sourceball.extractall() move(self.name, self.orig_dir(self.name))
def __init__(self, name=None, tarball=""): if tarball: tmp_dir = mkdtemp() with pwd(tmp_dir): sourceball_name = copy(tarball, split(tarball)[1]) log.debug("sourceball_name " + sourceball_name) sourceball = tarfile.open(sourceball_name) extract_dir = base_dir(sourceball) if name and not name == extract_dir: log.debug("hahahahhaah") raise ExecutionException("tarball is not target directory") if not name: name = extract_dir super(SourceBall, self).__init__(name) if tarball: self.cfg["tarball_source"] = tarball with pwd(self.parent): sourceball.extractall() with pwd(self.branches_dir): sourceball.extractall() move(self.name, self.orig_dir(self.name)) with pwd(self.pkg_src_dir): move(join(tmp_dir, sourceball_name), sourceball_name) self.cfg["sourceball"] = sourceball_name self.set_cur_to("head")
def set_cur_to(self, *args): '''passes arbitrary args to darcs get and makes a branch out of it this is not really optimal, because it only does things temporary, we need to look at as systematic way to handle branching. looking at a potential git and a potential cvs module may help ''' if len(args) >= 2 and args[0] == 'new': branch_name = args[1] args = args[2:] else: branch_name = '' if len(args) == 1 and args[0] == 'head': log.debug('doing head') self.cfg['source'] = '.' elif len(args) == 2 and args[0] == 'branch': self.cfg['source'] = join(self.branches, args[1]) else: log.debug('creating new branch') index = str(hash(args)) index_dir = join(self.branches, index) with pwd(self.dir): if not exists(index_dir): log.debug('index dir should be ' + index_dir) self.get(self.source(), index_dir, *args) self.cfg['source'] = index_dir if branch_name: with pwd(self.branches_dir): self.get(index, branch_name) self.set_current_src()
def rpmbuild(self, param, package, profile=None): '''The work horse of building rpms Given a directive param for rpmbuild, the package, and a possible profile, it runs rpmbuild appropriatly. package is a directory name to a Package (Directory). profile is a Profile object, this parameter should only be used internally returns nothing ''' pkg = DirFactory(package) log.debug(package) log.debug(pkg) if profile: defines = join_defines(profile.dist_defines, dir_defines(self.dir)) else: defines = dir_defines(self.dir) log.debug('defines are ' + str(defines)) log.debug('spec file is ' + pkg.spec_file) cmd = ['rpmbuild'] + defines + ['-v', param, pkg.spec_file] log.debug('cmd is ' + str(cmd)) with pwd(pkg.dir): with log_file('rpmbuild.log') as rpm_out: with pwd(join(self.dir, 'SPECS')): p = Popen(cmd, stdout=rpm_out, stderr=rpm_out) log.info('building %s... please wait' % pkg.spec_file) p.wait() log.debug(p.returncode)
def __init__(self, name=None, fedora_name=None): if fedora_name: if not name: name = fedora_name tmp_dir = mkdtemp() with pwd(tmp_dir): env = copy(environ) env['CVSROOT'] = CVSROOT cmd = ['cvs', 'co', fedora_name] with log_file('cvs.log') as cvs_log: p = Popen(cmd, stdout=cvs_log, stderr=cvs_log, env=env) log.info('Fetching full source from CVS... please wait...') p.communicate() with pwd(join(fedora_name, 'common')): with file('branches', 'r') as branches: branches = (branch for branch in branches if ':' in branch) branches = (branch.split(':')[0] for branch in branches) branches = list(branches) with pwd(fedora_name): p_branches = os.listdir('.') v_branches = (branch for branch in branches if branch in p_branches) v_branches = list(v_branches) for branch in v_branches: dir_name = fedora_name + '.' + branch cvs_branch = join(fedora_name, branch) cmd = ['git', 'cvsimport', '-o', branch, '-C', dir_name, cvs_branch] with log_file('cvs.log') as cvs_log: p = Popen(cmd, stdout=cvs_log, stderr=cvs_log, env=env) log.info('Fetching partial source from CVS for %s... please wait...' % branch) p.communicate() non_devel_v_branches = (branch for branch in v_branches if branch is not 'devel') for branch in non_devel_v_branches: dir_name = fedora_name + '.' + branch cvs_branch = join(fedora_name, branch) refspec = branch + ':' + branch cmd = ['git', 'fetch', join(tmp_dir, dir_name), refspec] with log_file('cvs.log') as cvs_log: with pwd(fedora_name + '.devel'): p = Popen(cmd, stdout=cvs_log, stderr=cvs_log, env=env) log.info('Combining CVS sources for %s ... please wait...' % branch) p.communicate() move(join(tmp_dir, fedora_name + '.devel'), name) move(join(tmp_dir, fedora_name), join(name, '.fedora_cvs')) move(join(tmp_dir, 'cvs.log'), join(name, 'cvs.log')) super(RpmCvsPackage, self).__init__(name) if fedora_name: self.cfg['branches'] = v_branches self.cfg['pkg_name'] = fedora_name
def generate_patch(self, patch_name): prefix = self.next_patch_num + "." patch_name = patch_name + ".patch" if not patch_name.endswith(".patch") else patch_name patch_name = prefix + patch_name log.debug("patch_name " + patch_name) with self.patches_applied("orig"): with self.do_diff(self.branch("orig"), ".") as diff: with pwd(self.dir): with file(join(self.pkg_src_dir, patch_name), "w") as patch_file: diff(patch_file) # We have to do this last step because of the context manager # it assumes that this last patch has also been applied patch_name = join(self.pkg_src_dir, patch_name) with pwd(self.branch_dir("orig")): self.apply_patch(patch_name)
def checkout(self, *args): with pwd(self.source_dir): with log_file('git.log') as git_out: # NB: args is a tuple, must be converted p = Popen(['git', 'checkout'] + list(args), stdout=git_out, stderr=git_out) log.info('git checkout %s, please wait....' % str(args)) p.communicate()
def import_other_patch(self, patch_file, patch_level): if type(patch_level) is int: patch_level = str(patch_level) cmd = ["patch", "-p" + patch_level] with pwd(self.dir): self.do_patch(patch_file, cmd) patch_name = basename(patch_file) self.generate_patch(patch_name)
def install_tag(self, tag): '''assuming a package that supports tagging, install a specific tag tag is the name of the tag in the DVCS ''' with pwd(self.package.dir): with self.package.tag(tag): self.install()
def verify_patches(self, *args): test_file = join(self.pkg_src_dir, "test.patch.") with self.patches_applied(*args): with self.do_diff(self.branch("orig"), ".") as diff: with pwd(self.dir): with file(test_file, "w") as patch_file: diff(patch_file) clean = False if len(file(test_file).read()) else True log.info("Verified clean patches: " + str(clean)) return clean
def whatis_sysdir(dir): ''' given a dir, determine it's type''' with pwd(dir): cfg = ConfigObj('.devshell') try: type = cfg['type'] log.debug('is type ' + type) return type except KeyError, e: return 'directory'
def do_diff(self, src, tgt): cmd = ["diff", "-r", "-u", "-X", DIFF_EXCLUDES_FILE, src, tgt] with pwd(self.pkg_src_dir): with log_file("diff.log") as diff_out: def command(fd): p = Popen(cmd, stdout=fd, stderr=diff_out) log.info("generating patch %s, please wait..." % fd.name) p.communicate() yield command
def add_sourceball(self, sourceball): '''copies a tarball into the package tarball is a path to some tarball ''' with pwd(self.pkg.dir): pkg_src = self.sourceball('', sourceball) pkg_src.set_buildsystem(self.builder._type) name = pkg_src.name self.pkg.add_source(name) return close_later(pkg_src)
def fetch_sourceballs(self, profile=None, regen=True): pkg_srcen = self.sources pkg_srcen = (DirFactory(pkg_src) for pkg_src in pkg_srcen) self.cfg['sourceballen'] = list() with pwd(self.dir): for pkg_src in pkg_srcen: if regen: pkg_src.setup_sourceball(self.ver(profile)) symlink(pkg_src.sourceball_loc, pkg_src.sourceball) self.cfg['sourceballen'].append(pkg_src.sourceball) pkg_src.close()
def add_vcs(self, url, tgt, *args): '''creates a darcs variant of a cabal package using darcs source url is a url to some darcs repo tgt is the local name of the darcs repo ''' with pwd(self.pkg.dir): pkg_src = self.revision_control(tgt, url, *args) pkg_src.set_buildsystem(self.builder._type) name = pkg_src.name self.pkg.add_source(name) return close_later(pkg_src)
def fetch_build(self, package): '''fetches the built source tree from the execution of rpmbuild for review This is usefull to see if rpmbuild did anything funny in execution of the spec file. Hopefully you'll never need this. The results end up in the top level directory of the given package package is a directory name to a Package (Directory). ''' pkg = DirFactory(package) with pwd(self.dir): source = pkg.cfg['source'] move(join('BUILD', source), join(pkg.dir, 'results'))
def configure_from_system(self, branch): '''sets up a profile based on system profiles branch is a branch name from the fedora-cvs ''' self.cfg['branch'] = branch d = distro[branch] self.cfg['koji_target'] = d[TARGET] self.cfg['distval'] = d[DISTVAL] self.cfg['distvar'] = d[DISTVAR] self.cfg['dist'] = d[DIST] with pwd(self.dir): copytree(MOCK_CFG_DIR, self.mock_cfg_dir)
def setup_sourceball(self, ver, delete_old=False): '''gets darcs to spit out a sourceball to be used in rpm making by other modules''' log.debug('someone set us up the bomb') if delete_old: with pwd(self.pkg_src_dir): rm(self.sourceball) name = self.upstream_name date = self.date full_name = '%s-%s.%sdarcs' % (name, ver, date) log.debug('full name is ' + full_name) sourceball = full_name + '.tar.gz' with pwd(self.source_dir): with log_file('darcs.log') as darcs_out: log.debug('we get signal') p = Popen(['darcs', 'dist', '-d', full_name], stdout=darcs_out, stderr=darcs_out) log.info('generating tarball %s.tar.gz, please wait...' % full_name) p.communicate() with pwd(self.pkg_src_dir): move(join(self.source_dir, sourceball), sourceball) self.cfg['sourceball'] = sourceball
def setup_sourceball(self, ver, delete_old=False): '''gets darcs to spit out a sourceball to be used in rpm making by other modules''' log.debug('someone set us up the bomb') if delete_old: with pwd(self.pkg_src_dir): rm(self.sourceball) name = self.upstream_name date = self.date full_name = '%s-%s.%s' % (name, ver, self.alphatag) log.debug('full name is ' + full_name) sourceball = full_name + '.tar.gz' with pwd(self.source_dir): with log_file('git.log') as git_out: p1 = Popen(['git', 'archive', '--format=tar', "--prefix=%s/" % full_name, "HEAD"], stdout=PIPE, stderr=git_out) log.info('generating tarball %s.tar.gz, please wait...' % full_name) with pwd(self.pkg_src_dir): with file(sourceball, "wb") as sourceball_file: p2 = Popen(['gzip'], stdin=p1.stdout, stdout=sourceball_file, stderr=git_out) p2.communicate() self.cfg['sourceball'] = sourceball self.cfg['full_name'] = full_name
def prepare_sourceballs(self, install=False, force=False, *args): tmp_dir = mkdtemp() pkg_srcen = self.pkg.sources pkg_srcen = (DirFactory(pkg_src) for pkg_src in pkg_srcen) with self.pkg.in_dir(): for pkg_src in pkg_srcen: with pwd(tmp_dir): sourceball = SourceBall(pkg_src.full_name, pkg_src.sourceball_loc) builder = self.builder(sourceball) builder.prepare(install, force, *args) rm(sourceball.pkg_src_dir) tar_file = tarfile.open(pkg_src.sourceball_loc, 'w:gz') tar_file.add(pkg_src.full_name) tar_file.close() rm(tmp_dir)
def setup_source(self, package): '''Given a package, set's it up in the buildroot for being built with rpmbuild package is a directory name to a Package (Directory). returns nothing ''' pkg = DirFactory(package) with pwd(pkg.dir): symlink(pkg.spec_file, join(self.dir, 'SPECS', pkg.spec_file)) pkg.fetch_sourceballs() for source in pkg.spec_all_sources(): log.debug(source) symlink(source, join(self.dir, 'SOURCES', source))
def __init__(self, name=None, url=None, *args): if url: #split chokes on URLs that end in a / url = url[:-1] if url.endswith('/') else url tgt = basename(url) if not name: name = tgt self.get(url, tgt, *args) super(Darcs, self).__init__(name) if url: self.cfg['vc_url'] = url self.cfg['upstream_name'] = tgt self.cfg['source'] = '.' with pwd(self.branches_dir): self.get(self.dir, 'orig') self.set_current_src()
def fetch_rpms(self, target_dir): '''fetches all rpm files from the buildroot to another target directory As a module author, it is a good idea to use this to clean up afterwards. This module pulls all RPMs said module may have made as well as any other RPMs from earlier incantations of other modules. Please be courteous to other module authors and run this liberally. target_dir is a path to the destination for all the rpms ''' target_dir = abspath(target_dir) with pwd(self.dir): for path, dirs, files in walk('.'): for f in files: if f.endswith('.rpm'): move(join(path, f), join(target_dir, f))
def set_current_src(self): '''sets the current internal state to reflect the current head chances are, the user should rarely mess with this. this may change, because rather than using .patch files for rpm handling, we may ask the user to commit all the changes to darcs, and then have devshell generate .patch files automatically instead ''' with pwd(self.source_dir): p = Popen(['darcs', 'changes', '--xml-output', '--last=1'], stdout = PIPE, stderr = PIPE) change = p.communicate()[0] hash = hash_re.search(change).groups()[0] date = date_re.search(change).groups()[0] self.cfg['head'] = (hash, date)
def build(self, package): pkg = DirFactory(package) srpm_name = pkg.get_srpm_name(self.profile) mock_cfg = self.profile.mock_cfg result_dir = self.profile.result_dir config_dir = self.profile.mock_cfg_dir log.debug("mock_cfg is " + mock_cfg) log.debug("result_dir is " + result_dir) log.debug("config_dir is " + config_dir) cmd = ["mock", "-r", mock_cfg, "--configdir=%s" % config_dir, "--resultdir=%s" % result_dir, srpm_name] log.debug("cmd is " + str(cmd)) with pwd(result_dir): with log_file("mock.log") as mock_out: p = Popen(cmd, stdout=mock_out, stderr=mock_out) log.info("mock compiling %s... please wait" % srpm_name) p.communicate()
def add_spec(self, spec_file): '''add's a spec file to the package, and sets the canonical package name based on the spec file, possibly renaming the spec file to match within fedora guidelines''' log.debug('spec_file is %s' % spec_file) log.debug('spec_file_name is %s' % self.name + '.spec') #TODO: get the spec file name, copy # Then get the actual package name and set pkg_name to the right one spec_file = abspath(spec_file) spec_fname = basename(spec_file) with pwd(self.dir): try: copy(spec_file, spec_fname) rpm = RPMSpec(spec_fname) self.cfg['pkg_name'] = rpm.name() if not spec_fname == self.spec_file: move(spec_fname, self.spec_file) except IOError, e: log.error(str(e)) raise ExecutionException(e, 'spec-file could not be added')
def set_current_src(self): '''sets the current internal state to reflect the current head chances are, the user should rarely mess with this. this may change, because rather than using .patch files for rpm handling, we may ask the user to commit all the changes to darcs, and then have devshell generate .patch files automatically instead ''' with pwd(self.source_dir): p = Popen(['git', 'log', '--pretty=format:"%H%n%ad"', 'HEAD^..'], stdout = PIPE, stderr = PIPE) change = p.communicate()[0] change = change.split('\n') # for some reason, there are extra quotation marks that shouldn't be there # so we have to strip them away hash = change[0][1:] date = dateutil.parser.parse(change[1][:-1]).strftime('%Y%m%d%H%M%S') print hash, date self.cfg['head'] = (hash, date) self.cfg['alphatag'] = date[:8] + 'git.' + hash[:12]
def in_branches_dir(self): with pwd(self.branches_dir): yield
def rpm_obj(self, *args): with pwd(self.dir): yield RPMSpec(*args)
def del_source(self, source): self.rem_source(source) with pwd(self.dir): rm(source)