def create_meta_pkg(packagelist, distro, distro_name, metapackage, deps, os_platform, arch, staging_dir, wet_distro): workdir = staging_dir metadir = os.path.join(workdir, 'meta') if not os.path.exists(metadir): os.makedirs(metadir) debdir = os.path.join(metadir, 'DEBIAN') if not os.path.exists(debdir): os.makedirs(debdir) control_file = os.path.join(debdir, 'control') deb_name = "ros-%s-%s"%(distro_name, debianize_name(metapackage)) deb_version = "1.0.0-s%d~%s"%(time.mktime(time.gmtime()), os_platform) ros_depends = [] missing = False for stack in deps: if stack in distro.released_stacks or wet_distro.get_package_list(): stack_deb_name = "ros-%s-%s"%(distro_name, debianize_name(stack)) if stack_deb_name in packagelist: stack_deb_version = packagelist[stack_deb_name]['Version'] ros_depends.append('%s (= %s)'%(stack_deb_name, stack_deb_version)) else: debug("WARNING: Variant %s depends on non-built deb, %s"%(metapackage, stack)) missing = True else: debug("WARNING: Variant %s depends on non-exist stack, %s"%(metapackage, stack)) missing = True if missing: return None ros_depends_str = ', '.join(ros_depends) with open(control_file, 'w') as f: f.write(""" Package: %(deb_name)s Version: %(deb_version)s Architecture: %(arch)s Maintainer: The ROS community <*****@*****.**> Installed-Size: Depends: %(ros_depends_str)s Section: unknown Priority: optional WG-rosdistro: %(distro_name)s Description: Meta package for %(metapackage)s variant of ROS. """%locals()) if not missing: dest_deb = os.path.join(workdir, "%(deb_name)s_%(deb_version)s_%(arch)s.deb"%locals()) subprocess.check_call(['dpkg-deb', '--nocheck', '--build', metadir, dest_deb], stderr=subprocess.STDOUT) else: dest_deb = None shutil.rmtree(metadir) return dest_deb
def gen_metapkgs(distro, os_platform, arch, staging_dir, repo_fqdn, force=False): distro_name = distro.release_name # Retrieve the package list from the shadow repo packageurl = repo_url(repo_fqdn) + "/dists/%(os_platform)s/main/binary-%(arch)s/Packages" % locals() try: packagetxt = urllib2.urlopen(packageurl).read() except urllib2.URLError as ex: print "Failed to open url: %s with error %s" % (packageurl, ex) raise ex except: print "Failed to open url: %s" % (packageurl) raise packagelist = parse_deb_packages(packagetxt) debs = [] missing = [] _, _, missing_excluded, missing_excluded_dep = list_missing.get_missing(distro, os_platform, arch, repo=repo_url(repo_fqdn)) missing_ok = missing_excluded.union(missing_excluded_dep) # if (metapkg missing) or (metapkg missing deps), then create # modify create to version-lock deps wet_distro = rosdistro.Rosdistro(distro_name) # Build the new meta packages for (v, d) in distro.variants.iteritems(): deb_name = "ros-%s-%s" % (distro_name, debianize_name(v)) # If the metapkg is in the packagelist AND already has the right deps, we leave it: if deb_name in packagelist: list_deps = set([x.split()[0].strip() for x in packagelist[deb_name]['Depends'].split(',')]) mp_deps = set(["ros-%s-%s" % (distro_name, debianize_name(x)) for x in set(d.stack_names) - missing_ok]) if list_deps == mp_deps: debug("Metapackage %s already has correct deps" % deb_name) continue # Else, we create the new metapkg mp = create_meta_pkg(packagelist, distro, distro_name, v, set(d.stack_names) - missing_ok, os_platform, arch, staging_dir, wet_distro) if mp: debs.append(mp) else: missing.append(v) # We should always need to build the special "all" metapackage mp = create_meta_pkg(packagelist, distro, distro_name, "all", set(distro.released_stacks.keys()) - missing_ok, os_platform, arch, staging_dir, wet_distro) if mp: debs.append(mp) else: missing.append('all') upload_binary_debs(debs, distro_name, os_platform, arch, repo_fqdn) if missing: raise StackBuildFailure("Did not generate all metapkgs: %s." % missing)
def create_meta_pkg(packagelist, distro, distro_name, metapackage, deps, os_platform, arch, staging_dir): workdir = staging_dir metadir = os.path.join(workdir, 'meta') if not os.path.exists(metadir): os.makedirs(metadir) debdir = os.path.join(metadir, 'DEBIAN') if not os.path.exists(debdir): os.makedirs(debdir) control_file = os.path.join(debdir, 'control') deb_name = "ros-%s-%s" % (distro_name, debianize_name(metapackage)) deb_version = "1.0.0-s%d~%s" % (time.mktime(time.gmtime()), os_platform) ros_depends = [] missing = False for stack in deps: if stack in distro.released_stacks: stack_deb_name = "ros-%s-%s" % (distro_name, debianize_name(stack)) if stack_deb_name in packagelist: stack_deb_version = packagelist[stack_deb_name]['Version'] ros_depends.append('%s (= %s)' % (stack_deb_name, stack_deb_version)) else: print >> sys.stderr, "Variant %s depends on non-built deb, %s" % (metapackage, stack) missing = True else: print >> sys.stderr, "Variant %s depends on non-exist stack, %s" % (metapackage, stack) missing = True ros_depends_str = ', '.join(ros_depends) with open(control_file, 'w') as f: f.write(""" Package: %(deb_name)s Version: %(deb_version)s Architecture: %(arch)s Maintainer: The ROS community <*****@*****.**> Installed-Size: Depends: %(ros_depends_str)s Section: unknown Priority: optional WG-rosdistro: %(distro_name)s Description: Meta package for %(metapackage)s variant of ROS. """ % locals()) if not missing: dest_deb = os.path.join(workdir, "%(deb_name)s_%(deb_version)s_%(arch)s.deb" % locals()) subprocess.check_call(['dpkg-deb', '--nocheck', '--build', metadir, dest_deb]) else: dest_deb = None shutil.rmtree(metadir) return dest_deb
def get_missing(distro, os_platform, arch, repo=SHADOW_REPO, lock_version=True): distro_name = distro.release_name # Load the list of exclusions excludes_uri = "https://code.ros.org/svn/release/trunk/distros/%s.excludes"%(distro_name) excludes = ExclusionList(excludes_uri, distro_name, os_platform, arch) # Find all the deps in the distro for this stack deps = compute_deps(distro, 'ALL') # These stacks are not actually relased, so we treat them as implicitly excluded missing_primary = set(distro.stacks.keys()) - set(distro.released_stacks.keys()) missing_dep = set() missing_excluded = set(distro.stacks.keys()) - set(distro.released_stacks.keys()) missing_excluded_dep = set() # Build the deps in order for (sn, sv) in deps: if not sv: missing_primary.add(sn) continue deb_name = "ros-%s-%s"%(distro_name, debianize_name(sn)) if lock_version: deb_version = debianize_version(sv, '\w*', os_platform) else: deb_version = '[0-9.]*-[st][0-9]+~[a-z]+' if not deb_in_repo(repo, deb_name, deb_version, os_platform, arch, use_regex=True): try: si = load_info(sn, sv) depends = set(si['depends']) except: # stack is missing, including its info depends = set() # subtract any depends that aren't in the distro b/c of catkin dry/wet line to_remove = [d for d in depends if not d in distro.stacks] for d in to_remove: depends.remove(d) if excludes.check(sn): missing_excluded.add(sn) missing_primary.add(sn) elif depends.isdisjoint(missing_primary.union(missing_dep)): missing_primary.add(sn) else: missing_dep.add(sn) if not depends.isdisjoint(missing_excluded.union(missing_excluded_dep)): missing_excluded_dep.add(sn) else: pass #print "IN", sn missing_primary -= missing_excluded missing_dep -= missing_excluded_dep return missing_primary, missing_dep, missing_excluded, missing_excluded_dep
def get_buildable(deps, distro_name, os_platform, arch, requested_stack_name, force): # have to recalculate buildable after each build as invalidation # may have occurred. We examine in order to minimize retreading. cache = {} # fresh Packages cache each time through for sn, sv in deps: deb_name = "ros-%s-%s" % (distro_name, debianize_name(sn)) deb_version = debianize_version(sv, "\w*", os_platform) in_repo = deb_in_repo(deb_name, deb_version, os_platform, arch, cache) if not in_repo: debug("selecting [%s] because [%s, %s] not in repo" % (sn, deb_name, deb_version)) return sn, sv elif force and sn == requested_stack_name: debug("forcing build of %s" % (requested_stack_name))
def compute_missing_depends(stack_name, distro, os_platform, arch, repo=SHADOW_REPO, lock_version=True): missing_deps = set() deps = compute_deps(distro, stack_name, ignore_catkinized = False) #don't include self deps = set([(sn, sv) for (sn, sv) in deps if not sn == stack_name]) for sn, sv in deps: deb_name = "ros-%s-%s"%(distro.release_name, debianize_name(sn)) # see if there's a dry version deb_version = '[0-9.-]*-[st][0-9]+~[a-z]+' if not deb_in_repo(repo, deb_name, deb_version, os_platform, arch, use_regex=True): # now test for wet version wet_deb_version = '[0-9.]*-[0-9a-z]+-[0-9]+-[0-9]+-\+0000' if not deb_in_repo(repo, deb_name, wet_deb_version, os_platform, arch, use_regex=True): missing_deps.add(deb_name) return missing_deps
def test_control_data(self): import roslib.stacks from roslib.stack_manifest import parse_file, stack_file from rosdeb import debianize_name from rosdeb.source_deb import control_data import rosdeb.rosutil stack_version = '1.2.3' for stack_name in ['ros', 'navigation', 'simulator_gazebo', 'geometry', 'common', 'pr2_common']: if stack_name == 'sandbox': continue print "TESTING", stack_name stack_xml = stack_file(stack_name) d1 = control_data(stack_name, stack_version, 'fakemd5', stack_xml) d2 = control_data(stack_name, stack_version, 'fakemd5') self.assertEquals(d1, d2) self.assertEquals(d1['stack'], stack_name) self.assertEquals(d1['package'], debianize_name(stack_name)) self.assertEquals(d1['version'], stack_version) self.assert_(d1['description-brief'] is not None) self.assert_(len(d1['description-brief']) <= 60) self.assert_(d1['rosdeps'] is not None) if stack_name == 'common_msgs': self.assertEquals(['ros'], d1['depends']) try: self.assertEquals(rosdeb.rosutil.IMPLICIT_DEPS, d1['rosdeps']['lucid']) except: self.fail(d1['rosdeps'])
def do_deb_build(distro_name, stack_name, stack_version, os_platform, arch, staging_dir, noupload, interactive, repo_fqdn): debug("Actually trying to build %s-%s..."%(stack_name, stack_version)) distro_tgz = os.path.join('/var/cache/pbuilder', "%s-%s-%s.tgz"%(os_platform, arch, TGZ_VERSION)) deb_name = "ros-%s-%s"%(distro_name, debianize_name(stack_name)) deb_version = debianize_version(stack_version, '0', os_platform) ros_file = "%s-%s"%(stack_name, stack_version) deb_file = "%s_%s"%(deb_name, deb_version) conf_file = os.path.join(os.path.dirname(rosdeb.__file__),'pbuilder.conf') # Make sure the distro chroot exists if not os.path.exists(distro_tgz): raise InternalBuildFailure("%s does not exist."%(distro_tgz)) # Download deb and tar.gz files: dsc_name = '%s.dsc'%(deb_file) tar_gz_name = '%s.tar.gz'%(deb_file) (dsc_file, tar_gz_file) = download_files(stack_name, stack_version, staging_dir, [dsc_name, tar_gz_name]) # Create hook and results directories hook_dir = os.path.join(staging_dir, 'hooks') results_dir = os.path.join(staging_dir, 'results') build_dir = os.path.join(staging_dir, 'pbuilder') if not os.path.exists(hook_dir): os.makedirs(hook_dir) if not os.path.exists(results_dir): os.makedirs(results_dir) if not os.path.exists(build_dir): os.makedirs(build_dir) # Hook script which will download our tar.bz2 into environment p = os.path.join(hook_dir, 'A50fetch') with open(p, 'w') as f: f.write("""#!/bin/sh set -o errexit apt-get install ca-certificates -y # not in default ubuntu anymore wget https://code.ros.org/svn/release/download/stacks/%(stack_name)s/%(stack_name)s-%(stack_version)s/%(stack_name)s-%(stack_version)s.tar.bz2 -O /tmp/buildd/%(stack_name)s-%(stack_version)s.tar.bz2 rosdep update chown -R pbuilder /tmp/buildd/.ros su pbuilder -c "rosdep resolve gtest" su pbuilder -c "cp -r /tmp/buildd/.ros /tmp" """%locals()) os.chmod(p, stat.S_IRWXU) # Hook script which makes sure we have updated our apt cache p = os.path.join(hook_dir, 'D50update') with open(p, 'w') as f: f.write("""#!/bin/bash set -o errexit apt-get update apt-get install -y python-rosdep rosdep init"""%locals()) os.chmod(p, stat.S_IRWXU) if interactive: # Hook scripts to make us interactive: p = os.path.join(hook_dir, 'B50interactive') with open(p, 'w') as f: f.write("""#!/bin/bash echo "Entering interactive environment. Exit when done to continue pbuilder operation." export ROS_DESTDIR=/tmp/buildd/%(deb_name)s-%(stack_version)s/debian/%(deb_name)s source /tmp/buildd/%(deb_name)s-%(stack_version)s/setup_deb.sh roscd %(stack_name)s bash </dev/tty echo "Resuming pbuilder" """%locals()) os.chmod(p, stat.S_IRWXU) # Hook scripts to make us interactive: p = os.path.join(hook_dir, 'C50interactive') with open(p, 'w') as f: f.write("""#!/bin/bash echo "Entering interactive environment. Exit when done to continue pbuilder operation." export ROS_DESTDIR=/tmp/buildd/%(deb_name)s-%(stack_version)s/debian/%(deb_name)s source /tmp/buildd/%(deb_name)s-%(stack_version)s/setup_deb.sh roscd %(stack_name)s bash </dev/tty echo "Resuming pbuilder" """%locals()) os.chmod(p, stat.S_IRWXU) if arch == 'amd64' or arch == 'armel' or arch == 'armhf': archcmd = [] else: archcmd = ['setarch', arch] # Actually build the deb. This results in the deb being located in results_dir debug("starting pbuilder build of %s-%s"%(stack_name, stack_version)) subprocess.check_call(archcmd+ ['sudo', 'pbuilder', '--build', '--basetgz', distro_tgz, '--configfile', conf_file, '--hookdir', hook_dir, '--buildresult', results_dir, '--binary-arch', '--buildplace', build_dir, dsc_file], stderr=subprocess.STDOUT) # Set up an RE to look for the debian file and find the build_version deb_version_wild = debianize_version(stack_version, '(\w*)', os_platform) deb_file_wild = "%s_%s_%s\.deb"%(deb_name, deb_version_wild, arch) build_version = None # Extract the version number we just built: files = os.listdir(results_dir) for f in files: M = re.match(deb_file_wild, f) if M: build_version = M.group(1) if not build_version: raise InternalBuildFailure("No deb-file generated matching template: %s"%deb_file_wild) deb_version_final = debianize_version(stack_version, build_version, os_platform) deb_file_final = "%s_%s"%(deb_name, deb_version_final) # Build a package db if we have to debug("starting package db build of %s-%s"%(stack_name, stack_version)) subprocess.check_call(['bash', '-c', 'cd %(staging_dir)s && dpkg-scanpackages . > %(results_dir)s/Packages'%locals()]) # Script to execute for deb verification # TODO: Add code to run all the unit-tests for the deb! verify_script = os.path.join(staging_dir, 'verify_script.sh') with open(verify_script, 'w') as f: f.write("""#!/bin/sh set -o errexit echo "deb file:%(staging_dir)s results/" > /etc/apt/sources.list.d/pbuild.list apt-get update apt-get install %(deb_name)s=%(deb_version_final)s -y --force-yes dpkg -l %(deb_name)s """%locals()) os.chmod(verify_script, stat.S_IRWXU) debug("starting verify script for %s-%s"%(stack_name, stack_version)) subprocess.check_call(archcmd + ['sudo', 'pbuilder', '--execute', '--basetgz', distro_tgz, '--configfile', conf_file, '--bindmounts', results_dir, '--buildplace', build_dir, verify_script], stderr=subprocess.STDOUT) # Upload the debs to the server base_files = ['%s_%s.changes'%(deb_file, arch)] # , "%s_%s.deb"%(deb_file_final, arch) files = [os.path.join(results_dir, x) for x in base_files] print "Generated debian change files: %s" % files if not noupload: invalidate_debs(deb_name, os_platform, arch, repo_fqdn) if not upload_debs(files, distro_name, os_platform, arch, repo_fqdn): print "Upload of debs failed!!!" return 1 return 0
def sourcedeb_url(distro, stack_name, os_platform): # have to setup locals for substitution distro_name = distro.release_name stack_version = distro.stacks[stack_name].version deb_name = "ros-%s-%s"%(distro_name, debianize_name(stack_name)) return SOURCEDEB_URI%locals()
def gen_metapkgs(distro, os_platform, arch, staging_dir, force=False): distro_name = distro.release_name # Retrieve the package list from the shadow repo packageurl = ( "http://packages.ros.org/ros-shadow/ubuntu/dists/%(os_platform)s/main/binary-%(arch)s/Packages" % locals() ) packagetxt = urllib2.urlopen(packageurl).read() packagelist = parse_deb_packages(packagetxt) debs = [] missing = [] missing_primary, missing_dep, missing_excluded, missing_excluded_dep = list_missing.get_missing( distro, os_platform, arch ) missing_ok = missing_excluded.union(missing_excluded_dep) # if (metapkg missing) or (metapkg missing deps), then create # modify create to version-lock deps # Build the new meta packages for (v, d) in distro.variants.iteritems(): deb_name = "ros-%s-%s" % (distro_name, debianize_name(v)) # If the metapkg is in the packagelist AND already has the right deps, we leave it: if deb_name in packagelist: list_deps = set([x.split()[0].strip() for x in packagelist[deb_name]["Depends"].split(",")]) mp_deps = set(["ros-%s-%s" % (distro_name, debianize_name(x)) for x in set(d.stack_names) - missing_ok]) if list_deps == mp_deps: debug("Metapackage %s already has correct deps" % deb_name) continue # Else, we create the new metapkg mp = create_meta_pkg( packagelist, distro, distro_name, v, set(d.stack_names) - missing_ok, os_platform, arch, staging_dir ) if mp: debs.append(mp) else: missing.append(v) # We should always need to build the special "all" metapackage mp = create_meta_pkg( packagelist, distro, distro_name, "all", set(distro.released_stacks.keys()) - missing_ok, os_platform, arch, staging_dir, ) if mp: debs.append(mp) else: missing.append("all") upload_debs(debs, distro_name, os_platform, arch) if missing: raise StackBuildFailure("Did not generate all metapkgs: %s." % missing)
def do_deb_build(distro_name, stack_name, stack_version, os_platform, arch, staging_dir, noupload, interactive): debug("Actually trying to build %s-%s..." % (stack_name, stack_version)) distro_tgz = os.path.join("/var/cache/pbuilder", "%s-%s-%s.tgz" % (os_platform, arch, TGZ_VERSION)) cache_dir = "/home/rosbuild/aptcache/%s-%s" % (os_platform, arch) deb_name = "ros-%s-%s" % (distro_name, debianize_name(stack_name)) deb_version = debianize_version(stack_version, "0", os_platform) ros_file = "%s-%s" % (stack_name, stack_version) deb_file = "%s_%s" % (deb_name, deb_version) conf_file = os.path.join(roslib.packages.get_pkg_dir("rosdeb"), "config", "pbuilder.conf") # Make sure the distro chroot exists if not os.path.exists(distro_tgz): raise InternalBuildFailure("%s does not exist." % (distro_tgz)) # Download deb and tar.gz files: dsc_name = "%s.dsc" % (deb_file) tar_gz_name = "%s.tar.gz" % (deb_file) (dsc_file, tar_gz_file) = download_files(stack_name, stack_version, staging_dir, [dsc_name, tar_gz_name]) # Create hook and results directories hook_dir = os.path.join(staging_dir, "hooks") results_dir = os.path.join(staging_dir, "results") build_dir = os.path.join(staging_dir, "pbuilder") if not os.path.exists(cache_dir): os.makedirs(cache_dir) if not os.path.exists(hook_dir): os.makedirs(hook_dir) if not os.path.exists(results_dir): os.makedirs(results_dir) if not os.path.exists(build_dir): os.makedirs(build_dir) # Hook script which will download our tar.bz2 into environment p = os.path.join(hook_dir, "A50fetch") with open(p, "w") as f: f.write( """#!/bin/sh set -o errexit wget https://code.ros.org/svn/release/download/stacks/%(stack_name)s/%(stack_name)s-%(stack_version)s/%(stack_name)s-%(stack_version)s.tar.bz2 -O /tmp/buildd/%(stack_name)s-%(stack_version)s.tar.bz2""" % locals() ) os.chmod(p, stat.S_IRWXU) # Hook script which makes sure we have updated our apt cache p = os.path.join(hook_dir, "D50update") with open(p, "w") as f: f.write( """#!/bin/sh set -o errexit apt-get update""" % locals() ) os.chmod(p, stat.S_IRWXU) if interactive: # Hook scripts to make us interactive: p = os.path.join(hook_dir, "B50interactive") with open(p, "w") as f: f.write( """#!/bin/sh echo "Entering interactive environment. Exit when done to continue pbuilder operation." export ROS_DESTDIR=/tmp/buildd/%(deb_name)s-%(stack_version)s/debian/%(deb_name)s source /tmp/buildd/%(deb_name)s-%(stack_version)s/setup_deb.sh roscd %(stack_name)s bash </dev/tty echo "Resuming pbuilder" """ % locals() ) os.chmod(p, stat.S_IRWXU) # Hook scripts to make us interactive: p = os.path.join(hook_dir, "C50interactive") with open(p, "w") as f: f.write( """#!/bin/sh echo "Entering interactive environment. Exit when done to continue pbuilder operation." export ROS_DESTDIR=/tmp/buildd/%(deb_name)s-%(stack_version)s/debian/%(deb_name)s source /tmp/buildd/%(deb_name)s-%(stack_version)s/setup_deb.sh roscd %(stack_name)s bash </dev/tty echo "Resuming pbuilder" """ % locals() ) os.chmod(p, stat.S_IRWXU) if arch == "amd64" or arch == "armel": archcmd = [] else: archcmd = ["setarch", arch] # Actually build the deb. This results in the deb being located in results_dir debug("starting pbuilder build of %s-%s" % (stack_name, stack_version)) subprocess.check_call( archcmd + [ "sudo", "pbuilder", "--build", "--basetgz", distro_tgz, "--configfile", conf_file, "--hookdir", hook_dir, "--buildresult", results_dir, "--binary-arch", "--buildplace", build_dir, "--aptcache", cache_dir, dsc_file, ], stderr=subprocess.STDOUT, ) # Set up an RE to look for the debian file and find the build_version deb_version_wild = debianize_version(stack_version, "(\w*)", os_platform) deb_file_wild = "%s_%s_%s\.deb" % (deb_name, deb_version_wild, arch) build_version = None # Extract the version number we just built: files = os.listdir(results_dir) for f in files: M = re.match(deb_file_wild, f) if M: build_version = M.group(1) if not build_version: raise InternalBuildFailure("No deb-file generated matching template: %s" % deb_file_wild) deb_version_final = debianize_version(stack_version, build_version, os_platform) deb_file_final = "%s_%s" % (deb_name, deb_version_final) # Build a package db if we have to debug("starting package db build of %s-%s" % (stack_name, stack_version)) subprocess.check_call( ["bash", "-c", "cd %(staging_dir)s && dpkg-scanpackages . > %(results_dir)s/Packages" % locals()] ) # Script to execute for deb verification # TODO: Add code to run all the unit-tests for the deb! verify_script = os.path.join(staging_dir, "verify_script.sh") with open(verify_script, "w") as f: f.write( """#!/bin/sh set -o errexit echo "deb file:%(staging_dir)s results/" > /etc/apt/sources.list.d/pbuild.list apt-get update apt-get install %(deb_name)s=%(deb_version_final)s -y --force-yes dpkg -l %(deb_name)s """ % locals() ) os.chmod(verify_script, stat.S_IRWXU) debug("starting verify script for %s-%s" % (stack_name, stack_version)) subprocess.check_call( archcmd + [ "sudo", "pbuilder", "--execute", "--basetgz", distro_tgz, "--configfile", conf_file, "--bindmounts", results_dir, "--buildplace", build_dir, "--aptcache", cache_dir, verify_script, ], stderr=subprocess.STDOUT, ) if not noupload: # Upload the debs to the server base_files = ["%s_%s.changes" % (deb_file, arch), "%s_%s.deb" % (deb_file_final, arch)] files = [os.path.join(results_dir, x) for x in base_files] debug("uploading debs for %s-%s to %s" % (stack_name, stack_version, REPO_HOSTNAME)) cmd = ["scp"] + files + ["%s:/var/packages/ros-shadow/ubuntu/incoming/%s" % (REPO_LOGIN, os_platform)] debug(" ".join(cmd)) subprocess.check_call(cmd, stderr=subprocess.STDOUT) debug("upload complete") # Assemble string for moving all files from incoming to queue (while lock is being held) move_str = "\n".join( [ "mv " + os.path.join("/var/packages/ros-shadow/ubuntu/incoming", os_platform, x) + " " + os.path.join("/var/packages/ros-shadow/ubuntu/queue", os_platform, x) for x in base_files ] ) # This script moves files into queue directory, removes all dependent debs, removes the existing deb, and then processes the incoming files remote_cmd = "TMPFILE=`mktemp` || exit 1 && cat > ${TMPFILE} && chmod +x ${TMPFILE} && ${TMPFILE}; ret=${?}; rm ${TMPFILE}; exit ${ret}" debug("running remote command [%s]" % (remote_cmd)) run_script = subprocess.Popen( ["ssh", REPO_LOGIN, remote_cmd], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) debug("getting depends to prepare invalidate script") invalidate = [deb_name] + get_depends(deb_name, os_platform, arch) debug("invalidating pre-existing and downstream: %s" % (invalidate)) invalidate_cmds = [ "reprepro -b /var/packages/ros-shadow/ubuntu -V -A %(arch)s removefilter %(os_platform)s 'Package (==%(deb_name_x)s)'" % locals() for deb_name_x in invalidate ] invalidate_str = "\n".join(invalidate_cmds) script_content = ( """ #!/bin/bash set -o errexit ( flock 200 # Move from incoming to queue %(move_str)s # Remove all debs that depend on this package %(invalidate_str)s # Load it into the repo reprepro -b /var/packages/ros-shadow/ubuntu -V processincoming %(os_platform)s ) 200>/var/lock/ros-shadow.lock """ % locals() ) # Actually run script and check result (o, e) = run_script.communicate(script_content) debug("waiting for invalidation script") res = run_script.wait() debug("invalidation script result: %s" % o) if res != 0: raise InternalBuildFailure("Could not run upload script:\n%s\n%s" % (o, e)) # The cache is no longer valid, we clear it so that we won't skip debs that have been invalidated rosdeb.repo._Packages_cache = {}
def do_deb_build(distro_name, stack_name, stack_version, os_platform, arch, staging_dir, noupload, interactive): print "Actually trying to build %s-%s..." % (stack_name, stack_version) project_name = stack_name.split('/')[-1].rstrip('.git') #pull down the git repo using git-buildpackage clone, this gets all the right tags subprocess.check_call(["/bin/bash", "-c", "cd %(staging_dir)s && gbp-clone %(stack_name)s" % locals()]) #update any submodules. subprocess.check_call(["/bin/bash", "-c", "cd %(staging_dir)s/%(project_name)s && git submodule update --init"% locals()]) lines = open(os.path.join(staging_dir,project_name,'debian/changelog'),'r').readlines() first_line = lines[0] w = first_line.split('(') left = w[0] right = w[-1].split(')') middle = right[0] right = right[-1] middle += '~' + os_platform line = left + '(' + middle + ')' + right lines[0] = line with open(os.path.join(staging_dir,project_name,'debian/changelog'),'w') as nchlog: nchlog.writelines(lines) subprocess.check_call(["/bin/bash", "-c", "cd %(staging_dir)s/%(project_name)s && git commit -a -m 'change to platform specific'"% locals()]) subprocess.check_call(["/bin/bash", "-c", "cd %(staging_dir)s/%(project_name)s && git-buildpackage -S -uc -us" % locals()]) distro_tgz = os.path.join('/var/cache/pbuilder', "%s-%s.tgz" % (os_platform, arch)) cache_dir = '/home/rosbuild/aptcache/%s-%s' % (os_platform, arch) deb_name = "ros-%s-%s" % (distro_name, debianize_name(stack_name)) deb_version = debianize_version(stack_version, '0', os_platform) ros_file = "%s-%s" % (stack_name, stack_version) deb_file = "%s_%s" % (deb_name, deb_version) conf_file = os.path.join(roslib.packages.get_pkg_dir('rosdeb'), 'config', 'pbuilder.conf') # Make sure the distro chroot exists if not os.path.exists(distro_tgz): raise InternalBuildFailure("%s does not exist." % (distro_tgz)) staging_dir_contents = os.listdir(staging_dir) dsc_files = [f for f in staging_dir_contents if ".dsc" in f] if len(dsc_files) != 1: raise InternalBuildFailure("Too many dsc files found %s" % dsc_files) dsc_file = os.path.join(staging_dir, dsc_files[0]) # Create hook and results directories hook_dir = os.path.join(staging_dir, 'hooks') results_dir = os.path.join(staging_dir, 'results') build_dir = os.path.join(staging_dir, 'pbuilder') if not os.path.exists(cache_dir): os.makedirs(cache_dir) if not os.path.exists(hook_dir): os.makedirs(hook_dir) if not os.path.exists(results_dir): os.makedirs(results_dir) if not os.path.exists(build_dir): os.makedirs(build_dir) # Hook script which makes sure we have updated our apt cache p = os.path.join(hook_dir, 'D50update') with open(p, 'w') as f: f.write("""#!/bin/sh set -o errexit apt-get update""" % locals()) os.chmod(p, stat.S_IRWXU) if arch == 'amd64': archcmd = [] else: archcmd = ['setarch', arch] # Actually build the deb. This results in the deb being located in results_dir print "starting pbuilder build of %s-%s" % (stack_name, stack_version) subprocess.check_call(archcmd + ['sudo', 'pbuilder', '--build', '--basetgz', distro_tgz, '--configfile', conf_file, '--hookdir', hook_dir, '--buildresult', results_dir, '--binary-arch', '--buildplace', build_dir, '--aptcache', cache_dir, dsc_file]) # Extract the version number we just built: files = os.listdir(results_dir) # Find debian file outputs deb_files_detected = [f for f in files if f.endswith('.deb')] deb_names = [d.split('_')[0] for d in deb_files_detected] if len(deb_files_detected) < 1: raise InternalBuildFailure("No deb-file generated") # Build a package db if we have to print "starting package db build of %s-%s" % (stack_name, stack_version) subprocess.check_call(['bash', '-c', 'cd %(staging_dir)s && dpkg-scanpackages . > %(results_dir)s/Packages' % locals()]) for d in deb_names: # Script to execute for deb verification # TODO: Add code to run all the unit-tests for the deb! verify_script = os.path.join(staging_dir, 'verify_script.sh') with open(verify_script, 'w') as f: f.write("""#!/bin/sh set -o errexit echo "deb file:%(staging_dir)s results/" > /etc/apt/sources.list.d/pbuild.list apt-get update apt-get install %(d)s -y --force-yes dpkg -l %(d)s """ % locals()) os.chmod(verify_script, stat.S_IRWXU) print "starting verify script for %s-%s" % (stack_name, stack_version) subprocess.check_call(archcmd + ['sudo', 'pbuilder', '--execute', '--basetgz', distro_tgz, '--configfile', conf_file, '--bindmounts', results_dir, '--buildplace', build_dir, '--aptcache', cache_dir, verify_script]) # Detect changes files change_files = [f for f in files if '.changes' in f] upload_files = [os.path.join(results_dir, x) for x in deb_files_detected] if not noupload: upload_debs(upload_files, 'ros-shadow-3rdparty', distro_name, os_platform, arch) else: print "No Upload option selected, I would have uploaded the files:", upload_files