def get_release_info(name, distro, prefix=None): """ Steps to check for a released version of the package 1) Look in the wet distro file for the package/stack name, if it's there, return the repo 2) Look in the dry distro file for the package/stack name, if it's there, return the repo 3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at what stack a given package belongs to 4) Look in the distro files again to see if the stack name is there, if it is, return the repo """ url = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' % distro try: wet_distro = yaml.load(urllib2.urlopen(url)) except: raise IOError("Could not load the %s rosdistro file from github.\n\ Are you sure you've selected a valid distro?\n\ I'm looking for the following file %s" % (distro, url)) dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro)) #Check to see if the name just exists in one of our rosdistro files rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix) if rosinstall: return rosinstall #If we didn't find the name, we need to try to find a stack for it doc_yaml = get_manifest_yaml(name, distro) for metapackage in doc_yaml.get('metapackages', []): meta_yaml = get_manifest_yaml(metapackage, distro) if meta_yaml['package_type'] == 'stack': rosinstall = get_release_rosinstall(metapackage, wet_distro, dry_distro, prefix) if rosinstall: return rosinstall return None
def get_release_info(name, distro, prefix=None): """ Steps to check for a released version of the package 1) Look in the wet distro file for the package/stack name, if it's there, return the repo 2) Look in the dry distro file for the package/stack name, if it's there, return the repo 3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at what stack a given package belongs to 4) Look in the distro files again to see if the stack name is there, if it is, return the repo """ url = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' % distro try: wet_distro = yaml.load(urllib2.urlopen(url)) except: raise IOError("Could not load the %s rosdistro file from github.\n\ Are you sure you've selected a valid distro?\n\ I'm looking for the following file %s" % (distro, url)) dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro)) # Check to see if the name just exists in one of our rosdistro files rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix) if rosinstall: return rosinstall # If we didn't find the name, we need to try to find a stack for it doc_yaml = get_manifest_yaml(name, distro) for metapackage in doc_yaml.get('metapackages', []): meta_yaml = get_manifest_yaml(metapackage, distro) if meta_yaml['package_type'] == 'stack': rosinstall = get_release_rosinstall( metapackage, wet_distro, dry_distro, prefix) if rosinstall: return rosinstall return None
def dry_get_versioned_dependency_tree(rosdistro): url = distro_uri(rosdistro) try: d = load_distro(url) except urllib2.URLError as ex: print("Loading distro from '%s'failed with URLError %s" % (url, ex), file=sys.stderr) raise dependency_tree = {} versions = {} maintainer_dict = {} for s in d.stacks: version = d.stacks[s].version versions[s] = version yaml_info = dry_get_stack_info(s, version) if 'depends' in yaml_info: dependency_tree[s] = yaml_info['depends'] else: dependency_tree[s] = [] maintainers = [] if 'maintainer' in yaml_info: maintainers = _extract_emails(yaml_info['maintainer']) if not maintainers and 'contact' in yaml_info: maintainers = _extract_emails(yaml_info['contact']) maintainer_dict[s] = maintainers return dependency_tree, versions, maintainer_dict
def dry_get_versioned_dependency_tree(rosdistro): url = distro_uri(rosdistro) try: d = load_distro(url) except urllib2.URLError as ex: print ("Loading distro from '%s'failed with URLError %s" % (url, ex), file=sys.stderr) raise dependency_tree = {} versions = {} maintainer_dict = {} for s in d.stacks: version = d.stacks[s].version versions[s] = version yaml_info = dry_get_stack_info(s, version) if 'depends' in yaml_info: dependency_tree[s] = yaml_info['depends'] else: dependency_tree[s] = [] maintainers = [] if 'maintainer' in yaml_info: maintainers = _extract_emails(yaml_info['maintainer']) if not maintainers and 'contact' in yaml_info: maintainers = _extract_emails(yaml_info['contact']) maintainer_dict[s] = maintainers return dependency_tree, versions, maintainer_dict
def compute_missing(distros, arches, fqdn, rosdistro, sourcerpm_only=False, wet_only=False): """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """ repo_url = 'http://%s/smd-ros-building' % fqdn rd = Rosdistro(rosdistro) # We take the intersection of repo-specific targets with default # targets. if distros: target_distros = distros else: target_distros = rd.get_target_distros() missing = {} for short_package_name in rd.get_package_list(): #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros)) # todo check if sourcerpm is present with the right version rpm_name = redhatify_package_name(rosdistro, short_package_name) expected_version = rd.get_version(short_package_name, full_version=True) missing[short_package_name] = [] for d in target_distros: d_ver = fedora_release_version(d) if not repo.rpm_in_repo(repo_url, rpm_name, str(expected_version) + "\.fc" + d_ver, d, 'SRPMS'): missing[short_package_name].append((d, 'SRPMS')) if not sourcerpm_only: for a in arches: if not repo.rpm_in_repo(repo_url, rpm_name, str(expected_version) + "\.fc" + d_ver, d, a): missing[short_package_name].append((d, a)) if not sourcerpm_only and not wet_only: #dry stacks # dry dependencies dist = load_distro(distro_uri(rosdistro)) distro_arches = [] for d in target_distros: for a in arches: distro_arches.append((d, a)) for s in dist.stacks: #print ("Analyzing DRY job [%s]" % s) expected_version = dry_get_stack_version(s, dist) # sanitize undeclared versions for string substitution if not expected_version: expected_version = '' missing[s] = [] # for each distro arch check if the RPM is present. If not trigger the build. for (d, a) in distro_arches: if not repo.rpm_in_repo(repo_url, redhatify_package_name(rosdistro, s), expected_version + "\.fc" + fedora_release_version(d), d, a): missing[s].append((d, a)) return missing
def single_deb_main(): from optparse import OptionParser parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME) parser.add_option("-d", "--dir", dest="staging_dir", default=None, help="directory to use for staging source debs", metavar="STAGING_DIR") parser.add_option("--force", dest="force", default=False, action="store_true") parser.add_option("--noupload", dest="noupload", default=False, action="store_true") parser.add_option("--fqdn", dest="fqdn", default='50.28.27.175', action="store") parser.add_option("--interactive", dest="interactive", default=False, action="store_true") parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER") (options, args) = parser.parse_args() if len(args) != 4: parser.error('invalid args') (distro_name, stack_name, os_platform, arch) = args distro = failure_message = warning_message = None if options.staging_dir is not None: staging_dir = options.staging_dir staging_dir = os.path.abspath(staging_dir) else: staging_dir = tempfile.mkdtemp() try: if distro_name not in rosdeb.targets.os_platform: raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys()))) target_platforms = rosdeb.targets.os_platform[distro_name] if os_platform not in target_platforms: raise BuildFailure("[%s] is not a known platformfor distro %s.\nSupported platforms are: %s" % (os_platform, distro_name, ' '.join(target_platforms))) if not os.path.exists(staging_dir): debug("creating staging dir: %s"%(staging_dir)) os.makedirs(staging_dir) uri = distro_uri(distro_name) debug("loading distro file from %s"%(uri)) distro = load_distro(uri) if stack_name == 'metapackages': (warning_message, failure_message) = gen_metapkgs_setup(options.staging_dir, distro, os_platform, arch, options.fqdn) else: build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive, options.fqdn) except StackBuildFailure, e: warning_message = "Warning Message:\n"+"="*80+'\n'+str(e)
def get_dry_names_packages(rosdistro): ''' Fetches a yaml file from the web and returns a list of pairs of the form [(short_pkg_name, pkg_dict), ...] for the dry (rosbuild) packages. ''' dry_yaml = yaml.load(urllib2.urlopen(distro_uri(rosdistro))) return [(name, d) for name, d in dry_yaml['stacks'].items() if name != '_rules']
def build_debs_main(): from optparse import OptionParser parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME) parser.add_option("-d", "--dir", dest="staging_dir", default=None, help="directory to use for staging source debs", metavar="STAGING_DIR") parser.add_option("--force", dest="force", default=False, action="store_true") parser.add_option("--noupload", dest="noupload", default=False, action="store_true") parser.add_option("--noramdisk", dest="ramdisk", default=True, action="store_false") parser.add_option("--interactive", dest="interactive", default=False, action="store_true") parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER") (options, args) = parser.parse_args() if len(args) != 4: parser.error('invalid args') (distro_name, stack_name, os_platform, arch) = args distro = failure_message = warning_message = None if options.staging_dir is not None: staging_dir = options.staging_dir staging_dir = os.path.abspath(staging_dir) else: staging_dir = tempfile.mkdtemp() try: if distro_name not in rosdeb.targets.os_platform: raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys()))) target_platforms = rosdeb.targets.os_platform[distro_name] if os_platform not in target_platforms: raise BuildFailure("[%s] is not a known platform.\nSupported platforms are: %s" % (os_platform, ' '.join(target_platforms))) if not os.path.exists(staging_dir): print "creating staging dir: %s" % (staging_dir) os.makedirs(staging_dir) distro = load_distro(distro_uri(distro_name)) if options.ramdisk: with TempRamFS(staging_dir, "20G"): build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive) else: build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive) except StackBuildFailure, e: warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
def dry_get_versioned_dependency_tree(rosdistro): d = load_distro(distro_uri(rosdistro)) dependency_tree = {} versions = {} for s in d.stacks: version = d.stacks[s].version versions[s] = version yaml_info = dry_get_stack_info(s, version) if 'depends' in yaml_info: dependency_tree[s] = yaml_info['depends'] else: dependency_tree[s] = [] return dependency_tree, versions
def compute_missing(job_params, sourcedeb_only=False): """ Compute what wet packages are missing from a repo based on the rosdistro files. """ repo_url = 'http://%s/repos/building' % job_params.fqdn rd = job_params.rd # We take the intersection of repo-specific targets with default # targets. target_distros = job_params.distros missing = {} for short_package_name in rd.get_packages(): deb_name = debianize_package_name(job_params.rosdistro, short_package_name) expected_version = rd.get_version(short_package_name) missing[short_package_name] = [] for d in target_distros: if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + d, d, arch='na', source=True): missing[short_package_name].append('%s_source' % d) if not sourcedeb_only: target_arches = job_params.arches[d] for a in target_arches: if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + ".*", d, a): missing[short_package_name].append('%s_%s' % (d, a)) if not sourcedeb_only: #dry stacks # dry dependencies dist = load_distro(distro_uri(rosdistro)) distro_arches = [] for d in target_distros: target_arches = job_params.arches[d] for a in target_arches: distro_arches.append((d, a)) for s in dist.stacks: #print ("Analyzing DRY job [%s]" % s) expected_version = dry_get_stack_version(s, dist) # sanitize undeclared versions for string substitution if not expected_version: expected_version = '' missing[s] = [] # for each distro arch check if the deb is present. If not trigger the build. for (d, a) in distro_arches: if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a): missing[s].append('%s_%s' % (d, a)) return missing
def translate(distro, translate_dir): d = load_distro(distro_uri(distro)) repo_list = d.get_stacks(True) for name, item in repo_list.iteritems(): if item.vcs_config.type == 'svn': rosinstall = [{item.vcs_config.type: \ {'local-name': item.name, 'uri': item.vcs_config.anon_dev}}] else: rosinstall = [{item.vcs_config.type: \ {'local-name': item.name, 'uri': item.vcs_config.anon_repo_uri, 'version': item.vcs_config.dev_branch}}] path = os.path.join(translate_dir, "%s.rosinstall" % item.name) with open(path, 'w+') as f: print "writing to %s" % path yaml.safe_dump(rosinstall, f, default_flow_style=False)
def _get_electric_rosinstall(name, prefix=None): """ Please delete me when you don't care at all about electric anymore """ dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri('electric')) if _is_dry(dry_distro, name): return get_dry_rosinstall(dry_distro, name, prefix=prefix) # If we didn't find the name, we need to try to find a stack for it doc_yaml = get_manifest_yaml(name, 'electric') for metapackage in doc_yaml.get('metapackages', []): meta_yaml = get_manifest_yaml(metapackage, 'electric') if meta_yaml['package_type'] == 'stack': if _is_dry(dry_distro, metapackage): return get_dry_rosinstall(dry_distro, metapackage, prefix=prefix) return None
def __init__(self, rosdistro_name): self.packages = {} from buildfarm.ros_distro import Rosdistro # for fuerte we still fetch the new groovy rosdistro to get a list of distros rd = Rosdistro(rosdistro_name if rosdistro_name != 'fuerte' else 'groovy') self.rosdistro_index = rd._index self.rosdistro_dist = rd._dist # load wet rosdistro packages if rosdistro_name == 'fuerte': from buildfarm.ros_distro_fuerte import Rosdistro as RosdistroFuerte rd = RosdistroFuerte(rosdistro_name) for pkg_name in rd.get_package_list(): version = rd.get_version(pkg_name, full_version=True) if version: self.packages[pkg_name] = RosdistroVersion(pkg_name, 'wet', version) # load dry rosdistro stacks if rosdistro_name == 'groovy': dry_yaml = yaml.load(urllib2.urlopen(distro_uri(rosdistro_name))) stacks = dry_yaml['stacks'] or {} for stack_name, d in stacks.items(): if stack_name == '_rules': continue version = d.get('version') if version: if stack_name in self.packages: logging.warn("Stack '%s' exists in dry (%s) as well as in wet (%s) distro. Ignoring dry package." % (stack_name, version, self.packages[stack_name].version)) continue self.packages[stack_name] = RosdistroVersion(stack_name, 'dry', version) # load variants variants = dry_yaml['variants'] or {} for variant in variants: if len(variant) != 1: logging.warn("Not length 1 dict in variant '%s': skipping" % variant) continue variant_name = variant.keys()[0] if variant_name in self.packages: logging.warn("Variant '%s' exists also as a package in %s. Ignoring variant." % (variant_name, self.packages[variant_name].type)) continue self.packages[variant_name] = RosdistroVersion(variant_name, 'variant', '1.0.0')
def get_missing_dry_packages(rosdistro, default_distros, arches, repo_url): missing = {} dist = load_distro(distro_uri(rosdistro)) distro_arches = [(d, a) for d in default_distros for a in arches] for s in dist.stacks: #print ("Analyzing DRY job [%s]" % s) expected_version = dry_get_stack_version(s, dist) # sanitize undeclared versions for string substitution if not expected_version: expected_version = '' missing[s] = [] # for each distro arch check if the deb is present. If not trigger the build. for (d, a) in distro_arches: if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version+".*", d, a): missing[s].append( '%s_%s' % (d, a) ) return missing
def _get_electric_rosinstall(name, prefix=None): """ Please delete me when you don't care at all about electric anymore """ dry_distro = rospkg_distro.load_distro( rospkg_distro.distro_uri('electric')) if _is_dry(dry_distro, name): return get_dry_rosinstall(dry_distro, name, prefix=prefix) # If we didn't find the name, we need to try to find a stack for it doc_yaml = get_manifest_yaml(name, 'electric') for metapackage in doc_yaml.get('metapackages', []): meta_yaml = get_manifest_yaml(metapackage, 'electric') if meta_yaml['package_type'] == 'stack': if _is_dry(dry_distro, metapackage): return get_dry_rosinstall(dry_distro, metapackage, prefix=prefix) return None
def _get_fuerte_rosinstall(name, prefix=None): """ Please delete me when fuerte is not supported anymore See REP137 about rosdistro files """ dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri("fuerte")) wet_distro = _get_fuerte_release() # Check to see if the name just exists in one of our rosdistro files rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix) if rosinstall: return rosinstall # If we didn't find the name, we need to try to find a stack for it doc_yaml = get_manifest_yaml(name, "fuerte") for metapackage in doc_yaml.get("metapackages", []): meta_yaml = get_manifest_yaml(metapackage, "fuerte") if meta_yaml["package_type"] == "stack": rosinstall = get_release_rosinstall(metapackage, wet_distro, dry_distro, prefix) if rosinstall: return rosinstall return None
def get_release_info(name, distro, prefix=None): """ Steps to check for a released version of the package 1) Look in the wet distro file for the package/stack name, if it's there, return the repo 2) Look in the dry distro file for the package/stack name, if it's there, return the repo 3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at what stack a given package belongs to 4) Look in the distro files again to see if the stack name is there, if it is, return the repo """ # fuerte is different. if distro == 'fuerte': return _get_fuerte_rosinstall(name, prefix=prefix) # electric is ancient. if distro == 'electric': return _get_electric_rosinstall(name, prefix=prefix) wet_distro = _get_rosdistro_release(distro) dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro)) # Check to see if the name just exists in one of our rosdistro files if _is_wet(wet_distro, name): return get_wet_rosinstall(wet_distro, name, prefix=prefix) if _is_dry(dry_distro, name): return get_dry_rosinstall(dry_distro, name, prefix=prefix) # If we didn't find the name, we need to try to find a stack for it doc_yaml = get_manifest_yaml(name, distro) for metapackage in doc_yaml.get('metapackages', []): meta_yaml = get_manifest_yaml(metapackage, distro) if meta_yaml['package_type'] == 'stack': if _is_dry(dry_distro, metapackage): return get_dry_rosinstall(dry_distro, metapackage, prefix=prefix) return None
def _get_fuerte_rosinstall(name, prefix=None): """ Please delete me when fuerte is not supported anymore See REP137 about rosdistro files """ dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri('fuerte')) wet_distro = _get_fuerte_release() # Check to see if the name just exists in one of our rosdistro files rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix) if rosinstall: return rosinstall # If we didn't find the name, we need to try to find a stack for it doc_yaml = get_manifest_yaml(name, 'fuerte') for metapackage in doc_yaml.get('metapackages', []): meta_yaml = get_manifest_yaml(metapackage, 'fuerte') if meta_yaml['package_type'] == 'stack': rosinstall = get_release_rosinstall(metapackage, wet_distro, dry_distro, prefix) if rosinstall: return rosinstall return None
def _bootstrap_from_rosdistro(self, rosdistro): if rosdistro == 'fuerte': from buildfarm.ros_distro_fuerte import Rosdistro else: from buildfarm.ros_distro import Rosdistro rd = Rosdistro(rosdistro) for name in rd.get_package_list(): self.add(name, 'rosdistro', 'wet', rd.get_version(name, full_version=True)) if rosdistro not in ['fuerte', 'groovy', 'hydro', 'indigo']: return dry_yaml = yaml.load(urllib2.urlopen(distro_uri(rosdistro))) for name, d in dry_yaml['stacks'].items(): if name == '_rules': continue self.add(name, 'rosdistro', 'dry', d.get('version')) for variant in dry_yaml['variants']: if len(variant) != 1: logging.warn("Not length 1 dict in variant %s: skipping" % \ variant) continue name = variant.keys()[0] self.add(name, 'rosdistro', 'variant', '1.0.0')
def compute_missing(distros, arches, fqdn, rosdistro, sourcepkg_only=False, platform='ubuntu'): """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """ repo_url = 'http://%s/repos/building/%s' % (fqdn, platform) # TODO Building Repo Workaround if platform == 'ubuntu': repo_url = 'http://%s/repos/building' % fqdn elif platform == 'fedora': repo_url = 'http://%s/smd-ros-building/%s' % (fqdn, platform) # End Workaround if rosdistro != 'fuerte': from ros_distro import Rosdistro else: from ros_distro_fuerte import Rosdistro rd = Rosdistro(rosdistro) # We take the intersection of repo-specific targets with default # targets. if distros: target_distros = distros else: target_distros = rd.get_target_distros()[platform] missing = {} for short_package_name in rd.get_package_list(): #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros)) # todo check if sourcepkg is present with the right version pkg_name = debianize_package_name(rosdistro, short_package_name) expected_version = rd.get_version(short_package_name, full_version=True) # Don't report packages as missing if their version is None if not expected_version: print("Skipping package %s with no version" % short_package_name) continue missing[short_package_name] = [] for d in target_distros: if platform == 'fedora': version_extra = ".fc" + str(get_fedora_ver(d)) else: version_extra = ".*" if not repo.pkg_in_repo(repo_url, pkg_name, get_full_version(expected_version, d, platform), d, arch='na', source=True, platform=platform): missing[short_package_name].append('%s_source' % d) if not sourcepkg_only: for a in arches: if not repo.pkg_in_repo(repo_url, pkg_name, str(expected_version) + version_extra, d, a, platform=platform): missing[short_package_name].append('%s_%s' % (d, a)) if not sourcepkg_only and rosdistro == 'groovy' and not platform == 'fedora': #dry stacks # dry dependencies dist = load_distro(distro_uri(rosdistro)) distro_arches = [] for d in target_distros: for a in arches: distro_arches.append((d, a)) for s in dist.stacks: #print ("Analyzing DRY job [%s]" % s) expected_version = dry_get_stack_version(s, dist) # Don't report packages as missing if their version is None if not expected_version: print("Skipping package %s with no version" % s) continue missing[s] = [] # for each distro arch check if the pkg is present. If not trigger the build. for (d, a) in distro_arches: if not repo.pkg_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a, platform=platform): missing[s].append('%s_%s' % (d, a)) return missing
def test_distro_uri(): from rospkg.distro import distro_uri assert distro_uri('groovy') == "http://svn.code.sf.net/p/ros-dry-releases/code/trunk/distros/groovy.rosdistro"
def get_distro(distro_name): return rosdistro.load_distro(rosdistro.distro_uri(distro_name))
def single_deb_main(): from optparse import OptionParser parser = OptionParser( usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME) parser.add_option("-d", "--dir", dest="staging_dir", default=None, help="directory to use for staging source debs", metavar="STAGING_DIR") parser.add_option("--force", dest="force", default=False, action="store_true") parser.add_option("--noupload", dest="noupload", default=False, action="store_true") parser.add_option("--fqdn", dest="fqdn", default='50.28.27.175', action="store") parser.add_option("--interactive", dest="interactive", default=False, action="store_true") parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER") (options, args) = parser.parse_args() if len(args) != 4: parser.error('invalid args') (distro_name, stack_name, os_platform, arch) = args distro = failure_message = warning_message = None if options.staging_dir is not None: staging_dir = options.staging_dir staging_dir = os.path.abspath(staging_dir) else: staging_dir = tempfile.mkdtemp() try: if distro_name not in rosdeb.targets.os_platform: raise BuildFailure( "[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys()))) target_platforms = rosdeb.targets.os_platform[distro_name] if os_platform not in target_platforms: raise BuildFailure( "[%s] is not a known platformfor distro %s.\nSupported platforms are: %s" % (os_platform, distro_name, ' '.join(target_platforms))) if not os.path.exists(staging_dir): debug("creating staging dir: %s" % (staging_dir)) os.makedirs(staging_dir) uri = distro_uri(distro_name) debug("loading distro file from %s" % (uri)) distro = load_distro(uri) if stack_name == 'metapackages': (warning_message, failure_message) = gen_metapkgs_setup(options.staging_dir, distro, os_platform, arch, options.fqdn) else: build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive, options.fqdn) except StackBuildFailure, e: warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
def single_deb_main(): from optparse import OptionParser parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME) parser.add_option("-d", "--dir", dest="staging_dir", default=None, help="directory to use for staging source debs", metavar="STAGING_DIR") parser.add_option("--force", dest="force", default=False, action="store_true") parser.add_option("--noupload", dest="noupload", default=False, action="store_true") parser.add_option("--fqdn", dest="fqdn", default='repos.ros.org', action="store") parser.add_option("--interactive", dest="interactive", default=False, action="store_true") parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER") (options, args) = parser.parse_args() if len(args) != 4: parser.error('invalid args') (distro_name, stack_name, os_platform, arch) = args distro = failure_message = warning_message = None if options.staging_dir is not None: staging_dir = os.path.abspath(options.staging_dir) else: staging_dir = tempfile.mkdtemp() try: if distro_name not in rosdeb.targets.os_platform: raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys()))) target_platforms = rosdeb.targets.os_platform[distro_name] if os_platform not in target_platforms: raise BuildFailure("[%s] is not a known platform or distro %s.\nSupported platforms are: %s" % (os_platform, distro_name, ' '.join(target_platforms))) if not os.path.exists(staging_dir): debug("creating staging dir: %s" % (staging_dir)) os.makedirs(staging_dir) uri = distro_uri(distro_name) debug("loading distro file from %s" % (uri)) distro = load_distro(uri) if stack_name == 'metapackages': (warning_message, failure_message) = gen_metapkgs_setup(options.staging_dir, distro, os_platform, arch, options.fqdn) else: build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive, options.fqdn) except StackBuildFailure as e: warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e) except BuildFailure as e: failure_message = "Failure Message:\n" + "=" * 80 + '\n' + str(e) except Exception as e: failure_message = "Internal failure release system setting up the staging dir. Please notify [email protected]:\n%s\n\n%s" % (e, traceback.format_exc(e)) finally: # if we created our own staging dir, we are responsible for cleaning it up if options.staging_dir is None: shutil.rmtree(staging_dir) if failure_message or warning_message: debug("FAILURE: %s" % failure_message) debug("WARNING: %s" % warning_message) #if not options.interactive: #failure_message = "%s\n%s\n%s" % (failure_message, warning_message, os.environ.get('BUILD_URL', '')) #if options.smtp and stack_name != 'metapackages' and distro is not None: #stack_version = distro.stacks[stack_name].version #control = download_control(stack_name, stack_version) #if 'contact' in control and distro_name != 'diamondback': # DISABLE SENDING OF EMAIL from script. This can be done better by jenkins. #to_addr = control['contact'] #subject = 'debian build [%s-%s-%s-%s] failed'%(distro_name, stack_name, os_platform, arch) # send_email(options.smtp, EMAIL_FROM_ADDR, to_addr, subject, failure_message) if stack_name == 'metapackages': debug("Not failing because build is a metapackage") return True return False return True
def compute_missing(distros, arches, fqdn, rosdistro, sourcedeb_only=False): """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """ repo_url = 'http://%s/repos/building' % fqdn if rosdistro != 'fuerte': from ros_distro import Rosdistro else: from ros_distro_fuerte import Rosdistro rd = Rosdistro(rosdistro) # We take the intersection of repo-specific targets with default # targets. if distros: target_distros = distros else: target_distros = rd.get_target_distros() missing = {} for short_package_name in rd.get_package_list(): #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros)) # todo check if sourcedeb is present with the right version deb_name = debianize_package_name(rosdistro, short_package_name) expected_version = rd.get_version(short_package_name, full_version=True) # Don't report packages as missing if their version is None if not expected_version: print("Skipping package %s with no version" % short_package_name) continue missing[short_package_name] = [] for d in target_distros: if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + d, d, arch='na', source=True): missing[short_package_name].append('%s_source' % d) if not sourcedeb_only: for a in arches: if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + ".*", d, a): missing[short_package_name].append('%s_%s' % (d, a)) if not sourcedeb_only: #dry stacks # dry dependencies dist = load_distro(distro_uri(rosdistro)) distro_arches = [] for d in target_distros: for a in arches: distro_arches.append((d, a)) for s in dist.stacks: #print ("Analyzing DRY job [%s]" % s) expected_version = dry_get_stack_version(s, dist) # Don't report packages as missing if their version is None if not expected_version: print("Skipping package %s with no version" % s) continue missing[s] = [] # for each distro arch check if the deb is present. If not trigger the build. for (d, a) in distro_arches: if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a): missing[s].append('%s_%s' % (d, a)) return missing
def compute_missing(distros, fqdn, rosdistro): """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """ URL_PROTOTYPE = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' repo_url = 'http://%s/repos/building' % fqdn print('Fetching "%s"' % (URL_PROTOTYPE % rosdistro)) repo_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % rosdistro)) # What ROS distro are we configuring? if 'release-name' not in repo_map: print('No "release-name" key in yaml file') sys.exit(1) if repo_map['release-name'] != rosdistro: print('release-name mismatch (%s != %s)' % (repo_map['release-name'], rosdistro)) sys.exit(1) if 'repositories' not in repo_map: print('No "repositories" key in yaml file') if 'type' not in repo_map or repo_map['type'] != 'gbp': print('Wrong type value in yaml file') sys.exit(1) # Figure out default distros. Command-line arg takes precedence; if # it's not specified, then read targets.yaml. if distros: default_distros = distros else: print('Fetching "%s"' % (URL_PROTOTYPE % 'targets')) targets_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % 'targets')) my_targets = [x for x in targets_map if rosdistro in x] if len(my_targets) != 1: print('Must have exactly one entry for rosdistro "%s" in targets.yaml' % rosdistro) sys.exit(1) default_distros = my_targets[0][rosdistro] arches = ['amd64', 'i386'] # We take the intersection of repo-specific targets with default # targets. missing = {} for short_package_name, r in repo_map['repositories'].items(): if 'url' not in r: print('"url" key missing for repository "%s"; skipping' % r) continue url = r['url'] if 'target' not in r or r['target'] == 'all': target_distros = default_distros else: target_distros = list(set(r['target']) & set(default_distros)) #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros)) # todo check if sourcedeb is present with the right version deb_name = debianize_package_name(rosdistro, short_package_name) if not 'version' in r: print('"version" key missing for repository %s; skipping' % r) continue expected_version = r['version'] if not expected_version: expected_version = '' missing[short_package_name] = [] for d in target_distros: if not repo.deb_in_repo(repo_url, deb_name, expected_version+".*", d, arch='na', source=True): missing[short_package_name].append('%s_source' % d) for a in arches: if not repo.deb_in_repo(repo_url, deb_name, expected_version+".*", d, a): missing[short_package_name].append('%s_%s' % (d, a)) # if not trigger sourcedeb # else if binaries don't exist trigger them for d in target_distros: for a in arches: pass#missing[short_package_name] = ['source'] #dry stacks # dry dependencies dist = load_distro(distro_uri(rosdistro)) distro_arches = [] for d in default_distros: for a in arches: distro_arches.append( (d, a) ) for s in dist.stacks: #print ("Analyzing DRY job [%s]" % s) expected_version = dry_get_stack_version(s, dist) # sanitize undeclared versions for string substitution if not expected_version: expected_version = '' missing[s] = [] # for each distro arch check if the deb is present. If not trigger the build. for (d, a) in distro_arches: if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version+".*", d, a): missing[s].append( '%s_%s' % (d, a) ) return missing
def compute_missing(distros, fqdn, rosdistro): """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """ URL_PROTOTYPE = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' repo_url = 'http://%s/repos/building' % fqdn print('Fetching "%s"' % (URL_PROTOTYPE % rosdistro)) repo_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % rosdistro)) # What ROS distro are we configuring? if 'release-name' not in repo_map: print('No "release-name" key in yaml file') sys.exit(1) if repo_map['release-name'] != rosdistro: print('release-name mismatch (%s != %s)' % (repo_map['release-name'], rosdistro)) sys.exit(1) if 'repositories' not in repo_map: print('No "repositories" key in yaml file') if 'type' not in repo_map or repo_map['type'] != 'gbp': print('Wrong type value in yaml file') sys.exit(1) # Figure out default distros. Command-line arg takes precedence; if # it's not specified, then read targets.yaml. if distros: default_distros = distros else: print('Fetching "%s"' % (URL_PROTOTYPE % 'targets')) targets_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % 'targets')) my_targets = [x for x in targets_map if rosdistro in x] if len(my_targets) != 1: print( 'Must have exactly one entry for rosdistro "%s" in targets.yaml' % rosdistro) sys.exit(1) default_distros = my_targets[0][rosdistro] arches = ['amd64', 'i386'] # We take the intersection of repo-specific targets with default # targets. missing = {} for short_package_name, r in repo_map['repositories'].items(): if 'url' not in r: print('"url" key missing for repository "%s"; skipping' % r) continue url = r['url'] if 'target' not in r or r['target'] == 'all': target_distros = default_distros else: target_distros = list(set(r['target']) & set(default_distros)) #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros)) # todo check if sourcedeb is present with the right version deb_name = debianize_package_name(rosdistro, short_package_name) if not 'version' in r: print('"version" key missing for repository %s; skipping' % r) continue expected_version = r['version'] if not expected_version: expected_version = '' missing[short_package_name] = [] for d in target_distros: if not repo.deb_in_repo(repo_url, deb_name, expected_version + ".*", d, arch='na', source=True): missing[short_package_name].append('%s_source' % d) for a in arches: if not repo.deb_in_repo(repo_url, deb_name, expected_version + ".*", d, a): missing[short_package_name].append('%s_%s' % (d, a)) # if not trigger sourcedeb # else if binaries don't exist trigger them for d in target_distros: for a in arches: pass #missing[short_package_name] = ['source'] #dry stacks # dry dependencies dist = load_distro(distro_uri(rosdistro)) distro_arches = [] for d in default_distros: for a in arches: distro_arches.append((d, a)) for s in dist.stacks: #print ("Analyzing DRY job [%s]" % s) expected_version = dry_get_stack_version(s, dist) # sanitize undeclared versions for string substitution if not expected_version: expected_version = '' missing[s] = [] # for each distro arch check if the deb is present. If not trigger the build. for (d, a) in distro_arches: if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a): missing[s].append('%s_%s' % (d, a)) return missing
def single_deb_main(): from optparse import OptionParser parser = OptionParser( usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME) parser.add_option("-d", "--dir", dest="staging_dir", default=None, help="directory to use for staging source debs", metavar="STAGING_DIR") parser.add_option("--force", dest="force", default=False, action="store_true") parser.add_option("--noupload", dest="noupload", default=False, action="store_true") parser.add_option("--fqdn", dest="fqdn", default='repos.ros.org', action="store") parser.add_option("--interactive", dest="interactive", default=False, action="store_true") parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER") (options, args) = parser.parse_args() if len(args) != 4: parser.error('invalid args') (distro_name, stack_name, os_platform, arch) = args distro = failure_message = warning_message = None if options.staging_dir is not None: staging_dir = os.path.abspath(options.staging_dir) else: staging_dir = tempfile.mkdtemp() try: if distro_name not in rosdeb.targets.os_platform: raise BuildFailure( "[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys()))) target_platforms = rosdeb.targets.os_platform[distro_name] if os_platform not in target_platforms: raise BuildFailure( "[%s] is not a known platform or distro %s.\nSupported platforms are: %s" % (os_platform, distro_name, ' '.join(target_platforms))) if not os.path.exists(staging_dir): debug("creating staging dir: %s" % (staging_dir)) os.makedirs(staging_dir) uri = distro_uri(distro_name) debug("loading distro file from %s" % (uri)) distro = load_distro(uri) if stack_name == 'metapackages': (warning_message, failure_message) = gen_metapkgs_setup(options.staging_dir, distro, os_platform, arch, options.fqdn) else: build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive, options.fqdn) except StackBuildFailure as e: warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e) except BuildFailure as e: failure_message = "Failure Message:\n" + "=" * 80 + '\n' + str(e) except Exception as e: failure_message = "Internal failure release system setting up the staging dir. Please notify [email protected]:\n%s\n\n%s" % ( e, traceback.format_exc(e)) finally: # if we created our own staging dir, we are responsible for cleaning it up if options.staging_dir is None: shutil.rmtree(staging_dir) if failure_message or warning_message: debug("FAILURE: %s" % failure_message) debug("WARNING: %s" % warning_message) #if not options.interactive: #failure_message = "%s\n%s\n%s" % (failure_message, warning_message, os.environ.get('BUILD_URL', '')) #if options.smtp and stack_name != 'metapackages' and distro is not None: #stack_version = distro.stacks[stack_name].version #control = download_control(stack_name, stack_version) #if 'contact' in control and distro_name != 'diamondback': # DISABLE SENDING OF EMAIL from script. This can be done better by jenkins. #to_addr = control['contact'] #subject = 'debian build [%s-%s-%s-%s] failed'%(distro_name, stack_name, os_platform, arch) # send_email(options.smtp, EMAIL_FROM_ADDR, to_addr, subject, failure_message) if stack_name == 'metapackages': debug("Not failing because build is a metapackage") return True return False return True
def test_distro_uri(): from rospkg.distro import distro_uri assert distro_uri( 'electric' ) == "https://code.ros.org/svn/release/trunk/distros/electric.rosdistro"
def test_distro_uri(): from rospkg.distro import distro_uri assert distro_uri( 'groovy' ) == "http://ros-dry-releases.googlecode.com/svn/trunk/distros/groovy.rosdistro"
def test_distro_uri(): from rospkg.distro import distro_uri assert distro_uri("groovy") == "http://ros-dry-releases.googlecode.com/svn/trunk/distros/groovy.rosdistro"
def test_distro_uri(): from rospkg.distro import distro_uri assert distro_uri('electric') == "https://code.ros.org/svn/release/trunk/distros/electric.rosdistro"
def get_dry_yaml(): return yaml.load(urllib2.urlopen(distro_uri('groovy')))
def compute_missing(distros, arches, fqdn, rosdistro, sourcedeb_only=False): """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """ repo_url = 'http://%s/repos/building' % fqdn if rosdistro != 'fuerte': from ros_distro import Rosdistro else: from ros_distro_fuerte import Rosdistro rd = Rosdistro(rosdistro) # We take the intersection of repo-specific targets with default # targets. if distros: target_distros = distros else: target_distros = rd.get_target_distros() missing = {} for short_package_name in rd.get_package_list(): #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros)) # todo check if sourcedeb is present with the right version deb_name = debianize_package_name(rosdistro, short_package_name) expected_version = rd.get_version(short_package_name, full_version=True) # Don't report packages as missing if their version is None if not expected_version: print("Skipping package %s with no version" % short_package_name) continue missing[short_package_name] = [] for d in target_distros: if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + d, d, arch='na', source=True): missing[short_package_name].append('%s_source' % d) if not sourcedeb_only: for a in arches: if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + ".*", d, a): missing[short_package_name].append('%s_%s' % (d, a)) if not sourcedeb_only and rosdistro == 'groovy': #dry stacks # dry dependencies dist = load_distro(distro_uri(rosdistro)) distro_arches = [] for d in target_distros: for a in arches: distro_arches.append((d, a)) for s in dist.stacks: #print ("Analyzing DRY job [%s]" % s) expected_version = dry_get_stack_version(s, dist) # Don't report packages as missing if their version is None if not expected_version: print("Skipping package %s with no version" % s) continue missing[s] = [] # for each distro arch check if the deb is present. If not trigger the build. for (d, a) in distro_arches: if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a): missing[s].append('%s_%s' % (d, a)) return missing