示例#1
0
def get_release_info(name, distro, prefix=None):
    """
    Steps to check for a released version of the package
    1) Look in the wet distro file for the package/stack name, if it's there, return the repo
    2) Look in the dry distro file for the package/stack name, if it's there, return the repo
    3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
    what stack a given package belongs to
    4) Look in the distro files again to see if the stack name is there, if it is, return the repo
    """
    url = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' % distro
    try:
        wet_distro = yaml.load(urllib2.urlopen(url))
    except:
        raise IOError("Could not load the %s rosdistro file from github.\n\
Are you sure you've selected a valid distro?\n\
I'm looking for the following file %s" % (distro, url))

    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))

    #Check to see if the name just exists in one of our rosdistro files
    rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
    if rosinstall:
        return rosinstall

    #If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, distro)
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, distro)
        if meta_yaml['package_type'] == 'stack':
            rosinstall = get_release_rosinstall(metapackage, wet_distro, dry_distro, prefix)
            if rosinstall:
                return rosinstall

    return None
示例#2
0
def get_release_info(name, distro, prefix=None):
    """
    Steps to check for a released version of the package
    1) Look in the wet distro file for the package/stack name, if it's there, return the repo
    2) Look in the dry distro file for the package/stack name, if it's there, return the repo
    3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
    what stack a given package belongs to
    4) Look in the distro files again to see if the stack name is there, if it is, return the repo
    """
    url = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' % distro
    try:
        wet_distro = yaml.load(urllib2.urlopen(url))
    except:
        raise IOError("Could not load the %s rosdistro file from github.\n\
Are you sure you've selected a valid distro?\n\
I'm looking for the following file %s" % (distro, url))

    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))

    # Check to see if the name just exists in one of our rosdistro files
    rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
    if rosinstall:
        return rosinstall

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, distro)
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, distro)
        if meta_yaml['package_type'] == 'stack':
            rosinstall = get_release_rosinstall(
                metapackage, wet_distro, dry_distro, prefix)
            if rosinstall:
                return rosinstall

    return None
示例#3
0
def dry_get_versioned_dependency_tree(rosdistro):
    url = distro_uri(rosdistro)
    try:
        d = load_distro(url)
    except urllib2.URLError as ex:
        print("Loading distro from '%s'failed with URLError %s" % (url, ex),
              file=sys.stderr)
        raise
    dependency_tree = {}
    versions = {}
    maintainer_dict = {}
    for s in d.stacks:
        version = d.stacks[s].version
        versions[s] = version
        yaml_info = dry_get_stack_info(s, version)
        if 'depends' in yaml_info:
            dependency_tree[s] = yaml_info['depends']
        else:
            dependency_tree[s] = []
        maintainers = []
        if 'maintainer' in yaml_info:
            maintainers = _extract_emails(yaml_info['maintainer'])
        if not maintainers and 'contact' in yaml_info:
            maintainers = _extract_emails(yaml_info['contact'])
        maintainer_dict[s] = maintainers
    return dependency_tree, versions, maintainer_dict
示例#4
0
def dry_get_versioned_dependency_tree(rosdistro):
    url = distro_uri(rosdistro)
    try:
        d = load_distro(url)
    except urllib2.URLError as ex:
        print ("Loading distro from '%s'failed with URLError %s" % (url, ex), file=sys.stderr)
        raise
    dependency_tree = {}
    versions = {}
    maintainer_dict = {}
    for s in d.stacks:
        version = d.stacks[s].version
        versions[s] = version
        yaml_info = dry_get_stack_info(s, version)
        if 'depends' in yaml_info:
            dependency_tree[s] = yaml_info['depends']
        else:
            dependency_tree[s] = []
        maintainers = []
        if 'maintainer' in yaml_info:
            maintainers = _extract_emails(yaml_info['maintainer'])
        if not maintainers and 'contact' in yaml_info:
            maintainers = _extract_emails(yaml_info['contact'])
        maintainer_dict[s] = maintainers
    return dependency_tree, versions, maintainer_dict
示例#5
0
def compute_missing(distros, arches, fqdn, rosdistro, sourcerpm_only=False, wet_only=False):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    repo_url = 'http://%s/smd-ros-building' % fqdn

    rd = Rosdistro(rosdistro)
    # We take the intersection of repo-specific targets with default
    # targets.

    if distros:
        target_distros = distros
    else:
        target_distros = rd.get_target_distros()

    missing = {}
    for short_package_name in rd.get_package_list():
        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))

        # todo check if sourcerpm is present with the right version
        rpm_name = redhatify_package_name(rosdistro, short_package_name)
        expected_version = rd.get_version(short_package_name, full_version=True)

        missing[short_package_name] = []
        for d in target_distros:
            d_ver = fedora_release_version(d)
            if not repo.rpm_in_repo(repo_url, rpm_name, str(expected_version) + "\.fc" + d_ver, d, 'SRPMS'):
                missing[short_package_name].append((d, 'SRPMS'))
            if not sourcerpm_only:
                for a in arches:
                    if not repo.rpm_in_repo(repo_url, rpm_name, str(expected_version) + "\.fc" + d_ver, d, a):
                        missing[short_package_name].append((d, a))

    if not sourcerpm_only and not wet_only:
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            for a in arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)

            # sanitize undeclared versions for string substitution
            if not expected_version:
                expected_version = ''
            missing[s] = []
            # for each distro arch check if the RPM is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.rpm_in_repo(repo_url, redhatify_package_name(rosdistro, s), expected_version + "\.fc" + fedora_release_version(d), d, a):
                    missing[s].append((d, a))

    return missing
示例#6
0
def single_deb_main():

    from optparse import OptionParser
    parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)

    parser.add_option("-d", "--dir",
                      dest="staging_dir", default=None,
                      help="directory to use for staging source debs", metavar="STAGING_DIR")
    parser.add_option("--force",
                      dest="force", default=False, action="store_true")
    parser.add_option("--noupload",
                      dest="noupload", default=False, action="store_true")
    parser.add_option("--fqdn",
                      dest="fqdn", default='50.28.27.175', action="store")
    parser.add_option("--interactive",
                      dest="interactive", default=False, action="store_true")
    parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER")

    (options, args) = parser.parse_args()

    if len(args) != 4:
        parser.error('invalid args')

    (distro_name, stack_name, os_platform, arch) = args
    distro = failure_message = warning_message = None

    if options.staging_dir is not None:
        staging_dir    = options.staging_dir
        staging_dir = os.path.abspath(staging_dir)
    else:
        staging_dir = tempfile.mkdtemp()


    try:
        if distro_name not in rosdeb.targets.os_platform:
            raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
        target_platforms = rosdeb.targets.os_platform[distro_name]
        if os_platform not in target_platforms:
            raise BuildFailure("[%s] is not a known platformfor distro %s.\nSupported platforms are: %s" % (os_platform, distro_name, ' '.join(target_platforms)))

        if not os.path.exists(staging_dir):
            debug("creating staging dir: %s"%(staging_dir))
            os.makedirs(staging_dir)

        uri = distro_uri(distro_name)
        debug("loading distro file from %s"%(uri))
        distro = load_distro(uri)

        if stack_name == 'metapackages':
            (warning_message, failure_message) = gen_metapkgs_setup(options.staging_dir, distro, os_platform, arch, options.fqdn)
        else:
            build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive, options.fqdn)

    except StackBuildFailure, e:
        warning_message = "Warning Message:\n"+"="*80+'\n'+str(e)
示例#7
0
def get_dry_names_packages(rosdistro):
    '''
    Fetches a yaml file from the web and returns a list of pairs of the form

    [(short_pkg_name, pkg_dict), ...]

    for the dry (rosbuild) packages.
    '''
    
    dry_yaml = yaml.load(urllib2.urlopen(distro_uri(rosdistro)))
    return [(name, d) for name, d in dry_yaml['stacks'].items() if name != '_rules']
示例#8
0
def get_dry_names_packages(rosdistro):
    '''
    Fetches a yaml file from the web and returns a list of pairs of the form

    [(short_pkg_name, pkg_dict), ...]

    for the dry (rosbuild) packages.
    '''

    dry_yaml = yaml.load(urllib2.urlopen(distro_uri(rosdistro)))
    return [(name, d) for name, d in dry_yaml['stacks'].items()
            if name != '_rules']
def build_debs_main():

    from optparse import OptionParser
    parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)

    parser.add_option("-d", "--dir",
                      dest="staging_dir", default=None,
                      help="directory to use for staging source debs", metavar="STAGING_DIR")
    parser.add_option("--force",
                      dest="force", default=False, action="store_true")
    parser.add_option("--noupload",
                      dest="noupload", default=False, action="store_true")
    parser.add_option("--noramdisk",
                      dest="ramdisk", default=True, action="store_false")
    parser.add_option("--interactive",
                      dest="interactive", default=False, action="store_true")
    parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER")

    (options, args) = parser.parse_args()

    if len(args) != 4:
        parser.error('invalid args')
        
    (distro_name, stack_name, os_platform, arch) = args
    distro = failure_message = warning_message = None

    if options.staging_dir is not None:
        staging_dir = options.staging_dir
        staging_dir = os.path.abspath(staging_dir)
    else:
        staging_dir = tempfile.mkdtemp()

    try:
        if distro_name not in rosdeb.targets.os_platform:
            raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
        target_platforms = rosdeb.targets.os_platform[distro_name]
        if os_platform not in target_platforms:
            raise BuildFailure("[%s] is not a known platform.\nSupported platforms are: %s" % (os_platform, ' '.join(target_platforms)))

        if not os.path.exists(staging_dir):
            print "creating staging dir: %s" % (staging_dir)
            os.makedirs(staging_dir)

        distro = load_distro(distro_uri(distro_name))

        if options.ramdisk:
            with TempRamFS(staging_dir, "20G"):
                build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive)
        else:
            build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive)

    except StackBuildFailure, e:
        warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
示例#10
0
def dry_get_versioned_dependency_tree(rosdistro):
    d = load_distro(distro_uri(rosdistro))    
    dependency_tree = {}
    versions = {}
    for s in d.stacks:
        version = d.stacks[s].version
        versions[s] = version
        yaml_info = dry_get_stack_info(s, version)
        if 'depends' in yaml_info:
            dependency_tree[s] = yaml_info['depends']
        else:
            dependency_tree[s] = []
    return dependency_tree, versions
示例#11
0
def dry_get_versioned_dependency_tree(rosdistro):
    d = load_distro(distro_uri(rosdistro))
    dependency_tree = {}
    versions = {}
    for s in d.stacks:
        version = d.stacks[s].version
        versions[s] = version
        yaml_info = dry_get_stack_info(s, version)
        if 'depends' in yaml_info:
            dependency_tree[s] = yaml_info['depends']
        else:
            dependency_tree[s] = []
    return dependency_tree, versions
示例#12
0
def compute_missing(job_params, sourcedeb_only=False):
    """ Compute what wet packages are missing from a repo based on the rosdistro files. """

    repo_url = 'http://%s/repos/building' % job_params.fqdn

    rd = job_params.rd
    # We take the intersection of repo-specific targets with default
    # targets.

    target_distros = job_params.distros

    missing = {}
    for short_package_name in rd.get_packages():
        deb_name = debianize_package_name(job_params.rosdistro, short_package_name)
        expected_version = rd.get_version(short_package_name)

        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + d, d, arch='na', source=True):
                missing[short_package_name].append('%s_source' % d)
            if not sourcedeb_only:
                target_arches = job_params.arches[d]
                for a in target_arches:
                    if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + ".*", d, a):
                        missing[short_package_name].append('%s_%s' % (d, a))

    if not sourcedeb_only:
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            target_arches = job_params.arches[d]
            for a in target_arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)

            # sanitize undeclared versions for string substitution
            if not expected_version:
                expected_version = ''
            missing[s] = []
            # for each distro arch check if the deb is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a):
                    missing[s].append('%s_%s' % (d, a))

    return missing
示例#13
0
def translate(distro, translate_dir):
    d = load_distro(distro_uri(distro))
    repo_list = d.get_stacks(True)
    for name, item in repo_list.iteritems():
        if item.vcs_config.type == 'svn':
            rosinstall = [{item.vcs_config.type: \
                           {'local-name': item.name,
                            'uri': item.vcs_config.anon_dev}}]
        else:
            rosinstall = [{item.vcs_config.type: \
                           {'local-name': item.name,
                            'uri': item.vcs_config.anon_repo_uri,
                            'version': item.vcs_config.dev_branch}}]

        path = os.path.join(translate_dir, "%s.rosinstall" % item.name)
        with open(path, 'w+') as f:
            print "writing to %s" % path
            yaml.safe_dump(rosinstall, f, default_flow_style=False)
示例#14
0
def _get_electric_rosinstall(name, prefix=None):
    """
    Please delete me when you don't care at all about electric anymore
    """
    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri('electric'))

    if _is_dry(dry_distro, name):
        return get_dry_rosinstall(dry_distro, name, prefix=prefix)

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, 'electric')
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, 'electric')
        if meta_yaml['package_type'] == 'stack':
            if _is_dry(dry_distro, metapackage):
                return get_dry_rosinstall(dry_distro, metapackage, prefix=prefix)

    return None
示例#15
0
    def __init__(self, rosdistro_name):
        self.packages = {}
        from buildfarm.ros_distro import Rosdistro
        # for fuerte we still fetch the new groovy rosdistro to get a list of distros
        rd = Rosdistro(rosdistro_name if rosdistro_name != 'fuerte' else 'groovy')
        self.rosdistro_index = rd._index
        self.rosdistro_dist = rd._dist

        # load wet rosdistro packages
        if rosdistro_name == 'fuerte':
            from buildfarm.ros_distro_fuerte import Rosdistro as RosdistroFuerte
            rd = RosdistroFuerte(rosdistro_name)

        for pkg_name in rd.get_package_list():
            version = rd.get_version(pkg_name, full_version=True)
            if version:
                self.packages[pkg_name] = RosdistroVersion(pkg_name, 'wet', version)

        # load dry rosdistro stacks
        if rosdistro_name == 'groovy':
            dry_yaml = yaml.load(urllib2.urlopen(distro_uri(rosdistro_name)))
            stacks = dry_yaml['stacks'] or {}
            for stack_name, d in stacks.items():
                if stack_name == '_rules':
                    continue
                version = d.get('version')
                if version:
                    if stack_name in self.packages:
                        logging.warn("Stack '%s' exists in dry (%s) as well as in wet (%s) distro. Ignoring dry package." % (stack_name, version, self.packages[stack_name].version))
                        continue
                    self.packages[stack_name] = RosdistroVersion(stack_name, 'dry', version)

            # load variants
            variants = dry_yaml['variants'] or {}
            for variant in variants:
                if len(variant) != 1:
                    logging.warn("Not length 1 dict in variant '%s': skipping" % variant)
                    continue
                variant_name = variant.keys()[0]
                if variant_name in self.packages:
                    logging.warn("Variant '%s' exists also as a package in %s. Ignoring variant." % (variant_name, self.packages[variant_name].type))
                    continue
                self.packages[variant_name] = RosdistroVersion(variant_name, 'variant', '1.0.0')
示例#16
0
    def __init__(self, rosdistro_name):
        self.packages = {}
        from buildfarm.ros_distro import Rosdistro
        # for fuerte we still fetch the new groovy rosdistro to get a list of distros
        rd = Rosdistro(rosdistro_name if rosdistro_name != 'fuerte' else 'groovy')
        self.rosdistro_index = rd._index
        self.rosdistro_dist = rd._dist

        # load wet rosdistro packages
        if rosdistro_name == 'fuerte':
            from buildfarm.ros_distro_fuerte import Rosdistro as RosdistroFuerte
            rd = RosdistroFuerte(rosdistro_name)

        for pkg_name in rd.get_package_list():
            version = rd.get_version(pkg_name, full_version=True)
            if version:
                self.packages[pkg_name] = RosdistroVersion(pkg_name, 'wet', version)

        # load dry rosdistro stacks
        if rosdistro_name == 'groovy':
            dry_yaml = yaml.load(urllib2.urlopen(distro_uri(rosdistro_name)))
            stacks = dry_yaml['stacks'] or {}
            for stack_name, d in stacks.items():
                if stack_name == '_rules':
                    continue
                version = d.get('version')
                if version:
                    if stack_name in self.packages:
                        logging.warn("Stack '%s' exists in dry (%s) as well as in wet (%s) distro. Ignoring dry package." % (stack_name, version, self.packages[stack_name].version))
                        continue
                    self.packages[stack_name] = RosdistroVersion(stack_name, 'dry', version)

            # load variants
            variants = dry_yaml['variants'] or {}
            for variant in variants:
                if len(variant) != 1:
                    logging.warn("Not length 1 dict in variant '%s': skipping" % variant)
                    continue
                variant_name = variant.keys()[0]
                if variant_name in self.packages:
                    logging.warn("Variant '%s' exists also as a package in %s. Ignoring variant." % (variant_name, self.packages[variant_name].type))
                    continue
                self.packages[variant_name] = RosdistroVersion(variant_name, 'variant', '1.0.0')
示例#17
0
def get_missing_dry_packages(rosdistro, default_distros, arches, repo_url):
    missing = {}
    dist = load_distro(distro_uri(rosdistro))

    distro_arches = [(d, a) for d in default_distros for a in arches]

    for s in dist.stacks:
        #print ("Analyzing DRY job [%s]" % s)
        expected_version = dry_get_stack_version(s, dist)
        
        # sanitize undeclared versions for string substitution
        if not expected_version:
            expected_version = ''
        missing[s] = []
        # for each distro arch check if the deb is present. If not trigger the build. 
        for (d, a) in distro_arches:
            if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version+".*", d, a):
                missing[s].append( '%s_%s' % (d, a) )
    return missing
示例#18
0
def _get_electric_rosinstall(name, prefix=None):
    """
    Please delete me when you don't care at all about electric anymore
    """
    dry_distro = rospkg_distro.load_distro(
        rospkg_distro.distro_uri('electric'))

    if _is_dry(dry_distro, name):
        return get_dry_rosinstall(dry_distro, name, prefix=prefix)

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, 'electric')
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, 'electric')
        if meta_yaml['package_type'] == 'stack':
            if _is_dry(dry_distro, metapackage):
                return get_dry_rosinstall(dry_distro,
                                          metapackage,
                                          prefix=prefix)

    return None
示例#19
0
def _get_fuerte_rosinstall(name, prefix=None):
    """
    Please delete me when fuerte is not supported anymore
    See REP137 about rosdistro files
    """
    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri("fuerte"))
    wet_distro = _get_fuerte_release()
    # Check to see if the name just exists in one of our rosdistro files
    rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
    if rosinstall:
        return rosinstall

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, "fuerte")
    for metapackage in doc_yaml.get("metapackages", []):
        meta_yaml = get_manifest_yaml(metapackage, "fuerte")
        if meta_yaml["package_type"] == "stack":
            rosinstall = get_release_rosinstall(metapackage, wet_distro, dry_distro, prefix)
            if rosinstall:
                return rosinstall

    return None
示例#20
0
def get_release_info(name, distro, prefix=None):
    """
    Steps to check for a released version of the package
    1) Look in the wet distro file for the package/stack name, if it's there, return the repo
    2) Look in the dry distro file for the package/stack name, if it's there, return the repo
    3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
    what stack a given package belongs to
    4) Look in the distro files again to see if the stack name is there, if it is, return the repo
    """

    # fuerte is different.
    if distro == 'fuerte':
        return _get_fuerte_rosinstall(name, prefix=prefix)

    # electric is ancient.
    if distro == 'electric':
        return _get_electric_rosinstall(name, prefix=prefix)

    wet_distro = _get_rosdistro_release(distro)
    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))

    # Check to see if the name just exists in one of our rosdistro files
    if _is_wet(wet_distro, name):
        return get_wet_rosinstall(wet_distro, name, prefix=prefix)
    if _is_dry(dry_distro, name):
        return get_dry_rosinstall(dry_distro, name, prefix=prefix)

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, distro)
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, distro)
        if meta_yaml['package_type'] == 'stack':
            if _is_dry(dry_distro, metapackage):
                return get_dry_rosinstall(dry_distro,
                                          metapackage,
                                          prefix=prefix)

    return None
示例#21
0
def _get_fuerte_rosinstall(name, prefix=None):
    """
    Please delete me when fuerte is not supported anymore
    See REP137 about rosdistro files
    """
    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri('fuerte'))
    wet_distro = _get_fuerte_release()
    # Check to see if the name just exists in one of our rosdistro files
    rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
    if rosinstall:
        return rosinstall

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, 'fuerte')
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, 'fuerte')
        if meta_yaml['package_type'] == 'stack':
            rosinstall = get_release_rosinstall(metapackage, wet_distro,
                                                dry_distro, prefix)
            if rosinstall:
                return rosinstall

    return None
示例#22
0
 def _bootstrap_from_rosdistro(self, rosdistro):
     if rosdistro == 'fuerte':
         from buildfarm.ros_distro_fuerte import Rosdistro
     else:
         from buildfarm.ros_distro import Rosdistro
     rd = Rosdistro(rosdistro)
     for name in rd.get_package_list():
         self.add(name, 'rosdistro', 'wet',
                  rd.get_version(name, full_version=True))
     if rosdistro not in ['fuerte', 'groovy', 'hydro', 'indigo']:
         return
     dry_yaml = yaml.load(urllib2.urlopen(distro_uri(rosdistro)))
     for name, d in dry_yaml['stacks'].items():
         if name == '_rules':
             continue
         self.add(name, 'rosdistro', 'dry', d.get('version'))
     for variant in dry_yaml['variants']:
         if len(variant) != 1:
             logging.warn("Not length 1 dict in variant %s: skipping" % \
                              variant)
             continue
         name = variant.keys()[0]
         self.add(name, 'rosdistro', 'variant', '1.0.0')
示例#23
0
def get_release_info(name, distro, prefix=None):
    """
    Steps to check for a released version of the package
    1) Look in the wet distro file for the package/stack name, if it's there, return the repo
    2) Look in the dry distro file for the package/stack name, if it's there, return the repo
    3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
    what stack a given package belongs to
    4) Look in the distro files again to see if the stack name is there, if it is, return the repo
    """

    # fuerte is different.
    if distro == 'fuerte':
        return _get_fuerte_rosinstall(name, prefix=prefix)

    # electric is ancient.
    if distro == 'electric':
        return _get_electric_rosinstall(name, prefix=prefix)

    wet_distro = _get_rosdistro_release(distro)
    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))

    # Check to see if the name just exists in one of our rosdistro files
    if _is_wet(wet_distro, name):
        return get_wet_rosinstall(wet_distro, name, prefix=prefix)
    if _is_dry(dry_distro, name):
        return get_dry_rosinstall(dry_distro, name, prefix=prefix)

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, distro)
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, distro)
        if meta_yaml['package_type'] == 'stack':
            if _is_dry(dry_distro, metapackage):
                return get_dry_rosinstall(dry_distro, metapackage, prefix=prefix)

    return None
示例#24
0
def compute_missing(distros, arches, fqdn, rosdistro, sourcepkg_only=False, platform='ubuntu'):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    repo_url = 'http://%s/repos/building/%s' % (fqdn, platform)

    # TODO Building Repo Workaround
    if platform == 'ubuntu':
        repo_url = 'http://%s/repos/building' % fqdn
    elif platform == 'fedora':
        repo_url = 'http://%s/smd-ros-building/%s' % (fqdn, platform)
    # End Workaround

    if rosdistro != 'fuerte':
        from ros_distro import Rosdistro
    else:
        from ros_distro_fuerte import Rosdistro
    rd = Rosdistro(rosdistro)
    # We take the intersection of repo-specific targets with default
    # targets.

    if distros:
        target_distros = distros
    else:
        target_distros = rd.get_target_distros()[platform]

    missing = {}
    for short_package_name in rd.get_package_list():
        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))

        # todo check if sourcepkg is present with the right version
        pkg_name = debianize_package_name(rosdistro, short_package_name)
        expected_version = rd.get_version(short_package_name, full_version=True)

        # Don't report packages as missing if their version is None
        if not expected_version:
            print("Skipping package %s with no version" % short_package_name)
            continue

        missing[short_package_name] = []
        for d in target_distros:
            if platform == 'fedora':
                version_extra = ".fc" + str(get_fedora_ver(d))
            else:
                version_extra = ".*"
            if not repo.pkg_in_repo(repo_url, pkg_name, get_full_version(expected_version, d, platform), d, arch='na', source=True, platform=platform):
                missing[short_package_name].append('%s_source' % d)
            if not sourcepkg_only:
                for a in arches:
                    if not repo.pkg_in_repo(repo_url, pkg_name, str(expected_version) + version_extra, d, a, platform=platform):
                        missing[short_package_name].append('%s_%s' % (d, a))

    if not sourcepkg_only and rosdistro == 'groovy' and not platform == 'fedora':
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            for a in arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)
            # Don't report packages as missing if their version is None
            if not expected_version:
                print("Skipping package %s with no version" % s)
                continue
            missing[s] = []
            # for each distro arch check if the pkg is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.pkg_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a, platform=platform):
                    missing[s].append('%s_%s' % (d, a))

    return missing
示例#25
0
def test_distro_uri():
    from rospkg.distro import distro_uri
    assert distro_uri('groovy') == "http://svn.code.sf.net/p/ros-dry-releases/code/trunk/distros/groovy.rosdistro"
示例#26
0
def get_distro(distro_name):
    return rosdistro.load_distro(rosdistro.distro_uri(distro_name))
示例#27
0
def single_deb_main():

    from optparse import OptionParser
    parser = OptionParser(
        usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)

    parser.add_option("-d",
                      "--dir",
                      dest="staging_dir",
                      default=None,
                      help="directory to use for staging source debs",
                      metavar="STAGING_DIR")
    parser.add_option("--force",
                      dest="force",
                      default=False,
                      action="store_true")
    parser.add_option("--noupload",
                      dest="noupload",
                      default=False,
                      action="store_true")
    parser.add_option("--fqdn",
                      dest="fqdn",
                      default='50.28.27.175',
                      action="store")
    parser.add_option("--interactive",
                      dest="interactive",
                      default=False,
                      action="store_true")
    parser.add_option('--smtp',
                      dest="smtp",
                      default='pub1.willowgarage.com',
                      metavar="SMTP_SERVER")

    (options, args) = parser.parse_args()

    if len(args) != 4:
        parser.error('invalid args')

    (distro_name, stack_name, os_platform, arch) = args
    distro = failure_message = warning_message = None

    if options.staging_dir is not None:
        staging_dir = options.staging_dir
        staging_dir = os.path.abspath(staging_dir)
    else:
        staging_dir = tempfile.mkdtemp()

    try:
        if distro_name not in rosdeb.targets.os_platform:
            raise BuildFailure(
                "[%s] is not a known rosdistro.\nValid rosdistros are: %s" %
                (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
        target_platforms = rosdeb.targets.os_platform[distro_name]
        if os_platform not in target_platforms:
            raise BuildFailure(
                "[%s] is not a known platformfor distro %s.\nSupported platforms are: %s"
                % (os_platform, distro_name, ' '.join(target_platforms)))

        if not os.path.exists(staging_dir):
            debug("creating staging dir: %s" % (staging_dir))
            os.makedirs(staging_dir)

        uri = distro_uri(distro_name)
        debug("loading distro file from %s" % (uri))
        distro = load_distro(uri)

        if stack_name == 'metapackages':
            (warning_message,
             failure_message) = gen_metapkgs_setup(options.staging_dir, distro,
                                                   os_platform, arch,
                                                   options.fqdn)
        else:
            build_debs(distro, stack_name, os_platform, arch, staging_dir,
                       options.force, options.noupload, options.interactive,
                       options.fqdn)

    except StackBuildFailure, e:
        warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
示例#28
0
def single_deb_main():

    from optparse import OptionParser
    parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)

    parser.add_option("-d", "--dir",
                      dest="staging_dir", default=None,
                      help="directory to use for staging source debs", metavar="STAGING_DIR")
    parser.add_option("--force",
                      dest="force", default=False, action="store_true")
    parser.add_option("--noupload",
                      dest="noupload", default=False, action="store_true")
    parser.add_option("--fqdn",
                      dest="fqdn", default='repos.ros.org', action="store")
    parser.add_option("--interactive",
                      dest="interactive", default=False, action="store_true")
    parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER")

    (options, args) = parser.parse_args()

    if len(args) != 4:
        parser.error('invalid args')

    (distro_name, stack_name, os_platform, arch) = args
    distro = failure_message = warning_message = None

    if options.staging_dir is not None:
        staging_dir = os.path.abspath(options.staging_dir)
    else:
        staging_dir = tempfile.mkdtemp()

    try:
        if distro_name not in rosdeb.targets.os_platform:
            raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
        target_platforms = rosdeb.targets.os_platform[distro_name]
        if os_platform not in target_platforms:
            raise BuildFailure("[%s] is not a known platform or distro %s.\nSupported platforms are: %s" % (os_platform, distro_name, ' '.join(target_platforms)))

        if not os.path.exists(staging_dir):
            debug("creating staging dir: %s" % (staging_dir))
            os.makedirs(staging_dir)

        uri = distro_uri(distro_name)
        debug("loading distro file from %s" % (uri))
        distro = load_distro(uri)

        if stack_name == 'metapackages':
            (warning_message, failure_message) = gen_metapkgs_setup(options.staging_dir, distro, os_platform, arch, options.fqdn)
        else:
            build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive, options.fqdn)

    except StackBuildFailure as e:
        warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
    except BuildFailure as e:
        failure_message = "Failure Message:\n" + "=" * 80 + '\n' + str(e)
    except Exception as e:
        failure_message = "Internal failure release system setting up the staging dir. Please notify [email protected]:\n%s\n\n%s" % (e, traceback.format_exc(e))
    finally:
        # if we created our own staging dir, we are responsible for cleaning it up
        if options.staging_dir is None:
            shutil.rmtree(staging_dir)

    if failure_message or warning_message:
        debug("FAILURE: %s" % failure_message)
        debug("WARNING: %s" % warning_message)

        #if not options.interactive:
            #failure_message = "%s\n%s\n%s" % (failure_message, warning_message, os.environ.get('BUILD_URL', ''))
            #if options.smtp and stack_name != 'metapackages' and distro is not None:
                #stack_version = distro.stacks[stack_name].version
                #control = download_control(stack_name, stack_version)
                #if 'contact' in control and distro_name != 'diamondback':
                    # DISABLE SENDING OF EMAIL from script. This can be done better by jenkins.
                    #to_addr = control['contact']
                    #subject = 'debian build [%s-%s-%s-%s] failed'%(distro_name, stack_name, os_platform, arch)
                    # send_email(options.smtp, EMAIL_FROM_ADDR, to_addr, subject, failure_message)

        if stack_name == 'metapackages':
            debug("Not failing because build is a metapackage")
            return True
        return False
    return True
示例#29
0
def compute_missing(distros, arches, fqdn, rosdistro, sourcedeb_only=False):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    repo_url = 'http://%s/repos/building' % fqdn

    if rosdistro != 'fuerte':
        from ros_distro import Rosdistro
    else:
        from ros_distro_fuerte import Rosdistro
    rd = Rosdistro(rosdistro)
    # We take the intersection of repo-specific targets with default
    # targets.

    if distros:
        target_distros = distros
    else:
        target_distros = rd.get_target_distros()

    missing = {}
    for short_package_name in rd.get_package_list():
        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))

        # todo check if sourcedeb is present with the right version
        deb_name = debianize_package_name(rosdistro, short_package_name)
        expected_version = rd.get_version(short_package_name, full_version=True)

        # Don't report packages as missing if their version is None
        if not expected_version:
            print("Skipping package %s with no version" % short_package_name)
            continue

        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + d, d, arch='na', source=True):
                missing[short_package_name].append('%s_source' % d)
            if not sourcedeb_only:
                for a in arches:
                    if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + ".*", d, a):
                        missing[short_package_name].append('%s_%s' % (d, a))

    if not sourcedeb_only:
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            for a in arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)
            # Don't report packages as missing if their version is None
            if not expected_version:
                print("Skipping package %s with no version" % s)
                continue
            missing[s] = []
            # for each distro arch check if the deb is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a):
                    missing[s].append('%s_%s' % (d, a))

    return missing
def get_distro(distro_name):
    return rosdistro.load_distro(rosdistro.distro_uri(distro_name))
示例#31
0
def compute_missing(distros, fqdn, rosdistro):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    URL_PROTOTYPE = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml'


    repo_url = 'http://%s/repos/building' % fqdn

    print('Fetching "%s"' % (URL_PROTOTYPE % rosdistro))
    repo_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % rosdistro))


    # What ROS distro are we configuring?
    if 'release-name' not in repo_map:
        print('No "release-name" key in yaml file')
        sys.exit(1)
    if repo_map['release-name'] != rosdistro:
        print('release-name mismatch (%s != %s)' % (repo_map['release-name'], rosdistro))
        sys.exit(1)    
    if 'repositories' not in repo_map:
        print('No "repositories" key in yaml file')
    if 'type' not in repo_map or repo_map['type'] != 'gbp':
        print('Wrong type value in yaml file')
        sys.exit(1)

    # Figure out default distros.  Command-line arg takes precedence; if
    # it's not specified, then read targets.yaml.
    if distros:
        default_distros = distros
    else:
        print('Fetching "%s"' % (URL_PROTOTYPE % 'targets'))
        targets_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % 'targets'))
        my_targets = [x for x in targets_map if rosdistro in x]
        if len(my_targets) != 1:
            print('Must have exactly one entry for rosdistro "%s" in targets.yaml' % rosdistro)
            sys.exit(1)
        default_distros = my_targets[0][rosdistro]

    arches = ['amd64', 'i386']

    # We take the intersection of repo-specific targets with default
    # targets.
    missing = {}
    for short_package_name, r in repo_map['repositories'].items():
        if 'url' not in r:
            print('"url" key missing for repository "%s"; skipping' % r)
            continue
        url = r['url']
        if 'target' not in r or r['target'] == 'all':
            target_distros = default_distros
        else:
            target_distros = list(set(r['target']) & set(default_distros))

        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))
        
        # todo check if sourcedeb is present with the right version
        deb_name = debianize_package_name(rosdistro, short_package_name)
        if not 'version' in r:
            print('"version" key missing for repository %s; skipping' % r)
            continue
        expected_version = r['version']
        if not expected_version:
            expected_version = ''
        
        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url, deb_name, expected_version+".*", d, arch='na', source=True):
                missing[short_package_name].append('%s_source' % d)
            for a in arches:
                if not repo.deb_in_repo(repo_url, deb_name, expected_version+".*", d, a):
                    missing[short_package_name].append('%s_%s' % (d, a))

                                               
        # if not trigger sourcedeb

        # else if binaries don't exist trigger them
        for d in target_distros:
            for a in arches:
                pass#missing[short_package_name] = ['source']
        


        

    #dry stacks
    # dry dependencies
    dist = load_distro(distro_uri(rosdistro))

    distro_arches = []
    for d in default_distros:
        for a in arches:
            distro_arches.append( (d, a) )

    for s in dist.stacks:
        #print ("Analyzing DRY job [%s]" % s)
        expected_version = dry_get_stack_version(s, dist)
        
        # sanitize undeclared versions for string substitution
        if not expected_version:
            expected_version = ''
        missing[s] = []
        # for each distro arch check if the deb is present. If not trigger the build. 
        for (d, a) in distro_arches:
            if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version+".*", d, a):
                missing[s].append( '%s_%s' % (d, a) )


    return missing
示例#32
0
def compute_missing(distros, fqdn, rosdistro):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    URL_PROTOTYPE = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml'

    repo_url = 'http://%s/repos/building' % fqdn

    print('Fetching "%s"' % (URL_PROTOTYPE % rosdistro))
    repo_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % rosdistro))

    # What ROS distro are we configuring?
    if 'release-name' not in repo_map:
        print('No "release-name" key in yaml file')
        sys.exit(1)
    if repo_map['release-name'] != rosdistro:
        print('release-name mismatch (%s != %s)' %
              (repo_map['release-name'], rosdistro))
        sys.exit(1)
    if 'repositories' not in repo_map:
        print('No "repositories" key in yaml file')
    if 'type' not in repo_map or repo_map['type'] != 'gbp':
        print('Wrong type value in yaml file')
        sys.exit(1)

    # Figure out default distros.  Command-line arg takes precedence; if
    # it's not specified, then read targets.yaml.
    if distros:
        default_distros = distros
    else:
        print('Fetching "%s"' % (URL_PROTOTYPE % 'targets'))
        targets_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % 'targets'))
        my_targets = [x for x in targets_map if rosdistro in x]
        if len(my_targets) != 1:
            print(
                'Must have exactly one entry for rosdistro "%s" in targets.yaml'
                % rosdistro)
            sys.exit(1)
        default_distros = my_targets[0][rosdistro]

    arches = ['amd64', 'i386']

    # We take the intersection of repo-specific targets with default
    # targets.
    missing = {}
    for short_package_name, r in repo_map['repositories'].items():
        if 'url' not in r:
            print('"url" key missing for repository "%s"; skipping' % r)
            continue
        url = r['url']
        if 'target' not in r or r['target'] == 'all':
            target_distros = default_distros
        else:
            target_distros = list(set(r['target']) & set(default_distros))

        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))

        # todo check if sourcedeb is present with the right version
        deb_name = debianize_package_name(rosdistro, short_package_name)
        if not 'version' in r:
            print('"version" key missing for repository %s; skipping' % r)
            continue
        expected_version = r['version']
        if not expected_version:
            expected_version = ''

        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url,
                                    deb_name,
                                    expected_version + ".*",
                                    d,
                                    arch='na',
                                    source=True):
                missing[short_package_name].append('%s_source' % d)
            for a in arches:
                if not repo.deb_in_repo(repo_url, deb_name,
                                        expected_version + ".*", d, a):
                    missing[short_package_name].append('%s_%s' % (d, a))

        # if not trigger sourcedeb

        # else if binaries don't exist trigger them
        for d in target_distros:
            for a in arches:
                pass  #missing[short_package_name] = ['source']

    #dry stacks
    # dry dependencies
    dist = load_distro(distro_uri(rosdistro))

    distro_arches = []
    for d in default_distros:
        for a in arches:
            distro_arches.append((d, a))

    for s in dist.stacks:
        #print ("Analyzing DRY job [%s]" % s)
        expected_version = dry_get_stack_version(s, dist)

        # sanitize undeclared versions for string substitution
        if not expected_version:
            expected_version = ''
        missing[s] = []
        # for each distro arch check if the deb is present. If not trigger the build.
        for (d, a) in distro_arches:
            if not repo.deb_in_repo(repo_url,
                                    debianize_package_name(rosdistro, s),
                                    expected_version + ".*", d, a):
                missing[s].append('%s_%s' % (d, a))

    return missing
示例#33
0
def single_deb_main():

    from optparse import OptionParser
    parser = OptionParser(
        usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)

    parser.add_option("-d",
                      "--dir",
                      dest="staging_dir",
                      default=None,
                      help="directory to use for staging source debs",
                      metavar="STAGING_DIR")
    parser.add_option("--force",
                      dest="force",
                      default=False,
                      action="store_true")
    parser.add_option("--noupload",
                      dest="noupload",
                      default=False,
                      action="store_true")
    parser.add_option("--fqdn",
                      dest="fqdn",
                      default='repos.ros.org',
                      action="store")
    parser.add_option("--interactive",
                      dest="interactive",
                      default=False,
                      action="store_true")
    parser.add_option('--smtp',
                      dest="smtp",
                      default='pub1.willowgarage.com',
                      metavar="SMTP_SERVER")

    (options, args) = parser.parse_args()

    if len(args) != 4:
        parser.error('invalid args')

    (distro_name, stack_name, os_platform, arch) = args
    distro = failure_message = warning_message = None

    if options.staging_dir is not None:
        staging_dir = os.path.abspath(options.staging_dir)
    else:
        staging_dir = tempfile.mkdtemp()

    try:
        if distro_name not in rosdeb.targets.os_platform:
            raise BuildFailure(
                "[%s] is not a known rosdistro.\nValid rosdistros are: %s" %
                (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
        target_platforms = rosdeb.targets.os_platform[distro_name]
        if os_platform not in target_platforms:
            raise BuildFailure(
                "[%s] is not a known platform or distro %s.\nSupported platforms are: %s"
                % (os_platform, distro_name, ' '.join(target_platforms)))

        if not os.path.exists(staging_dir):
            debug("creating staging dir: %s" % (staging_dir))
            os.makedirs(staging_dir)

        uri = distro_uri(distro_name)
        debug("loading distro file from %s" % (uri))
        distro = load_distro(uri)

        if stack_name == 'metapackages':
            (warning_message,
             failure_message) = gen_metapkgs_setup(options.staging_dir, distro,
                                                   os_platform, arch,
                                                   options.fqdn)
        else:
            build_debs(distro, stack_name, os_platform, arch, staging_dir,
                       options.force, options.noupload, options.interactive,
                       options.fqdn)

    except StackBuildFailure as e:
        warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
    except BuildFailure as e:
        failure_message = "Failure Message:\n" + "=" * 80 + '\n' + str(e)
    except Exception as e:
        failure_message = "Internal failure release system setting up the staging dir. Please notify [email protected]:\n%s\n\n%s" % (
            e, traceback.format_exc(e))
    finally:
        # if we created our own staging dir, we are responsible for cleaning it up
        if options.staging_dir is None:
            shutil.rmtree(staging_dir)

    if failure_message or warning_message:
        debug("FAILURE: %s" % failure_message)
        debug("WARNING: %s" % warning_message)

        #if not options.interactive:
        #failure_message = "%s\n%s\n%s" % (failure_message, warning_message, os.environ.get('BUILD_URL', ''))
        #if options.smtp and stack_name != 'metapackages' and distro is not None:
        #stack_version = distro.stacks[stack_name].version
        #control = download_control(stack_name, stack_version)
        #if 'contact' in control and distro_name != 'diamondback':
        # DISABLE SENDING OF EMAIL from script. This can be done better by jenkins.
        #to_addr = control['contact']
        #subject = 'debian build [%s-%s-%s-%s] failed'%(distro_name, stack_name, os_platform, arch)
        # send_email(options.smtp, EMAIL_FROM_ADDR, to_addr, subject, failure_message)

        if stack_name == 'metapackages':
            debug("Not failing because build is a metapackage")
            return True
        return False
    return True
示例#34
0
def test_distro_uri():
    from rospkg.distro import distro_uri
    assert distro_uri(
        'electric'
    ) == "https://code.ros.org/svn/release/trunk/distros/electric.rosdistro"
示例#35
0
def test_distro_uri():
    from rospkg.distro import distro_uri
    assert distro_uri(
        'groovy'
    ) == "http://ros-dry-releases.googlecode.com/svn/trunk/distros/groovy.rosdistro"
示例#36
0
def test_distro_uri():
    from rospkg.distro import distro_uri

    assert distro_uri("groovy") == "http://ros-dry-releases.googlecode.com/svn/trunk/distros/groovy.rosdistro"
示例#37
0
def test_distro_uri():
    from rospkg.distro import distro_uri
    assert distro_uri('electric') == "https://code.ros.org/svn/release/trunk/distros/electric.rosdistro"
示例#38
0
def get_dry_yaml():
    return yaml.load(urllib2.urlopen(distro_uri('groovy')))
def compute_missing(distros, arches, fqdn, rosdistro, sourcedeb_only=False):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    repo_url = 'http://%s/repos/building' % fqdn

    if rosdistro != 'fuerte':
        from ros_distro import Rosdistro
    else:
        from ros_distro_fuerte import Rosdistro
    rd = Rosdistro(rosdistro)
    # We take the intersection of repo-specific targets with default
    # targets.

    if distros:
        target_distros = distros
    else:
        target_distros = rd.get_target_distros()

    missing = {}
    for short_package_name in rd.get_package_list():
        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))

        # todo check if sourcedeb is present with the right version
        deb_name = debianize_package_name(rosdistro, short_package_name)
        expected_version = rd.get_version(short_package_name,
                                          full_version=True)

        # Don't report packages as missing if their version is None
        if not expected_version:
            print("Skipping package %s with no version" % short_package_name)
            continue

        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url,
                                    deb_name,
                                    str(expected_version) + d,
                                    d,
                                    arch='na',
                                    source=True):
                missing[short_package_name].append('%s_source' % d)
            if not sourcedeb_only:
                for a in arches:
                    if not repo.deb_in_repo(repo_url, deb_name,
                                            str(expected_version) + ".*", d,
                                            a):
                        missing[short_package_name].append('%s_%s' % (d, a))

    if not sourcedeb_only and rosdistro == 'groovy':
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            for a in arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)
            # Don't report packages as missing if their version is None
            if not expected_version:
                print("Skipping package %s with no version" % s)
                continue
            missing[s] = []
            # for each distro arch check if the deb is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.deb_in_repo(repo_url,
                                        debianize_package_name(rosdistro, s),
                                        expected_version + ".*", d, a):
                    missing[s].append('%s_%s' % (d, a))

    return missing
示例#40
0
def compute_missing(job_params, sourcedeb_only=False):
    """ Compute what wet packages are missing from a repo based on the rosdistro files. """

    repo_url = 'http://%s/repos/building' % job_params.fqdn

    rd = job_params.rd
    # We take the intersection of repo-specific targets with default
    # targets.

    target_distros = job_params.distros

    missing = {}
    for short_package_name in rd.get_packages():
        deb_name = debianize_package_name(job_params.rosdistro,
                                          short_package_name)
        expected_version = rd.get_version(short_package_name)

        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url,
                                    deb_name,
                                    str(expected_version) + d,
                                    d,
                                    arch='na',
                                    source=True):
                missing[short_package_name].append('%s_source' % d)
            if not sourcedeb_only:
                target_arches = job_params.arches[d]
                for a in target_arches:
                    if not repo.deb_in_repo(repo_url, deb_name,
                                            str(expected_version) + ".*", d,
                                            a):
                        missing[short_package_name].append('%s_%s' % (d, a))

    if not sourcedeb_only:
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            target_arches = job_params.arches[d]
            for a in target_arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)

            # sanitize undeclared versions for string substitution
            if not expected_version:
                expected_version = ''
            missing[s] = []
            # for each distro arch check if the deb is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.deb_in_repo(repo_url,
                                        debianize_package_name(rosdistro, s),
                                        expected_version + ".*", d, a):
                    missing[s].append('%s_%s' % (d, a))

    return missing