예제 #1
0
def test_load_distro_bad_data():
    from rospkg import ResourceNotFound
    from rospkg.distro import load_distro, InvalidDistro
    try:
        load_distro('bad')
        assert False
    except ResourceNotFound: pass
    for i in range(1, 10):
        filename = 'bad%s.rosdistro'%(i)
        try:
            d = get_test_path()
            p = os.path.join(d, filename)
            load_distro(p)
            assert False, "should have raised: %s"%(filename)
        except InvalidDistro: pass
예제 #2
0
def get_release_info(name, distro, prefix=None):
    """
    Steps to check for a released version of the package
    1) Look in the wet distro file for the package/stack name, if it's there, return the repo
    2) Look in the dry distro file for the package/stack name, if it's there, return the repo
    3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
    what stack a given package belongs to
    4) Look in the distro files again to see if the stack name is there, if it is, return the repo
    """
    url = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' % distro
    try:
        wet_distro = yaml.load(urllib2.urlopen(url))
    except:
        raise IOError("Could not load the %s rosdistro file from github.\n\
Are you sure you've selected a valid distro?\n\
I'm looking for the following file %s" % (distro, url))

    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))

    # Check to see if the name just exists in one of our rosdistro files
    rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
    if rosinstall:
        return rosinstall

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, distro)
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, distro)
        if meta_yaml['package_type'] == 'stack':
            rosinstall = get_release_rosinstall(
                metapackage, wet_distro, dry_distro, prefix)
            if rosinstall:
                return rosinstall

    return None
예제 #3
0
def test_load_distro_diamondback():
    from rospkg.distro import load_distro, Distro
    d = get_test_path()
    p = os.path.join(d, 'diamondback.rosdistro')
    distro = load_distro(p)
    assert isinstance(distro, Distro)

    assert distro.release_name == 'diamondback', distro.release_name
    assert distro.version == 'r8596', distro.version
    assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
    assert set(distro.variants.keys()) == set(diamondback_variants)
    assert set(distro.stacks.keys()) == set(diamondback_stacks), set(
        distro.stacks.keys()) ^ set(diamondback_stacks)

    assert distro.variants['ros-base'].extends == []
    retval = distro.variants['ros-base'].get_stack_names(True)
    assert retval == ['ros', 'ros_comm'], retval
    assert distro.variants['ros-base'].get_stack_names(False) == [
        'ros', 'ros_comm'
    ]
    assert set(distro.variants['ros-full'].get_stack_names(True)) == set(
        ['ros', 'ros_comm', 'rx', 'documentation'])

    assert distro.stacks['common'].version == '1.3.3'
    assert distro.stacks['common'].vcs_config.get_branch(
        'devel',
        True) == ('https://code.ros.org/svn/ros-pkg/stacks/common/trunk', None)
예제 #4
0
def dry_get_versioned_dependency_tree(rosdistro):
    url = distro_uri(rosdistro)
    try:
        d = load_distro(url)
    except urllib2.URLError as ex:
        print("Loading distro from '%s'failed with URLError %s" % (url, ex),
              file=sys.stderr)
        raise
    dependency_tree = {}
    versions = {}
    maintainer_dict = {}
    for s in d.stacks:
        version = d.stacks[s].version
        versions[s] = version
        yaml_info = dry_get_stack_info(s, version)
        if 'depends' in yaml_info:
            dependency_tree[s] = yaml_info['depends']
        else:
            dependency_tree[s] = []
        maintainers = []
        if 'maintainer' in yaml_info:
            maintainers = _extract_emails(yaml_info['maintainer'])
        if not maintainers and 'contact' in yaml_info:
            maintainers = _extract_emails(yaml_info['contact'])
        maintainer_dict[s] = maintainers
    return dependency_tree, versions, maintainer_dict
예제 #5
0
def dry_get_versioned_dependency_tree(rosdistro):
    url = distro_uri(rosdistro)
    try:
        d = load_distro(url)
    except urllib2.URLError as ex:
        print ("Loading distro from '%s'failed with URLError %s" % (url, ex), file=sys.stderr)
        raise
    dependency_tree = {}
    versions = {}
    maintainer_dict = {}
    for s in d.stacks:
        version = d.stacks[s].version
        versions[s] = version
        yaml_info = dry_get_stack_info(s, version)
        if 'depends' in yaml_info:
            dependency_tree[s] = yaml_info['depends']
        else:
            dependency_tree[s] = []
        maintainers = []
        if 'maintainer' in yaml_info:
            maintainers = _extract_emails(yaml_info['maintainer'])
        if not maintainers and 'contact' in yaml_info:
            maintainers = _extract_emails(yaml_info['contact'])
        maintainer_dict[s] = maintainers
    return dependency_tree, versions, maintainer_dict
예제 #6
0
def test_load_distro_diamondback():
    from rospkg.distro import load_distro, Distro

    d = get_test_path()
    p = os.path.join(d, "diamondback.rosdistro")
    distro = load_distro(p)
    assert isinstance(distro, Distro)

    assert distro.release_name == "diamondback", distro.release_name
    assert distro.version == "r8596", distro.version
    assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
    assert set(distro.variants.keys()) == set(diamondback_variants)
    assert set(distro.stacks.keys()) == set(diamondback_stacks), set(distro.stacks.keys()) ^ set(diamondback_stacks)

    assert distro.variants["ros-base"].extends == []
    retval = distro.variants["ros-base"].get_stack_names(True)
    assert retval == ["ros", "ros_comm"], retval
    assert distro.variants["ros-base"].get_stack_names(False) == ["ros", "ros_comm"]
    assert set(distro.variants["ros-full"].get_stack_names(True)) == set(["ros", "ros_comm", "rx", "documentation"])

    assert distro.stacks["common"].version == "1.3.3"
    assert distro.stacks["common"].vcs_config.get_branch("devel", True) == (
        "https://code.ros.org/svn/ros-pkg/stacks/common/trunk",
        None,
    )
예제 #7
0
def get_release_info(name, distro, prefix=None):
    """
    Steps to check for a released version of the package
    1) Look in the wet distro file for the package/stack name, if it's there, return the repo
    2) Look in the dry distro file for the package/stack name, if it's there, return the repo
    3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
    what stack a given package belongs to
    4) Look in the distro files again to see if the stack name is there, if it is, return the repo
    """
    url = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' % distro
    try:
        wet_distro = yaml.load(urllib2.urlopen(url))
    except:
        raise IOError("Could not load the %s rosdistro file from github.\n\
Are you sure you've selected a valid distro?\n\
I'm looking for the following file %s" % (distro, url))

    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))

    #Check to see if the name just exists in one of our rosdistro files
    rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
    if rosinstall:
        return rosinstall

    #If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, distro)
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, distro)
        if meta_yaml['package_type'] == 'stack':
            rosinstall = get_release_rosinstall(metapackage, wet_distro, dry_distro, prefix)
            if rosinstall:
                return rosinstall

    return None
예제 #8
0
def test_distro_to_rosinstall():
    from rospkg.distro import distro_to_rosinstall, load_distro
    d = get_test_path()
    distro = load_distro(os.path.join(d, 'simple.rosdistro'))
    data = distro_to_rosinstall(distro,
                                'devel',
                                variant_name='base',
                                implicit=False,
                                released_only=True,
                                anonymous=True)
    # should only have a single stack
    assert len(data) == 1
    url = 'https://simple.com/svn/trunk/stack1'
    assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
    data = distro_to_rosinstall(distro,
                                'devel',
                                variant_name=None,
                                implicit=False,
                                released_only=True,
                                anonymous=True)
    assert len(data) == 1
    assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
    data = distro_to_rosinstall(distro,
                                'devel',
                                variant_name=None,
                                implicit=False,
                                released_only=False,
                                anonymous=True)
    assert len(data) == 1
    assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
예제 #9
0
def test_load_distro_bad_data():
    from rospkg import ResourceNotFound
    from rospkg.distro import load_distro, InvalidDistro
    try:
        load_distro('bad')
        assert False
    except ResourceNotFound: pass
    for i in range(1, 10):
        if i == 4:
            # currently non-existent stacks in variants are not determinable
            continue
        filename = 'bad%s.rosdistro'%(i)
        try:
            d = get_test_path()
            p = os.path.join(d, filename)
            load_distro(p)
            assert False, "should have raised: %s"%(filename)
        except InvalidDistro: pass
예제 #10
0
def test_load_distro_variants():
    # test with no and empty variants (issue found in fuerte bringup)
    from rospkg.distro import load_distro
    d = get_test_path()
    for name in ['no_variants.rosdistro', 'empty_variants.rosdistro']:
        p = os.path.join(d, name)
        distro = load_distro(p)
        assert distro.release_name == 'simple', distro.release_name
        assert distro.variants.keys() == []
예제 #11
0
def test_load_distro_variants():
    # test with no and empty variants (issue found in fuerte bringup)
    from rospkg.distro import load_distro
    d = get_test_path()
    for name in ['no_variants.rosdistro', 'empty_variants.rosdistro']:
        p = os.path.join(d, name)
        distro = load_distro(p)
        assert distro.release_name == 'simple', distro.release_name
        assert distro.variants.keys() == []
예제 #12
0
def compute_missing(distros, arches, fqdn, rosdistro, sourcerpm_only=False, wet_only=False):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    repo_url = 'http://%s/smd-ros-building' % fqdn

    rd = Rosdistro(rosdistro)
    # We take the intersection of repo-specific targets with default
    # targets.

    if distros:
        target_distros = distros
    else:
        target_distros = rd.get_target_distros()

    missing = {}
    for short_package_name in rd.get_package_list():
        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))

        # todo check if sourcerpm is present with the right version
        rpm_name = redhatify_package_name(rosdistro, short_package_name)
        expected_version = rd.get_version(short_package_name, full_version=True)

        missing[short_package_name] = []
        for d in target_distros:
            d_ver = fedora_release_version(d)
            if not repo.rpm_in_repo(repo_url, rpm_name, str(expected_version) + "\.fc" + d_ver, d, 'SRPMS'):
                missing[short_package_name].append((d, 'SRPMS'))
            if not sourcerpm_only:
                for a in arches:
                    if not repo.rpm_in_repo(repo_url, rpm_name, str(expected_version) + "\.fc" + d_ver, d, a):
                        missing[short_package_name].append((d, a))

    if not sourcerpm_only and not wet_only:
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            for a in arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)

            # sanitize undeclared versions for string substitution
            if not expected_version:
                expected_version = ''
            missing[s] = []
            # for each distro arch check if the RPM is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.rpm_in_repo(repo_url, redhatify_package_name(rosdistro, s), expected_version + "\.fc" + fedora_release_version(d), d, a):
                    missing[s].append((d, a))

    return missing
예제 #13
0
def single_deb_main():

    from optparse import OptionParser
    parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)

    parser.add_option("-d", "--dir",
                      dest="staging_dir", default=None,
                      help="directory to use for staging source debs", metavar="STAGING_DIR")
    parser.add_option("--force",
                      dest="force", default=False, action="store_true")
    parser.add_option("--noupload",
                      dest="noupload", default=False, action="store_true")
    parser.add_option("--fqdn",
                      dest="fqdn", default='50.28.27.175', action="store")
    parser.add_option("--interactive",
                      dest="interactive", default=False, action="store_true")
    parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER")

    (options, args) = parser.parse_args()

    if len(args) != 4:
        parser.error('invalid args')

    (distro_name, stack_name, os_platform, arch) = args
    distro = failure_message = warning_message = None

    if options.staging_dir is not None:
        staging_dir    = options.staging_dir
        staging_dir = os.path.abspath(staging_dir)
    else:
        staging_dir = tempfile.mkdtemp()


    try:
        if distro_name not in rosdeb.targets.os_platform:
            raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
        target_platforms = rosdeb.targets.os_platform[distro_name]
        if os_platform not in target_platforms:
            raise BuildFailure("[%s] is not a known platformfor distro %s.\nSupported platforms are: %s" % (os_platform, distro_name, ' '.join(target_platforms)))

        if not os.path.exists(staging_dir):
            debug("creating staging dir: %s"%(staging_dir))
            os.makedirs(staging_dir)

        uri = distro_uri(distro_name)
        debug("loading distro file from %s"%(uri))
        distro = load_distro(uri)

        if stack_name == 'metapackages':
            (warning_message, failure_message) = gen_metapkgs_setup(options.staging_dir, distro, os_platform, arch, options.fqdn)
        else:
            build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive, options.fqdn)

    except StackBuildFailure, e:
        warning_message = "Warning Message:\n"+"="*80+'\n'+str(e)
예제 #14
0
def test_load_distro_bad_data():
    from rospkg import ResourceNotFound
    from rospkg.distro import load_distro, InvalidDistro
    try:
        load_distro('bad')
        assert False
    except ResourceNotFound:
        pass
    for i in range(1, 10):
        if i == 4:
            # currently non-existent stacks in variants are not determinable
            continue
        filename = 'bad%s.rosdistro' % (i)
        try:
            d = get_test_path()
            p = os.path.join(d, filename)
            load_distro(p)
            assert False, "should have raised: %s" % (filename)
        except InvalidDistro:
            pass
def build_debs_main():

    from optparse import OptionParser
    parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)

    parser.add_option("-d", "--dir",
                      dest="staging_dir", default=None,
                      help="directory to use for staging source debs", metavar="STAGING_DIR")
    parser.add_option("--force",
                      dest="force", default=False, action="store_true")
    parser.add_option("--noupload",
                      dest="noupload", default=False, action="store_true")
    parser.add_option("--noramdisk",
                      dest="ramdisk", default=True, action="store_false")
    parser.add_option("--interactive",
                      dest="interactive", default=False, action="store_true")
    parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER")

    (options, args) = parser.parse_args()

    if len(args) != 4:
        parser.error('invalid args')
        
    (distro_name, stack_name, os_platform, arch) = args
    distro = failure_message = warning_message = None

    if options.staging_dir is not None:
        staging_dir = options.staging_dir
        staging_dir = os.path.abspath(staging_dir)
    else:
        staging_dir = tempfile.mkdtemp()

    try:
        if distro_name not in rosdeb.targets.os_platform:
            raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
        target_platforms = rosdeb.targets.os_platform[distro_name]
        if os_platform not in target_platforms:
            raise BuildFailure("[%s] is not a known platform.\nSupported platforms are: %s" % (os_platform, ' '.join(target_platforms)))

        if not os.path.exists(staging_dir):
            print "creating staging dir: %s" % (staging_dir)
            os.makedirs(staging_dir)

        distro = load_distro(distro_uri(distro_name))

        if options.ramdisk:
            with TempRamFS(staging_dir, "20G"):
                build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive)
        else:
            build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive)

    except StackBuildFailure, e:
        warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
예제 #16
0
def dry_get_versioned_dependency_tree(rosdistro):
    d = load_distro(distro_uri(rosdistro))
    dependency_tree = {}
    versions = {}
    for s in d.stacks:
        version = d.stacks[s].version
        versions[s] = version
        yaml_info = dry_get_stack_info(s, version)
        if 'depends' in yaml_info:
            dependency_tree[s] = yaml_info['depends']
        else:
            dependency_tree[s] = []
    return dependency_tree, versions
예제 #17
0
def dry_get_versioned_dependency_tree(rosdistro):
    d = load_distro(distro_uri(rosdistro))    
    dependency_tree = {}
    versions = {}
    for s in d.stacks:
        version = d.stacks[s].version
        versions[s] = version
        yaml_info = dry_get_stack_info(s, version)
        if 'depends' in yaml_info:
            dependency_tree[s] = yaml_info['depends']
        else:
            dependency_tree[s] = []
    return dependency_tree, versions
예제 #18
0
def compute_missing(job_params, sourcedeb_only=False):
    """ Compute what wet packages are missing from a repo based on the rosdistro files. """

    repo_url = 'http://%s/repos/building' % job_params.fqdn

    rd = job_params.rd
    # We take the intersection of repo-specific targets with default
    # targets.

    target_distros = job_params.distros

    missing = {}
    for short_package_name in rd.get_packages():
        deb_name = debianize_package_name(job_params.rosdistro, short_package_name)
        expected_version = rd.get_version(short_package_name)

        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + d, d, arch='na', source=True):
                missing[short_package_name].append('%s_source' % d)
            if not sourcedeb_only:
                target_arches = job_params.arches[d]
                for a in target_arches:
                    if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + ".*", d, a):
                        missing[short_package_name].append('%s_%s' % (d, a))

    if not sourcedeb_only:
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            target_arches = job_params.arches[d]
            for a in target_arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)

            # sanitize undeclared versions for string substitution
            if not expected_version:
                expected_version = ''
            missing[s] = []
            # for each distro arch check if the deb is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a):
                    missing[s].append('%s_%s' % (d, a))

    return missing
예제 #19
0
def test_distro_to_rosinstall():
    from rospkg.distro import distro_to_rosinstall, load_distro
    d = get_test_path()
    distro = load_distro(os.path.join(d, 'simple.rosdistro'))
    data = distro_to_rosinstall(distro, 'devel', variant_name='base', implicit=False, released_only=True, anonymous=True)
    # should only have a single stack
    assert len(data) == 1
    url = 'https://simple.com/svn/trunk/stack1'
    assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
    data = distro_to_rosinstall(distro, 'devel', variant_name=None, implicit=False, released_only=True, anonymous=True)
    assert len(data) == 1
    assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
    data = distro_to_rosinstall(distro, 'devel', variant_name=None, implicit=False, released_only=False, anonymous=True)
    assert len(data) == 1
    assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
예제 #20
0
def test_load_distro_simple():
    from rospkg.distro import load_distro, Distro
    d = get_test_path()
    p = os.path.join(d, 'simple.rosdistro')
    distro = load_distro(p)
    assert isinstance(distro, Distro)
    
    assert distro.release_name == 'simple', distro.release_name
    assert distro.version == '1', distro.version
    assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
    assert distro.variants.keys() == ['base']
    assert distro.stacks.keys() == ['stack1']

    stack1 = distro.stacks['stack1']
    assert stack1.vcs_config.get_branch('devel', False) == ('https://simple.com/svn/trunk/stack1', None)
    assert stack1.vcs_config.get_branch('distro', False) == ('https://simple.com/svn/tags/distros/simple/stacks/stack1', None)
    assert stack1.vcs_config.get_branch('release', False) == ('https://simple.com/svn/tags/stacks/stack1/stack1-0.3.0', None)
예제 #21
0
def test_load_distro_simple():
    from rospkg.distro import load_distro, Distro
    d = get_test_path()
    p = os.path.join(d, 'simple.rosdistro')
    distro = load_distro(p)
    assert isinstance(distro, Distro)

    assert distro.release_name == 'simple', distro.release_name
    assert distro.version == '1', distro.version
    assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
    assert set(distro.variants.keys()) == set(['base'])
    assert set(distro.stacks.keys()) == set(['stack1'])

    stack1 = distro.stacks['stack1']
    assert stack1.vcs_config.get_branch('devel', False) == ('https://simple.com/svn/trunk/stack1', None)
    assert stack1.vcs_config.get_branch('distro', False) == ('https://simple.com/svn/tags/distros/simple/stacks/stack1', None)
    assert stack1.vcs_config.get_branch('release', False) == ('https://simple.com/svn/tags/stacks/stack1/stack1-0.3.0', None)
예제 #22
0
def _get_electric_rosinstall(name, prefix=None):
    """
    Please delete me when you don't care at all about electric anymore
    """
    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri('electric'))

    if _is_dry(dry_distro, name):
        return get_dry_rosinstall(dry_distro, name, prefix=prefix)

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, 'electric')
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, 'electric')
        if meta_yaml['package_type'] == 'stack':
            if _is_dry(dry_distro, metapackage):
                return get_dry_rosinstall(dry_distro, metapackage, prefix=prefix)

    return None
예제 #23
0
def test_distro_to_rosinstall():
    from rospkg.distro import distro_to_rosinstall, load_distro

    d = get_test_path()
    distro = load_distro(os.path.join(d, "simple.rosdistro"))
    data = distro_to_rosinstall(
        distro, "devel", variant_name="base", implicit=False, released_only=True, anonymous=True
    )
    # should only have a single stack
    assert len(data) == 1
    url = "https://simple.com/svn/trunk/stack1"
    assert data[0] == dict(svn={"uri": url, "local-name": "stack1"})
    data = distro_to_rosinstall(distro, "devel", variant_name=None, implicit=False, released_only=True, anonymous=True)
    assert len(data) == 1
    assert data[0] == dict(svn={"uri": url, "local-name": "stack1"})
    data = distro_to_rosinstall(distro, "devel", variant_name=None, implicit=False, released_only=False, anonymous=True)
    assert len(data) == 1
    assert data[0] == dict(svn={"uri": url, "local-name": "stack1"})
예제 #24
0
def translate(distro, translate_dir):
    d = load_distro(distro_uri(distro))
    repo_list = d.get_stacks(True)
    for name, item in repo_list.iteritems():
        if item.vcs_config.type == 'svn':
            rosinstall = [{item.vcs_config.type: \
                           {'local-name': item.name,
                            'uri': item.vcs_config.anon_dev}}]
        else:
            rosinstall = [{item.vcs_config.type: \
                           {'local-name': item.name,
                            'uri': item.vcs_config.anon_repo_uri,
                            'version': item.vcs_config.dev_branch}}]

        path = os.path.join(translate_dir, "%s.rosinstall" % item.name)
        with open(path, 'w+') as f:
            print "writing to %s" % path
            yaml.safe_dump(rosinstall, f, default_flow_style=False)
예제 #25
0
def get_missing_dry_packages(rosdistro, default_distros, arches, repo_url):
    missing = {}
    dist = load_distro(distro_uri(rosdistro))

    distro_arches = [(d, a) for d in default_distros for a in arches]

    for s in dist.stacks:
        #print ("Analyzing DRY job [%s]" % s)
        expected_version = dry_get_stack_version(s, dist)
        
        # sanitize undeclared versions for string substitution
        if not expected_version:
            expected_version = ''
        missing[s] = []
        # for each distro arch check if the deb is present. If not trigger the build. 
        for (d, a) in distro_arches:
            if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version+".*", d, a):
                missing[s].append( '%s_%s' % (d, a) )
    return missing
예제 #26
0
def _get_electric_rosinstall(name, prefix=None):
    """
    Please delete me when you don't care at all about electric anymore
    """
    dry_distro = rospkg_distro.load_distro(
        rospkg_distro.distro_uri('electric'))

    if _is_dry(dry_distro, name):
        return get_dry_rosinstall(dry_distro, name, prefix=prefix)

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, 'electric')
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, 'electric')
        if meta_yaml['package_type'] == 'stack':
            if _is_dry(dry_distro, metapackage):
                return get_dry_rosinstall(dry_distro,
                                          metapackage,
                                          prefix=prefix)

    return None
예제 #27
0
def test_load_distro_diamondback():
    from rospkg.distro import load_distro, Distro
    d = get_test_path()
    p = os.path.join(d, 'diamondback.rosdistro')
    distro = load_distro(p)
    assert isinstance(distro, Distro)

    assert distro.release_name == 'diamondback', distro.release_name
    assert distro.version == 'r8596', distro.version
    assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
    assert set(distro.variants.keys()) == set(diamondback_variants)
    assert set(distro.stacks.keys()) == set(diamondback_stacks), set(distro.stacks.keys()) ^ set(diamondback_stacks)

    assert distro.variants['ros-base'].extends == []
    retval = distro.variants['ros-base'].get_stack_names(True)
    assert retval == ['ros', 'ros_comm'], retval
    assert distro.variants['ros-base'].get_stack_names(False) == ['ros', 'ros_comm']    
    assert set(distro.variants['ros-full'].get_stack_names(True)) == set(['ros', 'ros_comm', 'rx', 'documentation'])
    
    assert distro.stacks['common'].version == '1.3.3'
    assert distro.stacks['common'].vcs_config.get_branch('devel', True) == ('https://code.ros.org/svn/ros-pkg/stacks/common/trunk', None)
예제 #28
0
def _get_fuerte_rosinstall(name, prefix=None):
    """
    Please delete me when fuerte is not supported anymore
    See REP137 about rosdistro files
    """
    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri("fuerte"))
    wet_distro = _get_fuerte_release()
    # Check to see if the name just exists in one of our rosdistro files
    rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
    if rosinstall:
        return rosinstall

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, "fuerte")
    for metapackage in doc_yaml.get("metapackages", []):
        meta_yaml = get_manifest_yaml(metapackage, "fuerte")
        if meta_yaml["package_type"] == "stack":
            rosinstall = get_release_rosinstall(metapackage, wet_distro, dry_distro, prefix)
            if rosinstall:
                return rosinstall

    return None
예제 #29
0
def get_release_info(name, distro, prefix=None):
    """
    Steps to check for a released version of the package
    1) Look in the wet distro file for the package/stack name, if it's there, return the repo
    2) Look in the dry distro file for the package/stack name, if it's there, return the repo
    3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
    what stack a given package belongs to
    4) Look in the distro files again to see if the stack name is there, if it is, return the repo
    """

    # fuerte is different.
    if distro == 'fuerte':
        return _get_fuerte_rosinstall(name, prefix=prefix)

    # electric is ancient.
    if distro == 'electric':
        return _get_electric_rosinstall(name, prefix=prefix)

    wet_distro = _get_rosdistro_release(distro)
    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))

    # Check to see if the name just exists in one of our rosdistro files
    if _is_wet(wet_distro, name):
        return get_wet_rosinstall(wet_distro, name, prefix=prefix)
    if _is_dry(dry_distro, name):
        return get_dry_rosinstall(dry_distro, name, prefix=prefix)

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, distro)
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, distro)
        if meta_yaml['package_type'] == 'stack':
            if _is_dry(dry_distro, metapackage):
                return get_dry_rosinstall(dry_distro,
                                          metapackage,
                                          prefix=prefix)

    return None
예제 #30
0
def _get_fuerte_rosinstall(name, prefix=None):
    """
    Please delete me when fuerte is not supported anymore
    See REP137 about rosdistro files
    """
    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri('fuerte'))
    wet_distro = _get_fuerte_release()
    # Check to see if the name just exists in one of our rosdistro files
    rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
    if rosinstall:
        return rosinstall

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, 'fuerte')
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, 'fuerte')
        if meta_yaml['package_type'] == 'stack':
            rosinstall = get_release_rosinstall(metapackage, wet_distro,
                                                dry_distro, prefix)
            if rosinstall:
                return rosinstall

    return None
예제 #31
0
def get_release_info(name, distro, prefix=None):
    """
    Steps to check for a released version of the package
    1) Look in the wet distro file for the package/stack name, if it's there, return the repo
    2) Look in the dry distro file for the package/stack name, if it's there, return the repo
    3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
    what stack a given package belongs to
    4) Look in the distro files again to see if the stack name is there, if it is, return the repo
    """

    # fuerte is different.
    if distro == 'fuerte':
        return _get_fuerte_rosinstall(name, prefix=prefix)

    # electric is ancient.
    if distro == 'electric':
        return _get_electric_rosinstall(name, prefix=prefix)

    wet_distro = _get_rosdistro_release(distro)
    dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))

    # Check to see if the name just exists in one of our rosdistro files
    if _is_wet(wet_distro, name):
        return get_wet_rosinstall(wet_distro, name, prefix=prefix)
    if _is_dry(dry_distro, name):
        return get_dry_rosinstall(dry_distro, name, prefix=prefix)

    # If we didn't find the name, we need to try to find a stack for it
    doc_yaml = get_manifest_yaml(name, distro)
    for metapackage in doc_yaml.get('metapackages', []):
        meta_yaml = get_manifest_yaml(metapackage, distro)
        if meta_yaml['package_type'] == 'stack':
            if _is_dry(dry_distro, metapackage):
                return get_dry_rosinstall(dry_distro, metapackage, prefix=prefix)

    return None
예제 #32
0
def test_load_distro_simple():
    from rospkg.distro import load_distro, Distro

    d = get_test_path()
    p = os.path.join(d, "simple.rosdistro")
    distro = load_distro(p)
    assert isinstance(distro, Distro)

    assert distro.release_name == "simple", distro.release_name
    assert distro.version == "1", distro.version
    assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
    assert distro.variants.keys() == ["base"]
    assert distro.stacks.keys() == ["stack1"]

    stack1 = distro.stacks["stack1"]
    assert stack1.vcs_config.get_branch("devel", False) == ("https://simple.com/svn/trunk/stack1", None)
    assert stack1.vcs_config.get_branch("distro", False) == (
        "https://simple.com/svn/tags/distros/simple/stacks/stack1",
        None,
    )
    assert stack1.vcs_config.get_branch("release", False) == (
        "https://simple.com/svn/tags/stacks/stack1/stack1-0.3.0",
        None,
    )
예제 #33
0
def compute_missing(distros, fqdn, rosdistro):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    URL_PROTOTYPE = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml'


    repo_url = 'http://%s/repos/building' % fqdn

    print('Fetching "%s"' % (URL_PROTOTYPE % rosdistro))
    repo_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % rosdistro))


    # What ROS distro are we configuring?
    if 'release-name' not in repo_map:
        print('No "release-name" key in yaml file')
        sys.exit(1)
    if repo_map['release-name'] != rosdistro:
        print('release-name mismatch (%s != %s)' % (repo_map['release-name'], rosdistro))
        sys.exit(1)    
    if 'repositories' not in repo_map:
        print('No "repositories" key in yaml file')
    if 'type' not in repo_map or repo_map['type'] != 'gbp':
        print('Wrong type value in yaml file')
        sys.exit(1)

    # Figure out default distros.  Command-line arg takes precedence; if
    # it's not specified, then read targets.yaml.
    if distros:
        default_distros = distros
    else:
        print('Fetching "%s"' % (URL_PROTOTYPE % 'targets'))
        targets_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % 'targets'))
        my_targets = [x for x in targets_map if rosdistro in x]
        if len(my_targets) != 1:
            print('Must have exactly one entry for rosdistro "%s" in targets.yaml' % rosdistro)
            sys.exit(1)
        default_distros = my_targets[0][rosdistro]

    arches = ['amd64', 'i386']

    # We take the intersection of repo-specific targets with default
    # targets.
    missing = {}
    for short_package_name, r in repo_map['repositories'].items():
        if 'url' not in r:
            print('"url" key missing for repository "%s"; skipping' % r)
            continue
        url = r['url']
        if 'target' not in r or r['target'] == 'all':
            target_distros = default_distros
        else:
            target_distros = list(set(r['target']) & set(default_distros))

        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))
        
        # todo check if sourcedeb is present with the right version
        deb_name = debianize_package_name(rosdistro, short_package_name)
        if not 'version' in r:
            print('"version" key missing for repository %s; skipping' % r)
            continue
        expected_version = r['version']
        if not expected_version:
            expected_version = ''
        
        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url, deb_name, expected_version+".*", d, arch='na', source=True):
                missing[short_package_name].append('%s_source' % d)
            for a in arches:
                if not repo.deb_in_repo(repo_url, deb_name, expected_version+".*", d, a):
                    missing[short_package_name].append('%s_%s' % (d, a))

                                               
        # if not trigger sourcedeb

        # else if binaries don't exist trigger them
        for d in target_distros:
            for a in arches:
                pass#missing[short_package_name] = ['source']
        


        

    #dry stacks
    # dry dependencies
    dist = load_distro(distro_uri(rosdistro))

    distro_arches = []
    for d in default_distros:
        for a in arches:
            distro_arches.append( (d, a) )

    for s in dist.stacks:
        #print ("Analyzing DRY job [%s]" % s)
        expected_version = dry_get_stack_version(s, dist)
        
        # sanitize undeclared versions for string substitution
        if not expected_version:
            expected_version = ''
        missing[s] = []
        # for each distro arch check if the deb is present. If not trigger the build. 
        for (d, a) in distro_arches:
            if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version+".*", d, a):
                missing[s].append( '%s_%s' % (d, a) )


    return missing
예제 #34
0
def compute_missing(distros, arches, fqdn, rosdistro, sourcedeb_only=False):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    repo_url = 'http://%s/repos/building' % fqdn

    if rosdistro != 'fuerte':
        from ros_distro import Rosdistro
    else:
        from ros_distro_fuerte import Rosdistro
    rd = Rosdistro(rosdistro)
    # We take the intersection of repo-specific targets with default
    # targets.

    if distros:
        target_distros = distros
    else:
        target_distros = rd.get_target_distros()

    missing = {}
    for short_package_name in rd.get_package_list():
        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))

        # todo check if sourcedeb is present with the right version
        deb_name = debianize_package_name(rosdistro, short_package_name)
        expected_version = rd.get_version(short_package_name, full_version=True)

        # Don't report packages as missing if their version is None
        if not expected_version:
            print("Skipping package %s with no version" % short_package_name)
            continue

        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + d, d, arch='na', source=True):
                missing[short_package_name].append('%s_source' % d)
            if not sourcedeb_only:
                for a in arches:
                    if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + ".*", d, a):
                        missing[short_package_name].append('%s_%s' % (d, a))

    if not sourcedeb_only:
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            for a in arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)
            # Don't report packages as missing if their version is None
            if not expected_version:
                print("Skipping package %s with no version" % s)
                continue
            missing[s] = []
            # for each distro arch check if the deb is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a):
                    missing[s].append('%s_%s' % (d, a))

    return missing
예제 #35
0
def compute_missing(job_params, sourcedeb_only=False):
    """ Compute what wet packages are missing from a repo based on the rosdistro files. """

    repo_url = 'http://%s/repos/building' % job_params.fqdn

    rd = job_params.rd
    # We take the intersection of repo-specific targets with default
    # targets.

    target_distros = job_params.distros

    missing = {}
    for short_package_name in rd.get_packages():
        deb_name = debianize_package_name(job_params.rosdistro,
                                          short_package_name)
        expected_version = rd.get_version(short_package_name)

        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url,
                                    deb_name,
                                    str(expected_version) + d,
                                    d,
                                    arch='na',
                                    source=True):
                missing[short_package_name].append('%s_source' % d)
            if not sourcedeb_only:
                target_arches = job_params.arches[d]
                for a in target_arches:
                    if not repo.deb_in_repo(repo_url, deb_name,
                                            str(expected_version) + ".*", d,
                                            a):
                        missing[short_package_name].append('%s_%s' % (d, a))

    if not sourcedeb_only:
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            target_arches = job_params.arches[d]
            for a in target_arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)

            # sanitize undeclared versions for string substitution
            if not expected_version:
                expected_version = ''
            missing[s] = []
            # for each distro arch check if the deb is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.deb_in_repo(repo_url,
                                        debianize_package_name(rosdistro, s),
                                        expected_version + ".*", d, a):
                    missing[s].append('%s_%s' % (d, a))

    return missing
예제 #36
0
def single_deb_main():

    from optparse import OptionParser
    parser = OptionParser(
        usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)

    parser.add_option("-d",
                      "--dir",
                      dest="staging_dir",
                      default=None,
                      help="directory to use for staging source debs",
                      metavar="STAGING_DIR")
    parser.add_option("--force",
                      dest="force",
                      default=False,
                      action="store_true")
    parser.add_option("--noupload",
                      dest="noupload",
                      default=False,
                      action="store_true")
    parser.add_option("--fqdn",
                      dest="fqdn",
                      default='repos.ros.org',
                      action="store")
    parser.add_option("--interactive",
                      dest="interactive",
                      default=False,
                      action="store_true")
    parser.add_option('--smtp',
                      dest="smtp",
                      default='pub1.willowgarage.com',
                      metavar="SMTP_SERVER")

    (options, args) = parser.parse_args()

    if len(args) != 4:
        parser.error('invalid args')

    (distro_name, stack_name, os_platform, arch) = args
    distro = failure_message = warning_message = None

    if options.staging_dir is not None:
        staging_dir = os.path.abspath(options.staging_dir)
    else:
        staging_dir = tempfile.mkdtemp()

    try:
        if distro_name not in rosdeb.targets.os_platform:
            raise BuildFailure(
                "[%s] is not a known rosdistro.\nValid rosdistros are: %s" %
                (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
        target_platforms = rosdeb.targets.os_platform[distro_name]
        if os_platform not in target_platforms:
            raise BuildFailure(
                "[%s] is not a known platform or distro %s.\nSupported platforms are: %s"
                % (os_platform, distro_name, ' '.join(target_platforms)))

        if not os.path.exists(staging_dir):
            debug("creating staging dir: %s" % (staging_dir))
            os.makedirs(staging_dir)

        uri = distro_uri(distro_name)
        debug("loading distro file from %s" % (uri))
        distro = load_distro(uri)

        if stack_name == 'metapackages':
            (warning_message,
             failure_message) = gen_metapkgs_setup(options.staging_dir, distro,
                                                   os_platform, arch,
                                                   options.fqdn)
        else:
            build_debs(distro, stack_name, os_platform, arch, staging_dir,
                       options.force, options.noupload, options.interactive,
                       options.fqdn)

    except StackBuildFailure as e:
        warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
    except BuildFailure as e:
        failure_message = "Failure Message:\n" + "=" * 80 + '\n' + str(e)
    except Exception as e:
        failure_message = "Internal failure release system setting up the staging dir. Please notify [email protected]:\n%s\n\n%s" % (
            e, traceback.format_exc(e))
    finally:
        # if we created our own staging dir, we are responsible for cleaning it up
        if options.staging_dir is None:
            shutil.rmtree(staging_dir)

    if failure_message or warning_message:
        debug("FAILURE: %s" % failure_message)
        debug("WARNING: %s" % warning_message)

        #if not options.interactive:
        #failure_message = "%s\n%s\n%s" % (failure_message, warning_message, os.environ.get('BUILD_URL', ''))
        #if options.smtp and stack_name != 'metapackages' and distro is not None:
        #stack_version = distro.stacks[stack_name].version
        #control = download_control(stack_name, stack_version)
        #if 'contact' in control and distro_name != 'diamondback':
        # DISABLE SENDING OF EMAIL from script. This can be done better by jenkins.
        #to_addr = control['contact']
        #subject = 'debian build [%s-%s-%s-%s] failed'%(distro_name, stack_name, os_platform, arch)
        # send_email(options.smtp, EMAIL_FROM_ADDR, to_addr, subject, failure_message)

        if stack_name == 'metapackages':
            debug("Not failing because build is a metapackage")
            return True
        return False
    return True
예제 #37
0
def compute_missing(distros, arches, fqdn, rosdistro, sourcedeb_only=False):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    repo_url = 'http://%s/repos/building' % fqdn

    if rosdistro != 'fuerte':
        from ros_distro import Rosdistro
    else:
        from ros_distro_fuerte import Rosdistro
    rd = Rosdistro(rosdistro)
    # We take the intersection of repo-specific targets with default
    # targets.

    if distros:
        target_distros = distros
    else:
        target_distros = rd.get_target_distros()

    missing = {}
    for short_package_name in rd.get_package_list():
        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))

        # todo check if sourcedeb is present with the right version
        deb_name = debianize_package_name(rosdistro, short_package_name)
        expected_version = rd.get_version(short_package_name,
                                          full_version=True)

        # Don't report packages as missing if their version is None
        if not expected_version:
            print("Skipping package %s with no version" % short_package_name)
            continue

        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url,
                                    deb_name,
                                    str(expected_version) + d,
                                    d,
                                    arch='na',
                                    source=True):
                missing[short_package_name].append('%s_source' % d)
            if not sourcedeb_only:
                for a in arches:
                    if not repo.deb_in_repo(repo_url, deb_name,
                                            str(expected_version) + ".*", d,
                                            a):
                        missing[short_package_name].append('%s_%s' % (d, a))

    if not sourcedeb_only and rosdistro == 'groovy':
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            for a in arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)
            # Don't report packages as missing if their version is None
            if not expected_version:
                print("Skipping package %s with no version" % s)
                continue
            missing[s] = []
            # for each distro arch check if the deb is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.deb_in_repo(repo_url,
                                        debianize_package_name(rosdistro, s),
                                        expected_version + ".*", d, a):
                    missing[s].append('%s_%s' % (d, a))

    return missing
예제 #38
0
def compute_missing(distros, arches, fqdn, rosdistro, sourcepkg_only=False, platform='ubuntu'):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    repo_url = 'http://%s/repos/building/%s' % (fqdn, platform)

    # TODO Building Repo Workaround
    if platform == 'ubuntu':
        repo_url = 'http://%s/repos/building' % fqdn
    elif platform == 'fedora':
        repo_url = 'http://%s/smd-ros-building/%s' % (fqdn, platform)
    # End Workaround

    if rosdistro != 'fuerte':
        from ros_distro import Rosdistro
    else:
        from ros_distro_fuerte import Rosdistro
    rd = Rosdistro(rosdistro)
    # We take the intersection of repo-specific targets with default
    # targets.

    if distros:
        target_distros = distros
    else:
        target_distros = rd.get_target_distros()[platform]

    missing = {}
    for short_package_name in rd.get_package_list():
        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))

        # todo check if sourcepkg is present with the right version
        pkg_name = debianize_package_name(rosdistro, short_package_name)
        expected_version = rd.get_version(short_package_name, full_version=True)

        # Don't report packages as missing if their version is None
        if not expected_version:
            print("Skipping package %s with no version" % short_package_name)
            continue

        missing[short_package_name] = []
        for d in target_distros:
            if platform == 'fedora':
                version_extra = ".fc" + str(get_fedora_ver(d))
            else:
                version_extra = ".*"
            if not repo.pkg_in_repo(repo_url, pkg_name, get_full_version(expected_version, d, platform), d, arch='na', source=True, platform=platform):
                missing[short_package_name].append('%s_source' % d)
            if not sourcepkg_only:
                for a in arches:
                    if not repo.pkg_in_repo(repo_url, pkg_name, str(expected_version) + version_extra, d, a, platform=platform):
                        missing[short_package_name].append('%s_%s' % (d, a))

    if not sourcepkg_only and rosdistro == 'groovy' and not platform == 'fedora':
        #dry stacks
        # dry dependencies
        dist = load_distro(distro_uri(rosdistro))

        distro_arches = []
        for d in target_distros:
            for a in arches:
                distro_arches.append((d, a))

        for s in dist.stacks:
            #print ("Analyzing DRY job [%s]" % s)
            expected_version = dry_get_stack_version(s, dist)
            # Don't report packages as missing if their version is None
            if not expected_version:
                print("Skipping package %s with no version" % s)
                continue
            missing[s] = []
            # for each distro arch check if the pkg is present. If not trigger the build.
            for (d, a) in distro_arches:
                if not repo.pkg_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a, platform=platform):
                    missing[s].append('%s_%s' % (d, a))

    return missing
예제 #39
0
def get_distro(distro_name):
    return rosdistro.load_distro(rosdistro.distro_uri(distro_name))
예제 #40
0
def single_deb_main():

    from optparse import OptionParser
    parser = OptionParser(
        usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)

    parser.add_option("-d",
                      "--dir",
                      dest="staging_dir",
                      default=None,
                      help="directory to use for staging source debs",
                      metavar="STAGING_DIR")
    parser.add_option("--force",
                      dest="force",
                      default=False,
                      action="store_true")
    parser.add_option("--noupload",
                      dest="noupload",
                      default=False,
                      action="store_true")
    parser.add_option("--fqdn",
                      dest="fqdn",
                      default='50.28.27.175',
                      action="store")
    parser.add_option("--interactive",
                      dest="interactive",
                      default=False,
                      action="store_true")
    parser.add_option('--smtp',
                      dest="smtp",
                      default='pub1.willowgarage.com',
                      metavar="SMTP_SERVER")

    (options, args) = parser.parse_args()

    if len(args) != 4:
        parser.error('invalid args')

    (distro_name, stack_name, os_platform, arch) = args
    distro = failure_message = warning_message = None

    if options.staging_dir is not None:
        staging_dir = options.staging_dir
        staging_dir = os.path.abspath(staging_dir)
    else:
        staging_dir = tempfile.mkdtemp()

    try:
        if distro_name not in rosdeb.targets.os_platform:
            raise BuildFailure(
                "[%s] is not a known rosdistro.\nValid rosdistros are: %s" %
                (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
        target_platforms = rosdeb.targets.os_platform[distro_name]
        if os_platform not in target_platforms:
            raise BuildFailure(
                "[%s] is not a known platformfor distro %s.\nSupported platforms are: %s"
                % (os_platform, distro_name, ' '.join(target_platforms)))

        if not os.path.exists(staging_dir):
            debug("creating staging dir: %s" % (staging_dir))
            os.makedirs(staging_dir)

        uri = distro_uri(distro_name)
        debug("loading distro file from %s" % (uri))
        distro = load_distro(uri)

        if stack_name == 'metapackages':
            (warning_message,
             failure_message) = gen_metapkgs_setup(options.staging_dir, distro,
                                                   os_platform, arch,
                                                   options.fqdn)
        else:
            build_debs(distro, stack_name, os_platform, arch, staging_dir,
                       options.force, options.noupload, options.interactive,
                       options.fqdn)

    except StackBuildFailure, e:
        warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
예제 #41
0
def single_deb_main():

    from optparse import OptionParser
    parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)

    parser.add_option("-d", "--dir",
                      dest="staging_dir", default=None,
                      help="directory to use for staging source debs", metavar="STAGING_DIR")
    parser.add_option("--force",
                      dest="force", default=False, action="store_true")
    parser.add_option("--noupload",
                      dest="noupload", default=False, action="store_true")
    parser.add_option("--fqdn",
                      dest="fqdn", default='repos.ros.org', action="store")
    parser.add_option("--interactive",
                      dest="interactive", default=False, action="store_true")
    parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER")

    (options, args) = parser.parse_args()

    if len(args) != 4:
        parser.error('invalid args')

    (distro_name, stack_name, os_platform, arch) = args
    distro = failure_message = warning_message = None

    if options.staging_dir is not None:
        staging_dir = os.path.abspath(options.staging_dir)
    else:
        staging_dir = tempfile.mkdtemp()

    try:
        if distro_name not in rosdeb.targets.os_platform:
            raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
        target_platforms = rosdeb.targets.os_platform[distro_name]
        if os_platform not in target_platforms:
            raise BuildFailure("[%s] is not a known platform or distro %s.\nSupported platforms are: %s" % (os_platform, distro_name, ' '.join(target_platforms)))

        if not os.path.exists(staging_dir):
            debug("creating staging dir: %s" % (staging_dir))
            os.makedirs(staging_dir)

        uri = distro_uri(distro_name)
        debug("loading distro file from %s" % (uri))
        distro = load_distro(uri)

        if stack_name == 'metapackages':
            (warning_message, failure_message) = gen_metapkgs_setup(options.staging_dir, distro, os_platform, arch, options.fqdn)
        else:
            build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive, options.fqdn)

    except StackBuildFailure as e:
        warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
    except BuildFailure as e:
        failure_message = "Failure Message:\n" + "=" * 80 + '\n' + str(e)
    except Exception as e:
        failure_message = "Internal failure release system setting up the staging dir. Please notify [email protected]:\n%s\n\n%s" % (e, traceback.format_exc(e))
    finally:
        # if we created our own staging dir, we are responsible for cleaning it up
        if options.staging_dir is None:
            shutil.rmtree(staging_dir)

    if failure_message or warning_message:
        debug("FAILURE: %s" % failure_message)
        debug("WARNING: %s" % warning_message)

        #if not options.interactive:
            #failure_message = "%s\n%s\n%s" % (failure_message, warning_message, os.environ.get('BUILD_URL', ''))
            #if options.smtp and stack_name != 'metapackages' and distro is not None:
                #stack_version = distro.stacks[stack_name].version
                #control = download_control(stack_name, stack_version)
                #if 'contact' in control and distro_name != 'diamondback':
                    # DISABLE SENDING OF EMAIL from script. This can be done better by jenkins.
                    #to_addr = control['contact']
                    #subject = 'debian build [%s-%s-%s-%s] failed'%(distro_name, stack_name, os_platform, arch)
                    # send_email(options.smtp, EMAIL_FROM_ADDR, to_addr, subject, failure_message)

        if stack_name == 'metapackages':
            debug("Not failing because build is a metapackage")
            return True
        return False
    return True
def get_distro(distro_name):
    return rosdistro.load_distro(rosdistro.distro_uri(distro_name))
예제 #43
0
def compute_missing(distros, fqdn, rosdistro):
    """ Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """

    URL_PROTOTYPE = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml'

    repo_url = 'http://%s/repos/building' % fqdn

    print('Fetching "%s"' % (URL_PROTOTYPE % rosdistro))
    repo_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % rosdistro))

    # What ROS distro are we configuring?
    if 'release-name' not in repo_map:
        print('No "release-name" key in yaml file')
        sys.exit(1)
    if repo_map['release-name'] != rosdistro:
        print('release-name mismatch (%s != %s)' %
              (repo_map['release-name'], rosdistro))
        sys.exit(1)
    if 'repositories' not in repo_map:
        print('No "repositories" key in yaml file')
    if 'type' not in repo_map or repo_map['type'] != 'gbp':
        print('Wrong type value in yaml file')
        sys.exit(1)

    # Figure out default distros.  Command-line arg takes precedence; if
    # it's not specified, then read targets.yaml.
    if distros:
        default_distros = distros
    else:
        print('Fetching "%s"' % (URL_PROTOTYPE % 'targets'))
        targets_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % 'targets'))
        my_targets = [x for x in targets_map if rosdistro in x]
        if len(my_targets) != 1:
            print(
                'Must have exactly one entry for rosdistro "%s" in targets.yaml'
                % rosdistro)
            sys.exit(1)
        default_distros = my_targets[0][rosdistro]

    arches = ['amd64', 'i386']

    # We take the intersection of repo-specific targets with default
    # targets.
    missing = {}
    for short_package_name, r in repo_map['repositories'].items():
        if 'url' not in r:
            print('"url" key missing for repository "%s"; skipping' % r)
            continue
        url = r['url']
        if 'target' not in r or r['target'] == 'all':
            target_distros = default_distros
        else:
            target_distros = list(set(r['target']) & set(default_distros))

        #print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))

        # todo check if sourcedeb is present with the right version
        deb_name = debianize_package_name(rosdistro, short_package_name)
        if not 'version' in r:
            print('"version" key missing for repository %s; skipping' % r)
            continue
        expected_version = r['version']
        if not expected_version:
            expected_version = ''

        missing[short_package_name] = []
        for d in target_distros:
            if not repo.deb_in_repo(repo_url,
                                    deb_name,
                                    expected_version + ".*",
                                    d,
                                    arch='na',
                                    source=True):
                missing[short_package_name].append('%s_source' % d)
            for a in arches:
                if not repo.deb_in_repo(repo_url, deb_name,
                                        expected_version + ".*", d, a):
                    missing[short_package_name].append('%s_%s' % (d, a))

        # if not trigger sourcedeb

        # else if binaries don't exist trigger them
        for d in target_distros:
            for a in arches:
                pass  #missing[short_package_name] = ['source']

    #dry stacks
    # dry dependencies
    dist = load_distro(distro_uri(rosdistro))

    distro_arches = []
    for d in default_distros:
        for a in arches:
            distro_arches.append((d, a))

    for s in dist.stacks:
        #print ("Analyzing DRY job [%s]" % s)
        expected_version = dry_get_stack_version(s, dist)

        # sanitize undeclared versions for string substitution
        if not expected_version:
            expected_version = ''
        missing[s] = []
        # for each distro arch check if the deb is present. If not trigger the build.
        for (d, a) in distro_arches:
            if not repo.deb_in_repo(repo_url,
                                    debianize_package_name(rosdistro, s),
                                    expected_version + ".*", d, a):
                missing[s].append('%s_%s' % (d, a))

    return missing