def test_load_distro_bad_data():
from rospkg import ResourceNotFound
from rospkg.distro import load_distro, InvalidDistro
try:
load_distro('bad')
assert False
except ResourceNotFound: pass
for i in range(1, 10):
filename = 'bad%s.rosdistro'%(i)
try:
d = get_test_path()
p = os.path.join(d, filename)
load_distro(p)
assert False, "should have raised: %s"%(filename)
except InvalidDistro: pass
def get_release_info(name, distro, prefix=None):
"""
Steps to check for a released version of the package
1) Look in the wet distro file for the package/stack name, if it's there, return the repo
2) Look in the dry distro file for the package/stack name, if it's there, return the repo
3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
what stack a given package belongs to
4) Look in the distro files again to see if the stack name is there, if it is, return the repo
"""
url = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' % distro
try:
wet_distro = yaml.load(urllib2.urlopen(url))
except:
raise IOError("Could not load the %s rosdistro file from github.\n\
Are you sure you've selected a valid distro?\n\
I'm looking for the following file %s" % (distro, url))
dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))
# Check to see if the name just exists in one of our rosdistro files
rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
if rosinstall:
return rosinstall
# If we didn't find the name, we need to try to find a stack for it
doc_yaml = get_manifest_yaml(name, distro)
for metapackage in doc_yaml.get('metapackages', []):
meta_yaml = get_manifest_yaml(metapackage, distro)
if meta_yaml['package_type'] == 'stack':
rosinstall = get_release_rosinstall(
metapackage, wet_distro, dry_distro, prefix)
if rosinstall:
return rosinstall
return None
def test_load_distro_diamondback():
from rospkg.distro import load_distro, Distro
d = get_test_path()
p = os.path.join(d, 'diamondback.rosdistro')
distro = load_distro(p)
assert isinstance(distro, Distro)
assert distro.release_name == 'diamondback', distro.release_name
assert distro.version == 'r8596', distro.version
assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
assert set(distro.variants.keys()) == set(diamondback_variants)
assert set(distro.stacks.keys()) == set(diamondback_stacks), set(
distro.stacks.keys()) ^ set(diamondback_stacks)
assert distro.variants['ros-base'].extends == []
retval = distro.variants['ros-base'].get_stack_names(True)
assert retval == ['ros', 'ros_comm'], retval
assert distro.variants['ros-base'].get_stack_names(False) == [
'ros', 'ros_comm'
]
assert set(distro.variants['ros-full'].get_stack_names(True)) == set(
['ros', 'ros_comm', 'rx', 'documentation'])
assert distro.stacks['common'].version == '1.3.3'
assert distro.stacks['common'].vcs_config.get_branch(
'devel',
True) == ('https://code.ros.org/svn/ros-pkg/stacks/common/trunk', None)
def dry_get_versioned_dependency_tree(rosdistro):
url = distro_uri(rosdistro)
try:
d = load_distro(url)
except urllib2.URLError as ex:
print("Loading distro from '%s'failed with URLError %s" % (url, ex),
file=sys.stderr)
raise
dependency_tree = {}
versions = {}
maintainer_dict = {}
for s in d.stacks:
version = d.stacks[s].version
versions[s] = version
yaml_info = dry_get_stack_info(s, version)
if 'depends' in yaml_info:
dependency_tree[s] = yaml_info['depends']
else:
dependency_tree[s] = []
maintainers = []
if 'maintainer' in yaml_info:
maintainers = _extract_emails(yaml_info['maintainer'])
if not maintainers and 'contact' in yaml_info:
maintainers = _extract_emails(yaml_info['contact'])
maintainer_dict[s] = maintainers
return dependency_tree, versions, maintainer_dict
def dry_get_versioned_dependency_tree(rosdistro):
url = distro_uri(rosdistro)
try:
d = load_distro(url)
except urllib2.URLError as ex:
print ("Loading distro from '%s'failed with URLError %s" % (url, ex), file=sys.stderr)
raise
dependency_tree = {}
versions = {}
maintainer_dict = {}
for s in d.stacks:
version = d.stacks[s].version
versions[s] = version
yaml_info = dry_get_stack_info(s, version)
if 'depends' in yaml_info:
dependency_tree[s] = yaml_info['depends']
else:
dependency_tree[s] = []
maintainers = []
if 'maintainer' in yaml_info:
maintainers = _extract_emails(yaml_info['maintainer'])
if not maintainers and 'contact' in yaml_info:
maintainers = _extract_emails(yaml_info['contact'])
maintainer_dict[s] = maintainers
return dependency_tree, versions, maintainer_dict
def test_load_distro_diamondback():
from rospkg.distro import load_distro, Distro
d = get_test_path()
p = os.path.join(d, "diamondback.rosdistro")
distro = load_distro(p)
assert isinstance(distro, Distro)
assert distro.release_name == "diamondback", distro.release_name
assert distro.version == "r8596", distro.version
assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
assert set(distro.variants.keys()) == set(diamondback_variants)
assert set(distro.stacks.keys()) == set(diamondback_stacks), set(distro.stacks.keys()) ^ set(diamondback_stacks)
assert distro.variants["ros-base"].extends == []
retval = distro.variants["ros-base"].get_stack_names(True)
assert retval == ["ros", "ros_comm"], retval
assert distro.variants["ros-base"].get_stack_names(False) == ["ros", "ros_comm"]
assert set(distro.variants["ros-full"].get_stack_names(True)) == set(["ros", "ros_comm", "rx", "documentation"])
assert distro.stacks["common"].version == "1.3.3"
assert distro.stacks["common"].vcs_config.get_branch("devel", True) == (
"https://code.ros.org/svn/ros-pkg/stacks/common/trunk",
None,
)
def get_release_info(name, distro, prefix=None):
"""
Steps to check for a released version of the package
1) Look in the wet distro file for the package/stack name, if it's there, return the repo
2) Look in the dry distro file for the package/stack name, if it's there, return the repo
3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
what stack a given package belongs to
4) Look in the distro files again to see if the stack name is there, if it is, return the repo
"""
url = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml' % distro
try:
wet_distro = yaml.load(urllib2.urlopen(url))
except:
raise IOError("Could not load the %s rosdistro file from github.\n\
Are you sure you've selected a valid distro?\n\
I'm looking for the following file %s" % (distro, url))
dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))
#Check to see if the name just exists in one of our rosdistro files
rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
if rosinstall:
return rosinstall
#If we didn't find the name, we need to try to find a stack for it
doc_yaml = get_manifest_yaml(name, distro)
for metapackage in doc_yaml.get('metapackages', []):
meta_yaml = get_manifest_yaml(metapackage, distro)
if meta_yaml['package_type'] == 'stack':
rosinstall = get_release_rosinstall(metapackage, wet_distro, dry_distro, prefix)
if rosinstall:
return rosinstall
return None
def test_distro_to_rosinstall():
from rospkg.distro import distro_to_rosinstall, load_distro
d = get_test_path()
distro = load_distro(os.path.join(d, 'simple.rosdistro'))
data = distro_to_rosinstall(distro,
'devel',
variant_name='base',
implicit=False,
released_only=True,
anonymous=True)
# should only have a single stack
assert len(data) == 1
url = 'https://simple.com/svn/trunk/stack1'
assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
data = distro_to_rosinstall(distro,
'devel',
variant_name=None,
implicit=False,
released_only=True,
anonymous=True)
assert len(data) == 1
assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
data = distro_to_rosinstall(distro,
'devel',
variant_name=None,
implicit=False,
released_only=False,
anonymous=True)
assert len(data) == 1
assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
def test_load_distro_bad_data():
from rospkg import ResourceNotFound
from rospkg.distro import load_distro, InvalidDistro
try:
load_distro('bad')
assert False
except ResourceNotFound: pass
for i in range(1, 10):
if i == 4:
# currently non-existent stacks in variants are not determinable
continue
filename = 'bad%s.rosdistro'%(i)
try:
d = get_test_path()
p = os.path.join(d, filename)
load_distro(p)
assert False, "should have raised: %s"%(filename)
except InvalidDistro: pass
def test_load_distro_variants():
# test with no and empty variants (issue found in fuerte bringup)
from rospkg.distro import load_distro
d = get_test_path()
for name in ['no_variants.rosdistro', 'empty_variants.rosdistro']:
p = os.path.join(d, name)
distro = load_distro(p)
assert distro.release_name == 'simple', distro.release_name
assert distro.variants.keys() == []
def test_load_distro_variants():
# test with no and empty variants (issue found in fuerte bringup)
from rospkg.distro import load_distro
d = get_test_path()
for name in ['no_variants.rosdistro', 'empty_variants.rosdistro']:
p = os.path.join(d, name)
distro = load_distro(p)
assert distro.release_name == 'simple', distro.release_name
assert distro.variants.keys() == []
def compute_missing(distros, arches, fqdn, rosdistro, sourcerpm_only=False, wet_only=False):
""" Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """
repo_url = 'http://%s/smd-ros-building' % fqdn
rd = Rosdistro(rosdistro)
# We take the intersection of repo-specific targets with default
# targets.
if distros:
target_distros = distros
else:
target_distros = rd.get_target_distros()
missing = {}
for short_package_name in rd.get_package_list():
#print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))
# todo check if sourcerpm is present with the right version
rpm_name = redhatify_package_name(rosdistro, short_package_name)
expected_version = rd.get_version(short_package_name, full_version=True)
missing[short_package_name] = []
for d in target_distros:
d_ver = fedora_release_version(d)
if not repo.rpm_in_repo(repo_url, rpm_name, str(expected_version) + "\.fc" + d_ver, d, 'SRPMS'):
missing[short_package_name].append((d, 'SRPMS'))
if not sourcerpm_only:
for a in arches:
if not repo.rpm_in_repo(repo_url, rpm_name, str(expected_version) + "\.fc" + d_ver, d, a):
missing[short_package_name].append((d, a))
if not sourcerpm_only and not wet_only:
#dry stacks
# dry dependencies
dist = load_distro(distro_uri(rosdistro))
distro_arches = []
for d in target_distros:
for a in arches:
distro_arches.append((d, a))
for s in dist.stacks:
#print ("Analyzing DRY job [%s]" % s)
expected_version = dry_get_stack_version(s, dist)
# sanitize undeclared versions for string substitution
if not expected_version:
expected_version = ''
missing[s] = []
# for each distro arch check if the RPM is present. If not trigger the build.
for (d, a) in distro_arches:
if not repo.rpm_in_repo(repo_url, redhatify_package_name(rosdistro, s), expected_version + "\.fc" + fedora_release_version(d), d, a):
missing[s].append((d, a))
return missing
def single_deb_main():
from optparse import OptionParser
parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)
parser.add_option("-d", "--dir",
dest="staging_dir", default=None,
help="directory to use for staging source debs", metavar="STAGING_DIR")
parser.add_option("--force",
dest="force", default=False, action="store_true")
parser.add_option("--noupload",
dest="noupload", default=False, action="store_true")
parser.add_option("--fqdn",
dest="fqdn", default='50.28.27.175', action="store")
parser.add_option("--interactive",
dest="interactive", default=False, action="store_true")
parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER")
(options, args) = parser.parse_args()
if len(args) != 4:
parser.error('invalid args')
(distro_name, stack_name, os_platform, arch) = args
distro = failure_message = warning_message = None
if options.staging_dir is not None:
staging_dir = options.staging_dir
staging_dir = os.path.abspath(staging_dir)
else:
staging_dir = tempfile.mkdtemp()
try:
if distro_name not in rosdeb.targets.os_platform:
raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
target_platforms = rosdeb.targets.os_platform[distro_name]
if os_platform not in target_platforms:
raise BuildFailure("[%s] is not a known platformfor distro %s.\nSupported platforms are: %s" % (os_platform, distro_name, ' '.join(target_platforms)))
if not os.path.exists(staging_dir):
debug("creating staging dir: %s"%(staging_dir))
os.makedirs(staging_dir)
uri = distro_uri(distro_name)
debug("loading distro file from %s"%(uri))
distro = load_distro(uri)
if stack_name == 'metapackages':
(warning_message, failure_message) = gen_metapkgs_setup(options.staging_dir, distro, os_platform, arch, options.fqdn)
else:
build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive, options.fqdn)
except StackBuildFailure, e:
warning_message = "Warning Message:\n"+"="*80+'\n'+str(e)
def test_load_distro_bad_data():
from rospkg import ResourceNotFound
from rospkg.distro import load_distro, InvalidDistro
try:
load_distro('bad')
assert False
except ResourceNotFound:
pass
for i in range(1, 10):
if i == 4:
# currently non-existent stacks in variants are not determinable
continue
filename = 'bad%s.rosdistro' % (i)
try:
d = get_test_path()
p = os.path.join(d, filename)
load_distro(p)
assert False, "should have raised: %s" % (filename)
except InvalidDistro:
pass
def build_debs_main():
from optparse import OptionParser
parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)
parser.add_option("-d", "--dir",
dest="staging_dir", default=None,
help="directory to use for staging source debs", metavar="STAGING_DIR")
parser.add_option("--force",
dest="force", default=False, action="store_true")
parser.add_option("--noupload",
dest="noupload", default=False, action="store_true")
parser.add_option("--noramdisk",
dest="ramdisk", default=True, action="store_false")
parser.add_option("--interactive",
dest="interactive", default=False, action="store_true")
parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER")
(options, args) = parser.parse_args()
if len(args) != 4:
parser.error('invalid args')
(distro_name, stack_name, os_platform, arch) = args
distro = failure_message = warning_message = None
if options.staging_dir is not None:
staging_dir = options.staging_dir
staging_dir = os.path.abspath(staging_dir)
else:
staging_dir = tempfile.mkdtemp()
try:
if distro_name not in rosdeb.targets.os_platform:
raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
target_platforms = rosdeb.targets.os_platform[distro_name]
if os_platform not in target_platforms:
raise BuildFailure("[%s] is not a known platform.\nSupported platforms are: %s" % (os_platform, ' '.join(target_platforms)))
if not os.path.exists(staging_dir):
print "creating staging dir: %s" % (staging_dir)
os.makedirs(staging_dir)
distro = load_distro(distro_uri(distro_name))
if options.ramdisk:
with TempRamFS(staging_dir, "20G"):
build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive)
else:
build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive)
except StackBuildFailure, e:
warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
def dry_get_versioned_dependency_tree(rosdistro):
d = load_distro(distro_uri(rosdistro))
dependency_tree = {}
versions = {}
for s in d.stacks:
version = d.stacks[s].version
versions[s] = version
yaml_info = dry_get_stack_info(s, version)
if 'depends' in yaml_info:
dependency_tree[s] = yaml_info['depends']
else:
dependency_tree[s] = []
return dependency_tree, versions
def dry_get_versioned_dependency_tree(rosdistro):
d = load_distro(distro_uri(rosdistro))
dependency_tree = {}
versions = {}
for s in d.stacks:
version = d.stacks[s].version
versions[s] = version
yaml_info = dry_get_stack_info(s, version)
if 'depends' in yaml_info:
dependency_tree[s] = yaml_info['depends']
else:
dependency_tree[s] = []
return dependency_tree, versions
def compute_missing(job_params, sourcedeb_only=False):
""" Compute what wet packages are missing from a repo based on the rosdistro files. """
repo_url = 'http://%s/repos/building' % job_params.fqdn
rd = job_params.rd
# We take the intersection of repo-specific targets with default
# targets.
target_distros = job_params.distros
missing = {}
for short_package_name in rd.get_packages():
deb_name = debianize_package_name(job_params.rosdistro, short_package_name)
expected_version = rd.get_version(short_package_name)
missing[short_package_name] = []
for d in target_distros:
if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + d, d, arch='na', source=True):
missing[short_package_name].append('%s_source' % d)
if not sourcedeb_only:
target_arches = job_params.arches[d]
for a in target_arches:
if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + ".*", d, a):
missing[short_package_name].append('%s_%s' % (d, a))
if not sourcedeb_only:
#dry stacks
# dry dependencies
dist = load_distro(distro_uri(rosdistro))
distro_arches = []
for d in target_distros:
target_arches = job_params.arches[d]
for a in target_arches:
distro_arches.append((d, a))
for s in dist.stacks:
#print ("Analyzing DRY job [%s]" % s)
expected_version = dry_get_stack_version(s, dist)
# sanitize undeclared versions for string substitution
if not expected_version:
expected_version = ''
missing[s] = []
# for each distro arch check if the deb is present. If not trigger the build.
for (d, a) in distro_arches:
if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a):
missing[s].append('%s_%s' % (d, a))
return missing
def test_distro_to_rosinstall():
from rospkg.distro import distro_to_rosinstall, load_distro
d = get_test_path()
distro = load_distro(os.path.join(d, 'simple.rosdistro'))
data = distro_to_rosinstall(distro, 'devel', variant_name='base', implicit=False, released_only=True, anonymous=True)
# should only have a single stack
assert len(data) == 1
url = 'https://simple.com/svn/trunk/stack1'
assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
data = distro_to_rosinstall(distro, 'devel', variant_name=None, implicit=False, released_only=True, anonymous=True)
assert len(data) == 1
assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
data = distro_to_rosinstall(distro, 'devel', variant_name=None, implicit=False, released_only=False, anonymous=True)
assert len(data) == 1
assert data[0] == dict(svn={'uri': url, 'local-name': 'stack1'})
def test_load_distro_simple():
from rospkg.distro import load_distro, Distro
d = get_test_path()
p = os.path.join(d, 'simple.rosdistro')
distro = load_distro(p)
assert isinstance(distro, Distro)
assert distro.release_name == 'simple', distro.release_name
assert distro.version == '1', distro.version
assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
assert distro.variants.keys() == ['base']
assert distro.stacks.keys() == ['stack1']
stack1 = distro.stacks['stack1']
assert stack1.vcs_config.get_branch('devel', False) == ('https://simple.com/svn/trunk/stack1', None)
assert stack1.vcs_config.get_branch('distro', False) == ('https://simple.com/svn/tags/distros/simple/stacks/stack1', None)
assert stack1.vcs_config.get_branch('release', False) == ('https://simple.com/svn/tags/stacks/stack1/stack1-0.3.0', None)
def test_load_distro_simple():
from rospkg.distro import load_distro, Distro
d = get_test_path()
p = os.path.join(d, 'simple.rosdistro')
distro = load_distro(p)
assert isinstance(distro, Distro)
assert distro.release_name == 'simple', distro.release_name
assert distro.version == '1', distro.version
assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
assert set(distro.variants.keys()) == set(['base'])
assert set(distro.stacks.keys()) == set(['stack1'])
stack1 = distro.stacks['stack1']
assert stack1.vcs_config.get_branch('devel', False) == ('https://simple.com/svn/trunk/stack1', None)
assert stack1.vcs_config.get_branch('distro', False) == ('https://simple.com/svn/tags/distros/simple/stacks/stack1', None)
assert stack1.vcs_config.get_branch('release', False) == ('https://simple.com/svn/tags/stacks/stack1/stack1-0.3.0', None)
def _get_electric_rosinstall(name, prefix=None):
"""
Please delete me when you don't care at all about electric anymore
"""
dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri('electric'))
if _is_dry(dry_distro, name):
return get_dry_rosinstall(dry_distro, name, prefix=prefix)
# If we didn't find the name, we need to try to find a stack for it
doc_yaml = get_manifest_yaml(name, 'electric')
for metapackage in doc_yaml.get('metapackages', []):
meta_yaml = get_manifest_yaml(metapackage, 'electric')
if meta_yaml['package_type'] == 'stack':
if _is_dry(dry_distro, metapackage):
return get_dry_rosinstall(dry_distro, metapackage, prefix=prefix)
return None
def test_distro_to_rosinstall():
from rospkg.distro import distro_to_rosinstall, load_distro
d = get_test_path()
distro = load_distro(os.path.join(d, "simple.rosdistro"))
data = distro_to_rosinstall(
distro, "devel", variant_name="base", implicit=False, released_only=True, anonymous=True
)
# should only have a single stack
assert len(data) == 1
url = "https://simple.com/svn/trunk/stack1"
assert data[0] == dict(svn={"uri": url, "local-name": "stack1"})
data = distro_to_rosinstall(distro, "devel", variant_name=None, implicit=False, released_only=True, anonymous=True)
assert len(data) == 1
assert data[0] == dict(svn={"uri": url, "local-name": "stack1"})
data = distro_to_rosinstall(distro, "devel", variant_name=None, implicit=False, released_only=False, anonymous=True)
assert len(data) == 1
assert data[0] == dict(svn={"uri": url, "local-name": "stack1"})
def translate(distro, translate_dir):
d = load_distro(distro_uri(distro))
repo_list = d.get_stacks(True)
for name, item in repo_list.iteritems():
if item.vcs_config.type == 'svn':
rosinstall = [{item.vcs_config.type: \
{'local-name': item.name,
'uri': item.vcs_config.anon_dev}}]
else:
rosinstall = [{item.vcs_config.type: \
{'local-name': item.name,
'uri': item.vcs_config.anon_repo_uri,
'version': item.vcs_config.dev_branch}}]
path = os.path.join(translate_dir, "%s.rosinstall" % item.name)
with open(path, 'w+') as f:
print "writing to %s" % path
yaml.safe_dump(rosinstall, f, default_flow_style=False)
def get_missing_dry_packages(rosdistro, default_distros, arches, repo_url):
missing = {}
dist = load_distro(distro_uri(rosdistro))
distro_arches = [(d, a) for d in default_distros for a in arches]
for s in dist.stacks:
#print ("Analyzing DRY job [%s]" % s)
expected_version = dry_get_stack_version(s, dist)
# sanitize undeclared versions for string substitution
if not expected_version:
expected_version = ''
missing[s] = []
# for each distro arch check if the deb is present. If not trigger the build.
for (d, a) in distro_arches:
if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version+".*", d, a):
missing[s].append( '%s_%s' % (d, a) )
return missing
def _get_electric_rosinstall(name, prefix=None):
"""
Please delete me when you don't care at all about electric anymore
"""
dry_distro = rospkg_distro.load_distro(
rospkg_distro.distro_uri('electric'))
if _is_dry(dry_distro, name):
return get_dry_rosinstall(dry_distro, name, prefix=prefix)
# If we didn't find the name, we need to try to find a stack for it
doc_yaml = get_manifest_yaml(name, 'electric')
for metapackage in doc_yaml.get('metapackages', []):
meta_yaml = get_manifest_yaml(metapackage, 'electric')
if meta_yaml['package_type'] == 'stack':
if _is_dry(dry_distro, metapackage):
return get_dry_rosinstall(dry_distro,
metapackage,
prefix=prefix)
return None
def test_load_distro_diamondback():
from rospkg.distro import load_distro, Distro
d = get_test_path()
p = os.path.join(d, 'diamondback.rosdistro')
distro = load_distro(p)
assert isinstance(distro, Distro)
assert distro.release_name == 'diamondback', distro.release_name
assert distro.version == 'r8596', distro.version
assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
assert set(distro.variants.keys()) == set(diamondback_variants)
assert set(distro.stacks.keys()) == set(diamondback_stacks), set(distro.stacks.keys()) ^ set(diamondback_stacks)
assert distro.variants['ros-base'].extends == []
retval = distro.variants['ros-base'].get_stack_names(True)
assert retval == ['ros', 'ros_comm'], retval
assert distro.variants['ros-base'].get_stack_names(False) == ['ros', 'ros_comm']
assert set(distro.variants['ros-full'].get_stack_names(True)) == set(['ros', 'ros_comm', 'rx', 'documentation'])
assert distro.stacks['common'].version == '1.3.3'
assert distro.stacks['common'].vcs_config.get_branch('devel', True) == ('https://code.ros.org/svn/ros-pkg/stacks/common/trunk', None)
def _get_fuerte_rosinstall(name, prefix=None):
"""
Please delete me when fuerte is not supported anymore
See REP137 about rosdistro files
"""
dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri("fuerte"))
wet_distro = _get_fuerte_release()
# Check to see if the name just exists in one of our rosdistro files
rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
if rosinstall:
return rosinstall
# If we didn't find the name, we need to try to find a stack for it
doc_yaml = get_manifest_yaml(name, "fuerte")
for metapackage in doc_yaml.get("metapackages", []):
meta_yaml = get_manifest_yaml(metapackage, "fuerte")
if meta_yaml["package_type"] == "stack":
rosinstall = get_release_rosinstall(metapackage, wet_distro, dry_distro, prefix)
if rosinstall:
return rosinstall
return None
def get_release_info(name, distro, prefix=None):
"""
Steps to check for a released version of the package
1) Look in the wet distro file for the package/stack name, if it's there, return the repo
2) Look in the dry distro file for the package/stack name, if it's there, return the repo
3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
what stack a given package belongs to
4) Look in the distro files again to see if the stack name is there, if it is, return the repo
"""
# fuerte is different.
if distro == 'fuerte':
return _get_fuerte_rosinstall(name, prefix=prefix)
# electric is ancient.
if distro == 'electric':
return _get_electric_rosinstall(name, prefix=prefix)
wet_distro = _get_rosdistro_release(distro)
dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))
# Check to see if the name just exists in one of our rosdistro files
if _is_wet(wet_distro, name):
return get_wet_rosinstall(wet_distro, name, prefix=prefix)
if _is_dry(dry_distro, name):
return get_dry_rosinstall(dry_distro, name, prefix=prefix)
# If we didn't find the name, we need to try to find a stack for it
doc_yaml = get_manifest_yaml(name, distro)
for metapackage in doc_yaml.get('metapackages', []):
meta_yaml = get_manifest_yaml(metapackage, distro)
if meta_yaml['package_type'] == 'stack':
if _is_dry(dry_distro, metapackage):
return get_dry_rosinstall(dry_distro,
metapackage,
prefix=prefix)
return None
def _get_fuerte_rosinstall(name, prefix=None):
"""
Please delete me when fuerte is not supported anymore
See REP137 about rosdistro files
"""
dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri('fuerte'))
wet_distro = _get_fuerte_release()
# Check to see if the name just exists in one of our rosdistro files
rosinstall = get_release_rosinstall(name, wet_distro, dry_distro, prefix)
if rosinstall:
return rosinstall
# If we didn't find the name, we need to try to find a stack for it
doc_yaml = get_manifest_yaml(name, 'fuerte')
for metapackage in doc_yaml.get('metapackages', []):
meta_yaml = get_manifest_yaml(metapackage, 'fuerte')
if meta_yaml['package_type'] == 'stack':
rosinstall = get_release_rosinstall(metapackage, wet_distro,
dry_distro, prefix)
if rosinstall:
return rosinstall
return None
def get_release_info(name, distro, prefix=None):
"""
Steps to check for a released version of the package
1) Look in the wet distro file for the package/stack name, if it's there, return the repo
2) Look in the dry distro file for the package/stack name, if it's there, return the repo
3) Look in the manifest.yaml generated by the documentation indexer to take a best guess at
what stack a given package belongs to
4) Look in the distro files again to see if the stack name is there, if it is, return the repo
"""
# fuerte is different.
if distro == 'fuerte':
return _get_fuerte_rosinstall(name, prefix=prefix)
# electric is ancient.
if distro == 'electric':
return _get_electric_rosinstall(name, prefix=prefix)
wet_distro = _get_rosdistro_release(distro)
dry_distro = rospkg_distro.load_distro(rospkg_distro.distro_uri(distro))
# Check to see if the name just exists in one of our rosdistro files
if _is_wet(wet_distro, name):
return get_wet_rosinstall(wet_distro, name, prefix=prefix)
if _is_dry(dry_distro, name):
return get_dry_rosinstall(dry_distro, name, prefix=prefix)
# If we didn't find the name, we need to try to find a stack for it
doc_yaml = get_manifest_yaml(name, distro)
for metapackage in doc_yaml.get('metapackages', []):
meta_yaml = get_manifest_yaml(metapackage, distro)
if meta_yaml['package_type'] == 'stack':
if _is_dry(dry_distro, metapackage):
return get_dry_rosinstall(dry_distro, metapackage, prefix=prefix)
return None
def test_load_distro_simple():
from rospkg.distro import load_distro, Distro
d = get_test_path()
p = os.path.join(d, "simple.rosdistro")
distro = load_distro(p)
assert isinstance(distro, Distro)
assert distro.release_name == "simple", distro.release_name
assert distro.version == "1", distro.version
assert yaml.load(open(p)) == distro.raw_data, distro.raw_data
assert distro.variants.keys() == ["base"]
assert distro.stacks.keys() == ["stack1"]
stack1 = distro.stacks["stack1"]
assert stack1.vcs_config.get_branch("devel", False) == ("https://simple.com/svn/trunk/stack1", None)
assert stack1.vcs_config.get_branch("distro", False) == (
"https://simple.com/svn/tags/distros/simple/stacks/stack1",
None,
)
assert stack1.vcs_config.get_branch("release", False) == (
"https://simple.com/svn/tags/stacks/stack1/stack1-0.3.0",
None,
)
def compute_missing(distros, fqdn, rosdistro):
""" Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """
URL_PROTOTYPE = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml'
repo_url = 'http://%s/repos/building' % fqdn
print('Fetching "%s"' % (URL_PROTOTYPE % rosdistro))
repo_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % rosdistro))
# What ROS distro are we configuring?
if 'release-name' not in repo_map:
print('No "release-name" key in yaml file')
sys.exit(1)
if repo_map['release-name'] != rosdistro:
print('release-name mismatch (%s != %s)' % (repo_map['release-name'], rosdistro))
sys.exit(1)
if 'repositories' not in repo_map:
print('No "repositories" key in yaml file')
if 'type' not in repo_map or repo_map['type'] != 'gbp':
print('Wrong type value in yaml file')
sys.exit(1)
# Figure out default distros. Command-line arg takes precedence; if
# it's not specified, then read targets.yaml.
if distros:
default_distros = distros
else:
print('Fetching "%s"' % (URL_PROTOTYPE % 'targets'))
targets_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % 'targets'))
my_targets = [x for x in targets_map if rosdistro in x]
if len(my_targets) != 1:
print('Must have exactly one entry for rosdistro "%s" in targets.yaml' % rosdistro)
sys.exit(1)
default_distros = my_targets[0][rosdistro]
arches = ['amd64', 'i386']
# We take the intersection of repo-specific targets with default
# targets.
missing = {}
for short_package_name, r in repo_map['repositories'].items():
if 'url' not in r:
print('"url" key missing for repository "%s"; skipping' % r)
continue
url = r['url']
if 'target' not in r or r['target'] == 'all':
target_distros = default_distros
else:
target_distros = list(set(r['target']) & set(default_distros))
#print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))
# todo check if sourcedeb is present with the right version
deb_name = debianize_package_name(rosdistro, short_package_name)
if not 'version' in r:
print('"version" key missing for repository %s; skipping' % r)
continue
expected_version = r['version']
if not expected_version:
expected_version = ''
missing[short_package_name] = []
for d in target_distros:
if not repo.deb_in_repo(repo_url, deb_name, expected_version+".*", d, arch='na', source=True):
missing[short_package_name].append('%s_source' % d)
for a in arches:
if not repo.deb_in_repo(repo_url, deb_name, expected_version+".*", d, a):
missing[short_package_name].append('%s_%s' % (d, a))
# if not trigger sourcedeb
# else if binaries don't exist trigger them
for d in target_distros:
for a in arches:
pass#missing[short_package_name] = ['source']
#dry stacks
# dry dependencies
dist = load_distro(distro_uri(rosdistro))
distro_arches = []
for d in default_distros:
for a in arches:
distro_arches.append( (d, a) )
for s in dist.stacks:
#print ("Analyzing DRY job [%s]" % s)
expected_version = dry_get_stack_version(s, dist)
# sanitize undeclared versions for string substitution
if not expected_version:
expected_version = ''
missing[s] = []
# for each distro arch check if the deb is present. If not trigger the build.
for (d, a) in distro_arches:
if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version+".*", d, a):
missing[s].append( '%s_%s' % (d, a) )
return missing
def compute_missing(distros, arches, fqdn, rosdistro, sourcedeb_only=False):
""" Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """
repo_url = 'http://%s/repos/building' % fqdn
if rosdistro != 'fuerte':
from ros_distro import Rosdistro
else:
from ros_distro_fuerte import Rosdistro
rd = Rosdistro(rosdistro)
# We take the intersection of repo-specific targets with default
# targets.
if distros:
target_distros = distros
else:
target_distros = rd.get_target_distros()
missing = {}
for short_package_name in rd.get_package_list():
#print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))
# todo check if sourcedeb is present with the right version
deb_name = debianize_package_name(rosdistro, short_package_name)
expected_version = rd.get_version(short_package_name, full_version=True)
# Don't report packages as missing if their version is None
if not expected_version:
print("Skipping package %s with no version" % short_package_name)
continue
missing[short_package_name] = []
for d in target_distros:
if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + d, d, arch='na', source=True):
missing[short_package_name].append('%s_source' % d)
if not sourcedeb_only:
for a in arches:
if not repo.deb_in_repo(repo_url, deb_name, str(expected_version) + ".*", d, a):
missing[short_package_name].append('%s_%s' % (d, a))
if not sourcedeb_only:
#dry stacks
# dry dependencies
dist = load_distro(distro_uri(rosdistro))
distro_arches = []
for d in target_distros:
for a in arches:
distro_arches.append((d, a))
for s in dist.stacks:
#print ("Analyzing DRY job [%s]" % s)
expected_version = dry_get_stack_version(s, dist)
# Don't report packages as missing if their version is None
if not expected_version:
print("Skipping package %s with no version" % s)
continue
missing[s] = []
# for each distro arch check if the deb is present. If not trigger the build.
for (d, a) in distro_arches:
if not repo.deb_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a):
missing[s].append('%s_%s' % (d, a))
return missing
def compute_missing(job_params, sourcedeb_only=False):
""" Compute what wet packages are missing from a repo based on the rosdistro files. """
repo_url = 'http://%s/repos/building' % job_params.fqdn
rd = job_params.rd
# We take the intersection of repo-specific targets with default
# targets.
target_distros = job_params.distros
missing = {}
for short_package_name in rd.get_packages():
deb_name = debianize_package_name(job_params.rosdistro,
short_package_name)
expected_version = rd.get_version(short_package_name)
missing[short_package_name] = []
for d in target_distros:
if not repo.deb_in_repo(repo_url,
deb_name,
str(expected_version) + d,
d,
arch='na',
source=True):
missing[short_package_name].append('%s_source' % d)
if not sourcedeb_only:
target_arches = job_params.arches[d]
for a in target_arches:
if not repo.deb_in_repo(repo_url, deb_name,
str(expected_version) + ".*", d,
a):
missing[short_package_name].append('%s_%s' % (d, a))
if not sourcedeb_only:
#dry stacks
# dry dependencies
dist = load_distro(distro_uri(rosdistro))
distro_arches = []
for d in target_distros:
target_arches = job_params.arches[d]
for a in target_arches:
distro_arches.append((d, a))
for s in dist.stacks:
#print ("Analyzing DRY job [%s]" % s)
expected_version = dry_get_stack_version(s, dist)
# sanitize undeclared versions for string substitution
if not expected_version:
expected_version = ''
missing[s] = []
# for each distro arch check if the deb is present. If not trigger the build.
for (d, a) in distro_arches:
if not repo.deb_in_repo(repo_url,
debianize_package_name(rosdistro, s),
expected_version + ".*", d, a):
missing[s].append('%s_%s' % (d, a))
return missing
def single_deb_main():
from optparse import OptionParser
parser = OptionParser(
usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)
parser.add_option("-d",
"--dir",
dest="staging_dir",
default=None,
help="directory to use for staging source debs",
metavar="STAGING_DIR")
parser.add_option("--force",
dest="force",
default=False,
action="store_true")
parser.add_option("--noupload",
dest="noupload",
default=False,
action="store_true")
parser.add_option("--fqdn",
dest="fqdn",
default='repos.ros.org',
action="store")
parser.add_option("--interactive",
dest="interactive",
default=False,
action="store_true")
parser.add_option('--smtp',
dest="smtp",
default='pub1.willowgarage.com',
metavar="SMTP_SERVER")
(options, args) = parser.parse_args()
if len(args) != 4:
parser.error('invalid args')
(distro_name, stack_name, os_platform, arch) = args
distro = failure_message = warning_message = None
if options.staging_dir is not None:
staging_dir = os.path.abspath(options.staging_dir)
else:
staging_dir = tempfile.mkdtemp()
try:
if distro_name not in rosdeb.targets.os_platform:
raise BuildFailure(
"[%s] is not a known rosdistro.\nValid rosdistros are: %s" %
(distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
target_platforms = rosdeb.targets.os_platform[distro_name]
if os_platform not in target_platforms:
raise BuildFailure(
"[%s] is not a known platform or distro %s.\nSupported platforms are: %s"
% (os_platform, distro_name, ' '.join(target_platforms)))
if not os.path.exists(staging_dir):
debug("creating staging dir: %s" % (staging_dir))
os.makedirs(staging_dir)
uri = distro_uri(distro_name)
debug("loading distro file from %s" % (uri))
distro = load_distro(uri)
if stack_name == 'metapackages':
(warning_message,
failure_message) = gen_metapkgs_setup(options.staging_dir, distro,
os_platform, arch,
options.fqdn)
else:
build_debs(distro, stack_name, os_platform, arch, staging_dir,
options.force, options.noupload, options.interactive,
options.fqdn)
except StackBuildFailure as e:
warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
except BuildFailure as e:
failure_message = "Failure Message:\n" + "=" * 80 + '\n' + str(e)
except Exception as e:
failure_message = "Internal failure release system setting up the staging dir. Please notify [email protected]:\n%s\n\n%s" % (
e, traceback.format_exc(e))
finally:
# if we created our own staging dir, we are responsible for cleaning it up
if options.staging_dir is None:
shutil.rmtree(staging_dir)
if failure_message or warning_message:
debug("FAILURE: %s" % failure_message)
debug("WARNING: %s" % warning_message)
#if not options.interactive:
#failure_message = "%s\n%s\n%s" % (failure_message, warning_message, os.environ.get('BUILD_URL', ''))
#if options.smtp and stack_name != 'metapackages' and distro is not None:
#stack_version = distro.stacks[stack_name].version
#control = download_control(stack_name, stack_version)
#if 'contact' in control and distro_name != 'diamondback':
# DISABLE SENDING OF EMAIL from script. This can be done better by jenkins.
#to_addr = control['contact']
#subject = 'debian build [%s-%s-%s-%s] failed'%(distro_name, stack_name, os_platform, arch)
# send_email(options.smtp, EMAIL_FROM_ADDR, to_addr, subject, failure_message)
if stack_name == 'metapackages':
debug("Not failing because build is a metapackage")
return True
return False
return True
def compute_missing(distros, arches, fqdn, rosdistro, sourcedeb_only=False):
""" Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """
repo_url = 'http://%s/repos/building' % fqdn
if rosdistro != 'fuerte':
from ros_distro import Rosdistro
else:
from ros_distro_fuerte import Rosdistro
rd = Rosdistro(rosdistro)
# We take the intersection of repo-specific targets with default
# targets.
if distros:
target_distros = distros
else:
target_distros = rd.get_target_distros()
missing = {}
for short_package_name in rd.get_package_list():
#print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))
# todo check if sourcedeb is present with the right version
deb_name = debianize_package_name(rosdistro, short_package_name)
expected_version = rd.get_version(short_package_name,
full_version=True)
# Don't report packages as missing if their version is None
if not expected_version:
print("Skipping package %s with no version" % short_package_name)
continue
missing[short_package_name] = []
for d in target_distros:
if not repo.deb_in_repo(repo_url,
deb_name,
str(expected_version) + d,
d,
arch='na',
source=True):
missing[short_package_name].append('%s_source' % d)
if not sourcedeb_only:
for a in arches:
if not repo.deb_in_repo(repo_url, deb_name,
str(expected_version) + ".*", d,
a):
missing[short_package_name].append('%s_%s' % (d, a))
if not sourcedeb_only and rosdistro == 'groovy':
#dry stacks
# dry dependencies
dist = load_distro(distro_uri(rosdistro))
distro_arches = []
for d in target_distros:
for a in arches:
distro_arches.append((d, a))
for s in dist.stacks:
#print ("Analyzing DRY job [%s]" % s)
expected_version = dry_get_stack_version(s, dist)
# Don't report packages as missing if their version is None
if not expected_version:
print("Skipping package %s with no version" % s)
continue
missing[s] = []
# for each distro arch check if the deb is present. If not trigger the build.
for (d, a) in distro_arches:
if not repo.deb_in_repo(repo_url,
debianize_package_name(rosdistro, s),
expected_version + ".*", d, a):
missing[s].append('%s_%s' % (d, a))
return missing
def compute_missing(distros, arches, fqdn, rosdistro, sourcepkg_only=False, platform='ubuntu'):
""" Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """
repo_url = 'http://%s/repos/building/%s' % (fqdn, platform)
# TODO Building Repo Workaround
if platform == 'ubuntu':
repo_url = 'http://%s/repos/building' % fqdn
elif platform == 'fedora':
repo_url = 'http://%s/smd-ros-building/%s' % (fqdn, platform)
# End Workaround
if rosdistro != 'fuerte':
from ros_distro import Rosdistro
else:
from ros_distro_fuerte import Rosdistro
rd = Rosdistro(rosdistro)
# We take the intersection of repo-specific targets with default
# targets.
if distros:
target_distros = distros
else:
target_distros = rd.get_target_distros()[platform]
missing = {}
for short_package_name in rd.get_package_list():
#print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))
# todo check if sourcepkg is present with the right version
pkg_name = debianize_package_name(rosdistro, short_package_name)
expected_version = rd.get_version(short_package_name, full_version=True)
# Don't report packages as missing if their version is None
if not expected_version:
print("Skipping package %s with no version" % short_package_name)
continue
missing[short_package_name] = []
for d in target_distros:
if platform == 'fedora':
version_extra = ".fc" + str(get_fedora_ver(d))
else:
version_extra = ".*"
if not repo.pkg_in_repo(repo_url, pkg_name, get_full_version(expected_version, d, platform), d, arch='na', source=True, platform=platform):
missing[short_package_name].append('%s_source' % d)
if not sourcepkg_only:
for a in arches:
if not repo.pkg_in_repo(repo_url, pkg_name, str(expected_version) + version_extra, d, a, platform=platform):
missing[short_package_name].append('%s_%s' % (d, a))
if not sourcepkg_only and rosdistro == 'groovy' and not platform == 'fedora':
#dry stacks
# dry dependencies
dist = load_distro(distro_uri(rosdistro))
distro_arches = []
for d in target_distros:
for a in arches:
distro_arches.append((d, a))
for s in dist.stacks:
#print ("Analyzing DRY job [%s]" % s)
expected_version = dry_get_stack_version(s, dist)
# Don't report packages as missing if their version is None
if not expected_version:
print("Skipping package %s with no version" % s)
continue
missing[s] = []
# for each distro arch check if the pkg is present. If not trigger the build.
for (d, a) in distro_arches:
if not repo.pkg_in_repo(repo_url, debianize_package_name(rosdistro, s), expected_version + ".*", d, a, platform=platform):
missing[s].append('%s_%s' % (d, a))
return missing
def get_distro(distro_name):
return rosdistro.load_distro(rosdistro.distro_uri(distro_name))
def single_deb_main():
from optparse import OptionParser
parser = OptionParser(
usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)
parser.add_option("-d",
"--dir",
dest="staging_dir",
default=None,
help="directory to use for staging source debs",
metavar="STAGING_DIR")
parser.add_option("--force",
dest="force",
default=False,
action="store_true")
parser.add_option("--noupload",
dest="noupload",
default=False,
action="store_true")
parser.add_option("--fqdn",
dest="fqdn",
default='50.28.27.175',
action="store")
parser.add_option("--interactive",
dest="interactive",
default=False,
action="store_true")
parser.add_option('--smtp',
dest="smtp",
default='pub1.willowgarage.com',
metavar="SMTP_SERVER")
(options, args) = parser.parse_args()
if len(args) != 4:
parser.error('invalid args')
(distro_name, stack_name, os_platform, arch) = args
distro = failure_message = warning_message = None
if options.staging_dir is not None:
staging_dir = options.staging_dir
staging_dir = os.path.abspath(staging_dir)
else:
staging_dir = tempfile.mkdtemp()
try:
if distro_name not in rosdeb.targets.os_platform:
raise BuildFailure(
"[%s] is not a known rosdistro.\nValid rosdistros are: %s" %
(distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
target_platforms = rosdeb.targets.os_platform[distro_name]
if os_platform not in target_platforms:
raise BuildFailure(
"[%s] is not a known platformfor distro %s.\nSupported platforms are: %s"
% (os_platform, distro_name, ' '.join(target_platforms)))
if not os.path.exists(staging_dir):
debug("creating staging dir: %s" % (staging_dir))
os.makedirs(staging_dir)
uri = distro_uri(distro_name)
debug("loading distro file from %s" % (uri))
distro = load_distro(uri)
if stack_name == 'metapackages':
(warning_message,
failure_message) = gen_metapkgs_setup(options.staging_dir, distro,
os_platform, arch,
options.fqdn)
else:
build_debs(distro, stack_name, os_platform, arch, staging_dir,
options.force, options.noupload, options.interactive,
options.fqdn)
except StackBuildFailure, e:
warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
def single_deb_main():
from optparse import OptionParser
parser = OptionParser(usage="usage: %prog <distro> <stack> <os-platform> <arch>", prog=NAME)
parser.add_option("-d", "--dir",
dest="staging_dir", default=None,
help="directory to use for staging source debs", metavar="STAGING_DIR")
parser.add_option("--force",
dest="force", default=False, action="store_true")
parser.add_option("--noupload",
dest="noupload", default=False, action="store_true")
parser.add_option("--fqdn",
dest="fqdn", default='repos.ros.org', action="store")
parser.add_option("--interactive",
dest="interactive", default=False, action="store_true")
parser.add_option('--smtp', dest="smtp", default='pub1.willowgarage.com', metavar="SMTP_SERVER")
(options, args) = parser.parse_args()
if len(args) != 4:
parser.error('invalid args')
(distro_name, stack_name, os_platform, arch) = args
distro = failure_message = warning_message = None
if options.staging_dir is not None:
staging_dir = os.path.abspath(options.staging_dir)
else:
staging_dir = tempfile.mkdtemp()
try:
if distro_name not in rosdeb.targets.os_platform:
raise BuildFailure("[%s] is not a known rosdistro.\nValid rosdistros are: %s" % (distro_name, ' '.join(rosdeb.targets.os_platform.keys())))
target_platforms = rosdeb.targets.os_platform[distro_name]
if os_platform not in target_platforms:
raise BuildFailure("[%s] is not a known platform or distro %s.\nSupported platforms are: %s" % (os_platform, distro_name, ' '.join(target_platforms)))
if not os.path.exists(staging_dir):
debug("creating staging dir: %s" % (staging_dir))
os.makedirs(staging_dir)
uri = distro_uri(distro_name)
debug("loading distro file from %s" % (uri))
distro = load_distro(uri)
if stack_name == 'metapackages':
(warning_message, failure_message) = gen_metapkgs_setup(options.staging_dir, distro, os_platform, arch, options.fqdn)
else:
build_debs(distro, stack_name, os_platform, arch, staging_dir, options.force, options.noupload, options.interactive, options.fqdn)
except StackBuildFailure as e:
warning_message = "Warning Message:\n" + "=" * 80 + '\n' + str(e)
except BuildFailure as e:
failure_message = "Failure Message:\n" + "=" * 80 + '\n' + str(e)
except Exception as e:
failure_message = "Internal failure release system setting up the staging dir. Please notify [email protected]:\n%s\n\n%s" % (e, traceback.format_exc(e))
finally:
# if we created our own staging dir, we are responsible for cleaning it up
if options.staging_dir is None:
shutil.rmtree(staging_dir)
if failure_message or warning_message:
debug("FAILURE: %s" % failure_message)
debug("WARNING: %s" % warning_message)
#if not options.interactive:
#failure_message = "%s\n%s\n%s" % (failure_message, warning_message, os.environ.get('BUILD_URL', ''))
#if options.smtp and stack_name != 'metapackages' and distro is not None:
#stack_version = distro.stacks[stack_name].version
#control = download_control(stack_name, stack_version)
#if 'contact' in control and distro_name != 'diamondback':
# DISABLE SENDING OF EMAIL from script. This can be done better by jenkins.
#to_addr = control['contact']
#subject = 'debian build [%s-%s-%s-%s] failed'%(distro_name, stack_name, os_platform, arch)
# send_email(options.smtp, EMAIL_FROM_ADDR, to_addr, subject, failure_message)
if stack_name == 'metapackages':
debug("Not failing because build is a metapackage")
return True
return False
return True
def get_distro(distro_name):
return rosdistro.load_distro(rosdistro.distro_uri(distro_name))
def compute_missing(distros, fqdn, rosdistro):
""" Compute what packages are missing from a repo based on the rosdistro files, both wet and dry. """
URL_PROTOTYPE = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml'
repo_url = 'http://%s/repos/building' % fqdn
print('Fetching "%s"' % (URL_PROTOTYPE % rosdistro))
repo_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % rosdistro))
# What ROS distro are we configuring?
if 'release-name' not in repo_map:
print('No "release-name" key in yaml file')
sys.exit(1)
if repo_map['release-name'] != rosdistro:
print('release-name mismatch (%s != %s)' %
(repo_map['release-name'], rosdistro))
sys.exit(1)
if 'repositories' not in repo_map:
print('No "repositories" key in yaml file')
if 'type' not in repo_map or repo_map['type'] != 'gbp':
print('Wrong type value in yaml file')
sys.exit(1)
# Figure out default distros. Command-line arg takes precedence; if
# it's not specified, then read targets.yaml.
if distros:
default_distros = distros
else:
print('Fetching "%s"' % (URL_PROTOTYPE % 'targets'))
targets_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % 'targets'))
my_targets = [x for x in targets_map if rosdistro in x]
if len(my_targets) != 1:
print(
'Must have exactly one entry for rosdistro "%s" in targets.yaml'
% rosdistro)
sys.exit(1)
default_distros = my_targets[0][rosdistro]
arches = ['amd64', 'i386']
# We take the intersection of repo-specific targets with default
# targets.
missing = {}
for short_package_name, r in repo_map['repositories'].items():
if 'url' not in r:
print('"url" key missing for repository "%s"; skipping' % r)
continue
url = r['url']
if 'target' not in r or r['target'] == 'all':
target_distros = default_distros
else:
target_distros = list(set(r['target']) & set(default_distros))
#print ('Analyzing WET stack "%s" for "%s"' % (r['url'], target_distros))
# todo check if sourcedeb is present with the right version
deb_name = debianize_package_name(rosdistro, short_package_name)
if not 'version' in r:
print('"version" key missing for repository %s; skipping' % r)
continue
expected_version = r['version']
if not expected_version:
expected_version = ''
missing[short_package_name] = []
for d in target_distros:
if not repo.deb_in_repo(repo_url,
deb_name,
expected_version + ".*",
d,
arch='na',
source=True):
missing[short_package_name].append('%s_source' % d)
for a in arches:
if not repo.deb_in_repo(repo_url, deb_name,
expected_version + ".*", d, a):
missing[short_package_name].append('%s_%s' % (d, a))
# if not trigger sourcedeb
# else if binaries don't exist trigger them
for d in target_distros:
for a in arches:
pass #missing[short_package_name] = ['source']
#dry stacks
# dry dependencies
dist = load_distro(distro_uri(rosdistro))
distro_arches = []
for d in default_distros:
for a in arches:
distro_arches.append((d, a))
for s in dist.stacks:
#print ("Analyzing DRY job [%s]" % s)
expected_version = dry_get_stack_version(s, dist)
# sanitize undeclared versions for string substitution
if not expected_version:
expected_version = ''
missing[s] = []
# for each distro arch check if the deb is present. If not trigger the build.
for (d, a) in distro_arches:
if not repo.deb_in_repo(repo_url,
debianize_package_name(rosdistro, s),
expected_version + ".*", d, a):
missing[s].append('%s_%s' % (d, a))
return missing