def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
example:
tasks:
- buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
- install:
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
for remote in ctx.cluster.remotes.iterkeys():
for install_config in lookup_configs(ctx, ctx.config):
gitbuilder = install._get_gitbuilder_project(
ctx, remote, install_config)
(tag, branch, sha1) = get_tag_branch_sha1(gitbuilder)
check_call("flock --close /tmp/buildpackages " + "make -C " + d +
" " + os.environ['HOME'] + "/.ssh_agent",
shell=True)
url = gitbuilder.base_url
target = os.path.dirname(urlparse.urlparse(url).path.strip('/'))
target = os.path.dirname(target) + '-' + sha1
openstack = OpenStack()
if 'cloud.ovh.net' in os.environ['OS_AUTH_URL']:
select = '^(vps|eg)-'
else:
select = ''
build_flavor = openstack.flavor(config['machine'], select)
http_flavor = openstack.flavor(
{
'disk': 10, # GB
'ram': 1024, # MB
'cpus': 1,
},
select)
cmd = (
". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close /tmp/buildpackages-" + sha1 + " make -C " + d +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + gitbuilder.pkg_type + " CEPH_OS_TYPE=" +
gitbuilder.os_type +
# os_version is from the remote and will be 7.1.23 for CentOS 7
# instead of the expected 7.0 for all 7.* CentOS
" CEPH_OS_VERSION=" + gitbuilder._get_version() +
" CEPH_DIST=" + gitbuilder.distro + " CEPH_ARCH=" +
gitbuilder.arch + " CEPH_SHA1=" + sha1 + " CEPH_TAG=" +
(tag or '') + " CEPH_BRANCH=" + (branch or '') +
" CEPH_FLAVOR=" + gitbuilder.flavor + " GITBUILDER_URL=" +
url + " BUILD_FLAVOR=" + build_flavor + " HTTP_FLAVOR=" +
http_flavor + " " + target + " ")
log.info("buildpackages: " + cmd)
check_call(cmd, shell=True)
teuth_config.gitbuilder_host = openstack.get_ip(
'packages-repository', '')
log.info('Finished buildpackages')
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
good_machine:
disk: 40 # GB
ram: 48000 # MB
cpus: 16
min_machine:
disk: 40 # GB
ram: 8000 # MB
cpus: 1
example:
tasks:
- buildpackages:
good_machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
min_machine:
disk: 40 # GB
ram: 8000 # MB
cpus: 1
- install:
When a buildpackages task is already included, the values it contains can be
overriden with:
overrides:
buildpackages:
good_machine:
disk: 20 # GB
ram: 2000 # MB
cpus: 2
min_machine:
disk: 10 # GB
ram: 1000 # MB
cpus: 1
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
overrides = ctx.config.get('overrides', {})
misc.deep_merge(config, overrides.get('buildpackages', {}))
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
os_type = misc.get_distro(ctx)
os_version = misc.get_distro_version(ctx)
arch = ctx.config.get('arch', OpenStack().get_default_arch())
dist = LocalGitbuilderProject()._get_distro(distro=os_type,
version=os_version)
pkg_type = get_pkg_type(os_type)
misc.sh(
"flock --close /tmp/buildpackages " +
"make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent")
for (flavor, tag, branch, sha1) in lookup_configs(ctx, ctx.config):
if tag:
sha1 = get_sha1(tag)
elif branch:
sha1 = get_sha1(branch)
log.info("building flavor = " + flavor + "," +
" tag = " + tag + "," +
" branch = " + branch + "," +
" sha1 = " + sha1)
target = ('ceph-' +
pkg_type + '-' +
dist + '-' +
arch + '-' +
flavor + '-' +
sha1)
openstack = OpenStack()
openstack.set_provider()
if openstack.provider == 'ovh':
select = '^(vps|hg)-.*ssd'
else:
select = ''
network = openstack.net()
if network != "":
network = " OPENSTACK_NETWORK='" + network + "' "
openstack.image(os_type, os_version, arch) # create if it does not exist
build_flavor = openstack.flavor_range(
config['min_machine'], config['good_machine'], arch, select)
default_arch = openstack.get_default_arch()
http_flavor = openstack.flavor({
'disk': 30, # GB
'ram': 1024, # MB
'cpus': 1,
}, default_arch, select)
lock = "/tmp/buildpackages-" + sha1 + "-" + os_type + "-" + os_version
cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close " + lock +
" make -C " + d +
network +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + pkg_type +
" CEPH_OS_TYPE=" + os_type +
" CEPH_OS_VERSION=" + os_version +
" CEPH_DIST=" + dist +
" CEPH_ARCH=" + arch +
" CEPH_SHA1=" + sha1 +
" CEPH_TAG=" + tag +
" CEPH_BRANCH=" + branch +
" CEPH_FLAVOR=" + flavor +
" BUILD_FLAVOR=" + build_flavor +
" HTTP_FLAVOR=" + http_flavor +
" HTTP_ARCH=" + default_arch +
" " + target +
" ")
log.info("buildpackages: " + cmd)
misc.sh(cmd)
teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
log.info('Finished buildpackages')
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
example:
tasks:
- buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
- install:
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
for remote in ctx.cluster.remotes.iterkeys():
for install_config in lookup_configs(ctx, ctx.config):
gitbuilder = install._get_gitbuilder_project(
ctx, remote, install_config)
(tag, branch, sha1) = get_tag_branch_sha1(gitbuilder)
check_call(
"flock --close /tmp/buildpackages " +
"make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent",
shell=True)
url = gitbuilder.base_url
target = os.path.dirname(urlparse.urlparse(url).path.strip('/'))
target = os.path.dirname(target) + '-' + sha1
openstack = OpenStack()
if 'cloud.ovh.net' in os.environ['OS_AUTH_URL']:
select = '^(vps|eg)-'
else:
select = ''
build_flavor = openstack.flavor(config['machine'], select)
http_flavor = openstack.flavor({
'disk': 10, # GB
'ram': 1024, # MB
'cpus': 1,
}, select)
cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close /tmp/buildpackages-" + sha1 +
" make -C " + d +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + gitbuilder.pkg_type +
" CEPH_OS_TYPE=" + gitbuilder.os_type +
# os_version is from the remote and will be 7.1.23 for CentOS 7
# instead of the expected 7.0 for all 7.* CentOS
" CEPH_OS_VERSION=" + gitbuilder._get_version() +
" CEPH_DIST=" + gitbuilder.distro +
" CEPH_ARCH=" + gitbuilder.arch +
" CEPH_SHA1=" + sha1 +
" CEPH_TAG=" + (tag or '') +
" CEPH_BRANCH=" + (branch or '') +
" GITBUILDER_URL=" + url +
" BUILD_FLAVOR=" + build_flavor +
" HTTP_FLAVOR=" + http_flavor +
" " + target +
" ")
log.info("buildpackages: " + cmd)
check_call(cmd, shell=True)
teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
log.info('Finished buildpackages')
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
good_machine:
disk: 40 # GB
ram: 48000 # MB
cpus: 16
min_machine:
disk: 40 # GB
ram: 8000 # MB
cpus: 1
example:
tasks:
- buildpackages:
good_machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
min_machine:
disk: 40 # GB
ram: 8000 # MB
cpus: 1
- install:
When a buildpackages task is already included, the values it contains can be
overriden with:
overrides:
buildpackages:
good_machine:
disk: 20 # GB
ram: 2000 # MB
cpus: 2
min_machine:
disk: 10 # GB
ram: 1000 # MB
cpus: 1
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
overrides = ctx.config.get('overrides', {})
misc.deep_merge(config, overrides.get('buildpackages', {}))
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
os_type = misc.get_distro(ctx)
os_version = misc.get_distro_version(ctx)
arch = ctx.config.get('arch', OpenStack().get_default_arch())
dist = LocalGitbuilderProject()._get_distro(distro=os_type,
version=os_version)
pkg_type = get_pkg_type(os_type)
misc.sh("flock --close /tmp/buildpackages " + "make -C " + d + " " +
os.environ['HOME'] + "/.ssh_agent")
for (flavor, tag, branch, sha1) in lookup_configs(ctx, ctx.config):
if tag:
sha1 = get_sha1(tag)
elif branch:
sha1 = get_sha1(branch)
log.info("building flavor = " + flavor + "," + " tag = " + tag + "," +
" branch = " + branch + "," + " sha1 = " + sha1)
self_name = 'teuthology'
key_name = 'teuthology'
pkg_repo = 'packages-repository'
security_group = 'teuthology'
if teuth_config.openstack.has_key('selfname'):
self_name = teuth_config.openstack['selfname']
if teuth_config.openstack.has_key('keypair'):
key_name = teuth_config.openstack['keypair']
if teuth_config.openstack.has_key('package_repo'):
pkg_repo = teuth_config.openstack['package_repo']
if teuth_config.openstack.has_key('server_group'):
security_group = teuth_config.openstack['server_group']
target = (self_name + '-ceph-' + pkg_type + '-' + dist + '-' + arch +
'-' + flavor + '-' + sha1)
openstack = OpenStack()
openstack.set_provider()
network = openstack.net()
if network != "":
network = " OPENSTACK_NETWORK='" + network + "' "
openstack.image(os_type, os_version,
arch) # create if it does not exist
build_flavor = openstack.flavor_range(config['min_machine'],
config['good_machine'], arch)
default_arch = openstack.get_default_arch()
http_flavor = openstack.flavor(
{
'disk': 30, # GB
'ram': 1024, # MB
'cpus': 1,
},
default_arch)
lock = "/tmp/buildpackages-" + sha1 + "-" + os_type + "-" + os_version
cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close " + lock + " make -C " + d + network +
" SELFNAME=" + self_name + " KEY_NAME=" + key_name +
" PKG_REPO=" + pkg_repo + " SEC_GROUP=" + security_group +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + pkg_type + " CEPH_OS_TYPE=" + os_type +
" CEPH_OS_VERSION=" + os_version + " CEPH_DIST=" + dist +
" CEPH_ARCH=" + arch + " CEPH_SHA1=" + sha1 + " CEPH_TAG=" +
tag + " CEPH_BRANCH=" + branch + " CEPH_FLAVOR=" + flavor +
" BUILD_FLAVOR=" + build_flavor + " HTTP_FLAVOR=" +
http_flavor + " HTTP_ARCH=" + default_arch +
" BUILDPACKAGES_CANONICAL_TAGS=" +
("true" if teuth_config.canonical_tags else "false") + " " +
target + " ")
log.info("Executing the following make command to build {} packages. " \
"Note that some values in the command, like CEPH_GIT_URL " \
"and BUILDPACKAGES_CANONICAL_TAGS, may differ from similar " \
"command-line parameter values. This is because " \
"the values used by this task are taken from the teuthology " \
"configuration file. If in doubt, tear down your teuthology " \
"instance and start again from scratch.".format(pkg_type))
log.info("buildpackages make command: " + cmd)
misc.sh(cmd)
teuth_config.gitbuilder_host = openstack.get_ip(pkg_repo, '')
log.info('Finished buildpackages')
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
example:
tasks:
- buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
- install:
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
os_type = misc.get_distro(ctx)
os_version = misc.get_distro_version(ctx)
arch = ctx.config.get('arch', 'x86_64')
dist = LocalGitbuilderProject()._get_distro(distro=os_type,
version=os_version)
pkg_type = get_pkg_type(os_type)
misc.sh(
"flock --close /tmp/buildpackages " +
"make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent")
for (flavor, tag, branch, sha1) in lookup_configs(ctx, ctx.config):
if tag:
sha1 = get_sha1(tag)
elif branch:
sha1 = get_sha1(branch)
log.info("building flavor = " + flavor + "," +
" tag = " + tag + "," +
" branch = " + branch + "," +
" sha1 = " + sha1)
target = ('ceph-' +
pkg_type + '-' +
dist + '-' +
arch + '-' +
flavor + '-' +
sha1)
openstack = OpenStack()
openstack.set_provider()
if openstack.provider == 'ovh':
select = '^(vps|eg)-'
else:
select = ''
openstack.image(os_type, os_version) # create if it does not exist
build_flavor = openstack.flavor(config['machine'], select)
http_flavor = openstack.flavor({
'disk': 40, # GB
'ram': 1024, # MB
'cpus': 1,
}, select)
cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close /tmp/buildpackages-" + sha1 +
" make -C " + d +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + pkg_type +
" CEPH_OS_TYPE=" + os_type +
" CEPH_OS_VERSION=" + os_version +
" CEPH_DIST=" + dist +
" CEPH_ARCH=" + arch +
" CEPH_SHA1=" + sha1 +
" CEPH_TAG=" + tag +
" CEPH_BRANCH=" + branch +
" CEPH_FLAVOR=" + flavor +
" BUILD_FLAVOR=" + build_flavor +
" HTTP_FLAVOR=" + http_flavor +
" " + target +
" ")
log.info("buildpackages: " + cmd)
misc.sh(cmd)
teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
log.info('Finished buildpackages')
