def test_flavor(self):
def get_sorted_flavors(self, arch, select):
return [
{
'Name': 'too_small',
'RAM': 2048,
'Disk': 50,
'VCPUs': 1,
},
]
with patch.multiple(
OpenStack,
get_sorted_flavors=get_sorted_flavors,
):
with pytest.raises(NoFlavorException):
hint = { 'ram': 1000, 'disk': 40, 'cpus': 2 }
OpenStack().flavor(hint, 'arch')
flavor = 'good-flavor'
def get_sorted_flavors(self, arch, select):
return [
{
'Name': flavor,
'RAM': 2048,
'Disk': 50,
'VCPUs': 2,
},
]
with patch.multiple(
OpenStack,
get_sorted_flavors=get_sorted_flavors,
):
hint = { 'ram': 1000, 'disk': 40, 'cpus': 2 }
assert flavor == OpenStack().flavor(hint, 'arch')
def stale_openstack(ctx):
targets = dict(map(lambda i: (i['ID'], i), OpenStack.list_instances()))
nodes = list_locks(keyed_by_name=True, locked=True)
stale_openstack_instances(ctx, targets, nodes)
stale_openstack_nodes(ctx, targets, nodes)
stale_openstack_volumes(ctx, OpenStack.list_volumes())
if not ctx.dry_run:
openstack_remove_again()
def test_cache_token_from_environment(self, m_sh):
OpenStack.token = None
o = OpenStack()
o.provider = 'ovh'
token = 'TOKEN VALUE'
os.environ['OS_TOKEN_VALUE'] = token
token_expires = int(time.time()) + OpenStack.token_cache_duration
os.environ['OS_TOKEN_EXPIRES'] = str(token_expires)
assert True == o.cache_token()
assert token == OpenStack.token
assert token_expires == OpenStack.token_expires
m_sh.assert_not_called()
def test_cache_token_from_environment(self, m_sh):
OpenStack.token = None
o = OpenStack()
o.provider = 'ovh'
token = 'TOKEN VALUE'
os.environ['OS_TOKEN_VALUE'] = token
token_expires = int(time.time()) + OpenStack.token_cache_duration
os.environ['OS_TOKEN_EXPIRES'] = str(token_expires)
assert True == o.cache_token()
assert token == OpenStack.token
assert token_expires == OpenStack.token_expires
m_sh.assert_not_called()
def test_interpret_hints(self):
defaults = {
'machine': {
'ram': 0,
'disk': 0,
'cpus': 0,
},
'volumes': {
'count': 0,
'size': 0,
},
}
expected_disk = 10 # first hint larger than the second
expected_ram = 20 # second hint larger than the first
expected_cpus = 0 # not set, hence zero by default
expected_count = 30 # second hint larger than the first
expected_size = 40 # does not exist in the first hint
hints = [
{
'machine': {
'ram': 2,
'disk': expected_disk,
},
'volumes': {
'count': 9,
'size': expected_size,
},
},
{
'machine': {
'ram': expected_ram,
'disk': 3,
},
'volumes': {
'count': expected_count,
},
},
]
hint = OpenStack().interpret_hints(defaults, hints)
assert hint == {
'machine': {
'ram': expected_ram,
'disk': expected_disk,
'cpus': expected_cpus,
},
'volumes': {
'count': expected_count,
'size': expected_size,
}
}
assert defaults == OpenStack().interpret_hints(defaults, None)
def test_cache_token_expired_environment(self, m_sh):
token = 'TOKEN VALUE'
m_sh.return_value = token
OpenStack.token = None
o = OpenStack()
o.provider = 'ovh'
os.environ['OS_TOKEN_VALUE'] = token
token_expires = int(time.time()) - 2000
os.environ['OS_TOKEN_EXPIRES'] = str(token_expires)
assert True == o.cache_token()
m_sh.assert_called_with('openstack -q token issue -c id -f value')
assert token == os.environ['OS_TOKEN_VALUE']
assert token == OpenStack.token
assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
assert time.time() < OpenStack.token_expires
def test_cache_token_expired_environment(self, m_sh):
token = 'TOKEN VALUE'
m_sh.return_value = token
OpenStack.token = None
o = OpenStack()
o.provider = 'ovh'
os.environ['OS_TOKEN_VALUE'] = token
token_expires = int(time.time()) - 2000
os.environ['OS_TOKEN_EXPIRES'] = str(token_expires)
assert True == o.cache_token()
m_sh.assert_called_with('openstack -q token issue -c id -f value')
assert token == os.environ['OS_TOKEN_VALUE']
assert token == OpenStack.token
assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
assert time.time() < OpenStack.token_expires
def stale_openstack_volumes(ctx, volumes):
now = datetime.datetime.now()
for volume in volumes:
volume_id = openstack_volume_id(volume)
try:
volume = json.loads(OpenStack().run("volume show -f json " +
volume_id))
except subprocess.CalledProcessError:
log.debug("stale-openstack: {id} disappeared, ignored".format(
id=volume_id))
continue
volume_name = openstack_volume_name(volume)
enforce_json_dictionary(volume)
created_at = datetime.datetime.strptime(volume['created_at'],
'%Y-%m-%dT%H:%M:%S.%f')
created = (now - created_at).total_seconds()
if created > config['max_job_time'] + OPENSTACK_DELAY:
log.info("stale-openstack: destroying volume {volume}({id})"
" because it was created {created} seconds ago"
" which is older than"
" max_job_time {max_job_time} + {delay}".format(
volume=volume_name,
id=volume_id,
created=created,
max_job_time=config['max_job_time'],
delay=OPENSTACK_DELAY))
if not ctx.dry_run:
openstack_delete_volume(volume_id)
continue
log.debug("stale-openstack: volume " + volume_id + " OK")
def test_get_provider(self):
auth = os.environ.get('OS_AUTH_URL', None)
os.environ['OS_AUTH_URL'] = 'cloud.ovh.net'
assert OpenStack().get_provider() == 'ovh'
if auth != None:
os.environ['OS_AUTH_URL'] = auth
else:
del os.environ['OS_AUTH_URL']
def test_cache_token(self, m_sh):
token = 'TOKEN VALUE'
m_sh.return_value = token
OpenStack.token = None
o = OpenStack()
#
# Only for OVH
#
o.provider = 'something'
assert False == o.cache_token()
o.provider = 'ovh'
#
# Set the environment with the token
#
assert 'OS_TOKEN_VALUE' not in os.environ
assert 'OS_TOKEN_EXPIRES' not in os.environ
assert True == o.cache_token()
m_sh.assert_called_with('openstack -q token issue -c id -f value')
assert token == os.environ['OS_TOKEN_VALUE']
assert token == OpenStack.token
assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
assert time.time() < OpenStack.token_expires
#
# Reset after it expires
#
token_expires = int(time.time()) - 2000
OpenStack.token_expires = token_expires
assert True == o.cache_token()
assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
assert time.time() < OpenStack.token_expires
def test_floating_ip(self):
if not self.can_create_floating_ips:
pytest.skip('unable to create floating ips')
expected = TeuthologyOpenStack.create_floating_ip()
ip = TeuthologyOpenStack.get_unassociated_floating_ip()
assert expected == ip
ip_id = TeuthologyOpenStack.get_floating_ip_id(ip)
OpenStack().run("ip floating delete " + ip_id)
def test_sorted_flavors(self, m_sh):
o = OpenStack()
select = '^(vps|hg)-.*ssd'
m_sh.return_value = TestOpenStack.flavors
flavors = o.get_sorted_flavors('arch', select)
assert [u'vps-ssd-1',
u'vps-ssd-2',
u'hg-7-ssd-flex',
u'hg-7-ssd',
u'vps-ssd-3',
u'hg-15-ssd-flex',
u'hg-15-ssd',
u'hg-30-ssd-flex',
u'hg-30-ssd',
u'hg-60-ssd-flex',
u'hg-60-ssd',
u'hg-120-ssd-flex',
u'hg-120-ssd',
] == [ f['Name'] for f in flavors ]
m_sh.assert_called_with("openstack --quiet flavor list -f json")
def test_sorted_flavors(self, m_sh):
o = OpenStack()
select = '^(vps|hg)-.*ssd'
m_sh.return_value = TestOpenStack.flavors
flavors = o.get_sorted_flavors('arch', select)
assert [u'vps-ssd-1',
u'vps-ssd-2',
u'hg-7-ssd-flex',
u'hg-7-ssd',
u'vps-ssd-3',
u'hg-15-ssd-flex',
u'hg-15-ssd',
u'hg-30-ssd-flex',
u'hg-30-ssd',
u'hg-60-ssd-flex',
u'hg-60-ssd',
u'hg-120-ssd-flex',
u'hg-120-ssd',
] == [ f['Name'] for f in flavors ]
m_sh.assert_called_with("openstack --quiet flavor list -f json")
def test_cache_token(self, m_sh):
token = 'TOKEN VALUE'
m_sh.return_value = token
OpenStack.token = None
o = OpenStack()
#
# Only for OVH
#
o.provider = 'something'
assert False == o.cache_token()
o.provider = 'ovh'
#
# Set the environment with the token
#
assert 'OS_TOKEN_VALUE' not in os.environ
assert 'OS_TOKEN_EXPIRES' not in os.environ
assert True == o.cache_token()
m_sh.assert_called_with('openstack -q token issue -c id -f value')
assert token == os.environ['OS_TOKEN_VALUE']
assert token == OpenStack.token
assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
assert time.time() < OpenStack.token_expires
#
# Reset after it expires
#
token_expires = int(time.time()) - 2000
OpenStack.token_expires = token_expires
assert True == o.cache_token()
assert time.time() < int(os.environ['OS_TOKEN_EXPIRES'])
assert time.time() < OpenStack.token_expires
def setup_class(self):
if 'OS_AUTH_URL' not in os.environ:
pytest.skip('no OS_AUTH_URL environment variable')
teuthology.log.setLevel(logging.DEBUG)
set_config_attr(argparse.Namespace())
ip = TeuthologyOpenStack.create_floating_ip()
if ip:
ip_id = TeuthologyOpenStack.get_floating_ip_id(ip)
OpenStack().run("ip floating delete " + ip_id)
self.can_create_floating_ips = True
else:
self.can_create_floating_ips = False
def openstack_remove_again():
"""
Volumes and servers with REMOVE-ME in the name are leftover
that failed to be removed. It is not uncommon for a failed removal
to succeed later on.
"""
sh("""
openstack server list --name REMOVE-ME --column ID --format value |
xargs --no-run-if-empty --max-args 1 -P20 openstack server delete --wait
true
""")
volumes = json.loads(OpenStack().run("volume list -f json --long"))
remove_me = [openstack_volume_id(v) for v in volumes
if 'REMOVE-ME' in openstack_volume_name(v)]
for i in remove_me:
log.info("Trying to remove stale volume %s" % i)
openstack_delete_volume(i)
def test_get_ip_neutron(self):
instance_id = '8e1fd70a-3065-46f8-9c30-84dc028c1834'
ip = '10.10.10.4'
def sh(cmd):
if 'neutron subnet-list' in cmd:
return """
[
{
"ip_version": 6,
"id": "c45b9661-b2ba-4817-9e3a-f8f63bf32989"
},
{
"ip_version": 4,
"id": "e03a3dbc-afc8-4b52-952e-7bf755397b50"
}
]
"""
elif 'neutron port-list' in cmd:
return ("""
[
{
"device_id": "915504ad-368b-4cce-be7c-4f8a83902e28",
"fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"10.10.10.1\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc::1\\"}"
},
{
"device_id": "{instance_id}",
"fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"{ip}\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc:f816:3eff:fe07:76c1\\"}"
},
{
"device_id": "17e4a968-4caa-4cee-8e4b-f950683a02bd",
"fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"10.10.10.5\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc:f816:3eff:fe9c:37f0\\"}"
}
]
""".replace('{instance_id}', instance_id).replace('{ip}', ip))
else:
raise Exception("unexpected " + cmd)
with patch.multiple(
misc,
sh=sh,
):
assert ip == OpenStack.get_ip_neutron(instance_id)
def test_get_ip_neutron(self):
instance_id = '8e1fd70a-3065-46f8-9c30-84dc028c1834'
ip = '10.10.10.4'
def sh(cmd):
if 'neutron subnet-list' in cmd:
return """
[
{
"ip_version": 6,
"id": "c45b9661-b2ba-4817-9e3a-f8f63bf32989"
},
{
"ip_version": 4,
"id": "e03a3dbc-afc8-4b52-952e-7bf755397b50"
}
]
"""
elif 'neutron port-list' in cmd:
return ("""
[
{
"device_id": "915504ad-368b-4cce-be7c-4f8a83902e28",
"fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"10.10.10.1\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc::1\\"}"
},
{
"device_id": "{instance_id}",
"fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"{ip}\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc:f816:3eff:fe07:76c1\\"}"
},
{
"device_id": "17e4a968-4caa-4cee-8e4b-f950683a02bd",
"fixed_ips": "{\\"subnet_id\\": \\"e03a3dbc-afc8-4b52-952e-7bf755397b50\\", \\"ip_address\\": \\"10.10.10.5\\"}\\n{\\"subnet_id\\": \\"c45b9661-b2ba-4817-9e3a-f8f63bf32989\\", \\"ip_address\\": \\"2607:f298:6050:9afc:f816:3eff:fe9c:37f0\\"}"
}
]
""".replace('{instance_id}', instance_id).
replace('{ip}', ip))
else:
raise Exception("unexpected " + cmd)
with patch.multiple(
misc,
sh=sh,
):
assert ip == OpenStack.get_ip_neutron(instance_id)
def test_get_os_url(self):
o = OpenStack()
#
# Only for OVH
#
o.provider = 'something'
assert "" == o.get_os_url("server ")
o.provider = 'ovh'
assert "" == o.get_os_url("unknown ")
type2cmd = {
'compute': ('server', 'flavor'),
'network': ('ip', 'security', 'network'),
'image': ('image',),
'volume': ('volume',),
}
os.environ['OS_REGION_NAME'] = 'REGION'
os.environ['OS_TENANT_ID'] = 'TENANT'
for (type, cmds) in type2cmd.iteritems():
for cmd in cmds:
assert ("//" + type) in o.get_os_url(cmd + " ")
for type in type2cmd.keys():
assert ("//" + type) in o.get_os_url("whatever ", type=type)
def attach_volumes(self, server_name, volumes):
"""
Create and attach volumes to the named OpenStack instance.
If attachment is failed, make another try.
"""
for i in range(volumes['count']):
volume_name = server_name + '-' + str(i)
volume_id = None
with safe_while(sleep=10, tries=3,
action="volume " + volume_name) as proceed:
while proceed():
try:
volume_id = self._create_volume(volume_name, volumes['size'])
self._await_volume_status(volume_id, 'available')
self._attach_volume(volume_id, server_name)
break
except Exception as e:
log.warning("%s" % e)
if volume_id:
OpenStack().volume_delete(volume_id)
def attach_volumes(self, name, volumes):
"""
Create and attach volumes to the named OpenStack instance.
"""
for i in range(volumes['count']):
volume_name = name + '-' + str(i)
try:
self.run("volume show -f json " + volume_name)
except subprocess.CalledProcessError as e:
if 'No volume with a name or ID' not in e.output:
raise e
# do not use OpenStack().run because its
# bugous for volume create as of openstackclient 3.2.0
# https://bugs.launchpad.net/python-openstackclient/+bug/1619726
misc.sh(
"openstack volume create -f json " +
config['openstack'].get('volume-create', '') + " " +
" --property ownedby=" + config.openstack['ip'] +
" --size " + str(volumes['size']) + " " +
volume_name)
with safe_while(sleep=2, tries=100,
action="volume " + volume_name) as proceed:
while proceed():
try:
r = OpenStack().run("volume show -f json " +
volume_name)
status = self.get_value(json.loads(r), 'status')
if status == 'available':
break
else:
log.info("volume " + volume_name +
" not available yet")
except subprocess.CalledProcessError:
log.info("volume " + volume_name +
" not information available yet")
# do not use OpenStack().run because its
# bugous for volume
misc.sh("openstack server add volume " + name + " " + volume_name)
def test_get_os_url(self):
o = OpenStack()
#
# Only for OVH
#
o.provider = 'something'
assert "" == o.get_os_url("server ")
o.provider = 'ovh'
assert "" == o.get_os_url("unknown ")
type2cmd = {
'compute': ('server', 'flavor'),
'network': ('ip', 'security', 'network'),
'image': ('image',),
'volume': ('volume',),
}
os.environ['OS_REGION_NAME'] = 'REGION'
os.environ['OS_TENANT_ID'] = 'TENANT'
for (type, cmds) in type2cmd.iteritems():
for cmd in cmds:
assert ("//" + type) in o.get_os_url(cmd + " ")
for type in type2cmd.keys():
assert ("//" + type) in o.get_os_url("whatever ", type=type)
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
example:
tasks:
- buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
- install:
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
for remote in ctx.cluster.remotes.iterkeys():
for install_config in lookup_configs(ctx, ctx.config):
gitbuilder = install._get_gitbuilder_project(
ctx, remote, install_config)
(tag, branch, sha1) = get_tag_branch_sha1(gitbuilder)
check_call(
"flock --close /tmp/buildpackages " +
"make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent",
shell=True)
url = gitbuilder.base_url
target = os.path.dirname(urlparse.urlparse(url).path.strip('/'))
target = os.path.dirname(target) + '-' + sha1
openstack = OpenStack()
if 'cloud.ovh.net' in os.environ['OS_AUTH_URL']:
select = '^(vps|eg)-'
else:
select = ''
build_flavor = openstack.flavor(config['machine'], select)
http_flavor = openstack.flavor({
'disk': 10, # GB
'ram': 1024, # MB
'cpus': 1,
}, select)
cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close /tmp/buildpackages-" + sha1 +
" make -C " + d +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + gitbuilder.pkg_type +
" CEPH_OS_TYPE=" + gitbuilder.os_type +
# os_version is from the remote and will be 7.1.23 for CentOS 7
# instead of the expected 7.0 for all 7.* CentOS
" CEPH_OS_VERSION=" + gitbuilder._get_version() +
" CEPH_DIST=" + gitbuilder.distro +
" CEPH_ARCH=" + gitbuilder.arch +
" CEPH_SHA1=" + sha1 +
" CEPH_TAG=" + (tag or '') +
" CEPH_BRANCH=" + (branch or '') +
" GITBUILDER_URL=" + url +
" BUILD_FLAVOR=" + build_flavor +
" HTTP_FLAVOR=" + http_flavor +
" " + target +
" ")
log.info("buildpackages: " + cmd)
check_call(cmd, shell=True)
teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
log.info('Finished buildpackages')
def test_flavor_range(self):
flavors = [
{
'Name': 'too_small',
'RAM': 2048,
'Disk': 50,
'VCPUs': 1,
},
]
def get_sorted_flavors(self, arch, select):
return flavors
min = { 'ram': 1000, 'disk': 40, 'cpus': 2 }
good = { 'ram': 4000, 'disk': 40, 'cpus': 2 }
#
# there are no flavors in the required range
#
with patch.multiple(
OpenStack,
get_sorted_flavors=get_sorted_flavors,
):
with pytest.raises(NoFlavorException):
OpenStack().flavor_range(min, good, 'arch')
#
# there is one flavor in the required range
#
flavors.append({
'Name': 'min',
'RAM': 2048,
'Disk': 40,
'VCPUs': 2,
})
with patch.multiple(
OpenStack,
get_sorted_flavors=get_sorted_flavors,
):
assert 'min' == OpenStack().flavor_range(min, good, 'arch')
#
# out of the two flavors in the required range, get the bigger one
#
flavors.append({
'Name': 'good',
'RAM': 3000,
'Disk': 40,
'VCPUs': 2,
})
with patch.multiple(
OpenStack,
get_sorted_flavors=get_sorted_flavors,
):
assert 'good' == OpenStack().flavor_range(min, good, 'arch')
#
# there is one flavor bigger or equal to good, get this one
#
flavors.append({
'Name': 'best',
'RAM': 4000,
'Disk': 40,
'VCPUs': 2,
})
with patch.multiple(
OpenStack,
get_sorted_flavors=get_sorted_flavors,
):
assert 'best' == OpenStack().flavor_range(min, good, 'arch')
#
# there are two flavors bigger or equal to good, get the smallest one
#
flavors.append({
'Name': 'too_big',
'RAM': 30000,
'Disk': 400,
'VCPUs': 20,
})
with patch.multiple(
OpenStack,
get_sorted_flavors=get_sorted_flavors,
):
assert 'best' == OpenStack().flavor_range(min, good, 'arch')
def openstack_delete_volume(id):
OpenStack().run("volume delete " + id + " || true")
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
example:
tasks:
- buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
- install:
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
for remote in ctx.cluster.remotes.iterkeys():
for install_config in lookup_configs(ctx, ctx.config):
gitbuilder = install._get_gitbuilder_project(
ctx, remote, install_config)
(tag, branch, sha1) = get_tag_branch_sha1(gitbuilder)
check_call("flock --close /tmp/buildpackages " + "make -C " + d +
" " + os.environ['HOME'] + "/.ssh_agent",
shell=True)
url = gitbuilder.base_url
target = os.path.dirname(urlparse.urlparse(url).path.strip('/'))
target = os.path.dirname(target) + '-' + sha1
openstack = OpenStack()
if 'cloud.ovh.net' in os.environ['OS_AUTH_URL']:
select = '^(vps|eg)-'
else:
select = ''
build_flavor = openstack.flavor(config['machine'], select)
http_flavor = openstack.flavor(
{
'disk': 10, # GB
'ram': 1024, # MB
'cpus': 1,
},
select)
cmd = (
". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close /tmp/buildpackages-" + sha1 + " make -C " + d +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + gitbuilder.pkg_type + " CEPH_OS_TYPE=" +
gitbuilder.os_type +
# os_version is from the remote and will be 7.1.23 for CentOS 7
# instead of the expected 7.0 for all 7.* CentOS
" CEPH_OS_VERSION=" + gitbuilder._get_version() +
" CEPH_DIST=" + gitbuilder.distro + " CEPH_ARCH=" +
gitbuilder.arch + " CEPH_SHA1=" + sha1 + " CEPH_TAG=" +
(tag or '') + " CEPH_BRANCH=" + (branch or '') +
" CEPH_FLAVOR=" + gitbuilder.flavor + " GITBUILDER_URL=" +
url + " BUILD_FLAVOR=" + build_flavor + " HTTP_FLAVOR=" +
http_flavor + " " + target + " ")
log.info("buildpackages: " + cmd)
check_call(cmd, shell=True)
teuth_config.gitbuilder_host = openstack.get_ip(
'packages-repository', '')
log.info('Finished buildpackages')
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
good_machine:
disk: 40 # GB
ram: 48000 # MB
cpus: 16
min_machine:
disk: 40 # GB
ram: 8000 # MB
cpus: 1
example:
tasks:
- buildpackages:
good_machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
min_machine:
disk: 40 # GB
ram: 8000 # MB
cpus: 1
- install:
When a buildpackages task is already included, the values it contains can be
overriden with:
overrides:
buildpackages:
good_machine:
disk: 20 # GB
ram: 2000 # MB
cpus: 2
min_machine:
disk: 10 # GB
ram: 1000 # MB
cpus: 1
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
overrides = ctx.config.get('overrides', {})
misc.deep_merge(config, overrides.get('buildpackages', {}))
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
os_type = misc.get_distro(ctx)
os_version = misc.get_distro_version(ctx)
arch = ctx.config.get('arch', OpenStack().get_default_arch())
dist = LocalGitbuilderProject()._get_distro(distro=os_type,
version=os_version)
pkg_type = get_pkg_type(os_type)
misc.sh(
"flock --close /tmp/buildpackages " +
"make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent")
for (flavor, tag, branch, sha1) in lookup_configs(ctx, ctx.config):
if tag:
sha1 = get_sha1(tag)
elif branch:
sha1 = get_sha1(branch)
log.info("building flavor = " + flavor + "," +
" tag = " + tag + "," +
" branch = " + branch + "," +
" sha1 = " + sha1)
target = ('ceph-' +
pkg_type + '-' +
dist + '-' +
arch + '-' +
flavor + '-' +
sha1)
openstack = OpenStack()
openstack.set_provider()
if openstack.provider == 'ovh':
select = '^(vps|hg)-.*ssd'
else:
select = ''
network = openstack.net()
if network != "":
network = " OPENSTACK_NETWORK='" + network + "' "
openstack.image(os_type, os_version, arch) # create if it does not exist
build_flavor = openstack.flavor_range(
config['min_machine'], config['good_machine'], arch, select)
default_arch = openstack.get_default_arch()
http_flavor = openstack.flavor({
'disk': 30, # GB
'ram': 1024, # MB
'cpus': 1,
}, default_arch, select)
lock = "/tmp/buildpackages-" + sha1 + "-" + os_type + "-" + os_version
cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close " + lock +
" make -C " + d +
network +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + pkg_type +
" CEPH_OS_TYPE=" + os_type +
" CEPH_OS_VERSION=" + os_version +
" CEPH_DIST=" + dist +
" CEPH_ARCH=" + arch +
" CEPH_SHA1=" + sha1 +
" CEPH_TAG=" + tag +
" CEPH_BRANCH=" + branch +
" CEPH_FLAVOR=" + flavor +
" BUILD_FLAVOR=" + build_flavor +
" HTTP_FLAVOR=" + http_flavor +
" HTTP_ARCH=" + default_arch +
" " + target +
" ")
log.info("buildpackages: " + cmd)
misc.sh(cmd)
teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
log.info('Finished buildpackages')
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
example:
tasks:
- buildpackages:
machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
- install:
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
os_type = misc.get_distro(ctx)
os_version = misc.get_distro_version(ctx)
arch = ctx.config.get('arch', 'x86_64')
dist = LocalGitbuilderProject()._get_distro(distro=os_type,
version=os_version)
pkg_type = get_pkg_type(os_type)
misc.sh(
"flock --close /tmp/buildpackages " +
"make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent")
for (flavor, tag, branch, sha1) in lookup_configs(ctx, ctx.config):
if tag:
sha1 = get_sha1(tag)
elif branch:
sha1 = get_sha1(branch)
log.info("building flavor = " + flavor + "," +
" tag = " + tag + "," +
" branch = " + branch + "," +
" sha1 = " + sha1)
target = ('ceph-' +
pkg_type + '-' +
dist + '-' +
arch + '-' +
flavor + '-' +
sha1)
openstack = OpenStack()
openstack.set_provider()
if openstack.provider == 'ovh':
select = '^(vps|eg)-'
else:
select = ''
openstack.image(os_type, os_version) # create if it does not exist
build_flavor = openstack.flavor(config['machine'], select)
http_flavor = openstack.flavor({
'disk': 40, # GB
'ram': 1024, # MB
'cpus': 1,
}, select)
cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close /tmp/buildpackages-" + sha1 +
" make -C " + d +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + pkg_type +
" CEPH_OS_TYPE=" + os_type +
" CEPH_OS_VERSION=" + os_version +
" CEPH_DIST=" + dist +
" CEPH_ARCH=" + arch +
" CEPH_SHA1=" + sha1 +
" CEPH_TAG=" + tag +
" CEPH_BRANCH=" + branch +
" CEPH_FLAVOR=" + flavor +
" BUILD_FLAVOR=" + build_flavor +
" HTTP_FLAVOR=" + http_flavor +
" " + target +
" ")
log.info("buildpackages: " + cmd)
misc.sh(cmd)
teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
log.info('Finished buildpackages')
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
good_machine:
disk: 40 # GB
ram: 48000 # MB
cpus: 16
min_machine:
disk: 40 # GB
ram: 8000 # MB
cpus: 1
example:
tasks:
- buildpackages:
good_machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
min_machine:
disk: 40 # GB
ram: 8000 # MB
cpus: 1
- install:
When a buildpackages task is already included, the values it contains can be
overriden with:
overrides:
buildpackages:
good_machine:
disk: 20 # GB
ram: 2000 # MB
cpus: 2
min_machine:
disk: 10 # GB
ram: 1000 # MB
cpus: 1
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
overrides = ctx.config.get('overrides', {})
misc.deep_merge(config, overrides.get('buildpackages', {}))
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
os_type = misc.get_distro(ctx)
os_version = misc.get_distro_version(ctx)
arch = ctx.config.get('arch', OpenStack().get_default_arch())
dist = LocalGitbuilderProject()._get_distro(distro=os_type,
version=os_version)
pkg_type = get_pkg_type(os_type)
misc.sh("flock --close /tmp/buildpackages " + "make -C " + d + " " +
os.environ['HOME'] + "/.ssh_agent")
for (flavor, tag, branch, sha1) in lookup_configs(ctx, ctx.config):
if tag:
sha1 = get_sha1(tag)
elif branch:
sha1 = get_sha1(branch)
log.info("building flavor = " + flavor + "," + " tag = " + tag + "," +
" branch = " + branch + "," + " sha1 = " + sha1)
self_name = 'teuthology'
key_name = 'teuthology'
pkg_repo = 'packages-repository'
security_group = 'teuthology'
if teuth_config.openstack.has_key('selfname'):
self_name = teuth_config.openstack['selfname']
if teuth_config.openstack.has_key('keypair'):
key_name = teuth_config.openstack['keypair']
if teuth_config.openstack.has_key('package_repo'):
pkg_repo = teuth_config.openstack['package_repo']
if teuth_config.openstack.has_key('server_group'):
security_group = teuth_config.openstack['server_group']
target = (self_name + '-ceph-' + pkg_type + '-' + dist + '-' + arch +
'-' + flavor + '-' + sha1)
openstack = OpenStack()
openstack.set_provider()
network = openstack.net()
if network != "":
network = " OPENSTACK_NETWORK='" + network + "' "
openstack.image(os_type, os_version,
arch) # create if it does not exist
build_flavor = openstack.flavor_range(config['min_machine'],
config['good_machine'], arch)
default_arch = openstack.get_default_arch()
http_flavor = openstack.flavor(
{
'disk': 30, # GB
'ram': 1024, # MB
'cpus': 1,
},
default_arch)
lock = "/tmp/buildpackages-" + sha1 + "-" + os_type + "-" + os_version
cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close " + lock + " make -C " + d + network +
" SELFNAME=" + self_name + " KEY_NAME=" + key_name +
" PKG_REPO=" + pkg_repo + " SEC_GROUP=" + security_group +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + pkg_type + " CEPH_OS_TYPE=" + os_type +
" CEPH_OS_VERSION=" + os_version + " CEPH_DIST=" + dist +
" CEPH_ARCH=" + arch + " CEPH_SHA1=" + sha1 + " CEPH_TAG=" +
tag + " CEPH_BRANCH=" + branch + " CEPH_FLAVOR=" + flavor +
" BUILD_FLAVOR=" + build_flavor + " HTTP_FLAVOR=" +
http_flavor + " HTTP_ARCH=" + default_arch +
" BUILDPACKAGES_CANONICAL_TAGS=" +
("true" if teuth_config.canonical_tags else "false") + " " +
target + " ")
log.info("Executing the following make command to build {} packages. " \
"Note that some values in the command, like CEPH_GIT_URL " \
"and BUILDPACKAGES_CANONICAL_TAGS, may differ from similar " \
"command-line parameter values. This is because " \
"the values used by this task are taken from the teuthology " \
"configuration file. If in doubt, tear down your teuthology " \
"instance and start again from scratch.".format(pkg_type))
log.info("buildpackages make command: " + cmd)
misc.sh(cmd)
teuth_config.gitbuilder_host = openstack.get_ip(pkg_repo, '')
log.info('Finished buildpackages')
