def _deploy(self):
with in_dir(self.path):
run(['ansible-playbook',
'play-hadoop.yml',
'addons/spark.yml'
])
def image_delete(identifier):
"""Delete an image from glance
:param identifier: identifier of the image
"""
logger.info('Deleting image {}'.format(identifier))
cmd = ['glance', 'image-delete', identifier]
run(cmd)
def _deploy(self):
with in_dir(self.path):
run(['ansible-playbook',
'-f', str(self.node_count),
'play-hadoop.yml',
'addons/pig.yml', 'addons/spark.yml',
'deploy.yml'
])
def _fetch(self, prefix):
with in_dir(prefix):
reponame = os.path.basename(self.repo)
if not os.path.exists(reponame):
run(['git', 'clone', '--recursive', self.repo])
return os.path.join(os.getcwd(), reponame)
def _prepare(self):
with in_dir(self.path):
run(['virtualenv', 'venv'])
new_env = self.eval_bash(['deactivate',
'source venv/bin/activate'])
with use_env(**new_env):
run(['pip', 'install', '-r', 'requirements.txt', '-U'])
return new_env
def image_create(path, name,
public=True,
disk_format='qcow2',
container_format='bare'):
"""Create an image
:param path: path to image on disk
:param name: name of the image to create
:param public: enable public access
:param disk_format: the format of the image on disk
:param container_format: format of image on glance
:returns: identifier of the created image
:rtype: str
"""
logger.info('Creating image {name} from {path}'.format(**locals()))
cmd = ['glance', 'image-create',
'--is-public', str(public),
'--disk-format', disk_format,
'--container-format', container_format,
'--name', name,
'--file', path
]
output = run(cmd, capture='both').out
id = openstack_parse_show(output, 'id')
logger.info('Created image {}'.format(id))
return id
def image_download(identifier, path):
"""Download an openstack image to a file
:param identifier: the identifier of the image
:param path: path to download file
"""
logger.info('Downloading {identifier} to {path}'.format(**locals()))
active = partial(operator.eq, 'active')
wait_for_image_property(identifier, 'state', active)
cmd = ['glance', 'image-download',
'--file', path,
'--progress',
identifier]
run(cmd)
def _launch(self):
with in_dir(self.path):
run(['vcl', 'boot', '-p', 'openstack', '-P',
self.name_prefix])
for i in xrange(12):
result = run(['ansible', 'all', '-m', 'ping', '-o',
'-f', str(self.node_count)],
raises=False)
if result.ret == 0:
return
else:
time.sleep(5)
msg = 'Timed out when waiting for nodes to come online'
raise BenchmarkError(msg)
def show(identifier):
"""Show the details of an instance
:param identifier: the instance identifier
:returns: the output of 'nova show'
:rtype: string
"""
cmd = ['nova', 'show', identifier]
return run(cmd, capture='both').out
def _clean_openstack(self):
with in_dir(self.path):
result = run(['vcl', 'list'], capture='stdout',
raises=False)
node_names = map(str.strip, result.out.split())
node_names = ['%s%s' % (self.name_prefix, n) for n in
node_names]
cmd = ['nova', 'delete'] + node_names
run(cmd, raises=False)
while True:
nova_list = run(['nova', 'list', '--fields', 'name'],
capture='stdout')
present = any([n in nova_list.out for n in node_names])
if present:
time.sleep(5)
else:
break
def boot(image, name=None, keyname=None, flavor='m1.small'):
logger.info('Booting {image} as {name} key {keyname} and flavor {flavor}'
.format(**locals()))
cmd = ['nova', 'boot',
'--image', image,
'--flavor', flavor
]\
+ (['--key-name', keyname] if keyname else [])\
+ [name or str(uuid.uuid4())]
output = run(cmd, capture='both').out
id = openstack_parse_show(output, 'id')
return id
def image_create(identifier, name):
"""Create a snapshot of a running instance
:param identifier: instance identifier
:param name: name of the image to create
:returns: the identifier of the image created
:rtype: string
"""
logger.info('Creating snapshot of {identifier} as {name}'
.format(**locals()))
cmd = ['nova', 'image-create', '--show', identifier, name]
output = run(cmd, capture='both').out
uuid = openstack_parse_show(output, 'id')
logger.info('Created snapshot {uuid}'.format(uuid=uuid))
return uuid
def eval_bash(self, commands):
"""Return a new environment obtained by evaluating these commands
:param commands: commands to evaluate in bash
:type paths: :class:`list` of :class:`str`
:returns: the new environment
:rtype: :class:`dict` of :class:`str` to :class:`str`
"""
cmds = ['%s >/dev/null 2>&1' % c for c in commands]
cmds += ['env']
script = '\n'.join(cmds)
new_env = dict()
result = run(['bash', '-c', script], capture='stdout')
for l in result.out.split('\n'):
line = l.strip()
if not line: continue
if '=' not in line: continue
k, v = line.split('=', 1)
new_env[k] = v
return new_env
def eval_bash(self, commands):
"""Return a new environment obtained by evaluating these commands
:param commands: commands to evaluate in bash
:type paths: :class:`list` of :class:`str`
:returns: the new environment
:rtype: :class:`dict` of :class:`str` to :class:`str`
"""
cmds = ['%s >/dev/null 2>&1' % c for c in commands]
cmds += ['env']
script = '\n'.join(cmds)
new_env = dict()
result = run(['bash', '-c', script], capture='stdout')
for l in result.out.split('\n'):
line = l.strip()
if not line: continue
if '=' not in line: continue
k, v = line.split('=', 1)
new_env[k] = v
return new_env
def image_show(identifier):
"""Show the details of an instance image.
"""
cmd = ['nova', 'image-show', identifier]
return run(cmd, capture='both').out
def image_show(identifier):
cmd = ['glance', 'image-show', identifier]
return run(cmd, capture='both')
def _run(self):
with in_dir(self.path):
run(['ansible-playbook',
'-f', str(self.node_count),
'run.yml'])
def delete(identifier):
logger.info('Deleting instance {}'.format(identifier))
cmd = ['nova', 'delete', identifier]
return run(cmd, capture='both')
def _run(self):
with in_dir(self.path):
run(['ansible-playbook',
'hibench.yml'])
