def install_post(self):
hosts = request.json.get('hosts')
install_calamari = request.json.get('calamari', False)
verbose_ansible = request.json.get('verbose', False)
extra_vars = util.get_install_extra_vars(request.json)
extra_vars['calamari'] = install_calamari
identifier = str(uuid4())
task = models.Task(
request=request,
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(
extra_vars=extra_vars,
tags="package-install",
verbose=verbose_ansible,
)
call_ansible.apply_async(
args=([('mons', hosts)], identifier),
kwargs=kwargs,
)
return task
def install(component, hosts, identifier, tags="package-install", extra_vars=None):
"""
``component``: What component is it going to get installed: mon, rgw, osd, calamari
``hosts``: A list of hosts to install, these host must be resolvable
``tags``: The tags as a comma-delimeted string that represents all the tags
this ansible call should follow. For example "package-install, other-tag"
``identifier``: The UUID identifer for the task object so this function can capture process
metadata and persist it to the database
"""
if not extra_vars:
extra_vars = dict()
component_title = "%s%s" % (component, '' if component.endswith('s') else 's')
hosts_file = util.generate_inventory_file(component_title, hosts, identifier)
command = process.make_ansible_command(hosts_file, identifier, tags=tags, extra_vars=extra_vars)
task = models.Task.query.filter_by(identifier=identifier).first()
task.command = ' '.join(command)
task.started = datetime.now()
# force a commit here so we can reference this command later if it fails
models.commit()
out, err, exit_code = process.run(command)
task.succeeded = not exit_code
task.exit_code = exit_code
task.stdout = out
task.stderr = err
task.ended = datetime.now()
models.commit()
def configure_post(self):
hosts = [request.json['host']]
monitor_hosts = util.parse_monitors(request.json["monitors"])
verbose_ansible = request.json.get('verbose', False)
# even with configuring we need to tell ceph-ansible
# if we're working with upstream ceph or red hat ceph storage
extra_vars = util.get_osd_configure_extra_vars(request.json)
if 'verbose' in extra_vars:
del extra_vars['verbose']
if 'conf' in extra_vars:
extra_vars['ceph_conf_overrides'] = request.json['conf']
del extra_vars['conf']
identifier = str(uuid4())
task = models.Task(
request=request,
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(
extra_vars=extra_vars,
skip_tags="package-install",
playbook="infrastructure-playbooks/osd-configure.yml",
verbose=verbose_ansible,
)
call_ansible.apply_async(args=([('osds', hosts),
('mons', monitor_hosts)], identifier),
kwargs=kwargs)
return task
def install_post(self):
hosts = request.json.get('hosts')
install_calamari = request.json.get('calamari', False)
verbose_ansible = request.json.get('verbose', False)
extra_vars = util.get_install_extra_vars(request.json)
extra_vars['calamari'] = install_calamari
identifier = str(uuid4())
task = models.Task(
request=request,
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(
extra_vars=extra_vars,
tags="package-install",
verbose=verbose_ansible,
)
call_ansible.apply_async(
args=([('mons', hosts)], identifier),
kwargs=kwargs,
)
return task
def configure_post(self):
hosts = [request.json['host']]
monitor_hosts = util.parse_monitors(request.json["monitors"])
verbose_ansible = request.json.get('verbose', False)
# even with configuring we need to tell ceph-ansible
# if we're working with upstream ceph or red hat ceph storage
extra_vars = util.get_osd_configure_extra_vars(request.json)
if 'verbose' in extra_vars:
del extra_vars['verbose']
if 'conf' in extra_vars:
extra_vars['ceph_conf_overrides'] = request.json['conf']
del extra_vars['conf']
identifier = str(uuid4())
task = models.Task(
request=request,
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(
extra_vars=extra_vars,
skip_tags="package-install",
playbook="osd-configure.yml",
verbose=verbose_ansible,
)
call_ansible.apply_async(
args=([('osds', hosts), ('mons', monitor_hosts)], identifier),
kwargs=kwargs
)
return task
def connection(app, request):
"""Session-wide test database."""
# Connect and create the temporary database
print "=" * 80
print "CREATING TEMPORARY DATABASE FOR TESTS"
print "=" * 80
# Bind and create the database tables
_db.clear()
engine_url = '%s/%s' % (BIND, DBNAME)
db_engine = create_engine(
engine_url,
encoding='utf-8',
poolclass=NullPool)
# AKA models.start()
_db.Session.bind = db_engine
_db.metadata.bind = _db.Session.bind
_db.Base.metadata.create_all(db_engine)
_db.commit()
_db.clear()
#connection = db_engine.connect()
def teardown():
_db.Base.metadata.drop_all(db_engine)
request.addfinalizer(teardown)
# Slap our test app on it
_db.app = app
return _db
def call_ansible(inventory,
identifier,
tags="",
skip_tags="",
extra_vars=None,
playbook="site.yml.sample",
**kw):
"""
This task builds an ansible-playbook command and runs it.
``inventory``: A list of tuples that details an ansible inventory. For example:
[('mons', ['mon1.host', 'mon2.host']), ('osds', ['osd1.host'])]
``tags``: The tags as a comma-delimeted string that represents all the tags
this ansible call should follow. For example "package-install, other-tag"
``skip_tags``: The tags as a comma-delimeted string that represents all the tags
this ansible call should skip. For example "package-install, other-tag"
``identifier``: The UUID identifer for the task object so this function can capture process
metadata and persist it to the database
``verbose``: Optional keyword argument, to flag the need for increased verbosity
when running ansible
"""
verbose = kw.get('verbose', False)
if not extra_vars:
extra_vars = dict()
hosts_file = util.generate_inventory_file(inventory, identifier)
command = process.make_ansible_command(hosts_file,
identifier,
tags=tags,
extra_vars=extra_vars,
skip_tags=skip_tags,
playbook=playbook,
verbose=verbose)
task = models.Task.query.filter_by(identifier=identifier).first()
task.command = ' '.join(command)
task.started = datetime.now()
# force a commit here so we can reference this command later if it fails
models.commit()
working_dir = util.get_ceph_ansible_path()
# ansible depends on relative pathing to figure out how to load
# plugins, among other things. Setting the current working directory
# for this subprocess call to the directory where the playbook resides
# allows ansible to properly find action plugins defined in ceph-ansible.
kwargs = dict(cwd=working_dir)
try:
out, err, exit_code = process.run(command, **kwargs)
except Exception as error:
task.succeeded = False
task.exit_code = -1
task.stderr = str(error)
logger.exception('failed to run command')
else:
task.succeeded = not exit_code
task.exit_code = exit_code
task.stdout = out
task.stderr = err
task.ended = datetime.now()
models.commit()
def configure_post(self):
monitor_mapping = dict(host=request.json['host'])
# Only add interface and address if they exist
for key in ['interface', 'address']:
try:
monitor_mapping[key] = request.json[key]
except KeyError:
pass
hosts = util.parse_monitors([monitor_mapping])
verbose_ansible = request.json.get('verbose', False)
monitors = request.json.get("monitors", [])
monitors = util.validate_monitors(monitors, request.json["host"])
# even with configuring we need to tell ceph-ansible
# if we're working with upstream ceph or red hat ceph storage
extra_vars = util.get_install_extra_vars(request.json)
# this update will take everything in the ``request.json`` body and
# just pass it in as extra-vars. That is the reason why optional values
# like "calamari" are not looked up explicitly. If they are passed in
# they will be used.
extra_vars.update(request.json)
if 'verbose' in extra_vars:
del extra_vars['verbose']
if 'conf' in extra_vars:
extra_vars['ceph_conf_overrides'] = request.json['conf']
del extra_vars['conf']
if monitors:
hosts.extend(util.parse_monitors(monitors))
del extra_vars['monitors']
if "cluster_name" in extra_vars:
extra_vars["cluster"] = extra_vars["cluster_name"]
del extra_vars["cluster_name"]
del extra_vars['host']
extra_vars.pop('interface', None)
extra_vars.pop('address', None)
identifier = str(uuid4())
task = models.Task(
request=request,
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(
extra_vars=extra_vars,
skip_tags="package-install",
verbose=verbose_ansible,
)
call_ansible.apply_async(
args=([('mons', hosts)], identifier),
kwargs=kwargs,
)
return task
def configure_post(self):
monitor_mapping = dict(host=request.json['host'])
# Only add interface and address if they exist
for key in ['interface', 'address']:
try:
monitor_mapping[key] = request.json[key]
except KeyError:
pass
hosts = util.parse_monitors([monitor_mapping])
verbose_ansible = request.json.get('verbose', False)
monitors = request.json.get("monitors", [])
monitors = util.validate_monitors(monitors, request.json["host"])
# even with configuring we need to tell ceph-ansible
# if we're working with upstream ceph or red hat ceph storage
extra_vars = util.get_install_extra_vars(request.json)
# this update will take everything in the ``request.json`` body and
# just pass it in as extra-vars. That is the reason why optional values
# like "calamari" are not looked up explicitly. If they are passed in
# they will be used.
extra_vars.update(request.json)
if 'verbose' in extra_vars:
del extra_vars['verbose']
if 'conf' in extra_vars:
extra_vars['ceph_conf_overrides'] = request.json['conf']
del extra_vars['conf']
if monitors:
hosts.extend(util.parse_monitors(monitors))
del extra_vars['monitors']
if "cluster_name" in extra_vars:
extra_vars["cluster"] = extra_vars["cluster_name"]
del extra_vars["cluster_name"]
del extra_vars['host']
extra_vars.pop('interface', None)
extra_vars.pop('address', None)
identifier = str(uuid4())
task = models.Task(
request=request,
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(
extra_vars=extra_vars,
skip_tags="package-install",
verbose=verbose_ansible,
)
call_ansible.apply_async(
args=([('mons', hosts)], identifier),
kwargs=kwargs,
)
return task
def call_ansible(inventory, identifier, tags="", skip_tags="", extra_vars=None, playbook="site.yml.sample", **kw):
"""
This task builds an ansible-playbook command and runs it.
``inventory``: A list of tuples that details an ansible inventory. For example:
[('mons', ['mon1.host', 'mon2.host']), ('osds', ['osd1.host'])]
``tags``: The tags as a comma-delimeted string that represents all the tags
this ansible call should follow. For example "package-install, other-tag"
``skip_tags``: The tags as a comma-delimeted string that represents all the tags
this ansible call should skip. For example "package-install, other-tag"
``identifier``: The UUID identifer for the task object so this function can capture process
metadata and persist it to the database
``verbose``: Optional keyword argument, to flag the need for increased verbosity
when running ansible
"""
verbose = kw.get('verbose', False)
if not extra_vars:
extra_vars = dict()
hosts_file = util.generate_inventory_file(inventory, identifier)
command = process.make_ansible_command(
hosts_file, identifier, tags=tags, extra_vars=extra_vars,
skip_tags=skip_tags, playbook=playbook, verbose=verbose
)
task = models.Task.query.filter_by(identifier=identifier).first()
task.command = ' '.join(command)
task.started = datetime.now()
# force a commit here so we can reference this command later if it fails
models.commit()
working_dir = util.get_ceph_ansible_path()
# ansible depends on relative pathing to figure out how to load
# plugins, among other things. Setting the current working directory
# for this subprocess call to the directory where the playbook resides
# allows ansible to properly find action plugins defined in ceph-ansible.
kwargs = dict(cwd=working_dir)
try:
out, err, exit_code = process.run(command, **kwargs)
except Exception as error:
task.succeeded = False
task.exit_code = -1
task.stderr = str(error)
logger.exception('failed to run command')
else:
task.succeeded = not exit_code
task.exit_code = exit_code
task.stdout = out
task.stderr = err
task.ended = datetime.now()
models.commit()
def install_post(self):
hosts = request.json.get('hosts')
identifier = str(uuid4())
task = models.Task(
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
install.apply_async(
('rgw', hosts, identifier),
)
return task
def call_ansible(component,
hosts,
identifier,
tags="",
skip_tags="",
extra_vars=None):
"""
This task builds an ansible-playbook command and runs it.
``component``: What component we're going to work with: mon, rgw, osd, calamari
``hosts``: A list of hosts to run ansible against, these host must be resolvable
``tags``: The tags as a comma-delimeted string that represents all the tags
this ansible call should follow. For example "package-install, other-tag"
``skip_tags``: The tags as a comma-delimeted string that represents all the tags
this ansible call should skip. For example "package-install, other-tag"
``identifier``: The UUID identifer for the task object so this function can capture process
metadata and persist it to the database
"""
if not extra_vars:
extra_vars = dict()
component_title = "%s%s" % (component,
'' if component.endswith('s') else 's')
hosts_file = util.generate_inventory_file(component_title, hosts,
identifier)
command = process.make_ansible_command(hosts_file,
identifier,
tags=tags,
extra_vars=extra_vars,
skip_tags=skip_tags)
task = models.Task.query.filter_by(identifier=identifier).first()
task.command = ' '.join(command)
task.started = datetime.now()
# force a commit here so we can reference this command later if it fails
models.commit()
working_dir = util.get_playbook_path()
working_dir = os.path.split(working_dir)[0]
# ansible depends on relative pathing to figure out how to load
# plugins, among other things. Setting the current working directory
# for this subprocess call to the directory where the playbook resides
# allows ansible to properly find action plugins defined in ceph-ansible.
kwargs = dict(cwd=working_dir)
out, err, exit_code = process.run(command, **kwargs)
task.succeeded = not exit_code
task.exit_code = exit_code
task.stdout = out
task.stderr = err
task.ended = datetime.now()
models.commit()
def run(self, args):
super(PopulateCommand, self).run(args)
out("LOADING ENVIRONMENT")
self.load_app()
out("BUILDING SCHEMA")
try:
out("STARTING A TRANSACTION...")
models.start()
models.Base.metadata.create_all(conf.sqlalchemy.engine)
except:
models.rollback()
out("ROLLING BACK... ")
raise
else:
out("COMMITING... ")
models.commit()
def run(self, args):
super(PopulateCommand, self).run(args)
out("LOADING ENVIRONMENT")
self.load_app()
out("BUILDING SCHEMA")
try:
out("STARTING A TRANSACTION...")
models.start()
models.Base.metadata.create_all(conf.sqlalchemy.engine)
except:
models.rollback()
out("ROLLING BACK... ")
raise
else:
out("COMMITING... ")
models.commit()
def install_post(self):
hosts = request.json.get('hosts')
identifier = str(uuid4())
task = models.Task(
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(tags="package-install")
call_ansible.apply_async(
args=('rgw', hosts, identifier),
kwargs=kwargs,
)
return task
def install(self):
hosts = request.json.get('hosts')
extra_vars = util.get_install_extra_vars(request.json)
identifier = str(uuid4())
task = models.Task(
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(extra_vars=extra_vars)
call_ansible.apply_async(
args=('agent', hosts, identifier),
kwargs=kwargs,
)
return task
def install(self):
hosts = request.json.get('hosts')
extra_vars = util.get_install_extra_vars(request.json)
identifier = str(uuid4())
task = models.Task(
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(extra_vars=extra_vars)
call_ansible.apply_async(
args=('agent', hosts, identifier),
kwargs=kwargs,
)
return task
def configure_post(self):
hosts = [request.json['host']]
# even with configuring we need to tell ceph-ansible
# if we're working with upstream ceph or red hat ceph storage
verbose_ansible = request.json.get('verbose', False)
extra_vars = util.get_install_extra_vars(request.json)
monitor_hosts = util.parse_monitors(request.json["monitors"])
# this update will take everything in the ``request.json`` body and
# just pass it in as extra-vars. That is the reason why optional values
# like "calamari" are not looked up explicitly. If they are passed in
# they will be used.
extra_vars.update(request.json)
if 'verbose' in extra_vars:
del extra_vars['verbose']
if 'conf' in extra_vars:
extra_vars['ceph_conf_overrides'] = request.json['conf']
del extra_vars['conf']
if "cluster_name" in extra_vars:
extra_vars["cluster"] = extra_vars["cluster_name"]
del extra_vars["cluster_name"]
del extra_vars['host']
extra_vars.pop('interface', None)
extra_vars.pop('address', None)
identifier = str(uuid4())
task = models.Task(
request=request,
identifier=identifier,
playbook="infrastructure-playbooks/rgw-standalone.yml",
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(
extra_vars=extra_vars,
skip_tags="package-install",
verbose=verbose_ansible,
)
call_ansible.apply_async(
args=([('rgws', hosts), ('mons', monitor_hosts)], identifier),
kwargs=kwargs,
)
return task
def install(self):
master = request.json.get('master', request.server_name)
logger.info('defining "%s" as the master host for the minion configuration', master)
hosts = request.json.get('hosts')
verbose_ansible = request.json.get('verbose', False)
extra_vars = util.get_install_extra_vars(request.json)
extra_vars['agent_master_host'] = master
identifier = str(uuid4())
task = models.Task(
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(extra_vars=extra_vars, verbose=verbose_ansible)
call_ansible.apply_async(
args=([('agents', hosts)], identifier),
kwargs=kwargs,
)
return task
def call_ansible(component, hosts, identifier, tags="", skip_tags="", extra_vars=None):
"""
This task builds an ansible-playbook command and runs it.
``component``: What component we're going to work with: mon, rgw, osd, calamari
``hosts``: A list of hosts to run ansible against, these host must be resolvable
``tags``: The tags as a comma-delimeted string that represents all the tags
this ansible call should follow. For example "package-install, other-tag"
``skip_tags``: The tags as a comma-delimeted string that represents all the tags
this ansible call should skip. For example "package-install, other-tag"
``identifier``: The UUID identifer for the task object so this function can capture process
metadata and persist it to the database
"""
if not extra_vars:
extra_vars = dict()
component_title = "%s%s" % (component, '' if component.endswith('s') else 's')
hosts_file = util.generate_inventory_file(component_title, hosts, identifier)
command = process.make_ansible_command(hosts_file, identifier, tags=tags, extra_vars=extra_vars, skip_tags=skip_tags)
task = models.Task.query.filter_by(identifier=identifier).first()
task.command = ' '.join(command)
task.started = datetime.now()
# force a commit here so we can reference this command later if it fails
models.commit()
working_dir = util.get_playbook_path()
working_dir = os.path.split(working_dir)[0]
# ansible depends on relative pathing to figure out how to load
# plugins, among other things. Setting the current working directory
# for this subprocess call to the directory where the playbook resides
# allows ansible to properly find action plugins defined in ceph-ansible.
kwargs = dict(cwd=working_dir)
out, err, exit_code = process.run(command, **kwargs)
task.succeeded = not exit_code
task.exit_code = exit_code
task.stdout = out
task.stderr = err
task.ended = datetime.now()
models.commit()
def configure_post(self):
hosts = [request.json['host']]
# even with configuring we need to tell ceph-ansible
# if we're working with upstream ceph or red hat ceph storage
extra_vars = util.get_install_extra_vars(request.json)
extra_vars['monitor_interface'] = request.json['monitor_interface']
extra_vars['fsid'] = request.json['fsid']
if request.json.get('monitor_secret'):
extra_vars['monitor_secret'] = request.json.get('monitor_secret')
identifier = str(uuid4())
task = models.Task(
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(extra_vars=extra_vars, skip_tags="package-install")
call_ansible.apply_async(
args=('mon', hosts, identifier),
kwargs=kwargs,
)
return task
def install(self):
master = request.json.get('master', request.server_name)
logger.info(
'defining "%s" as the master host for the minion configuration',
master)
hosts = request.json.get('hosts')
verbose_ansible = request.json.get('verbose', False)
extra_vars = util.get_install_extra_vars(request.json)
extra_vars['agent_master_host'] = master
identifier = str(uuid4())
task = models.Task(
identifier=identifier,
endpoint=request.path,
)
# we need an explicit commit here because the command may finish before
# we conclude this request
models.commit()
kwargs = dict(extra_vars=extra_vars, verbose=verbose_ansible)
call_ansible.apply_async(
args=([('agents', hosts)], identifier),
kwargs=kwargs,
)
return task
def setup(self):
models.Task(
identifier='aaaa',
endpoint='/api/test/',
)
models.commit()
def setup(self):
models.Task(
identifier='aaaa',
endpoint='/api/test/',
)
models.commit()
