def setup_profiler(binary, host):
if (osprofiler_notifier is None or profiler is None
or osprofiler_web is None or profiler_opts is None):
LOG.debug('osprofiler is not present')
return
if CONF.profiler.enabled:
_notifier = osprofiler_notifier.create(
"Messaging", messaging,
context.get_admin_context().to_dict(), rpc.TRANSPORT, "waterfall",
binary, host)
osprofiler_notifier.set(_notifier)
osprofiler_web.enable(CONF.profiler.hmac_keys)
LOG.warning(
_LW("OSProfiler is enabled.\nIt means that person who knows "
"any of hmac_keys that are specified in "
"/etc/waterfall/waterfall.conf can trace his requests. \n"
"In real life only operator can read this file so there "
"is no security issue. Note that even if person can "
"trigger profiler, only admin user can retrieve trace "
"information.\n"
"To disable OSprofiler set in waterfall.conf:\n"
"[profiler]\nenabled=false"))
else:
osprofiler_web.disable()
def purge(self, age_in_days):
"""Purge deleted rows older than a given age from waterfall tables."""
age_in_days = int(age_in_days)
if age_in_days <= 0:
print(_("Must supply a positive, non-zero value for age"))
sys.exit(1)
ctxt = context.get_admin_context()
db.purge_deleted_rows(ctxt, age_in_days)
def update_backup_host(self, currenthost, newhost):
"""Modify the host name associated with a backup.
Particularly to recover from cases where one has moved
their Waterfall Backup node, and not set backup_use_same_backend.
"""
ctxt = context.get_admin_context()
backups = objects.BackupList.get_all_by_host(ctxt, currenthost)
for bk in backups:
bk.host = newhost
bk.save()
def update_host(self, currenthost, newhost):
"""Modify the host name associated with a workflow.
Particularly to recover from cases where one has moved
their Waterfall Workflow node, or modified their backend_name in a
multi-backend config.
"""
ctxt = context.get_admin_context()
workflows = db.workflow_get_all_by_host(ctxt, currenthost)
for v in workflows:
db.workflow_update(ctxt, v['id'], {'host': newhost})
def __init__(self,
host,
binary,
topic,
manager,
report_interval=None,
periodic_interval=None,
periodic_fuzzy_delay=None,
service_name=None,
*args,
**kwargs):
super(Service, self).__init__()
if not rpc.initialized():
rpc.init(CONF)
self.host = host
self.binary = binary
self.topic = topic
self.manager_class_name = manager
manager_class = importutils.import_class(self.manager_class_name)
if CONF.profiler.enabled:
manager_class = profiler.trace_cls("rpc")(manager_class)
# NOTE(geguileo): We need to create the Service DB entry before we
# create the manager, otherwise capped versions for serializer and rpc
# client would used existing DB entries not including us, which could
# result in us using None (if it's the first time the service is run)
# or an old version (if this is a normal upgrade of a single service).
ctxt = context.get_admin_context()
#try:
# service_ref = objects.Service.get_by_args(ctxt, host, binary)
# service_ref.rpc_current_version = manager_class.RPC_API_VERSION
# obj_version = objects_base.OBJ_VERSIONS.get_current()
# service_ref.object_current_version = obj_version
# service_ref.save()
# self.service_id = service_ref.id
#except exception.NotFound:
# self._create_service_ref(ctxt, manager_class.RPC_API_VERSION)
self.manager = manager_class(host=self.host,
service_name=service_name,
*args,
**kwargs)
self.report_interval = report_interval
self.periodic_interval = periodic_interval
self.periodic_fuzzy_delay = periodic_fuzzy_delay
self.basic_config_check()
self.saved_args, self.saved_kwargs = args, kwargs
self.timers = []
setup_profiler(binary, host)
self.rpcserver = None
def report_state(self):
"""Update the state of this service in the datastore."""
if not self.manager.is_working():
# NOTE(dulek): If manager reports a problem we're not sending
# heartbeats - to indicate that service is actually down.
LOG.error(
_LE('Manager for service %(binary)s %(host)s is '
'reporting problems, not sending heartbeat. '
'Service will appear "down".'), {
'binary': self.binary,
'host': self.host
})
return
ctxt = context.get_admin_context()
zone = CONF.storage_availability_zone
try:
try:
service_ref = objects.Service.get_by_id(ctxt, self.service_id)
except exception.NotFound:
LOG.debug('The service database object disappeared, '
'recreating it.')
self._create_service_ref(ctxt)
service_ref = objects.Service.get_by_id(ctxt, self.service_id)
service_ref.report_count += 1
if zone != service_ref.availability_zone:
service_ref.availability_zone = zone
service_ref.save()
# TODO(termie): make this pattern be more elegant.
if getattr(self, 'model_disconnected', False):
self.model_disconnected = False
LOG.error(_LE('Recovered model server connection!'))
except db_exc.DBConnectionError:
if not getattr(self, 'model_disconnected', False):
self.model_disconnected = True
LOG.exception(_LE('model server went away'))
# NOTE(jsbryant) Other DB errors can happen in HA configurations.
# such errors shouldn't kill this thread, so we handle them here.
except db_exc.DBError:
if not getattr(self, 'model_disconnected', False):
self.model_disconnected = True
LOG.exception(_LE('DBError encountered: '))
except Exception:
if not getattr(self, 'model_disconnected', False):
self.model_disconnected = True
LOG.exception(_LE('Exception encountered: '))
def remove(self, binary, host_name):
"""Completely removes a service."""
ctxt = context.get_admin_context()
try:
svc = objects.Service.get_by_args(ctxt, host_name, binary)
svc.destroy()
except exception.ServiceNotFound as e:
print(
_("Host not found. Failed to remove %(service)s"
" on %(host)s.") % {
'service': binary,
'host': host_name
})
print(u"%s" % e.args)
return 2
print(
_("Service %(service)s on host %(host)s removed.") % {
'service': binary,
'host': host_name
})
def delete(self, workflow_id):
"""Delete a workflow, bypassing the check that it must be available."""
ctxt = context.get_admin_context()
workflow = objects.Workflow.get_by_id(ctxt, workflow_id)
host = vutils.extract_host(workflow.host) if workflow.host else None
if not host:
print(_("Workflow not yet assigned to host."))
print(_("Deleting workflow from database and skipping rpc."))
workflow.destroy()
return
if workflow.status == 'in-use':
print(_("Workflow is in-use."))
print(_("Detach workflow from instance and then try again."))
return
cctxt = self._rpc_client().prepare(server=host)
cctxt.cast(ctxt,
"delete_workflow",
workflow_id=workflow.id,
workflow=workflow)
def list(self, zone=None):
"""Show a list of all physical hosts.
Can be filtered by zone.
args: [zone]
"""
print(_("%(host)-25s\t%(zone)-15s") % {'host': 'host', 'zone': 'zone'})
ctxt = context.get_admin_context()
services = objects.ServiceList.get_all(ctxt)
if zone:
services = [s for s in services if s.availability_zone == zone]
hosts = []
for srv in services:
if not [h for h in hosts if h['host'] == srv['host']]:
hosts.append(srv)
for h in hosts:
print(
_("%(host)-25s\t%(availability_zone)-15s") % {
'host': h['host'],
'availability_zone': h['availability_zone']
})
def list(self):
"""List all backups.
List all backups (including ones in progress) and the host
on which the backup operation is running.
"""
ctxt = context.get_admin_context()
backups = objects.BackupList.get_all(ctxt)
hdr = "%-32s\t%-32s\t%-32s\t%-24s\t%-24s\t%-12s\t%-12s\t%-12s\t%-12s"
print(hdr %
(_('ID'), _('User ID'), _('Project ID'), _('Host'), _('Name'),
_('Container'), _('Status'), _('Size'), _('Object Count')))
res = "%-32s\t%-32s\t%-32s\t%-24s\t%-24s\t%-12s\t%-12s\t%-12d\t%-12d"
for backup in backups:
object_count = 0
if backup['object_count'] is not None:
object_count = backup['object_count']
print(res %
(backup['id'], backup['user_id'], backup['project_id'],
backup['host'], backup['display_name'], backup['container'],
backup['status'], backup['size'], object_count))
def list(self):
"""Show a list of all waterfall services."""
ctxt = context.get_admin_context()
services = objects.ServiceList.get_all(ctxt)
print_format = "%-16s %-36s %-16s %-10s %-5s %-20s %-12s %-15s"
print(print_format %
(_('Binary'), _('Host'), _('Zone'), _('Status'), _('State'),
_('Updated At'), _('RPC Version'), _('Object Version')))
for svc in services:
alive = utils.service_is_up(svc)
art = ":-)" if alive else "XXX"
status = 'enabled'
if svc.disabled:
status = 'disabled'
updated_at = svc.updated_at
if updated_at:
updated_at = timeutils.normalize_time(updated_at)
rpc_version = (svc.rpc_current_version
or rpc.LIBERTY_RPC_VERSIONS.get(svc.binary, ''))
object_version = (svc.object_current_version or 'liberty')
print(
print_format %
(svc.binary, svc.host.partition('.')[0], svc.availability_zone,
status, art, updated_at, rpc_version, object_version))
def periodic_tasks(self, raise_on_error=False):
"""Tasks to be run at a periodic interval."""
ctxt = context.get_admin_context()
self.manager.periodic_tasks(ctxt, raise_on_error=raise_on_error)