def load(self):
# type: () -> None
for k, v in self.mgr.get_store_prefix(HOST_CACHE_PREFIX).items():
host = k[len(HOST_CACHE_PREFIX):]
if host not in self.mgr.inventory:
self.mgr.log.warning('removing stray HostCache host record %s' % (
host))
self.mgr.set_store(k, None)
try:
j = json.loads(v)
if 'last_device_update' in j:
self.last_device_update[host] = str_to_datetime(j['last_device_update'])
else:
self.device_refresh_queue.append(host)
if 'last_device_change' in j:
self.last_device_change[host] = str_to_datetime(j['last_device_change'])
# for services, we ignore the persisted last_*_update
# and always trigger a new scrape on mgr restart.
self.daemon_refresh_queue.append(host)
self.network_refresh_queue.append(host)
self.daemons[host] = {}
self.osdspec_previews[host] = []
self.osdspec_last_applied[host] = {}
self.devices[host] = []
self.networks[host] = {}
self.daemon_config_deps[host] = {}
for name, d in j.get('daemons', {}).items():
self.daemons[host][name] = \
orchestrator.DaemonDescription.from_json(d)
for d in j.get('devices', []):
self.devices[host].append(inventory.Device.from_json(d))
self.networks[host] = j.get('networks_and_interfaces', {})
self.osdspec_previews[host] = j.get('osdspec_previews', {})
self.last_client_files[host] = j.get('last_client_files', {})
for name, ts in j.get('osdspec_last_applied', {}).items():
self.osdspec_last_applied[host][name] = str_to_datetime(ts)
for name, d in j.get('daemon_config_deps', {}).items():
self.daemon_config_deps[host][name] = {
'deps': d.get('deps', []),
'last_config': str_to_datetime(d['last_config']),
}
if 'last_host_check' in j:
self.last_host_check[host] = str_to_datetime(j['last_host_check'])
self.registry_login_queue.add(host)
self.scheduled_daemon_actions[host] = j.get('scheduled_daemon_actions', {})
self.agent_counter[host] = int(j.get('agent_counter', 1))
self.metadata_up_to_date[host] = False
self.agent_keys[host] = str(j.get('agent_keys', ''))
self.mgr.log.debug(
'HostCache.load: host %s has %d daemons, '
'%d devices, %d networks' % (
host, len(self.daemons[host]), len(self.devices[host]),
len(self.networks[host])))
except Exception as e:
self.mgr.log.warning('unable to load cached state for %s: %s' % (
host, e))
pass
def load(self):
# type: () -> None
for k, v in self.mgr.get_store_prefix(AGENT_CACHE_PREFIX).items():
host = k[len(AGENT_CACHE_PREFIX):]
if host not in self.mgr.inventory:
self.mgr.log.warning(
'removing stray AgentCache record for agent on %s' %
(host))
self.mgr.set_store(k, None)
try:
j = json.loads(v)
self.agent_config_deps[host] = {}
conf_deps = j.get('agent_config_deps', {})
if conf_deps:
conf_deps['last_config'] = str_to_datetime(
conf_deps['last_config'])
self.agent_config_deps[host] = conf_deps
self.agent_counter[host] = int(j.get('agent_counter', 1))
self.agent_timestamp[host] = str_to_datetime(
j.get('agent_timestamp', datetime_to_str(datetime_now())))
self.agent_keys[host] = str(j.get('agent_keys', ''))
agent_port = int(j.get('agent_ports', 0))
if agent_port:
self.agent_ports[host] = agent_port
except Exception as e:
self.mgr.log.warning(
'unable to load cached state for agent on host %s: %s' %
(host, e))
pass
def load(self):
# type: () -> None
for k, v in self.mgr.get_store_prefix(SPEC_STORE_PREFIX).items():
service_name = k[len(SPEC_STORE_PREFIX):]
try:
j = cast(Dict[str, dict], json.loads(v))
if ((self.mgr.migration_current or 0) < 3
and j['spec'].get('service_type') == 'nfs'):
self.mgr.log.debug(f'found legacy nfs spec {j}')
queue_migrate_nfs_spec(self.mgr, j)
spec = ServiceSpec.from_json(j['spec'])
created = str_to_datetime(cast(str, j['created']))
self._specs[service_name] = spec
self.spec_created[service_name] = created
if 'deleted' in j:
deleted = str_to_datetime(cast(str, j['deleted']))
self.spec_deleted[service_name] = deleted
if 'rank_map' in j and isinstance(j['rank_map'], dict):
self._rank_maps[service_name] = {}
for rank_str, m in j['rank_map'].items():
try:
rank = int(rank_str)
except ValueError:
logger.exception(
f"failed to parse rank in {j['rank_map']}")
continue
if isinstance(m, dict):
self._rank_maps[service_name][rank] = {}
for gen_str, name in m.items():
try:
gen = int(gen_str)
except ValueError:
logger.exception(
f"failed to parse gen in {j['rank_map']}"
)
continue
if isinstance(name, str) or m is None:
self._rank_maps[service_name][rank][
gen] = name
self.mgr.log.debug('SpecStore: loaded spec for %s' %
(service_name))
except Exception as e:
self.mgr.log.warning('unable to load spec for %s: %s' %
(service_name, e))
pass
def _refresh_host_daemons(self, host: str) -> Optional[str]:
try:
out, err, code = self._run_cephadm(host,
'mon',
'ls', [],
no_fsid=True)
if code:
return 'host %s cephadm ls returned %d: %s' % (host, code, err)
ls = json.loads(''.join(out))
except ValueError:
msg = 'host %s scrape failed: Cannot decode JSON' % host
self.log.exception('%s: \'%s\'' % (msg, ''.join(out)))
return msg
except Exception as e:
return 'host %s scrape failed: %s' % (host, e)
dm = {}
for d in ls:
if not d['style'].startswith('cephadm'):
continue
if d['fsid'] != self.mgr._cluster_fsid:
continue
if '.' not in d['name']:
continue
sd = orchestrator.DaemonDescription()
sd.last_refresh = datetime_now()
for k in [
'created', 'started', 'last_configured', 'last_deployed'
]:
v = d.get(k, None)
if v:
setattr(sd, k, str_to_datetime(d[k]))
sd.daemon_type = d['name'].split('.')[0]
sd.daemon_id = '.'.join(d['name'].split('.')[1:])
sd.hostname = host
sd.container_id = d.get('container_id')
if sd.container_id:
# shorten the hash
sd.container_id = sd.container_id[0:12]
sd.container_image_name = d.get('container_image_name')
sd.container_image_id = d.get('container_image_id')
sd.version = d.get('version')
if sd.daemon_type == 'osd':
sd.osdspec_affinity = self.mgr.osd_service.get_osdspec_affinity(
sd.daemon_id)
if 'state' in d:
sd.status_desc = d['state']
sd.status = {
'running': 1,
'stopped': 0,
'error': -1,
'unknown': -1,
}[d['state']]
else:
sd.status_desc = 'unknown'
sd.status = None
dm[sd.name()] = sd
self.log.debug('Refreshed host %s daemons (%d)' % (host, len(dm)))
self.mgr.cache.update_host_daemons(host, dm)
self.mgr.cache.save_host(host)
return None
def list(_) -> Optional[Dict]: # pylint: disable=no-self-argument
value: str = self.get_option(self.NAME)
if not value:
return None
data: MotdData = MotdData(**json.loads(value))
# Check if the MOTD has been expired.
if data.expires:
expires = str_to_datetime(data.expires)
if expires < datetime_now():
return None
return data._asdict()
def from_json(cls, data: dict) -> 'DaemonDescription':
c = data.copy()
event_strs = c.pop('events', [])
for k in ['last_refresh', 'created', 'started', 'last_deployed',
'last_configured']:
if k in c:
c[k] = str_to_datetime(c[k])
events = [OrchestratorEvent.from_json(e) for e in event_strs]
status_int = c.pop('status', None)
status = DaemonDescriptionStatus(status_int) if status_int is not None else None
return cls(events=events, status=status, **c)
def from_json(cls, inp: Optional[Dict[str, Any]], rm_util: RemoveUtil) -> Optional["OSD"]:
if not inp:
return None
for date_field in ['drain_started_at', 'drain_stopped_at', 'drain_done_at', 'process_started_at']:
if inp.get(date_field):
inp.update({date_field: str_to_datetime(inp.get(date_field, ''))})
inp.update({'remove_util': rm_util})
if 'nodename' in inp:
hostname = inp.pop('nodename')
inp['hostname'] = hostname
return cls(**inp)
def _refresh_host_daemons(self, host: str) -> Optional[str]:
try:
ls = self._run_cephadm_json(host, 'mon', 'ls', [], no_fsid=True)
except OrchestratorError as e:
return str(e)
dm = {}
for d in ls:
if not d['style'].startswith('cephadm'):
continue
if d['fsid'] != self.mgr._cluster_fsid:
continue
if '.' not in d['name']:
continue
sd = orchestrator.DaemonDescription()
sd.last_refresh = datetime_now()
for k in ['created', 'started', 'last_configured', 'last_deployed']:
v = d.get(k, None)
if v:
setattr(sd, k, str_to_datetime(d[k]))
sd.daemon_type = d['name'].split('.')[0]
sd.daemon_id = '.'.join(d['name'].split('.')[1:])
sd.hostname = host
sd.container_id = d.get('container_id')
if sd.container_id:
# shorten the hash
sd.container_id = sd.container_id[0:12]
sd.container_image_name = d.get('container_image_name')
sd.container_image_id = d.get('container_image_id')
sd.container_image_digests = d.get('container_image_digests')
sd.memory_usage = d.get('memory_usage')
sd.memory_request = d.get('memory_request')
sd.memory_limit = d.get('memory_limit')
sd._service_name = d.get('service_name')
sd.version = d.get('version')
sd.ports = d.get('ports')
sd.ip = d.get('ip')
if sd.daemon_type == 'osd':
sd.osdspec_affinity = self.mgr.osd_service.get_osdspec_affinity(sd.daemon_id)
if 'state' in d:
sd.status_desc = d['state']
sd.status = {
'running': DaemonDescriptionStatus.running,
'stopped': DaemonDescriptionStatus.stopped,
'error': DaemonDescriptionStatus.error,
'unknown': DaemonDescriptionStatus.error,
}[d['state']]
else:
sd.status_desc = 'unknown'
sd.status = None
dm[sd.name()] = sd
self.log.debug('Refreshed host %s daemons (%d)' % (host, len(dm)))
self.mgr.cache.update_host_daemons(host, dm)
self.mgr.cache.save_host(host)
return None
def from_json(cls, data: dict) -> 'ServiceDescription':
c = data.copy()
status = c.pop('status', {})
event_strs = c.pop('events', [])
spec = ServiceSpec.from_json(c)
c_status = status.copy()
for k in ['last_refresh', 'created']:
if k in c_status:
c_status[k] = str_to_datetime(c_status[k])
events = [OrchestratorEvent.from_json(e) for e in event_strs]
return cls(spec=spec, events=events, **c_status)
def load(self):
# type: () -> None
for k, v in self.mgr.get_store_prefix(SPEC_STORE_PREFIX).items():
service_name = k[len(SPEC_STORE_PREFIX):]
try:
j = cast(Dict[str, dict], json.loads(v))
spec = ServiceSpec.from_json(j['spec'])
created = str_to_datetime(cast(str, j['created']))
self._specs[service_name] = spec
self.spec_created[service_name] = created
if 'deleted' in v:
deleted = str_to_datetime(cast(str, j['deleted']))
self.spec_deleted[service_name] = deleted
self.mgr.log.debug('SpecStore: loaded spec for %s' %
(service_name))
except Exception as e:
self.mgr.log.warning('unable to load spec for %s: %s' %
(service_name, e))
pass
def from_json(cls, input):
# type: (Dict[str, Any]) -> Device
if not isinstance(input, dict):
raise ValueError('Device: Expected dict. Got `{}...`'.format(
json.dumps(input)[:10]))
ret = cls(
**{
key: (
input.get(key, None) if key != 'created' or not input.
get(key, None) else str_to_datetime(input.get(key, None)))
for key in Device.report_fields if key != 'human_readable_type'
})
if ret.rejected_reasons:
ret.rejected_reasons = sorted(ret.rejected_reasons)
return ret
def __init__(self, created: Union[str, datetime.datetime], kind: str,
subject: str, level: str, message: str) -> None:
if isinstance(created, str):
created = str_to_datetime(created)
self.created: datetime.datetime = created
assert kind in "service daemon".split()
self.kind: str = kind
# service name, or daemon danem or something
self.subject: str = subject
# Events are not meant for debugging. debugs should end in the log.
assert level in "INFO ERROR".split()
self.level = level
self.message: str = message
def test_datetime_now_1():
dt = str_to_datetime('2020-03-03T09:21:43.636153304Z')
dt_now = datetime_now()
assert type(dt_now) is datetime.datetime
assert dt_now.tzinfo is not None
assert dt < dt_now
def test_str_to_datetime_invalid_format_2():
with pytest.raises(ValueError):
str_to_datetime('2020-03-03')
def test_str_to_datetime_invalid_format_1():
with pytest.raises(ValueError):
str_to_datetime('2020-03-03 15:52:30.136257504')
def test_str_to_datetime_3():
dt = str_to_datetime('2020-03-03T15:52:30.136257504')
assert type(dt) is datetime.datetime
assert dt.tzinfo is not None
def test_str_to_datetime_1():
dt = str_to_datetime('2020-03-03T09:21:43.636153304Z')
assert type(dt) is datetime.datetime
assert dt.tzinfo is not None