def _check_scaling_allowed(self, cooldown):
metadata = self.metadata_get()
if metadata.get('scaling_in_progress'):
LOG.info("Can not perform scaling action: resource %s "
"is already in scaling.", self.name)
reason = _('due to scaling activity')
raise resource.NoActionRequired(res_name=self.name,
reason=reason)
cooldown = self._sanitize_cooldown(cooldown)
# if both cooldown and cooldown_end not in metadata
if all(k not in metadata for k in ('cooldown', 'cooldown_end')):
# Note: this is for supporting old version cooldown checking
metadata.pop('scaling_in_progress', None)
if metadata and cooldown != 0:
last_adjust = next(six.iterkeys(metadata))
if not timeutils.is_older_than(last_adjust, cooldown):
self._log_and_raise_no_action(cooldown)
elif 'cooldown_end' in metadata:
cooldown_end = next(six.iterkeys(metadata['cooldown_end']))
now = timeutils.utcnow().isoformat()
if now < cooldown_end:
self._log_and_raise_no_action(cooldown)
elif cooldown != 0:
# Note: this is also for supporting old version cooldown checking
last_adjust = next(six.iterkeys(metadata['cooldown']))
if not timeutils.is_older_than(last_adjust, cooldown):
self._log_and_raise_no_action(cooldown)
# Assumes _finished_scaling is called
# after the scaling operation completes
metadata['scaling_in_progress'] = True
self.metadata_set(metadata)
def _cooldown_inprogress(self):
inprogress = False
try:
# Negative values don't make sense, so they are clamped to zero
cooldown = max(0, self.properties[self.COOLDOWN])
except TypeError:
# If not specified, it will be None, same as cooldown == 0
cooldown = 0
metadata = self.metadata_get()
if metadata.get('scaling_in_progress'):
return True
if 'cooldown' not in metadata:
# Note: this is for supporting old version cooldown checking
if metadata and cooldown != 0:
last_adjust = next(six.iterkeys(metadata))
if not timeutils.is_older_than(last_adjust, cooldown):
inprogress = True
elif cooldown != 0:
last_adjust = next(six.iterkeys(metadata['cooldown']))
if not timeutils.is_older_than(last_adjust, cooldown):
inprogress = True
if not inprogress:
metadata['scaling_in_progress'] = True
self.metadata_set(metadata)
return inprogress
def _get_eligible_ovsvapp_agent(self, cluster_id, vcenter_id):
cluster_agents = []
agents = self.plugin.get_agents(
self.context,
filters={'agent_type': [ovsvapp_const.AGENT_TYPE_OVSVAPP]})
for agent in agents:
agent_cluster_id = agent['configurations'].get('cluster_id')
agent_vcenter_id = agent['configurations'].get('vcenter_id')
if (cluster_id != agent_cluster_id) or (
vcenter_id != agent_vcenter_id):
continue
cluster_agents.append(agent)
if not cluster_agents:
return
_agent = random.choice(cluster_agents)
recent_time = _agent['heartbeat_timestamp']
if not timeutils.is_older_than(recent_time,
cfg.CONF.agent_down_time):
return _agent
cluster_agents.remove(_agent)
for agent in cluster_agents:
delta = timeutils.delta_seconds(recent_time,
agent['heartbeat_timestamp'])
if delta > 0:
if not timeutils.is_older_than(agent['heartbeat_timestamp'],
cfg.CONF.agent_down_time):
return agent
def _is_scaling_allowed(self):
metadata = self.metadata_get()
if metadata.get('scaling_in_progress'):
return False
try:
# Negative values don't make sense, so they are clamped to zero
cooldown = max(0, self.properties[self.COOLDOWN])
except TypeError:
# If not specified, it will be None, same as cooldown == 0
cooldown = 0
if cooldown != 0:
try:
if 'cooldown' not in metadata:
# Note: this is for supporting old version cooldown logic
if metadata:
last_adjust = next(six.iterkeys(metadata))
if not timeutils.is_older_than(last_adjust, cooldown):
return False
else:
last_adjust = next(six.iterkeys(metadata['cooldown']))
if not timeutils.is_older_than(last_adjust, cooldown):
return False
except ValueError:
# occurs when metadata has only {scaling_in_progress: False}
pass
# Assumes _finished_scaling is called
# after the scaling operation completes
metadata['scaling_in_progress'] = True
self.metadata_set(metadata)
return True
def _test_is_older_than(self, fn):
strptime = datetime.datetime.strptime
with mock.patch('datetime.datetime') as datetime_mock:
datetime_mock.utcnow.return_value = self.skynet_self_aware_time
datetime_mock.strptime = strptime
expect_true = timeutils.is_older_than(fn(self.one_minute_before),
59)
self.assertTrue(expect_true)
expect_false = timeutils.is_older_than(fn(self.one_minute_before),
60)
self.assertFalse(expect_false)
expect_false = timeutils.is_older_than(fn(self.one_minute_before),
61)
self.assertFalse(expect_false)
def check_backlogged_hosting_devices(self):
""""Checks the status of backlogged hosting devices.
Skips newly spun up instances during their booting time as specified
in the boot time parameter.
:return A dict of the format:
{'reachable': [<hd_id>,..], 'dead': [<hd_id>,..]}
"""
response_dict = {'reachable': [], 'dead': []}
LOG.debug("Current Backlogged hosting devices: %s",
self.backlog_hosting_devices.keys())
for hd_id in self.backlog_hosting_devices.keys():
hd = self.backlog_hosting_devices[hd_id]['hd']
if not timeutils.is_older_than(hd['created_at'],
hd['booting_time']):
LOG.info(_LI("Hosting device: %(hd_id)s @ %(ip)s hasn't "
"passed minimum boot time. Skipping it. "),
{'hd_id': hd_id, 'ip': hd['management_ip_address']})
continue
LOG.info(_LI("Checking hosting device: %(hd_id)s @ %(ip)s for "
"reachability."), {'hd_id': hd_id,
'ip': hd['management_ip_address']})
if _is_pingable(hd['management_ip_address']):
hd.pop('backlog_insertion_ts', None)
del self.backlog_hosting_devices[hd_id]
response_dict['reachable'].append(hd_id)
LOG.info(_LI("Hosting device: %(hd_id)s @ %(ip)s is now "
"reachable. Adding it to response"),
{'hd_id': hd_id, 'ip': hd['management_ip_address']})
else:
LOG.info(_LI("Hosting device: %(hd_id)s @ %(ip)s still not "
"reachable "), {'hd_id': hd_id,
'ip': hd['management_ip_address']})
if timeutils.is_older_than(
hd['backlog_insertion_ts'],
cfg.CONF.cfg_agent.hosting_device_dead_timeout):
LOG.debug("Hosting device: %(hd_id)s @ %(ip)s hasn't "
"been reachable for the last %(time)d seconds. "
"Marking it dead.",
{'hd_id': hd_id,
'ip': hd['management_ip_address'],
'time': cfg.CONF.cfg_agent.
hosting_device_dead_timeout})
response_dict['dead'].append(hd_id)
hd.pop('backlog_insertion_ts', None)
del self.backlog_hosting_devices[hd_id]
LOG.debug("Response: %s", response_dict)
return response_dict
def _poll_shelved_instances(self, context):
if CONF.shelved_offload_time <= 0:
return
filters = {
'vm_state': vm_states.SHELVED,
'task_state': None,
'host': self.host
}
shelved_instances = objects.InstanceList.get_by_filters(
context,
filters=filters,
expected_attrs=['system_metadata'],
use_slave=True)
to_gc = []
for instance in shelved_instances:
sys_meta = instance.system_metadata
shelved_at = timeutils.parse_strtime(sys_meta['shelved_at'])
if timeutils.is_older_than(shelved_at, CONF.shelved_offload_time):
to_gc.append(instance)
for instance in to_gc:
try:
instance.task_state = task_states.SHELVING_OFFLOADING
instance.save(expected_task_state=(None, ))
self.shelve_offload_instance(context,
instance,
clean_shutdown=False)
except Exception:
LOG.exception(_LE('Periodic task failed to offload instance.'),
instance=instance)
def run_monitor(self, hosting_vnf):
mgmt_ips = hosting_vnf['mgmt_ip_addresses']
vdupolicies = hosting_vnf['monitoring_policy']['vdus']
vnf_delay = hosting_vnf['monitoring_policy'].get(
'monitoring_delay', self.boot_wait)
for vdu in vdupolicies.keys():
if hosting_vnf.get('dead') or (
hosting_vnf['vnf']['status']) == constants.PENDING_HEAL:
return
policy = vdupolicies[vdu]
for driver in policy.keys():
params = policy[driver].get('monitoring_params', {})
vdu_delay = params.get('monitoring_delay', vnf_delay)
if not timeutils.is_older_than(hosting_vnf['boot_at'],
vdu_delay):
continue
actions = policy[driver].get('actions', {})
params['mgmt_ip'] = mgmt_ips[vdu]
driver_return = self.monitor_call(driver, hosting_vnf['vnf'],
params)
LOG.debug('driver_return %s', driver_return)
if driver_return in actions:
action = actions[driver_return]
hosting_vnf['action_cb'](action, vdu_name=vdu)
def _check_del_instances(self, pools):
"""Scans the pool for deleted instances and checks deletion timers"""
# XXX: What do we do with instances stuck in deleting?
# For now, just return stuck instances to caller and we can figure
# out what to do with them later.
stuck_instances = []
del_instances = []
for resource, pool in pools.items():
del_instances += [i for i in pool if i.status == DELETING]
# clean out counters for old instances that have been deleted entirely
if self._delete_counters:
del_instance_ids = [i.id for i in del_instances]
for inst_id in copy.copy(self._delete_counters):
if inst_id not in del_instance_ids:
self._delete_counters.pop(inst_id)
for del_inst in del_instances:
if del_inst.id not in self._delete_counters:
self._delete_counters[del_inst.id] = timeutils.utcnow()
else:
if timeutils.is_older_than(self._delete_counters[del_inst.id],
self.delete_timeout):
LOG.error(_LE(
'Instance %s is stuck in %s for more than %s '
'seconds.'), i.id, DELETING, self.delete_timeout)
stuck_instances.append(del_inst)
return stuck_instances
def refresh_cluster_ssc(backend, na_server, vserver, synchronous=False):
"""Refresh cluster ssc for backend."""
if not isinstance(na_server, netapp_api.NaServer):
raise exception.InvalidInput(reason=_("Backend server not NaServer."))
delta_secs = getattr(backend, 'ssc_run_delta_secs', 1800)
if getattr(backend, 'ssc_job_running', None):
LOG.warning(_LW('ssc job in progress. Returning... '))
return
elif (getattr(backend, 'ssc_run_time', None) is None
or (backend.ssc_run_time
and timeutils.is_older_than(backend.ssc_run_time, delta_secs))):
if synchronous:
get_cluster_latest_ssc(backend, na_server, vserver)
else:
t = threading.Timer(0,
get_cluster_latest_ssc,
args=[backend, na_server, vserver])
t.start()
elif getattr(backend, 'refresh_stale_running', None):
LOG.warning(_LW('refresh stale ssc job in progress. Returning... '))
return
else:
if backend.stale_vols:
if synchronous:
refresh_cluster_stale_ssc(backend, na_server, vserver)
else:
t = threading.Timer(0,
refresh_cluster_stale_ssc,
args=[backend, na_server, vserver])
t.start()
def stale(client_id, last_access_time):
if timeutils.is_older_than(last_access_time, timeout):
LOG.debug(
'Removing stale RPC client: %s as it was last '
'accessed at %s', client_id, last_access_time)
return True
return False
def _poll_shelved_instances(self, context):
if CONF.shelved_offload_time <= 0:
return
filters = {'vm_state': vm_states.SHELVED,
'task_state': None,
'host': self.host}
shelved_instances = objects.InstanceList.get_by_filters(
context, filters=filters, expected_attrs=['system_metadata'],
use_slave=True)
to_gc = []
for instance in shelved_instances:
sys_meta = instance.system_metadata
shelved_at = timeutils.parse_strtime(sys_meta['shelved_at'])
if timeutils.is_older_than(shelved_at, CONF.shelved_offload_time):
to_gc.append(instance)
for instance in to_gc:
try:
instance.task_state = task_states.SHELVING_OFFLOADING
instance.save(expected_task_state=(None,))
self.shelve_offload_instance(context, instance,
clean_shutdown=False)
except Exception:
LOG.exception(_LE('Periodic task failed to offload instance.'),
instance=instance)
def refresh_cluster_ssc(backend, na_server, vserver, synchronous=False):
"""Refresh cluster ssc for backend."""
if not isinstance(na_server, netapp_api.NaServer):
raise exception.InvalidInput(reason=_("Backend server not NaServer."))
delta_secs = getattr(backend, 'ssc_run_delta_secs', 1800)
if getattr(backend, 'ssc_job_running', None):
LOG.warning(_LW('ssc job in progress. Returning... '))
return
elif (getattr(backend, 'ssc_run_time', None) is None or
(backend.ssc_run_time and
timeutils.is_older_than(backend.ssc_run_time, delta_secs))):
if synchronous:
get_cluster_latest_ssc(backend, na_server, vserver)
else:
t = threading.Timer(0, get_cluster_latest_ssc,
args=[backend, na_server, vserver])
t.start()
elif getattr(backend, 'refresh_stale_running', None):
LOG.warning(_LW('refresh stale ssc job in progress. Returning... '))
return
else:
if backend.stale_vols:
if synchronous:
refresh_cluster_stale_ssc(backend, na_server, vserver)
else:
t = threading.Timer(0, refresh_cluster_stale_ssc,
args=[backend, na_server, vserver])
t.start()
def _test_and_create_object(uuid):
try:
session = db_api.get_session()
with session.begin():
row = session.query(
models.DFLockedObjects).filter_by(object_uuid=uuid).one()
# test ttl
if row.lock and timeutils.is_older_than(
row.created_at, cfg.CONF.df.distributed_lock_ttl):
# reset the lock if it is timeout
LOG.warning(
'The lock for object %(id)s is reset '
'due to timeout.', {'id': uuid})
_lock_free_update(session,
uuid,
lock_state=True,
session_id=row.session_id)
except orm_exc.NoResultFound:
try:
session = db_api.get_session()
with session.begin():
_create_db_row(session, oid=uuid)
except db_exc.DBDuplicateEntry:
# the lock is concurrently created.
pass
def find_orphaned_instances(xenapi):
"""Find and return a list of orphaned instances."""
ctxt = context.get_admin_context(read_deleted="only")
orphaned_instances = []
for vm_ref, vm_rec in _get_applicable_vm_recs(xenapi):
try:
uuid = vm_rec['other_config']['nova_uuid']
instance = db.instance_get_by_uuid(ctxt, uuid)
except (KeyError, exception.InstanceNotFound):
# NOTE(jk0): Err on the side of caution here. If we don't know
# anything about the particular instance, ignore it.
print_xen_object("INFO: Ignoring VM", vm_rec, indent_level=0)
continue
# NOTE(jk0): This would be triggered if a VM was deleted but the
# actual deletion process failed somewhere along the line.
is_active_and_deleting = (instance.vm_state == "active" and
instance.task_state == "deleting")
# NOTE(jk0): A zombie VM is an instance that is not active and hasn't
# been updated in over the specified period.
is_zombie_vm = (instance.vm_state != "active"
and timeutils.is_older_than(instance.updated_at,
CONF.zombie_instance_updated_at_window))
if is_active_and_deleting or is_zombie_vm:
orphaned_instances.append((vm_ref, vm_rec, instance))
return orphaned_instances
def find_orphaned_instances(xenapi):
"""Find and return a list of orphaned instances."""
ctxt = context.get_admin_context(read_deleted="only")
orphaned_instances = []
for vm_ref, vm_rec in _get_applicable_vm_recs(xenapi):
try:
uuid = vm_rec['other_config']['nova_uuid']
instance = db.instance_get_by_uuid(ctxt, uuid)
except (KeyError, exception.InstanceNotFound):
# NOTE(jk0): Err on the side of caution here. If we don't know
# anything about the particular instance, ignore it.
print_xen_object("INFO: Ignoring VM", vm_rec, indent_level=0)
continue
# NOTE(jk0): This would be triggered if a VM was deleted but the
# actual deletion process failed somewhere along the line.
is_active_and_deleting = (instance.vm_state == "active"
and instance.task_state == "deleting")
# NOTE(jk0): A zombie VM is an instance that is not active and hasn't
# been updated in over the specified period.
is_zombie_vm = (instance.vm_state != "active"
and timeutils.is_older_than(
instance.updated_at,
CONF.zombie_instance_updated_at_window))
if is_active_and_deleting or is_zombie_vm:
orphaned_instances.append((vm_ref, vm_rec, instance))
return orphaned_instances
def _process_unfinished_notifications(self, context):
filters = {
'status':
[fields.NotificationStatus.ERROR, fields.NotificationStatus.NEW]
}
notifications_list = objects.NotificationList.get_all(context,
filters=filters)
for notification in notifications_list:
if (notification.status == fields.NotificationStatus.ERROR
or (notification.status == fields.NotificationStatus.NEW
and timeutils.is_older_than(
notification.generated_time,
CONF.retry_notification_new_status_interval))):
self._process_notification(context, notification)
# get updated notification from db after workflow execution
notification_db = objects.Notification.get_by_uuid(
context, notification.notification_uuid)
if notification_db.status == fields.NotificationStatus.ERROR:
# update notification status as failed
notification_status = fields.NotificationStatus.FAILED
update_data = {'status': notification_status}
notification_db.update(update_data)
notification_db.save()
LOG.error(
"Periodic task 'process_unfinished_notifications': "
"Notification %(notification_uuid)s exits with "
"status: %(status)s.", {
'notification_uuid': notification.notification_uuid,
'status': notification_status
})
def run_monitor(self, hosting_vnf):
mgmt_ips = hosting_vnf['management_ip_addresses']
vdupolicies = hosting_vnf['monitoring_policy']['vdus']
vnf_delay = hosting_vnf['monitoring_policy'].get(
'monitoring_delay', self.boot_wait)
for vdu in vdupolicies.keys():
if hosting_vnf.get('dead'):
return
policy = vdupolicies[vdu]
for driver in policy.keys():
params = policy[driver].get('monitoring_params', {})
vdu_delay = params.get('monitoring_delay', vnf_delay)
if not timeutils.is_older_than(
hosting_vnf['boot_at'],
vdu_delay):
continue
actions = policy[driver].get('actions', {})
if 'mgmt_ip' not in params:
params['mgmt_ip'] = mgmt_ips[vdu]
driver_return = self.monitor_call(driver,
hosting_vnf['vnf'],
params)
LOG.debug('driver_return %s', driver_return)
if driver_return in actions:
action = actions[driver_return]
hosting_vnf['action_cb'](action)
def _node_within_grace_period(self, node):
"""Check if current time is within the node_update_timeout grace period
:returns: True if current time is less than node_update_timeout since
last node update action. False otherwise.
"""
node_last_updated = node.updated_at or node.init_at
if timeutils.is_older_than(node_last_updated,
self.node_update_timeout):
# node was last updated more than node_update_timeout seconds ago
# -> we are outside the grace period
LOG.info(
"%s was updated at %s which is more "
"than %d secs ago. Mark node as unhealthy.", node.name,
node_last_updated, self.node_update_timeout)
return False
else:
# node was last updated less than node_update_timeout seconds ago
# -> we are inside the grace period
LOG.info(
"%s was updated at %s which is less "
"than %d secs ago. Mark node as healthy.", node.name,
node_last_updated, self.node_update_timeout)
return True
def _process_unfinished_notifications(self, context):
filters = {
'status': [fields.NotificationStatus.ERROR,
fields.NotificationStatus.NEW]
}
notifications_list = objects.NotificationList.get_all(context,
filters=filters)
for notification in notifications_list:
if (notification.status == fields.NotificationStatus.ERROR or
(notification.status == fields.NotificationStatus.NEW and
timeutils.is_older_than(
notification.generated_time,
CONF.retry_notification_new_status_interval))):
self._process_notification(context, notification)
# get updated notification from db after workflow execution
notification_db = objects.Notification.get_by_uuid(
context, notification.notification_uuid)
if notification_db.status == fields.NotificationStatus.ERROR:
# update notification status as failed
notification_status = fields.NotificationStatus.FAILED
update_data = {
'status': notification_status
}
notification_db.update(update_data)
notification_db.save()
LOG.error(
"Periodic task 'process_unfinished_notifications': "
"Notification %(notification_uuid)s exits with "
"status: %(status)s.",
{'notification_uuid': notification.notification_uuid,
'status': notification_status})
def describe_table(self, context, table_name):
table_info = self._table_info_repo.get(
context, table_name, ['status', 'last_update_date_time'])
if timeutils.is_older_than(table_info.last_update_date_time,
self._schema_operation_timeout):
if table_info.status == models.TableMeta.TABLE_STATUS_CREATING:
table_info.status = models.TableMeta.TABLE_STATUS_CREATE_FAILED
self._table_info_repo.update(context, table_info, ['status'])
LOG.debug(
"Table '{}' creation timed out."
" Setting status to {}".format(
table_info.name,
models.TableMeta.TABLE_STATUS_CREATE_FAILED)
)
if table_info.status == models.TableMeta.TABLE_STATUS_DELETING:
table_info.status = models.TableMeta.TABLE_STATUS_DELETE_FAILED
self._table_info_repo.update(context, table_info, ['status'])
LOG.debug(
"Table '{}' deletion timed out."
" Setting status to {}".format(
table_info.name,
models.TableMeta.TABLE_STATUS_DELETE_FAILED)
)
return models.TableMeta(
table_info.id,
table_info.schema,
table_info.status,
table_info.creation_date_time)
def _cache_valid(self, host):
cachevalid = False
if host in self.compute_nodes:
node_stats = self.compute_nodes.get(host)
if not timeutils.is_older_than(node_stats["vtime"], CONF.trusted_computing.attestation_auth_timeout):
cachevalid = True
return cachevalid
def _cache_valid(self, host):
cachevalid = False
if host in self.compute_nodes:
node_stats = self.compute_nodes.get(host)
if not timeutils.is_older_than(
node_stats['vtime'],
CONF.trusted_computing.attestation_auth_timeout):
cachevalid = True
return cachevalid
def _update_vgw(self):
config_vaule = CONF.vgw_info
config_value_str = '{' + config_vaule + '}'
config_value_dict = eval(config_value_str)
LOG.debug('begin update vgw_info')
for key, value in utils.vgw_update_time.items():
update_time = timeutils.parse_isotime(str(value))
if timeutils.is_older_than(update_time, 20):
LOG.debug('the server %s too long not updated' % key)
utils.remove_vgw_info(key, 'vgw_info', config_value_dict)
def parse_expiration_date(expiration_date):
if not expiration_date.endswith('Z'):
expiration_date += 'Z'
try:
expiration_time = timeutils.parse_isotime(expiration_date)
except ValueError:
raise exception.ValidationTimeStampError()
if timeutils.is_older_than(expiration_time, 0):
raise exception.ValidationExpirationError()
return expiration_time
def _cooldown_check(self, cooldown, last_adjust):
if not timeutils.is_older_than(last_adjust, cooldown):
LOG.info(
_LI("Can not perform scaling action: "
"resource %(name)s is in cooldown (%(cooldown)s).") % {
'name': self.name,
'cooldown': cooldown
})
reason = _('due to cooldown, ' 'cooldown %s') % cooldown
raise resource.NoActionRequired(res_name=self.name, reason=reason)
def parse_expiration_date(expiration_date):
if not expiration_date.endswith('Z'):
expiration_date += 'Z'
try:
expiration_time = timeutils.parse_isotime(expiration_date)
except ValueError:
raise exception.ValidationTimeStampError()
if timeutils.is_older_than(expiration_time, 0):
raise exception.ValidationExpirationError()
return expiration_time
def get_our_capabilities(self, include_children=True):
capabs = copy.deepcopy(self.my_cell_state.capabilities)
if include_children:
for cell in self.child_cells.values():
if timeutils.is_older_than(cell.last_seen, CONF.cells.mute_child_interval):
continue
for capab_name, values in cell.capabilities.items():
if capab_name not in capabs:
capabs[capab_name] = set([])
capabs[capab_name] |= values
return capabs
def is_engine_dead(ctx, engine_id, period_time=None):
# if engine didn't report its status for peirod_time, will consider it
# as a dead engine.
if period_time is None:
period_time = 2 * CONF.periodic_interval
eng = service_obj.Service.get(ctx, engine_id)
if not eng:
return True
if timeutils.is_older_than(eng.updated_at, period_time):
return True
return False
def is_engine_dead(ctx, engine_id, period_time=None):
# if engine didn't report its status for peirod_time, will consider it
# as a dead engine.
if period_time is None:
period_time = 2 * CONF.periodic_interval
eng = service_obj.Service.get(ctx, engine_id)
if not eng:
return True
if timeutils.is_older_than(eng.updated_at, period_time):
return True
return False
def service_registry_cleanup(self):
ctx = context.get_admin_context()
time_window = (2 * cfg.CONF.report_interval)
services = service_obj.Service.get_all(ctx)
for svc in services:
if svc['id'] == self.engine_id:
continue
if timeutils.is_older_than(svc['updated_at'], time_window):
# < time_line:
# hasn't been updated, assuming it's died.
LOG.info('Service %s was aborted', svc['id'])
service_obj.Service.delete(ctx, svc['id'])
def _parse_expiration_date(self, expiration_date):
if expiration_date is None:
return None
if not expiration_date.endswith("Z"):
expiration_date += "Z"
try:
expiration_time = timeutils.parse_isotime(expiration_date)
except ValueError:
raise exception.ValidationTimeStampError()
if timeutils.is_older_than(expiration_time, 0):
raise exception.ValidationExpirationError()
return expiration_time
def service_registry_cleanup(self):
ctx = context.get_admin_context()
time_window = (2 * cfg.CONF.report_interval)
services = service_obj.Service.get_all(ctx)
for svc in services:
if svc['id'] == self.engine_id:
continue
if timeutils.is_older_than(svc['updated_at'], time_window):
# < time_line:
# hasn't been updated, assuming it's died.
LOG.info(_LI('Service %s was aborted'), svc['id'])
service_obj.Service.delete(ctx, svc['id'])
def get_our_capabilities(self, include_children=True):
capabs = copy.deepcopy(self.my_cell_state.capabilities)
if include_children:
for cell in self.child_cells.values():
if timeutils.is_older_than(cell.last_seen,
CONF.cells.mute_child_interval):
continue
for capab_name, values in cell.capabilities.items():
if capab_name not in capabs:
capabs[capab_name] = set([])
capabs[capab_name] |= values
return capabs
def _is_scaling_allowed(self):
metadata = self.metadata_get()
# When scaling is in progress, heat-engine process restart,
# 'scaling_in_progress' will be always True, scaling will can not be
# triggered any more. So we need to recover this situation when scaling
# is timeout.
if metadata.get('scaling_in_progress') and not self._timeout():
return False
try:
# Negative values don't make sense, so they are clamped to zero
cooldown = max(0, self.properties[self.COOLDOWN])
except TypeError:
# If not specified, it will be None, same as cooldown == 0
cooldown = 0
if cooldown != 0:
try:
if 'cooldown' not in metadata:
# Note: this is for supporting old version cooldown logic
if metadata:
last_adjust = next(six.iterkeys(metadata))
if not timeutils.is_older_than(last_adjust, cooldown):
return False
else:
last_adjust = next(six.iterkeys(metadata['cooldown']))
if not timeutils.is_older_than(last_adjust, cooldown):
return False
except ValueError:
# occurs when metadata has only {scaling_in_progress: False}
pass
# Assumes _finished_scaling is called
# after the scaling operation completes
metadata['scaling_in_progress'] = True
self.metadata_set(metadata)
# Set last_adjust_time because we need to check it before adjust if
# 'scaling_in_progress' is True.
self.data_set('last_adjust_time', timeutils.utcnow().isoformat())
return True
def _is_scaling_allowed(self):
metadata = self.metadata_get()
# When scaling is in progress, heat-engine process restart,
# 'scaling_in_progress' will be always True, scaling will can not be
# triggered any more. So we need to recover this situation when scaling
# is timeout.
if metadata.get('scaling_in_progress') and not self._timeout():
return False
try:
# Negative values don't make sense, so they are clamped to zero
cooldown = max(0, self.properties[self.COOLDOWN])
except TypeError:
# If not specified, it will be None, same as cooldown == 0
cooldown = 0
if cooldown != 0:
try:
if 'cooldown' not in metadata:
# Note: this is for supporting old version cooldown logic
if metadata:
last_adjust = next(six.iterkeys(metadata))
if not timeutils.is_older_than(last_adjust, cooldown):
return False
else:
last_adjust = next(six.iterkeys(metadata['cooldown']))
if not timeutils.is_older_than(last_adjust, cooldown):
return False
except ValueError:
# occurs when metadata has only {scaling_in_progress: False}
pass
# Assumes _finished_scaling is called
# after the scaling operation completes
metadata['scaling_in_progress'] = True
self.metadata_set(metadata)
# Set last_adjust_time because we need to check it before adjust if
# 'scaling_in_progress' is True.
self.data_set('last_adjust_time', timeutils.utcnow().isoformat())
return True
def convert_with_links(cls, rpc_conductor, fields=None):
conductor = Conductor(**rpc_conductor.as_dict())
conductor.alive = not timeutils.is_older_than(
conductor.updated_at, CONF.conductor.heartbeat_timeout)
if fields is not None:
api_utils.check_for_invalid_fields(fields, conductor.as_dict())
conductor = cls._convert_with_links(conductor,
api.request.public_url,
fields=fields)
conductor.sanitize(fields)
return conductor
def is_ec2_timestamp_expired(request, expires=None):
"""Checks the timestamp or expiry time included in an EC2 request
and returns true if the request is expired
"""
query_time = None
timestamp = request.get('Timestamp')
expiry_time = request.get('Expires')
def parse_strtime(strtime):
if _ms_time_regex.match(strtime):
# NOTE(MotoKen): time format for aws-sdk-java contains millisecond
time_format = "%Y-%m-%dT%H:%M:%S.%fZ"
else:
time_format = "%Y-%m-%dT%H:%M:%SZ"
return timeutils.parse_strtime(strtime, time_format)
try:
if timestamp and expiry_time:
msg = _("Request must include either Timestamp or Expires,"
" but cannot contain both")
LOG.error(msg)
raise exception.InvalidRequest(msg)
elif expiry_time:
query_time = parse_strtime(expiry_time)
return timeutils.is_older_than(query_time, -1)
elif timestamp:
query_time = parse_strtime(timestamp)
# Check if the difference between the timestamp in the request
# and the time on our servers is larger than 5 minutes, the
# request is too old (or too new).
if query_time and expires:
return (timeutils.is_older_than(query_time, expires) or
timeutils.is_newer_than(query_time, expires))
return False
except ValueError:
LOG.exception(_("Timestamp is invalid: "))
return True
def validate_scaling_action(ctx, cluster_id, action):
"""Validate scaling action against actions table and policy cooldown.
:param ctx: An instance of the request context.
:param cluster_id: ID of the cluster the scaling action is targeting.
:param action: Scaling action being validated.
:return: None
:raises: An exception of ``ActionCooldown`` when the action being
validated is still in cooldown based off the policy or
``ActionConflict`` when a scaling action is already in the action
table.
"""
# Check for conflicting actions in the actions table.
conflicting_actions = Action._get_conflicting_scaling_actions(
ctx, cluster_id)
if conflicting_actions:
action_ids = [a.get('id', None) for a in conflicting_actions]
LOG.info(
"Unable to process %(action)s for cluster %(cluster_id)s "
"the action conflicts with %(conflicts)s", {
'action': action,
'cluster_id': cluster_id,
'conflicts': action_ids
})
raise exception.ActionConflict(type=action,
target=cluster_id,
actions=",".join(action_ids))
# Check to see if action cooldown should be observed.
bindings = cpo.ClusterPolicy.get_all(ctx,
cluster_id,
sort='priority',
filters={'enabled': True})
for pb in bindings:
policy = policy_mod.Policy.load(ctx, pb.policy_id)
if getattr(policy, 'cooldown', None) and policy.event == action:
if pb.last_op and not timeutils.is_older_than(
pb.last_op, policy.cooldown):
LOG.info(
"Unable to process %(action)s for cluster "
"%(cluster_id)s the actions policy %(policy)s "
"cooldown still in progress", {
'action': action,
'cluster_id': cluster_id,
'policy': pb.policy_id
})
raise exception.ActionCooldown(type=action,
cluster=cluster_id,
policy_id=pb.policy_id)
return
def _cooldown_inprogress(self):
inprogress = False
try:
# Negative values don't make sense, so they are clamped to zero
cooldown = max(0, self.properties[self.COOLDOWN])
except TypeError:
# If not specified, it will be None, same as cooldown == 0
cooldown = 0
metadata = self.metadata_get()
if metadata and cooldown != 0:
last_adjust = metadata.keys()[0]
if not timeutils.is_older_than(last_adjust, cooldown):
inprogress = True
return inprogress
def convert_with_links(rpc_conductor, fields=None, sanitize=True):
conductor = api_utils.object_to_dict(
rpc_conductor,
include_uuid=False,
fields=('hostname', 'conductor_group', 'drivers'),
link_resource='conductors',
link_resource_args=rpc_conductor.hostname)
conductor['alive'] = not timeutils.is_older_than(
rpc_conductor.updated_at, CONF.conductor.heartbeat_timeout)
if fields is not None:
api_utils.check_for_invalid_fields(fields, conductor)
if sanitize:
api_utils.sanitize_dict(conductor, fields)
return conductor
def _check_scaling_allowed(self):
metadata = self.metadata_get()
# WRS: If heat-engine is killed after setting scaling_in_progress
# and before clearing the flag, the cooldown is blocked forever.
# scaling_date provides a way of triggering a cleanup later
if metadata.get('scaling_in_progress'):
sd = metadata.get('scaling_date', None)
if sd is None:
LOG.info(
"Can not perform scaling action: resource %s "
"is already in scaling.", self.name)
reason = _('due to scaling activity')
raise resource.NoActionRequired(res_name=self.name,
reason=reason)
scale_max_time = CONF.cooldown.scaling_wait_time
if not timeutils.is_older_than(sd, scale_max_time):
LOG.info(
"Can not perform scaling action: resource %s "
"is already in scaling.", self.name)
reason = _('due to scaling activity')
raise resource.NoActionRequired(res_name=self.name,
reason=reason)
try:
# Negative values don't make sense, so they are clamped to zero
cooldown = max(0, self.properties[self.COOLDOWN])
except TypeError:
# If not specified, it will be None, same as cooldown == 0
cooldown = 0
if cooldown != 0:
try:
if 'cooldown' not in metadata:
# Note: this is for supporting old version cooldown logic
if metadata:
last_adjust = next(six.iterkeys(metadata))
self._cooldown_check(cooldown, last_adjust)
else:
last_adjust = next(six.iterkeys(metadata['cooldown']))
self._cooldown_check(cooldown, last_adjust)
except ValueError:
# occurs when metadata has only {scaling_in_progress: False}
pass
# Assumes _finished_scaling is called
# after the scaling operation completes
metadata['scaling_in_progress'] = True
metadata['scaling_date'] = timeutils.utcnow().isoformat()
self.metadata_set(metadata)
def __run__(self):
while (1):
time.sleep(self._status_check_intvl)
dead_hosting_devices = []
with self._lock:
for hosting_device in self._hosting_devices.values():
if hosting_device.get('dead', False):
continue
if not timeutils.is_older_than(
hosting_device['boot_at'],
hosting_device['boot_wait']):
continue
if not self.is_hosting_device_reachable(hosting_device):
dead_hosting_devices.append(hosting_device)
for hosting_device in dead_hosting_devices:
hosting_device['down_cb'](hosting_device)
def _timeout(self):
last_adjust_time = self.data().get('last_adjust_time')
if not last_adjust_time:
return False
timeout_seconds = (self.stack.timeout_mins *
60 if self.stack.timeout_mins else
cfg.CONF.stack_action_timeout)
try:
if timeutils.is_older_than(last_adjust_time, timeout_seconds):
return True
except ValueError:
pass
return False
def stop_node_recovery():
node_last_updated = node.updated_at or node.init_at
if not timeutils.is_older_than(node_last_updated,
self.node_update_timeout):
LOG.info(
"Node %s was updated at %s which is less than "
"%d secs ago. Skip node recovery from "
"NodePollUrlHealthCheck.", node.id, node_last_updated,
self.node_update_timeout)
return True
LOG.info("Node %s is reported as down (%d retries left)", node.id,
available_attemps)
time.sleep(retry_interval)
return False
def _poll_rescued_instances(self, context):
if CONF.rescue_timeout > 0:
filters = {'vm_state': vm_states.RESCUED,
'host': self.host}
rescued_instances = objects.InstanceList.get_by_filters(
context, filters, expected_attrs=["system_metadata"],
use_slave=True)
to_unrescue = []
for instance in rescued_instances:
if timeutils.is_older_than(instance.launched_at,
CONF.rescue_timeout):
to_unrescue.append(instance)
for instance in to_unrescue:
self.compute_api.unrescue(context, instance)
def _timeout(self):
last_adjust_time = self.data().get('last_adjust_time')
if not last_adjust_time:
return False
timeout_seconds = (self.stack.timeout_mins * 60
if self.stack.timeout_mins
else cfg.CONF.stack_action_timeout)
try:
if timeutils.is_older_than(last_adjust_time, timeout_seconds):
return True
except ValueError:
pass
return False
def __run__(self):
while(1):
time.sleep(self._status_check_intvl)
dead_hosting_devices = []
with self._lock:
for hosting_device in self._hosting_devices.values():
if hosting_device.get('dead', False):
continue
if not timeutils.is_older_than(
hosting_device['boot_at'],
hosting_device['boot_wait']):
continue
if not self.is_hosting_device_reachable(hosting_device):
dead_hosting_devices.append(hosting_device)
for hosting_device in dead_hosting_devices:
hosting_device['down_cb'](hosting_device)
def _weigh_object(self, cell, weight_properties):
"""Check cell against the last_seen timestamp that indicates the time
that the most recent capability or capacity update was received from
the given cell.
"""
last_seen = cell.last_seen
secs = CONF.cells.mute_child_interval
if timeutils.is_older_than(last_seen, secs):
# yep, that's a mute child; recommend highly that it be skipped!
LOG.warning("%(cell)s has not been seen since %(last_seen)s "
"and is being treated as mute.",
{'cell': cell, 'last_seen': last_seen})
return self.MUTE_WEIGH_VALUE
else:
return 0
def _poll_rescued_instances(self, context):
if CONF.rescue_timeout > 0:
filters = {'vm_state': vm_states.RESCUED, 'host': self.host}
rescued_instances = objects.InstanceList.get_by_filters(
context,
filters,
expected_attrs=["system_metadata"],
use_slave=True)
to_unrescue = []
for instance in rescued_instances:
if timeutils.is_older_than(instance.launched_at,
CONF.rescue_timeout):
to_unrescue.append(instance)
for instance in to_unrescue:
self.compute_api.unrescue(context, instance)
def _weigh_object(self, cell, weight_properties):
"""Check cell against the last_seen timestamp that indicates the time
that the most recent capability or capacity update was received from
the given cell.
"""
last_seen = cell.last_seen
secs = CONF.cells.mute_child_interval
if timeutils.is_older_than(last_seen, secs):
# yep, that's a mute child; recommend highly that it be skipped!
LOG.warning("%(cell)s has not been seen since %(last_seen)s "
"and is being treated as mute.",
{'cell': cell, 'last_seen': last_seen})
return self.MUTE_WEIGH_VALUE
else:
return 0
def _poll_rebooting_instances(self, context):
if CONF.reboot_timeout > 0:
filters = {'task_state':
[task_states.REBOOTING,
task_states.REBOOT_STARTED,
task_states.REBOOT_PENDING],
'host': self.host}
rebooting = objects.InstanceList.get_by_filters(
context, filters, expected_attrs=[], use_slave=True)
to_poll = []
for instance in rebooting:
if timeutils.is_older_than(instance.updated_at,
CONF.reboot_timeout):
to_poll.append(instance)
self.driver.poll_rebooting_instances(CONF.reboot_timeout, to_poll)
def _check_instance_build_time(self, context):
"""Ensure that instances are not stuck in build."""
timeout = CONF.instance_build_timeout
if timeout == 0:
return
filters = {'vm_state': vm_states.BUILDING,
'host': self.host}
building_insts = objects.InstanceList.get_by_filters(context,
filters, expected_attrs=[], use_slave=True)
for instance in building_insts:
if timeutils.is_older_than(instance.created_at, timeout):
self._set_instance_obj_error_state(context, instance)
LOG.warning(_LW("Instance build timed out. Set to error "
"state."), instance=instance)
def _wait(self, handle, started_at, timeout_in):
if timeutils.is_older_than(started_at, timeout_in):
exc = wc_base.WaitConditionTimeout(self, handle)
LOG.info('%(name)s Timed out (%(timeout)s)',
{'name': str(self), 'timeout': str(exc)})
raise exc
handle_status = handle.get_status()
if any(s != handle.STATUS_SUCCESS for s in handle_status):
failure = wc_base.WaitConditionFailure(self, handle)
LOG.info('%(name)s Failed (%(failure)s)',
{'name': str(self), 'failure': str(failure)})
raise failure
if len(handle_status) >= self.properties[self.COUNT]:
LOG.info("%s Succeeded", str(self))
return True
return False
def flush(self):
if not self.initial_timestamp:
return []
expired = self.retention_time and timeutils.is_older_than(self.initial_timestamp, self.retention_time)
full = self.size and self.aggregated_samples >= self.size
if full or expired:
x = list(self.samples.values())
# gauge aggregates need to be averages
for s in x:
if s.type == sample.TYPE_GAUGE:
key = self._get_unique_key(s)
s.volume /= self.counts[key]
self.samples.clear()
self.counts.clear()
self.aggregated_samples = 0
self.initial_timestamp = None
return x
return []
def _check_instance_build_time(self, context):
"""Ensure that instances are not stuck in build."""
timeout = CONF.instance_build_timeout
if timeout == 0:
return
filters = {'vm_state': vm_states.BUILDING, 'host': self.host}
building_insts = objects.InstanceList.get_by_filters(context,
filters,
expected_attrs=[],
use_slave=True)
for instance in building_insts:
if timeutils.is_older_than(instance.created_at, timeout):
self._set_instance_obj_error_state(context, instance)
LOG.warning(_LW("Instance build timed out. Set to error "
"state."),
instance=instance)
def _wait(self, handle, started_at, timeout_in):
if timeutils.is_older_than(started_at, timeout_in):
exc = wc_base.WaitConditionTimeout(self, handle)
LOG.info('%(name)s Timed out (%(timeout)s)',
{'name': str(self), 'timeout': str(exc)})
raise exc
handle_status = handle.get_status()
if any(s != handle.STATUS_SUCCESS for s in handle_status):
failure = wc_base.WaitConditionFailure(self, handle)
LOG.info('%(name)s Failed (%(failure)s)',
{'name': str(self), 'failure': str(failure)})
raise failure
if len(handle_status) >= self.properties[self.COUNT]:
LOG.info("%s Succeeded", str(self))
return True
return False
def is_engine_dead(ctx, engine_id, duration=None):
"""Check if an engine is dead.
If engine hasn't reported its status for the given duration, it is treated
as a dead engine.
:param ctx: A request context.
:param engine_id: The ID of the engine to test.
:param duration: The time duration in seconds.
"""
if not duration:
duration = 2 * cfg.CONF.periodic_interval
eng = service_obj.Service.get(ctx, engine_id)
if not eng:
return True
if timeutils.is_older_than(eng.updated_at, duration):
return True
return False
