def _check_scaling_allowed(self, cooldown): metadata = self.metadata_get() if metadata.get('scaling_in_progress'): LOG.info("Can not perform scaling action: resource %s " "is already in scaling.", self.name) reason = _('due to scaling activity') raise resource.NoActionRequired(res_name=self.name, reason=reason) cooldown = self._sanitize_cooldown(cooldown) # if both cooldown and cooldown_end not in metadata if all(k not in metadata for k in ('cooldown', 'cooldown_end')): # Note: this is for supporting old version cooldown checking metadata.pop('scaling_in_progress', None) if metadata and cooldown != 0: last_adjust = next(six.iterkeys(metadata)) if not timeutils.is_older_than(last_adjust, cooldown): self._log_and_raise_no_action(cooldown) elif 'cooldown_end' in metadata: cooldown_end = next(six.iterkeys(metadata['cooldown_end'])) now = timeutils.utcnow().isoformat() if now < cooldown_end: self._log_and_raise_no_action(cooldown) elif cooldown != 0: # Note: this is also for supporting old version cooldown checking last_adjust = next(six.iterkeys(metadata['cooldown'])) if not timeutils.is_older_than(last_adjust, cooldown): self._log_and_raise_no_action(cooldown) # Assumes _finished_scaling is called # after the scaling operation completes metadata['scaling_in_progress'] = True self.metadata_set(metadata)
def _cooldown_inprogress(self): inprogress = False try: # Negative values don't make sense, so they are clamped to zero cooldown = max(0, self.properties[self.COOLDOWN]) except TypeError: # If not specified, it will be None, same as cooldown == 0 cooldown = 0 metadata = self.metadata_get() if metadata.get('scaling_in_progress'): return True if 'cooldown' not in metadata: # Note: this is for supporting old version cooldown checking if metadata and cooldown != 0: last_adjust = next(six.iterkeys(metadata)) if not timeutils.is_older_than(last_adjust, cooldown): inprogress = True elif cooldown != 0: last_adjust = next(six.iterkeys(metadata['cooldown'])) if not timeutils.is_older_than(last_adjust, cooldown): inprogress = True if not inprogress: metadata['scaling_in_progress'] = True self.metadata_set(metadata) return inprogress
def _get_eligible_ovsvapp_agent(self, cluster_id, vcenter_id): cluster_agents = [] agents = self.plugin.get_agents( self.context, filters={'agent_type': [ovsvapp_const.AGENT_TYPE_OVSVAPP]}) for agent in agents: agent_cluster_id = agent['configurations'].get('cluster_id') agent_vcenter_id = agent['configurations'].get('vcenter_id') if (cluster_id != agent_cluster_id) or ( vcenter_id != agent_vcenter_id): continue cluster_agents.append(agent) if not cluster_agents: return _agent = random.choice(cluster_agents) recent_time = _agent['heartbeat_timestamp'] if not timeutils.is_older_than(recent_time, cfg.CONF.agent_down_time): return _agent cluster_agents.remove(_agent) for agent in cluster_agents: delta = timeutils.delta_seconds(recent_time, agent['heartbeat_timestamp']) if delta > 0: if not timeutils.is_older_than(agent['heartbeat_timestamp'], cfg.CONF.agent_down_time): return agent
def _is_scaling_allowed(self): metadata = self.metadata_get() if metadata.get('scaling_in_progress'): return False try: # Negative values don't make sense, so they are clamped to zero cooldown = max(0, self.properties[self.COOLDOWN]) except TypeError: # If not specified, it will be None, same as cooldown == 0 cooldown = 0 if cooldown != 0: try: if 'cooldown' not in metadata: # Note: this is for supporting old version cooldown logic if metadata: last_adjust = next(six.iterkeys(metadata)) if not timeutils.is_older_than(last_adjust, cooldown): return False else: last_adjust = next(six.iterkeys(metadata['cooldown'])) if not timeutils.is_older_than(last_adjust, cooldown): return False except ValueError: # occurs when metadata has only {scaling_in_progress: False} pass # Assumes _finished_scaling is called # after the scaling operation completes metadata['scaling_in_progress'] = True self.metadata_set(metadata) return True
def _test_is_older_than(self, fn): strptime = datetime.datetime.strptime with mock.patch('datetime.datetime') as datetime_mock: datetime_mock.utcnow.return_value = self.skynet_self_aware_time datetime_mock.strptime = strptime expect_true = timeutils.is_older_than(fn(self.one_minute_before), 59) self.assertTrue(expect_true) expect_false = timeutils.is_older_than(fn(self.one_minute_before), 60) self.assertFalse(expect_false) expect_false = timeutils.is_older_than(fn(self.one_minute_before), 61) self.assertFalse(expect_false)
def check_backlogged_hosting_devices(self): """"Checks the status of backlogged hosting devices. Skips newly spun up instances during their booting time as specified in the boot time parameter. :return A dict of the format: {'reachable': [<hd_id>,..], 'dead': [<hd_id>,..]} """ response_dict = {'reachable': [], 'dead': []} LOG.debug("Current Backlogged hosting devices: %s", self.backlog_hosting_devices.keys()) for hd_id in self.backlog_hosting_devices.keys(): hd = self.backlog_hosting_devices[hd_id]['hd'] if not timeutils.is_older_than(hd['created_at'], hd['booting_time']): LOG.info(_LI("Hosting device: %(hd_id)s @ %(ip)s hasn't " "passed minimum boot time. Skipping it. "), {'hd_id': hd_id, 'ip': hd['management_ip_address']}) continue LOG.info(_LI("Checking hosting device: %(hd_id)s @ %(ip)s for " "reachability."), {'hd_id': hd_id, 'ip': hd['management_ip_address']}) if _is_pingable(hd['management_ip_address']): hd.pop('backlog_insertion_ts', None) del self.backlog_hosting_devices[hd_id] response_dict['reachable'].append(hd_id) LOG.info(_LI("Hosting device: %(hd_id)s @ %(ip)s is now " "reachable. Adding it to response"), {'hd_id': hd_id, 'ip': hd['management_ip_address']}) else: LOG.info(_LI("Hosting device: %(hd_id)s @ %(ip)s still not " "reachable "), {'hd_id': hd_id, 'ip': hd['management_ip_address']}) if timeutils.is_older_than( hd['backlog_insertion_ts'], cfg.CONF.cfg_agent.hosting_device_dead_timeout): LOG.debug("Hosting device: %(hd_id)s @ %(ip)s hasn't " "been reachable for the last %(time)d seconds. " "Marking it dead.", {'hd_id': hd_id, 'ip': hd['management_ip_address'], 'time': cfg.CONF.cfg_agent. hosting_device_dead_timeout}) response_dict['dead'].append(hd_id) hd.pop('backlog_insertion_ts', None) del self.backlog_hosting_devices[hd_id] LOG.debug("Response: %s", response_dict) return response_dict
def _poll_shelved_instances(self, context): if CONF.shelved_offload_time <= 0: return filters = { 'vm_state': vm_states.SHELVED, 'task_state': None, 'host': self.host } shelved_instances = objects.InstanceList.get_by_filters( context, filters=filters, expected_attrs=['system_metadata'], use_slave=True) to_gc = [] for instance in shelved_instances: sys_meta = instance.system_metadata shelved_at = timeutils.parse_strtime(sys_meta['shelved_at']) if timeutils.is_older_than(shelved_at, CONF.shelved_offload_time): to_gc.append(instance) for instance in to_gc: try: instance.task_state = task_states.SHELVING_OFFLOADING instance.save(expected_task_state=(None, )) self.shelve_offload_instance(context, instance, clean_shutdown=False) except Exception: LOG.exception(_LE('Periodic task failed to offload instance.'), instance=instance)
def run_monitor(self, hosting_vnf): mgmt_ips = hosting_vnf['mgmt_ip_addresses'] vdupolicies = hosting_vnf['monitoring_policy']['vdus'] vnf_delay = hosting_vnf['monitoring_policy'].get( 'monitoring_delay', self.boot_wait) for vdu in vdupolicies.keys(): if hosting_vnf.get('dead') or ( hosting_vnf['vnf']['status']) == constants.PENDING_HEAL: return policy = vdupolicies[vdu] for driver in policy.keys(): params = policy[driver].get('monitoring_params', {}) vdu_delay = params.get('monitoring_delay', vnf_delay) if not timeutils.is_older_than(hosting_vnf['boot_at'], vdu_delay): continue actions = policy[driver].get('actions', {}) params['mgmt_ip'] = mgmt_ips[vdu] driver_return = self.monitor_call(driver, hosting_vnf['vnf'], params) LOG.debug('driver_return %s', driver_return) if driver_return in actions: action = actions[driver_return] hosting_vnf['action_cb'](action, vdu_name=vdu)
def _check_del_instances(self, pools): """Scans the pool for deleted instances and checks deletion timers""" # XXX: What do we do with instances stuck in deleting? # For now, just return stuck instances to caller and we can figure # out what to do with them later. stuck_instances = [] del_instances = [] for resource, pool in pools.items(): del_instances += [i for i in pool if i.status == DELETING] # clean out counters for old instances that have been deleted entirely if self._delete_counters: del_instance_ids = [i.id for i in del_instances] for inst_id in copy.copy(self._delete_counters): if inst_id not in del_instance_ids: self._delete_counters.pop(inst_id) for del_inst in del_instances: if del_inst.id not in self._delete_counters: self._delete_counters[del_inst.id] = timeutils.utcnow() else: if timeutils.is_older_than(self._delete_counters[del_inst.id], self.delete_timeout): LOG.error(_LE( 'Instance %s is stuck in %s for more than %s ' 'seconds.'), i.id, DELETING, self.delete_timeout) stuck_instances.append(del_inst) return stuck_instances
def refresh_cluster_ssc(backend, na_server, vserver, synchronous=False): """Refresh cluster ssc for backend.""" if not isinstance(na_server, netapp_api.NaServer): raise exception.InvalidInput(reason=_("Backend server not NaServer.")) delta_secs = getattr(backend, 'ssc_run_delta_secs', 1800) if getattr(backend, 'ssc_job_running', None): LOG.warning(_LW('ssc job in progress. Returning... ')) return elif (getattr(backend, 'ssc_run_time', None) is None or (backend.ssc_run_time and timeutils.is_older_than(backend.ssc_run_time, delta_secs))): if synchronous: get_cluster_latest_ssc(backend, na_server, vserver) else: t = threading.Timer(0, get_cluster_latest_ssc, args=[backend, na_server, vserver]) t.start() elif getattr(backend, 'refresh_stale_running', None): LOG.warning(_LW('refresh stale ssc job in progress. Returning... ')) return else: if backend.stale_vols: if synchronous: refresh_cluster_stale_ssc(backend, na_server, vserver) else: t = threading.Timer(0, refresh_cluster_stale_ssc, args=[backend, na_server, vserver]) t.start()
def stale(client_id, last_access_time): if timeutils.is_older_than(last_access_time, timeout): LOG.debug( 'Removing stale RPC client: %s as it was last ' 'accessed at %s', client_id, last_access_time) return True return False
def _poll_shelved_instances(self, context): if CONF.shelved_offload_time <= 0: return filters = {'vm_state': vm_states.SHELVED, 'task_state': None, 'host': self.host} shelved_instances = objects.InstanceList.get_by_filters( context, filters=filters, expected_attrs=['system_metadata'], use_slave=True) to_gc = [] for instance in shelved_instances: sys_meta = instance.system_metadata shelved_at = timeutils.parse_strtime(sys_meta['shelved_at']) if timeutils.is_older_than(shelved_at, CONF.shelved_offload_time): to_gc.append(instance) for instance in to_gc: try: instance.task_state = task_states.SHELVING_OFFLOADING instance.save(expected_task_state=(None,)) self.shelve_offload_instance(context, instance, clean_shutdown=False) except Exception: LOG.exception(_LE('Periodic task failed to offload instance.'), instance=instance)
def _test_and_create_object(uuid): try: session = db_api.get_session() with session.begin(): row = session.query( models.DFLockedObjects).filter_by(object_uuid=uuid).one() # test ttl if row.lock and timeutils.is_older_than( row.created_at, cfg.CONF.df.distributed_lock_ttl): # reset the lock if it is timeout LOG.warning( 'The lock for object %(id)s is reset ' 'due to timeout.', {'id': uuid}) _lock_free_update(session, uuid, lock_state=True, session_id=row.session_id) except orm_exc.NoResultFound: try: session = db_api.get_session() with session.begin(): _create_db_row(session, oid=uuid) except db_exc.DBDuplicateEntry: # the lock is concurrently created. pass
def find_orphaned_instances(xenapi): """Find and return a list of orphaned instances.""" ctxt = context.get_admin_context(read_deleted="only") orphaned_instances = [] for vm_ref, vm_rec in _get_applicable_vm_recs(xenapi): try: uuid = vm_rec['other_config']['nova_uuid'] instance = db.instance_get_by_uuid(ctxt, uuid) except (KeyError, exception.InstanceNotFound): # NOTE(jk0): Err on the side of caution here. If we don't know # anything about the particular instance, ignore it. print_xen_object("INFO: Ignoring VM", vm_rec, indent_level=0) continue # NOTE(jk0): This would be triggered if a VM was deleted but the # actual deletion process failed somewhere along the line. is_active_and_deleting = (instance.vm_state == "active" and instance.task_state == "deleting") # NOTE(jk0): A zombie VM is an instance that is not active and hasn't # been updated in over the specified period. is_zombie_vm = (instance.vm_state != "active" and timeutils.is_older_than(instance.updated_at, CONF.zombie_instance_updated_at_window)) if is_active_and_deleting or is_zombie_vm: orphaned_instances.append((vm_ref, vm_rec, instance)) return orphaned_instances
def find_orphaned_instances(xenapi): """Find and return a list of orphaned instances.""" ctxt = context.get_admin_context(read_deleted="only") orphaned_instances = [] for vm_ref, vm_rec in _get_applicable_vm_recs(xenapi): try: uuid = vm_rec['other_config']['nova_uuid'] instance = db.instance_get_by_uuid(ctxt, uuid) except (KeyError, exception.InstanceNotFound): # NOTE(jk0): Err on the side of caution here. If we don't know # anything about the particular instance, ignore it. print_xen_object("INFO: Ignoring VM", vm_rec, indent_level=0) continue # NOTE(jk0): This would be triggered if a VM was deleted but the # actual deletion process failed somewhere along the line. is_active_and_deleting = (instance.vm_state == "active" and instance.task_state == "deleting") # NOTE(jk0): A zombie VM is an instance that is not active and hasn't # been updated in over the specified period. is_zombie_vm = (instance.vm_state != "active" and timeutils.is_older_than( instance.updated_at, CONF.zombie_instance_updated_at_window)) if is_active_and_deleting or is_zombie_vm: orphaned_instances.append((vm_ref, vm_rec, instance)) return orphaned_instances
def _process_unfinished_notifications(self, context): filters = { 'status': [fields.NotificationStatus.ERROR, fields.NotificationStatus.NEW] } notifications_list = objects.NotificationList.get_all(context, filters=filters) for notification in notifications_list: if (notification.status == fields.NotificationStatus.ERROR or (notification.status == fields.NotificationStatus.NEW and timeutils.is_older_than( notification.generated_time, CONF.retry_notification_new_status_interval))): self._process_notification(context, notification) # get updated notification from db after workflow execution notification_db = objects.Notification.get_by_uuid( context, notification.notification_uuid) if notification_db.status == fields.NotificationStatus.ERROR: # update notification status as failed notification_status = fields.NotificationStatus.FAILED update_data = {'status': notification_status} notification_db.update(update_data) notification_db.save() LOG.error( "Periodic task 'process_unfinished_notifications': " "Notification %(notification_uuid)s exits with " "status: %(status)s.", { 'notification_uuid': notification.notification_uuid, 'status': notification_status })
def run_monitor(self, hosting_vnf): mgmt_ips = hosting_vnf['management_ip_addresses'] vdupolicies = hosting_vnf['monitoring_policy']['vdus'] vnf_delay = hosting_vnf['monitoring_policy'].get( 'monitoring_delay', self.boot_wait) for vdu in vdupolicies.keys(): if hosting_vnf.get('dead'): return policy = vdupolicies[vdu] for driver in policy.keys(): params = policy[driver].get('monitoring_params', {}) vdu_delay = params.get('monitoring_delay', vnf_delay) if not timeutils.is_older_than( hosting_vnf['boot_at'], vdu_delay): continue actions = policy[driver].get('actions', {}) if 'mgmt_ip' not in params: params['mgmt_ip'] = mgmt_ips[vdu] driver_return = self.monitor_call(driver, hosting_vnf['vnf'], params) LOG.debug('driver_return %s', driver_return) if driver_return in actions: action = actions[driver_return] hosting_vnf['action_cb'](action)
def _node_within_grace_period(self, node): """Check if current time is within the node_update_timeout grace period :returns: True if current time is less than node_update_timeout since last node update action. False otherwise. """ node_last_updated = node.updated_at or node.init_at if timeutils.is_older_than(node_last_updated, self.node_update_timeout): # node was last updated more than node_update_timeout seconds ago # -> we are outside the grace period LOG.info( "%s was updated at %s which is more " "than %d secs ago. Mark node as unhealthy.", node.name, node_last_updated, self.node_update_timeout) return False else: # node was last updated less than node_update_timeout seconds ago # -> we are inside the grace period LOG.info( "%s was updated at %s which is less " "than %d secs ago. Mark node as healthy.", node.name, node_last_updated, self.node_update_timeout) return True
def _process_unfinished_notifications(self, context): filters = { 'status': [fields.NotificationStatus.ERROR, fields.NotificationStatus.NEW] } notifications_list = objects.NotificationList.get_all(context, filters=filters) for notification in notifications_list: if (notification.status == fields.NotificationStatus.ERROR or (notification.status == fields.NotificationStatus.NEW and timeutils.is_older_than( notification.generated_time, CONF.retry_notification_new_status_interval))): self._process_notification(context, notification) # get updated notification from db after workflow execution notification_db = objects.Notification.get_by_uuid( context, notification.notification_uuid) if notification_db.status == fields.NotificationStatus.ERROR: # update notification status as failed notification_status = fields.NotificationStatus.FAILED update_data = { 'status': notification_status } notification_db.update(update_data) notification_db.save() LOG.error( "Periodic task 'process_unfinished_notifications': " "Notification %(notification_uuid)s exits with " "status: %(status)s.", {'notification_uuid': notification.notification_uuid, 'status': notification_status})
def describe_table(self, context, table_name): table_info = self._table_info_repo.get( context, table_name, ['status', 'last_update_date_time']) if timeutils.is_older_than(table_info.last_update_date_time, self._schema_operation_timeout): if table_info.status == models.TableMeta.TABLE_STATUS_CREATING: table_info.status = models.TableMeta.TABLE_STATUS_CREATE_FAILED self._table_info_repo.update(context, table_info, ['status']) LOG.debug( "Table '{}' creation timed out." " Setting status to {}".format( table_info.name, models.TableMeta.TABLE_STATUS_CREATE_FAILED) ) if table_info.status == models.TableMeta.TABLE_STATUS_DELETING: table_info.status = models.TableMeta.TABLE_STATUS_DELETE_FAILED self._table_info_repo.update(context, table_info, ['status']) LOG.debug( "Table '{}' deletion timed out." " Setting status to {}".format( table_info.name, models.TableMeta.TABLE_STATUS_DELETE_FAILED) ) return models.TableMeta( table_info.id, table_info.schema, table_info.status, table_info.creation_date_time)
def _cache_valid(self, host): cachevalid = False if host in self.compute_nodes: node_stats = self.compute_nodes.get(host) if not timeutils.is_older_than(node_stats["vtime"], CONF.trusted_computing.attestation_auth_timeout): cachevalid = True return cachevalid
def _cache_valid(self, host): cachevalid = False if host in self.compute_nodes: node_stats = self.compute_nodes.get(host) if not timeutils.is_older_than( node_stats['vtime'], CONF.trusted_computing.attestation_auth_timeout): cachevalid = True return cachevalid
def _update_vgw(self): config_vaule = CONF.vgw_info config_value_str = '{' + config_vaule + '}' config_value_dict = eval(config_value_str) LOG.debug('begin update vgw_info') for key, value in utils.vgw_update_time.items(): update_time = timeutils.parse_isotime(str(value)) if timeutils.is_older_than(update_time, 20): LOG.debug('the server %s too long not updated' % key) utils.remove_vgw_info(key, 'vgw_info', config_value_dict)
def parse_expiration_date(expiration_date): if not expiration_date.endswith('Z'): expiration_date += 'Z' try: expiration_time = timeutils.parse_isotime(expiration_date) except ValueError: raise exception.ValidationTimeStampError() if timeutils.is_older_than(expiration_time, 0): raise exception.ValidationExpirationError() return expiration_time
def _cooldown_check(self, cooldown, last_adjust): if not timeutils.is_older_than(last_adjust, cooldown): LOG.info( _LI("Can not perform scaling action: " "resource %(name)s is in cooldown (%(cooldown)s).") % { 'name': self.name, 'cooldown': cooldown }) reason = _('due to cooldown, ' 'cooldown %s') % cooldown raise resource.NoActionRequired(res_name=self.name, reason=reason)
def get_our_capabilities(self, include_children=True): capabs = copy.deepcopy(self.my_cell_state.capabilities) if include_children: for cell in self.child_cells.values(): if timeutils.is_older_than(cell.last_seen, CONF.cells.mute_child_interval): continue for capab_name, values in cell.capabilities.items(): if capab_name not in capabs: capabs[capab_name] = set([]) capabs[capab_name] |= values return capabs
def is_engine_dead(ctx, engine_id, period_time=None): # if engine didn't report its status for peirod_time, will consider it # as a dead engine. if period_time is None: period_time = 2 * CONF.periodic_interval eng = service_obj.Service.get(ctx, engine_id) if not eng: return True if timeutils.is_older_than(eng.updated_at, period_time): return True return False
def service_registry_cleanup(self): ctx = context.get_admin_context() time_window = (2 * cfg.CONF.report_interval) services = service_obj.Service.get_all(ctx) for svc in services: if svc['id'] == self.engine_id: continue if timeutils.is_older_than(svc['updated_at'], time_window): # < time_line: # hasn't been updated, assuming it's died. LOG.info('Service %s was aborted', svc['id']) service_obj.Service.delete(ctx, svc['id'])
def _parse_expiration_date(self, expiration_date): if expiration_date is None: return None if not expiration_date.endswith("Z"): expiration_date += "Z" try: expiration_time = timeutils.parse_isotime(expiration_date) except ValueError: raise exception.ValidationTimeStampError() if timeutils.is_older_than(expiration_time, 0): raise exception.ValidationExpirationError() return expiration_time
def service_registry_cleanup(self): ctx = context.get_admin_context() time_window = (2 * cfg.CONF.report_interval) services = service_obj.Service.get_all(ctx) for svc in services: if svc['id'] == self.engine_id: continue if timeutils.is_older_than(svc['updated_at'], time_window): # < time_line: # hasn't been updated, assuming it's died. LOG.info(_LI('Service %s was aborted'), svc['id']) service_obj.Service.delete(ctx, svc['id'])
def _is_scaling_allowed(self): metadata = self.metadata_get() # When scaling is in progress, heat-engine process restart, # 'scaling_in_progress' will be always True, scaling will can not be # triggered any more. So we need to recover this situation when scaling # is timeout. if metadata.get('scaling_in_progress') and not self._timeout(): return False try: # Negative values don't make sense, so they are clamped to zero cooldown = max(0, self.properties[self.COOLDOWN]) except TypeError: # If not specified, it will be None, same as cooldown == 0 cooldown = 0 if cooldown != 0: try: if 'cooldown' not in metadata: # Note: this is for supporting old version cooldown logic if metadata: last_adjust = next(six.iterkeys(metadata)) if not timeutils.is_older_than(last_adjust, cooldown): return False else: last_adjust = next(six.iterkeys(metadata['cooldown'])) if not timeutils.is_older_than(last_adjust, cooldown): return False except ValueError: # occurs when metadata has only {scaling_in_progress: False} pass # Assumes _finished_scaling is called # after the scaling operation completes metadata['scaling_in_progress'] = True self.metadata_set(metadata) # Set last_adjust_time because we need to check it before adjust if # 'scaling_in_progress' is True. self.data_set('last_adjust_time', timeutils.utcnow().isoformat()) return True
def convert_with_links(cls, rpc_conductor, fields=None): conductor = Conductor(**rpc_conductor.as_dict()) conductor.alive = not timeutils.is_older_than( conductor.updated_at, CONF.conductor.heartbeat_timeout) if fields is not None: api_utils.check_for_invalid_fields(fields, conductor.as_dict()) conductor = cls._convert_with_links(conductor, api.request.public_url, fields=fields) conductor.sanitize(fields) return conductor
def is_ec2_timestamp_expired(request, expires=None): """Checks the timestamp or expiry time included in an EC2 request and returns true if the request is expired """ query_time = None timestamp = request.get('Timestamp') expiry_time = request.get('Expires') def parse_strtime(strtime): if _ms_time_regex.match(strtime): # NOTE(MotoKen): time format for aws-sdk-java contains millisecond time_format = "%Y-%m-%dT%H:%M:%S.%fZ" else: time_format = "%Y-%m-%dT%H:%M:%SZ" return timeutils.parse_strtime(strtime, time_format) try: if timestamp and expiry_time: msg = _("Request must include either Timestamp or Expires," " but cannot contain both") LOG.error(msg) raise exception.InvalidRequest(msg) elif expiry_time: query_time = parse_strtime(expiry_time) return timeutils.is_older_than(query_time, -1) elif timestamp: query_time = parse_strtime(timestamp) # Check if the difference between the timestamp in the request # and the time on our servers is larger than 5 minutes, the # request is too old (or too new). if query_time and expires: return (timeutils.is_older_than(query_time, expires) or timeutils.is_newer_than(query_time, expires)) return False except ValueError: LOG.exception(_("Timestamp is invalid: ")) return True
def validate_scaling_action(ctx, cluster_id, action): """Validate scaling action against actions table and policy cooldown. :param ctx: An instance of the request context. :param cluster_id: ID of the cluster the scaling action is targeting. :param action: Scaling action being validated. :return: None :raises: An exception of ``ActionCooldown`` when the action being validated is still in cooldown based off the policy or ``ActionConflict`` when a scaling action is already in the action table. """ # Check for conflicting actions in the actions table. conflicting_actions = Action._get_conflicting_scaling_actions( ctx, cluster_id) if conflicting_actions: action_ids = [a.get('id', None) for a in conflicting_actions] LOG.info( "Unable to process %(action)s for cluster %(cluster_id)s " "the action conflicts with %(conflicts)s", { 'action': action, 'cluster_id': cluster_id, 'conflicts': action_ids }) raise exception.ActionConflict(type=action, target=cluster_id, actions=",".join(action_ids)) # Check to see if action cooldown should be observed. bindings = cpo.ClusterPolicy.get_all(ctx, cluster_id, sort='priority', filters={'enabled': True}) for pb in bindings: policy = policy_mod.Policy.load(ctx, pb.policy_id) if getattr(policy, 'cooldown', None) and policy.event == action: if pb.last_op and not timeutils.is_older_than( pb.last_op, policy.cooldown): LOG.info( "Unable to process %(action)s for cluster " "%(cluster_id)s the actions policy %(policy)s " "cooldown still in progress", { 'action': action, 'cluster_id': cluster_id, 'policy': pb.policy_id }) raise exception.ActionCooldown(type=action, cluster=cluster_id, policy_id=pb.policy_id) return
def _cooldown_inprogress(self): inprogress = False try: # Negative values don't make sense, so they are clamped to zero cooldown = max(0, self.properties[self.COOLDOWN]) except TypeError: # If not specified, it will be None, same as cooldown == 0 cooldown = 0 metadata = self.metadata_get() if metadata and cooldown != 0: last_adjust = metadata.keys()[0] if not timeutils.is_older_than(last_adjust, cooldown): inprogress = True return inprogress
def convert_with_links(rpc_conductor, fields=None, sanitize=True): conductor = api_utils.object_to_dict( rpc_conductor, include_uuid=False, fields=('hostname', 'conductor_group', 'drivers'), link_resource='conductors', link_resource_args=rpc_conductor.hostname) conductor['alive'] = not timeutils.is_older_than( rpc_conductor.updated_at, CONF.conductor.heartbeat_timeout) if fields is not None: api_utils.check_for_invalid_fields(fields, conductor) if sanitize: api_utils.sanitize_dict(conductor, fields) return conductor
def _check_scaling_allowed(self): metadata = self.metadata_get() # WRS: If heat-engine is killed after setting scaling_in_progress # and before clearing the flag, the cooldown is blocked forever. # scaling_date provides a way of triggering a cleanup later if metadata.get('scaling_in_progress'): sd = metadata.get('scaling_date', None) if sd is None: LOG.info( "Can not perform scaling action: resource %s " "is already in scaling.", self.name) reason = _('due to scaling activity') raise resource.NoActionRequired(res_name=self.name, reason=reason) scale_max_time = CONF.cooldown.scaling_wait_time if not timeutils.is_older_than(sd, scale_max_time): LOG.info( "Can not perform scaling action: resource %s " "is already in scaling.", self.name) reason = _('due to scaling activity') raise resource.NoActionRequired(res_name=self.name, reason=reason) try: # Negative values don't make sense, so they are clamped to zero cooldown = max(0, self.properties[self.COOLDOWN]) except TypeError: # If not specified, it will be None, same as cooldown == 0 cooldown = 0 if cooldown != 0: try: if 'cooldown' not in metadata: # Note: this is for supporting old version cooldown logic if metadata: last_adjust = next(six.iterkeys(metadata)) self._cooldown_check(cooldown, last_adjust) else: last_adjust = next(six.iterkeys(metadata['cooldown'])) self._cooldown_check(cooldown, last_adjust) except ValueError: # occurs when metadata has only {scaling_in_progress: False} pass # Assumes _finished_scaling is called # after the scaling operation completes metadata['scaling_in_progress'] = True metadata['scaling_date'] = timeutils.utcnow().isoformat() self.metadata_set(metadata)
def __run__(self): while (1): time.sleep(self._status_check_intvl) dead_hosting_devices = [] with self._lock: for hosting_device in self._hosting_devices.values(): if hosting_device.get('dead', False): continue if not timeutils.is_older_than( hosting_device['boot_at'], hosting_device['boot_wait']): continue if not self.is_hosting_device_reachable(hosting_device): dead_hosting_devices.append(hosting_device) for hosting_device in dead_hosting_devices: hosting_device['down_cb'](hosting_device)
def _timeout(self): last_adjust_time = self.data().get('last_adjust_time') if not last_adjust_time: return False timeout_seconds = (self.stack.timeout_mins * 60 if self.stack.timeout_mins else cfg.CONF.stack_action_timeout) try: if timeutils.is_older_than(last_adjust_time, timeout_seconds): return True except ValueError: pass return False
def stop_node_recovery(): node_last_updated = node.updated_at or node.init_at if not timeutils.is_older_than(node_last_updated, self.node_update_timeout): LOG.info( "Node %s was updated at %s which is less than " "%d secs ago. Skip node recovery from " "NodePollUrlHealthCheck.", node.id, node_last_updated, self.node_update_timeout) return True LOG.info("Node %s is reported as down (%d retries left)", node.id, available_attemps) time.sleep(retry_interval) return False
def _poll_rescued_instances(self, context): if CONF.rescue_timeout > 0: filters = {'vm_state': vm_states.RESCUED, 'host': self.host} rescued_instances = objects.InstanceList.get_by_filters( context, filters, expected_attrs=["system_metadata"], use_slave=True) to_unrescue = [] for instance in rescued_instances: if timeutils.is_older_than(instance.launched_at, CONF.rescue_timeout): to_unrescue.append(instance) for instance in to_unrescue: self.compute_api.unrescue(context, instance)
def __run__(self): while(1): time.sleep(self._status_check_intvl) dead_hosting_devices = [] with self._lock: for hosting_device in self._hosting_devices.values(): if hosting_device.get('dead', False): continue if not timeutils.is_older_than( hosting_device['boot_at'], hosting_device['boot_wait']): continue if not self.is_hosting_device_reachable(hosting_device): dead_hosting_devices.append(hosting_device) for hosting_device in dead_hosting_devices: hosting_device['down_cb'](hosting_device)
def _weigh_object(self, cell, weight_properties): """Check cell against the last_seen timestamp that indicates the time that the most recent capability or capacity update was received from the given cell. """ last_seen = cell.last_seen secs = CONF.cells.mute_child_interval if timeutils.is_older_than(last_seen, secs): # yep, that's a mute child; recommend highly that it be skipped! LOG.warning("%(cell)s has not been seen since %(last_seen)s " "and is being treated as mute.", {'cell': cell, 'last_seen': last_seen}) return self.MUTE_WEIGH_VALUE else: return 0
def _poll_rebooting_instances(self, context): if CONF.reboot_timeout > 0: filters = {'task_state': [task_states.REBOOTING, task_states.REBOOT_STARTED, task_states.REBOOT_PENDING], 'host': self.host} rebooting = objects.InstanceList.get_by_filters( context, filters, expected_attrs=[], use_slave=True) to_poll = [] for instance in rebooting: if timeutils.is_older_than(instance.updated_at, CONF.reboot_timeout): to_poll.append(instance) self.driver.poll_rebooting_instances(CONF.reboot_timeout, to_poll)
def _check_instance_build_time(self, context): """Ensure that instances are not stuck in build.""" timeout = CONF.instance_build_timeout if timeout == 0: return filters = {'vm_state': vm_states.BUILDING, 'host': self.host} building_insts = objects.InstanceList.get_by_filters(context, filters, expected_attrs=[], use_slave=True) for instance in building_insts: if timeutils.is_older_than(instance.created_at, timeout): self._set_instance_obj_error_state(context, instance) LOG.warning(_LW("Instance build timed out. Set to error " "state."), instance=instance)
def _wait(self, handle, started_at, timeout_in): if timeutils.is_older_than(started_at, timeout_in): exc = wc_base.WaitConditionTimeout(self, handle) LOG.info('%(name)s Timed out (%(timeout)s)', {'name': str(self), 'timeout': str(exc)}) raise exc handle_status = handle.get_status() if any(s != handle.STATUS_SUCCESS for s in handle_status): failure = wc_base.WaitConditionFailure(self, handle) LOG.info('%(name)s Failed (%(failure)s)', {'name': str(self), 'failure': str(failure)}) raise failure if len(handle_status) >= self.properties[self.COUNT]: LOG.info("%s Succeeded", str(self)) return True return False
def flush(self): if not self.initial_timestamp: return [] expired = self.retention_time and timeutils.is_older_than(self.initial_timestamp, self.retention_time) full = self.size and self.aggregated_samples >= self.size if full or expired: x = list(self.samples.values()) # gauge aggregates need to be averages for s in x: if s.type == sample.TYPE_GAUGE: key = self._get_unique_key(s) s.volume /= self.counts[key] self.samples.clear() self.counts.clear() self.aggregated_samples = 0 self.initial_timestamp = None return x return []
def is_engine_dead(ctx, engine_id, duration=None): """Check if an engine is dead. If engine hasn't reported its status for the given duration, it is treated as a dead engine. :param ctx: A request context. :param engine_id: The ID of the engine to test. :param duration: The time duration in seconds. """ if not duration: duration = 2 * cfg.CONF.periodic_interval eng = service_obj.Service.get(ctx, engine_id) if not eng: return True if timeutils.is_older_than(eng.updated_at, duration): return True return False