def simple_logging(debug=False): if debug: log_level = logging.DEBUG else: log_level = logging.INFO log = _log.setup_logging('shade') log.addHandler(logging.StreamHandler()) log.setLevel(log_level) # Suppress warning about keystoneauth loggers log = _log.setup_logging('keystoneauth.identity.base')
def simple_logging(debug=False): if debug: log_level = logging.DEBUG else: log_level = logging.INFO log = _log.setup_logging('shade') log.addHandler(logging.StreamHandler()) log.setLevel(log_level) # Suppress warning about keystoneauth loggers log = _log.setup_logging('keystoneauth.identity.base') log = _log.setup_logging('keystoneauth.identity.generic.base')
def _log_request_id(obj, request_id): # Add it, if passed in, even though we're going to pop in a second, # just to make the logic simpler if request_id is not None: obj['x_openstack_request_ids'] = [request_id] request_id = None request_ids = obj.pop('x_openstack_request_ids', None) if request_ids: request_id = request_ids[0] if request_id: # Log the request id and object id in a specific logger. This way # someone can turn it on if they're interested in this kind of tracing. log = _log.setup_logging('shade.request_ids') obj_id = None if isinstance(obj, dict): obj_id = obj.get('id', obj.get('uuid')) if obj_id: log.debug("Retrieved object %(id)s. Request ID %(request_id)s", { 'id': obj.get('id', obj.get('uuid')), 'request_id': request_id }) else: log.debug("Retrieved a response. Request ID %(request_id)s", {'request_id': request_id}) return obj
def find_best_address(addresses, family, public=False, cloud_public=True): do_check = public == cloud_public if not addresses: return None if len(addresses) == 1: return addresses[0] if len(addresses) > 1 and do_check: # We only want to do this check if the address is supposed to be # reachable. Otherwise we're just debug log spamming on every listing # of private ip addresses for address in addresses: # Return the first one that is reachable try: connect_socket = socket.socket(family, socket.SOCK_STREAM, 0) connect_socket.settimeout(1) connect_socket.connect((address, 22, 0, 0)) return address except Exception: pass # Give up and return the first - none work as far as we can tell if do_check: log = _log.setup_logging('shade') log.debug( 'The cloud returned multiple addresses, and none of them seem' ' to work. That might be what you wanted, but we have no clue' " what's going on, so we just picked one at random") return addresses[0]
class TaskManager(object): log = _log.setup_logging("shade.TaskManager") def __init__(self, client, name): self.name = name self._client = client def stop(self): """ This is a direct action passthrough TaskManager """ pass def run(self): """ This is a direct action passthrough TaskManager """ pass def submitTask(self, task, raw=False): """Submit and execute the given task. :param task: The task to execute. :param bool raw: If True, return the raw result as received from the underlying client call. """ self.log.debug("Manager %s running task %s" % (self.name, type(task).__name__)) start = time.time() task.run(self._client) end = time.time() self.log.debug("Manager %s ran task %s in %ss" % (self.name, type(task).__name__, (end - start))) return task.wait(raw)
def _iterate_timeout(timeout, message, wait=2): """Iterate and raise an exception on timeout. This is a generator that will continually yield and sleep for wait seconds, and if the timeout is reached, will raise an exception with <message>. """ log = _log.setup_logging('shade.iterate_timeout') try: # None as a wait winds up flowing well in the per-resource cache # flow. We could spread this logic around to all of the calling # points, but just having this treat None as "I don't have a value" # seems friendlier if wait is None: wait = 2 elif wait == 0: # wait should be < timeout, unless timeout is None wait = 0.1 if timeout is None else min(0.1, timeout) wait = float(wait) except ValueError: raise exc.OpenStackCloudException( "Wait value must be an int or float value. {wait} given" " instead".format(wait=wait)) start = time.time() count = 0 while (timeout is None) or (time.time() < start + timeout): count += 1 yield count log.debug('Waiting %s seconds', wait) time.sleep(wait) raise exc.OpenStackCloudTimeout(message)
def simple_logging(debug=False, http_debug=False): if http_debug: debug = True if debug: log_level = logging.DEBUG else: log_level = logging.INFO if http_debug: # Enable HTTP level tracing log = _log.setup_logging('keystoneauth') log.addHandler(logging.StreamHandler()) log.setLevel(log_level) # We only want extra shade HTTP tracing in http debug mode log = _log.setup_logging('shade.http') log.setLevel(log_level) else: # We only want extra shade HTTP tracing in http debug mode log = _log.setup_logging('shade.http') log.setLevel(logging.WARNING) # Simple case - we only care about request id log during debug log = _log.setup_logging('shade.request_ids') log.setLevel(log_level) log = _log.setup_logging('shade') log.addHandler(logging.StreamHandler()) log.setLevel(log_level) # Suppress warning about keystoneauth loggers log = _log.setup_logging('keystoneauth.identity.base') log = _log.setup_logging('keystoneauth.identity.generic.base')
def simple_logging(debug=False, http_debug=False): if http_debug: debug = True if debug: log_level = logging.DEBUG else: log_level = logging.INFO if http_debug: # Enable HTTP level tracing log = _log.setup_logging('keystoneauth') log.addHandler(logging.StreamHandler()) log.setLevel(log_level) log = _log.setup_logging('shade') log.addHandler(logging.StreamHandler()) log.setLevel(log_level) # Suppress warning about keystoneauth loggers log = _log.setup_logging('keystoneauth.identity.base') log = _log.setup_logging('keystoneauth.identity.generic.base')
def _call_client_and_retry(client, url, retry_on=None, call_retries=3, retry_wait=2, **kwargs): """Method to provide retry operations. Some APIs utilize HTTP errors on certian operations to indicate that the resource is presently locked, and as such this mechanism provides the ability to retry upon known error codes. :param object client: The client method, such as: ``self.baremetal_client.post`` :param string url: The URL to perform the operation upon. :param integer retry_on: A list of error codes that can be retried on. The method also supports a single integer to be defined. :param integer call_retries: The number of times to retry the call upon the error code defined by the 'retry_on' parameter. Default: 3 :param integer retry_wait: The time in seconds to wait between retry attempts. Default: 2 :returns: The object returned by the client call. """ # NOTE(TheJulia): This method, as of this note, does not have direct # unit tests, although is fairly well tested by the tests checking # retry logic in test_baremetal_node.py. log = _log.setup_logging('shade.http') if isinstance(retry_on, int): retry_on = [retry_on] count = 0 while (count < call_retries): count += 1 try: ret_val = client(url, **kwargs) except exc.OpenStackCloudHTTPError as e: if (retry_on is not None and e.response.status_code in retry_on): log.debug( 'Received retryable error {err}, waiting ' '{wait} seconds to retry', { 'err': e.response.status_code, 'wait': retry_wait }) time.sleep(retry_wait) continue else: raise # Break out of the loop, since the loop should only continue # when we encounter a known connection error. return ret_val
class TaskManager(object): log = _log.setup_logging(__name__) def __init__(self, client, name, result_filter_cb=None): self.name = name self._client = client if not result_filter_cb: self._result_filter_cb = _result_filter_cb else: self._result_filter_cb = result_filter_cb def stop(self): """ This is a direct action passthrough TaskManager """ pass def run(self): """ This is a direct action passthrough TaskManager """ pass def submit_task(self, task, raw=False): """Submit and execute the given task. :param task: The task to execute. :param bool raw: If True, return the raw result as received from the underlying client call. """ self.log.debug( "Manager %s running task %s" % (self.name, task.name)) start = time.time() task.run(self._client) end = time.time() self.log.debug( "Manager %s ran task %s in %ss" % ( self.name, task.name, (end - start))) return task.wait(raw) # Backwards compatibility submitTask = submit_task def submit_function( self, method, name=None, result_filter_cb=None, **kwargs): """ Allows submitting an arbitrary method for work. :param method: Method to run in the TaskManager. Can be either the name of a method to find on self.client, or a callable. """ if not result_filter_cb: result_filter_cb = self._result_filter_cb task_class = generate_task_class(method, name, result_filter_cb) return self.manager.submit_task(task_class(**kwargs))
def _log_request_id(obj, request_id): if request_id: # Log the request id and object id in a specific logger. This way # someone can turn it on if they're interested in this kind of tracing. log = _log.setup_logging('shade.request_ids') obj_id = None if isinstance(obj, dict): obj_id = obj.get('id', obj.get('uuid')) if obj_id: log.debug("Retrieved object %(id)s. Request ID %(request_id)s", {'id': obj.get('id', obj.get('uuid')), 'request_id': request_id}) else: log.debug("Retrieved a response. Request ID %(request_id)s", {'request_id': request_id}) return obj
def obj_list_to_dict(obj_list, request_id=None): """Enumerate through lists of objects and return lists of dictonaries. Some of the objects returned in OpenStack are actually lists of objects, and in order to expose the data structures as JSON, we need to facilitate the conversion to lists of dictonaries. """ new_list = [] if not request_id: request_id = getattr(obj_list, 'request_ids', [None])[0] if request_id: log = _log.setup_logging('shade.request_ids') log.debug("Retrieved a list. Request ID %(request_id)s", {'request_id': request_id}) for obj in obj_list: new_list.append(obj_to_dict(obj)) return new_list
def _log_request_id(obj, request_id): # Add it, if passed in, even though we're going to pop in a second, # just to make the logic simpler if request_id is not None: obj['x_openstack_request_ids'] = [request_id] request_id = None request_ids = obj.pop('x_openstack_request_ids', None) if request_ids: request_id = request_ids[0] if request_id: log = _log.setup_logging('shade.request_ids') # Log the request id and object id in a specific logger. This way # someone can turn it on if they're interested in this kind of tracing. log.debug("Retreived object {id}. Request ID {request_id}".format( id=obj.get('id', obj.get('uuid')), request_id=request_id)) return obj
def _get_supplemental_addresses(cloud, server): fixed_ip_mapping = {} for name, network in server['addresses'].items(): for address in network: if address['version'] == 6: continue if address.get('OS-EXT-IPS:type') == 'floating': # We have a floating IP that nova knows about, do nothing return server['addresses'] fixed_ip_mapping[address['addr']] = name try: # Don't bother doing this before the server is active, it's a waste # of an API call while polling for a server to come up if (cloud.has_service('network') and cloud._has_floating_ips() and server['status'] == 'ACTIVE'): for port in cloud.search_ports( filters=dict(device_id=server['id'])): # This SHOULD return one and only one FIP - but doing # it as a search/list lets the logic work regardless for fip in cloud.search_floating_ips( filters=dict(port_id=port['id'])): if fip['fixed_ip_address'] not in fixed_ip_mapping: log = _log.setup_logging('shade') log.debug( "The cloud returned floating ip %(fip)s attached" " to server %(server)s but the fixed ip associated" " with the floating ip in the neutron listing" " does not exist in the nova listing. Something" " is exceptionally broken.", dict(fip=fip['id'], server=server['id'])) fixed_net = fixed_ip_mapping[fip['fixed_ip_address']] server['addresses'][fixed_net].append( _make_address_dict(fip, port)) except exc.OpenStackCloudException: # If something goes wrong with a cloud call, that's cool - this is # an attempt to provide additional data and should not block forward # progress pass return server['addresses']
def _get_supplemental_addresses(cloud, server): fixed_ip_mapping = {} for name, network in server['addresses'].items(): for address in network: if address['version'] == 6: continue if address.get('OS-EXT-IPS:type') == 'floating': # We have a floating IP that nova knows about, do nothing return server['addresses'] fixed_ip_mapping[address['addr']] = name try: # Don't bother doing this before the server is active, it's a waste # of an API call while polling for a server to come up if (cloud.has_service('network') and cloud._has_floating_ips() and server['status'] == 'ACTIVE'): for port in cloud.search_ports( filters=dict(device_id=server['id'])): for fip in cloud.search_floating_ips( filters=dict(port_id=port['id'])): # This SHOULD return one and only one FIP - but doing # it as a search/list lets the logic work regardless if fip['fixed_ip_address'] not in fixed_ip_mapping: log = _log.setup_logging('shade') log.debug( "The cloud returned floating ip %(fip)s attached" " to server %(server)s but the fixed ip associated" " with the floating ip in the neutron listing" " does not exist in the nova listing. Something" " is exceptionally broken.", dict(fip=fip['id'], server=server['id'])) fixed_net = fixed_ip_mapping[fip['fixed_ip_address']] server['addresses'][fixed_net].append( _make_address_dict(fip, port)) except exc.OpenStackCloudException: # If something goes wrong with a cloud call, that's cool - this is # an attempt to provide additional data and should not block forward # progress pass return server['addresses']
def _log_response_extras(response): # Sometimes we get weird HTML errors. This is usually from load balancers # or other things. Log them to a special logger so that they can be # toggled indepdently - and at debug level so that a person logging # shade.* only gets them at debug. if response.headers.get('content-type') != 'text/html': return try: if int(response.headers.get('content-length', 0)) == 0: return except Exception: return logger = _log.setup_logging('shade.http') if response.reason: logger.debug( "Non-standard error '{reason}' returned from {url}:".format( reason=response.reason, url=response.url)) else: logger.debug( "Non-standard error returned from {url}:".format(url=response.url)) for response_line in response.text.split('\n'): logger.debug(response_line)
class TaskManager(object): log = _log.setup_logging("shade.TaskManager") def __init__(self, client, name): self.name = name self._client = client def stop(self): """ This is a direct action passthrough TaskManager """ pass def run(self): """ This is a direct action passthrough TaskManager """ pass def submitTask(self, task): self.log.debug("Manager %s running task %s" % (self.name, type(task).__name__)) start = time.time() task.run(self._client) end = time.time() self.log.debug("Manager %s ran task %s in %ss" % (self.name, type(task).__name__, (end - start))) return task.wait()
def _log_response_extras(response): # Sometimes we get weird HTML errors. This is usually from load balancers # or other things. Log them to a special logger so that they can be # toggled indepdently - and at debug level so that a person logging # shade.* only gets them at debug. if response.headers.get('content-type') != 'text/html': return try: if int(response.headers.get('content-length', 0)) == 0: return except Exception: return logger = _log.setup_logging('shade.http') if response.reason: logger.debug( "Non-standard error '{reason}' returned from {url}:".format( reason=response.reason, url=response.url)) else: logger.debug( "Non-standard error returned from {url}:".format( url=response.url)) for response_line in response.text.split('\n'): logger.debug(response_line)
def __init__(self, shade_logger, manager, *args, **kwargs): super(ShadeAdapter, self).__init__(*args, **kwargs) self.shade_logger = shade_logger self.manager = manager self.request_log = _log.setup_logging('shade.request_ids')
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import munch import ipaddress import six from shade import exc from shade import _log NON_CALLABLES = (six.string_types, bool, dict, int, float, list, type(None)) log = _log.setup_logging(__name__) def find_nova_addresses(addresses, ext_tag=None, key_name=None, version=4): ret = [] for (k, v) in iter(addresses.items()): if key_name is not None and k != key_name: # key_name is specified and it doesn't match the current network. # Continue with the next one continue for interface_spec in v: if ext_tag is not None: if 'OS-EXT-IPS:type' not in interface_spec: # ext_tag is specified, but this interface has no tag
def log_error(self, logger=None): if not logger: logger = _log.setup_logging('shade.exc') if self.inner_exception and self.inner_exception[1]: logger.error(self.orig_message, exc_info=self.inner_exception)
def log_error(self, logger=None): if not logger: logger = _log.setup_logging(__name__) if self.inner_exception and self.inner_exception[1]: logger.error(self.orig_message, exc_info=self.inner_exception)
def _filter_list(data, name_or_id, filters): """Filter a list by name/ID and arbitrary meta data. :param list data: The list of dictionary data to filter. It is expected that each dictionary contains an 'id' and 'name' key if a value for name_or_id is given. :param string name_or_id: The name or ID of the entity being filtered. Can be a glob pattern, such as 'nb01*'. :param filters: A dictionary of meta data to use for further filtering. Elements of this dictionary may, themselves, be dictionaries. Example:: { 'last_name': 'Smith', 'other': { 'gender': 'Female' } } OR A string containing a jmespath expression for further filtering. """ # The logger is shade.fmmatch to allow a user/operator to configure logging # not to communicate about fnmatch misses (they shouldn't be too spammy, # but one never knows) log = _log.setup_logging('shade.fnmatch') if name_or_id: # name_or_id might already be unicode name_or_id = _make_unicode(name_or_id) identifier_matches = [] bad_pattern = False try: fn_reg = re.compile(fnmatch.translate(name_or_id)) except sre_constants.error: # If the fnmatch re doesn't compile, then we don't care, # but log it in case the user DID pass a pattern but did # it poorly and wants to know what went wrong with their # search fn_reg = None for e in data: e_id = _make_unicode(e.get('id', None)) e_name = _make_unicode(e.get('name', None)) if ((e_id and e_id == name_or_id) or (e_name and e_name == name_or_id)): identifier_matches.append(e) else: # Only try fnmatch if we don't match exactly if not fn_reg: # If we don't have a pattern, skip this, but set the flag # so that we log the bad pattern bad_pattern = True continue if ((e_id and fn_reg.match(e_id)) or (e_name and fn_reg.match(e_name))): identifier_matches.append(e) if not identifier_matches and bad_pattern: log.debug("Bad pattern passed to fnmatch", exc_info=True) data = identifier_matches if not filters: return data if isinstance(filters, six.string_types): return jmespath.search(filters, data) def _dict_filter(f, d): if not d: return False for key in f.keys(): if isinstance(f[key], dict): if not _dict_filter(f[key], d.get(key, None)): return False elif d.get(key, None) != f[key]: return False return True filtered = [] for e in data: filtered.append(e) for key in filters.keys(): if isinstance(filters[key], dict): if not _dict_filter(filters[key], e.get(key, None)): filtered.pop() break elif e.get(key, None) != filters[key]: filtered.pop() break return filtered
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import munch import ipaddress import six from shade import exc from shade import _log NON_CALLABLES = (six.string_types, bool, dict, int, float, list, type(None)) log = _log.setup_logging(__name__) def find_nova_addresses(addresses, ext_tag=None, key_name=None, version=4): ret = [] for (k, v) in iter(addresses.items()): if key_name is not None and k != key_name: # key_name is specified and it doesn't match the current network. # Continue with the next one continue for interface_spec in v: if ext_tag is not None: if 'OS-EXT-IPS:type' not in interface_spec: # ext_tag is specified, but this interface has no tag
class TaskManager(object): log = _log.setup_logging('shade.task_manager') def __init__(self, client, name, result_filter_cb=None, workers=5, **kwargs): self.name = name self._client = client self._executor = concurrent.futures.ThreadPoolExecutor( max_workers=workers) if not result_filter_cb: self._result_filter_cb = _result_filter_cb else: self._result_filter_cb = result_filter_cb def set_client(self, client): self._client = client def stop(self): """ This is a direct action passthrough TaskManager """ self._executor.shutdown(wait=True) def run(self): """ This is a direct action passthrough TaskManager """ pass def submit_task(self, task, raw=False): """Submit and execute the given task. :param task: The task to execute. :param bool raw: If True, return the raw result as received from the underlying client call. """ return self.run_task(task=task, raw=raw) def _run_task_async(self, task, raw=False): self.log.debug("Manager %s submitting task %s", self.name, task.name) return self._executor.submit(self._run_task, task, raw=raw) def run_task(self, task, raw=False): if hasattr(task, 'run_async') and task.run_async: return self._run_task_async(task, raw=raw) else: return self._run_task(task, raw=raw) def _run_task(self, task, raw=False): self.log.debug("Manager %s running task %s", self.name, task.name) start = time.time() task.run(self._client) end = time.time() dt = end - start self.log.debug("Manager %s ran task %s in %ss", self.name, task.name, dt) self.post_run_task(dt, task) return task.wait(raw) def post_run_task(self, elasped_time, task): pass # Backwards compatibility submitTask = submit_task def submit_function(self, method, name=None, result_filter_cb=None, **kwargs): """ Allows submitting an arbitrary method for work. :param method: Method to run in the TaskManager. Can be either the name of a method to find on self.client, or a callable. """ if not result_filter_cb: result_filter_cb = self._result_filter_cb task_class = generate_task_class(method, name, result_filter_cb) return self._executor.submit_task(task_class(**kwargs))