def convert_record_to_dict(record, client_config): if record.name in client_config.get('log_namespace_blacklist', []): return None if not getattr(record, 'created'): time_string = datetime.datetime.utcnow().isoformat() else: time_string = time.strftime( '%Y-%m-%dT%H:%M:%S.', time.gmtime(record.created)) + ( '%0.3f' % record.msecs).replace('.', '').zfill(6) try: message = record.getMessage() tags_list = [] log_dict = { 'log_level': record.levelname, "namespace": record.name, 'server': client_config.get('server_name', 'unknown'), 'date': time_string, 'request_id': None } if PY3: log_dict['message'] = '%s' % message else: msg = message.encode('utf8') if isinstance(message, unicode) else message log_dict['message'] = '%s' % msg # TODO: Based on docs, that attribute exists if a formatter # already formatted the traceback, not sure if it is always # there. if client_config.get('logging_attach_exc_text'): exc_text = getattr(record, 'exc_text', '') if exc_text: log_dict['message'] += '\n%s' % exc_text # populate tags from extra for k, v in vars(record).iteritems(): if k not in EXCLUDED_LOG_VARS: try: tags_list.append(parse_tag(k, v)) if k == 'ae_primary_key': log_dict['primary_key'] = unicode(v) if k == 'ae_permanent': try: log_dict['permanent'] = asbool(v) except Exception: log_dict['permanent'] = True except Exception as e: log.info(u'Couldn\'t convert attached tag %s' % e) if tags_list: log_dict['tags'] = tags_list return log_dict except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e: # handle some weird case where record.getMessage() fails log.warning(e)
def convert_record_to_dict(record, client_config): if record.name in client_config.get('log_namespace_blacklist', []): return None if not getattr(record, 'created'): time_string = datetime.datetime.utcnow().isoformat() else: time_string = time.strftime( '%Y-%m-%dT%H:%M:%S.', time.gmtime(record.created)) + ('%0.3f' % record.msecs).replace('.','').zfill(6) try: message = record.getMessage() tags_list = [] log_dict = {'log_level': record.levelname, "namespace": record.name, 'server': client_config.get('server_name', 'unknown'), 'date': time_string, 'request_id': None} if PY3: log_dict['message'] = '%s' % message else: msg = message.encode('utf8') if isinstance(message, unicode) else message log_dict['message'] = '%s' % msg # TODO: Based on docs, that attribute exists if a formatter # already formatted the traceback, not sure if it is always # there. if client_config.get('logging_attach_exc_text'): exc_text = getattr(record, 'exc_text', '') if exc_text: log_dict['message'] += '\n%s' % exc_text # populate tags from extra for k, v in six.iteritems(vars(record)): if k not in EXCLUDED_LOG_VARS: try: tags_list.append(parse_tag(k, v)) if k == 'ae_primary_key': log_dict['primary_key'] = six.text_type(v) if k == 'ae_permanent': try: log_dict['permanent'] = asbool(v) except Exception: log_dict['permanent'] = True except Exception as e: log.info(u'Couldn\'t convert attached tag %s' % e) if tags_list: log_dict['tags'] = tags_list return log_dict except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e: # handle some weird case where record.getMessage() fails log.warning(e)
def convert_record_to_dict(record, client_config): if record.channel in client_config.get('log_namespace_blacklist', []): return None if not getattr(record, 'time'): time_string = datetime.datetime.utcnow().isoformat() else: time_string = record.time.isoformat() try: message = record.msg tags_list = [] log_dict = { 'log_level': record.level_name, "namespace": record.channel, 'server': client_config.get('server_name', 'unknown'), 'date': time_string, 'request_id': None } if PY3: log_dict['message'] = '%s' % message else: msg = message.encode('utf8') if isinstance(message, unicode) else message log_dict['message'] = '%s' % msg if client_config.get('logging_attach_exc_text'): pass # populate tags from extra for k, v in record.extra.iteritems(): if k not in EXCLUDED_LOG_VARS: try: tags_list.append(parse_tag(k, v)) if k == 'ae_primary_key': log_dict['primary_key'] = unicode(v) if k == 'ae_permanent': try: log_dict['permanent'] = asbool(v) except Exception: log_dict['permanent'] = True except Exception as e: log.info(u'Couldn\'t convert attached tag %s' % e) if tags_list: log_dict['tags'] = tags_list return log_dict except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e: # handle some weird case where record.getMessage() fails log.warning(e)
def convert_record_to_dict(record, client_config): if record.channel in client_config.get('log_namespace_blacklist', []): return None if not getattr(record, 'time'): time_string = datetime.datetime.utcnow().isoformat() else: time_string = record.time.isoformat() try: message = record.msg tags_list = [] log_dict = {'log_level': record.level_name, "namespace": record.channel, 'server': client_config.get('server_name', 'unknown'), 'date': time_string, 'request_id': None} if PY3: log_dict['message'] = '%s' % message else: msg = message.encode('utf8') if isinstance(message, unicode) else message log_dict['message'] = '%s' % msg if client_config.get('logging_attach_exc_text'): pass # populate tags from extra for k, v in record.extra.iteritems(): if k not in EXCLUDED_LOG_VARS: try: tags_list.append(parse_tag(k, v)) if k == 'ae_primary_key': log_dict['primary_key'] = unicode(v) if k == 'ae_permanent': try: log_dict['permanent'] = asbool(v) except Exception: log_dict['permanent'] = True except Exception as e: log.info(u'Couldn\'t convert attached tag %s' % e) if tags_list: log_dict['tags'] = tags_list return log_dict except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e: # handle some weird case where record.getMessage() fails log.warning(e)
def process_environ(self, environ, traceback=None, include_params=False, http_status=200): # form friendly to json encode parsed_environ = {} appenlight_info = {} if ('PATH_INFO' in environ or 'SERVER_NAME' in environ or 'wsgi.url_scheme' in environ): # dummy data for Request to work if 'wsgi.url_scheme' not in environ: environ['wsgi.url_scheme'] = 'http' environ['HTTP_HOST'] = "" try: req = Request(environ) except Exception: req = None else: req = None reserved_list = ('appenlight.client', 'appenlight.force_send', 'appenlight.log', 'appenlight.report', 'appenlight.force_logs', 'appenlight.tags', 'appenlight.extra', 'appenlight.post_vars') for key, value in environ.items(): if key.startswith('appenlight.') and key not in reserved_list: appenlight_info[key[11:]] = unicode(value) elif key == 'appenlight.tags': appenlight_info['tags'] = [] for k, v in value.iteritems(): try: appenlight_info['tags'].append(parse_tag(k, v)) except Exception as e: log.info(u'Couldn\'t convert attached tag %s' % e) elif key == 'appenlight.extra': appenlight_info['extra'] = [] for k, v in value.iteritems(): try: appenlight_info['extra'].append(parse_tag(k, v)) except Exception as e: log.info( u'Couldn\'t convert attached extra value %s' % e) else: whitelisted = key.startswith('HTTP') or key in self.config[ 'environ_keys_whitelist'] if http_status not in [404, '404'] and whitelisted: try: if isinstance(value, str): if PY3: parsed_environ[key] = value else: parsed_environ[key] = value.decode('utf8') else: parsed_environ[key] = unicode(value) except Exception: pass # provide better details for 500's try: parsed_environ['HTTP_METHOD'] = req.method except Exception: pass if include_params and req: try: parsed_environ['COOKIES'] = dict( [(k, v,) for k, v in req.cookies.items() if k in self.config['cookie_keys_whitelist']]) except Exception: parsed_environ['COOKIES'] = {} try: parsed_environ['GET'] = dict( [(k, req.GET.getall(k),) for k in req.GET]) except Exception: parsed_environ['GET'] = {} try: # handle situation where something already did seek() # on wsgi.input parsed_environ['POST'] = {} post_vars = req.environ.get('appenlight.post_vars', None) if post_vars is None: post_vars = req.POST.mixed() # handle werkzeug and multiple post values if hasattr(post_vars, 'to_dict'): post_vars = post_vars.to_dict(flat=False).items() # handle django request object elif hasattr(post_vars, 'lists'): post_vars = post_vars.lists() # handle everything else that should be dict-like else: post_vars = post_vars.items() # if django request object use "lists()" to get multiple values for k, v in post_vars: try: if isinstance(v, basestring): parsed_environ['POST'][k] = v else: try: parsed_environ['POST'][k] = [ unicode(val) for val in v] except Exception: parsed_environ['POST'][k] = unicode(v) except Exception as e: pass except Exception as e: parsed_environ['POST'] = {} # figure out real ip if environ.get("HTTP_X_FORWARDED_FOR"): remote_addr = environ.get("HTTP_X_FORWARDED_FOR").split(',')[ 0].strip() else: remote_addr = ( environ.get("HTTP_X_REAL_IP") or environ.get('REMOTE_ADDR')) parsed_environ['HTTP_USER_AGENT'] = environ.get("HTTP_USER_AGENT", '') parsed_environ['REMOTE_ADDR'] = remote_addr if 'message' in appenlight_info: appenlight_info['extra'].append( ('message', appenlight_info['message']), ) try: username = appenlight_info.get('username', environ.get('REMOTE_USER', u'')) appenlight_info['username'] = u'%s' % username except (UnicodeEncodeError, UnicodeDecodeError): appenlight_info['username'] = "******" try: appenlight_info['URL'] = req.url except (UnicodeEncodeError, UnicodeDecodeError): appenlight_info['URL'] = '/invalid-encoded-url' except Exception: appenlight_info['URL'] = 'unknown' return parsed_environ, appenlight_info
def py_log(self, environ, records=None, r_uuid=None, created_report=None): if not records: records = self.log_handlers_get_records() self.log_handlers_clear_records() if not environ.get('appenlight.force_logs') and \ (self.config['logging_on_error'] and created_report is None): return False count = 0 for record in records: count += 1 if record.name in self.config['log_namespace_blacklist']: continue if not getattr(record, 'created'): time_string = datetime.datetime.utcnow().isoformat() else: time_string = time.strftime( '%Y-%m-%dT%H:%M:%S.', time.gmtime(record.created)) + ( '%0.3f' % record.msecs).replace('.', '').zfill(6) try: message = record.getMessage() tags_list = [] log_dict = { 'log_level': record.levelname, "namespace": record.name, 'server': self.config['server_name'], 'date': time_string, 'request_id': r_uuid } if PY3: log_dict['message'] = '%s' % message else: msg = message.encode('utf8') if isinstance( message, unicode) else message log_dict['message'] = '%s' % msg # TODO: Based on docs, that attribute exists if a formatter # already formatted the traceback, not sure if it is always # there. if self.config['logging_attach_exc_text']: exc_text = getattr(record, 'exc_text', '') if exc_text: log_dict['message'] += '\n%s' % exc_text # populate tags from extra for k, v in vars(record).iteritems(): if k not in EXCLUDED_LOG_VARS: try: tags_list.append(parse_tag(k, v)) if k == 'ae_primary_key': log_dict['primary_key'] = unicode(v) if k == 'ae_permanent': try: log_dict['permanent'] = asbool(v) except Exception: log_dict['permanent'] = True except Exception as e: log.info(u'Couldn\'t convert attached tag %s' % e) if tags_list: log_dict['tags'] = tags_list self.transport.feed_log(log_dict) except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e: # handle some weird case where record.getMessage() fails log.warning(e) log.debug('add %s log entries to queue' % count) return True
def process_environ(self, environ, traceback=None, include_params=False, http_status=200): # form friendly to json encode parsed_environ = {} appenlight_info = {} if ('PATH_INFO' in environ or 'SERVER_NAME' in environ or 'wsgi.url_scheme' in environ): # dummy data for Request to work if 'wsgi.url_scheme' not in environ: environ['wsgi.url_scheme'] = 'http' environ['HTTP_HOST'] = "" try: req = Request(environ) except Exception: req = None else: req = None reserved_list = ('appenlight.client', 'appenlight.force_send', 'appenlight.log', 'appenlight.report', 'appenlight.force_logs', 'appenlight.tags', 'appenlight.extra', 'appenlight.post_vars') for key, value in environ.items(): if key.startswith('appenlight.') and key not in reserved_list: appenlight_info[key[11:]] = six.text_type(value) elif key == 'appenlight.tags': appenlight_info['tags'] = [] for k, v in six.iteritems(value): try: appenlight_info['tags'].append(parse_tag(k, v)) except Exception as e: log.info(u'Couldn\'t convert attached tag %s' % e) elif key == 'appenlight.extra': appenlight_info['extra'] = [] for k, v in six.iteritems(value): try: appenlight_info['extra'].append(parse_tag(k, v)) except Exception as e: log.info( u'Couldn\'t convert attached extra value %s' % e) else: whitelisted = key.startswith('HTTP') or key in self.config[ 'environ_keys_whitelist'] if http_status not in [404, '404'] and whitelisted: try: if isinstance(value, str): if PY3: parsed_environ[key] = value else: parsed_environ[key] = value.decode('utf8') else: parsed_environ[key] = six.text_type(value) except Exception: pass # provide better details for 500's try: parsed_environ['HTTP_METHOD'] = req.method except Exception: pass if include_params and req: try: parsed_environ['COOKIES'] = dict( [(k, v,) for k, v in req.cookies.items() if k in self.config['cookie_keys_whitelist']]) except Exception: parsed_environ['COOKIES'] = {} try: parsed_environ['GET'] = dict( [(k, req.GET.getall(k),) for k in req.GET]) except Exception: parsed_environ['GET'] = {} try: # handle situation where something already did seek() # on wsgi.input parsed_environ['POST'] = {} post_vars = req.environ.get('appenlight.post_vars', None) if post_vars is None: post_vars = req.POST.mixed() # handle werkzeug and multiple post values if hasattr(post_vars, 'to_dict'): post_vars = post_vars.to_dict(flat=False).items() # handle django request object elif hasattr(post_vars, 'lists'): post_vars = post_vars.lists() # handle everything else that should be dict-like else: post_vars = post_vars.items() # if django request object use "lists()" to get multiple values for k, v in post_vars: try: if isinstance(v, basestring): parsed_environ['POST'][k] = v else: try: parsed_environ['POST'][k] = [ six.text_type(val) for val in v] except Exception: parsed_environ['POST'][k] = six.text_type(v) except Exception as e: pass except Exception as e: parsed_environ['POST'] = {} # figure out real ip if environ.get("HTTP_X_FORWARDED_FOR"): remote_addr = environ.get("HTTP_X_FORWARDED_FOR").split(',')[ 0].strip() else: remote_addr = ( environ.get("HTTP_X_REAL_IP") or environ.get('REMOTE_ADDR')) parsed_environ['HTTP_USER_AGENT'] = environ.get("HTTP_USER_AGENT", '') parsed_environ['REMOTE_ADDR'] = remote_addr if 'message' in appenlight_info: appenlight_info['extra'].append( ('message', appenlight_info['message']), ) try: username = appenlight_info.get('username', environ.get('REMOTE_USER', u'')) appenlight_info['username'] = u'%s' % username except (UnicodeEncodeError, UnicodeDecodeError): appenlight_info['username'] = "******" try: appenlight_info['URL'] = req.url except (UnicodeEncodeError, UnicodeDecodeError): appenlight_info['URL'] = '/invalid-encoded-url' except Exception: appenlight_info['URL'] = 'unknown' return parsed_environ, appenlight_info