def test_logging5():
reset_unittests()
set_config(standard_logging_redirect=False)
x = logging.getLogger()
x.info("foobar")
assert UNIT_TESTS_STDERR == []
assert UNIT_TESTS_JSON == []
assert UNIT_TESTS_STDOUT == []
def test_json_only_keys1():
reset_unittests()
set_config(json_only_keys=["extra_context_key1", "extra_context_key2"])
x = get_logger("foo.bar")
x = x.bind(k1=1, k2="bar")
x.info("foo", k1=2, k3=2)
assert UNIT_TESTS_STDERR == []
assert UNIT_TESTS_JSON == []
_test_stdxxx(UNIT_TESTS_STDOUT, "INFO", "foo", "{k1=2 k2=bar k3=2}")
def test_extra_context():
reset_unittests()
set_config(extra_context_func=extra_context)
x = get_logger("foo.bar")
x = x.bind(k1=1, k2="bar")
x.info("foo", k1=2, k3=2)
assert UNIT_TESTS_STDERR == []
assert UNIT_TESTS_JSON == []
_test_stdxxx(
UNIT_TESTS_STDOUT, "INFO", "foo",
"{extra_context_key1=extra_context_value1 "
"extra_context_key2=extra_context_value2 k1=2 k2=bar k3=2}")
def main():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument("--transform-func",
default="jsonlog2elasticsearch.no_transform",
help="python function to transform each json line")
parser.add_argument("--debug",
action="store_true",
help="force debug mode")
parser.add_argument("--index-override-func",
default="no",
help="python function to override the ES_INDEX value")
parser.add_argument("ES_HOST", help="ES hostname/ip")
parser.add_argument("ES_PORT", help="ES port", type=int)
parser.add_argument("ES_INDEX", help="ES index name", type=str)
parser.add_argument("LOG_PATH", help="json log file fullpath")
args = parser.parse_args()
if args.debug:
set_config(minimal_level="DEBUG")
if not os.path.isfile(args.LOG_PATH):
touch(args.LOG_PATH)
transform_func = get_transform_func(args.transform_func)
if args.index_override_func == "no":
index_func = functools.partial(default_index_func, args.ES_INDEX)
else:
index_func = get_index_func(args.index_override_func)
pygtail.Pygtail._is_new_file = patched_is_new_file
pygtail.Pygtail._update_offset_file = patched_update_offset_file
pt = pygtail.Pygtail(filename=args.LOG_PATH, read_from_end=True)
es = elasticsearch.Elasticsearch(hosts=[{
"host": args.ES_HOST,
"port": args.ES_PORT,
"use_ssl": False
}])
signal.signal(signal.SIGTERM, signal_handler)
LOG.info("started")
while RUNNING:
while True:
try:
line = pt.next()
except StopIteration:
break
except FileNotFoundError:
touch(args.LOG_PATH)
break
else:
if process(line, transform_func, index_func):
commit(es)
commit(es, True)
LOG.debug("sleeping %i seconds..." % SLEEP_AFTER_EACH_ITERATION)
time.sleep(SLEEP_AFTER_EACH_ITERATION)
LOG.info("exited")
def __init__(
self,
services_to_add: List[Service] = [],
port: int = 0,
bind_host: str = "127.0.0.1",
log_configure_logger: bool = True,
log_minimal_level: str = "INFO",
log_fancy_output: Optional[bool] = None,
):
self.manager: Manager = Manager()
self.__wait_task = None
self.services_to_add = services_to_add
self.__shutdown_task = None
self.port = port
self.bind_host = bind_host
self.log_minimal_level = log_minimal_level
self.log_fancy_output = log_fancy_output
self.log_configure_logger = log_configure_logger
if self.log_configure_logger:
mflog.set_config(fancy_output=self.log_fancy_output,
minimal_level=self.log_minimal_level)
self.logger = mflog.get_logger("alwaysup.daemon")
except Exception:
return "default", "default"
app.add_middleware(
RateLimitMiddleware,
authenticate=auth_function,
backend=RedisBackend(
host=get_settings().ratelimit_redis_host,
port=get_settings().ratelimit_redis_port,
),
config={
r"^/login/check": [Rule(minute=20)],
},
)
templates = Jinja2Templates(directory=os.path.join(DIR, "templates"))
mflog.set_config()
@cache
def get_auth_controller() -> AuthenticationController:
auth_backend = make_auth_backend()
return AuthenticationController(auth_backend)
def check_redirect_url(url: str):
parsed_url = urlparse(url)
if parsed_url.netloc != "":
if not parsed_url.netloc.endswith("." + get_settings().domain):
raise HTTPException(
400,
"the redirection url: %s is not in the %s domain" %
USE_I18N = True
USE_L10N = True
USE_TZ = False
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.2/howto/static-files/
STATIC_URL = '/static/'
# ADDED BY METWORK/MFSERV/DJANGO PLUGIN TEMPLATE
# TO PROVIDE PREFIX BASED ROUTING
STATIC_URL = '/apicvf_django/apicvf_django/static/'
STATIC_ROOT = os.path.join(BASE_DIR, "apicvf_django/static")
# ADDED BY METWORK/MFSERV/DJANGO PLUGIN TEMPLATE
# TO PROVIDE DEBUG FEATURE
DEBUG = (os.environ.get('MFSERV_CURRENT_PLUGIN_DEBUG', '0') == '1')
import mflog
if DEBUG:
mflog.set_config(minimal_level="DEBUG")
else:
mflog.set_config()
ALLOWED_HOSTS = ['localhost', '127.0.0.1', '[::1]']
if not LOCAL_VM:
ALLOWED_HOSTS.append('apicf31-sidev')
def main():
global SYSLOG_THREAD, DISCARDED
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument("--transform-func",
default="jsonsyslog2elasticsearch.no_transform",
help="python function to transform each json line")
parser.add_argument("--debug",
action="store_true",
help="force debug mode")
parser.add_argument("--syslog-port",
type=int,
default=5144,
help="UDP port to listen for syslog messages "
"(warning: the default value: 5144 is not the "
"default syslog port)")
parser.add_argument("--syslog-host",
type=str,
default="0.0.0.0",
help="can be use to bind to a specific interface "
"(default: 0.0.0.0 => all)")
parser.add_argument(
"--internal-queue-size",
type=int,
default=10000,
help="internal queue maximum size (in messages) between syslog "
"receiver and elasticsearch upload (default: 10000)")
parser.add_argument(
"--internal-queue-timeout",
type=int,
default=3,
help="if the internal queue is full during this interval (in seconds "
", default to 3), then we will drop some messages ")
parser.add_argument("--index-override-func",
default="no",
help="python function to override the ES_INDEX value")
parser.add_argument("ES_HOST", help="ES hostname/ip")
parser.add_argument("ES_PORT", help="ES port", type=int)
parser.add_argument("ES_INDEX", help="ES index name", type=str)
args = parser.parse_args()
if args.debug:
set_config(minimal_level="DEBUG")
silent_elasticsearch_logger()
transform_func = get_transform_func(args.transform_func)
if args.index_override_func == "no":
index_func = functools.partial(default_index_func, args.ES_INDEX)
else:
index_func = get_index_func(args.index_override_func)
es = elasticsearch.Elasticsearch(hosts=[{
"host": args.ES_HOST,
"port": args.ES_PORT,
"use_ssl": False
}])
signal.signal(signal.SIGTERM, signal_handler)
q = queue.Queue(maxsize=args.internal_queue_size)
SYSLOG_THREAD = SysLogUDPReceiverThread(args.syslog_host, args.syslog_port,
q, args.internal_queue_timeout)
SYSLOG_THREAD.start()
last_queue_size_log = time.time()
while RUNNING:
now = time.time()
if (now - last_queue_size_log) > LOG_QUEUE_SIZE_EVERY:
size = q.qsize()
if size > 0:
LOG.info("Internal queue size: %i" % size)
if DISCARDED > 0:
LOG.warning(
"%i lines were discarded because of full queue during "
"last %i seconds" % (DISCARDED, LOG_QUEUE_SIZE_EVERY))
DISCARDED = 0
last_queue_size_log = now
try:
item = q.get(block=True, timeout=1)
except Exception:
commit(es, True)
continue
f = item.find('{')
if f == -1:
LOG.warning("can't find { char in read message: %s => ignoring",
item)
continue
json_string = item[f:]
if process(json_string, transform_func, index_func):
commit(es)
def __init__(self, app, raise_exception=False, debug=False):
self.app = app
self.raise_exception = raise_exception
self.debug = debug
if self.debug:
mflog.set_config("DEBUG")
