def __init__(self, ovh_token, **kwargs):
"""
OVH TCP Handler
:param ovh_token: OVH API Token
"""
self.ovh_token = ovh_token
GelfTcpHandler.__init__(self, **kwargs)
def init_logger(name=None):
logger = logging.getLogger(name)
job_id = get_job_id()
logger.setLevel(get_log_level(config.get('log', 'level')))
if 3 > len(logger.handlers):
fmt = config.get('log', 'LOG_FORMAT')
formatter = logging.Formatter(fmt.format(job_id=job_id))
# Standard handler
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
logger.addHandler(handler)
# File handler
log_file = config.get('log', 'LOG_FILE').format(job_id=job_id)
os.makedirs(os.path.dirname(log_file), exist_ok=True)
handler_file = RotatingFileHandler(filename=log_file, maxBytes=10 * 1024 * 1024, backupCount=7)
handler_file.setFormatter(formatter)
logger.addHandler(handler_file)
# Graylog handler
logger.addHandler(GelfTcpHandler(host=config.get('log', 'GRAYLOG_SERVER'),
debug=True,
port=config.get('log', 'GRAYLOG_TCP_PORT'),
trace_id=job_id,
source_tag='passor',
include_extra_fields=True,
))
return logger
def set_gelf_logger(log_level: int = logging.INFO,
transport_layer='TCP',
stdout=False,
include_extra_fields=True,
**gelf_kwargs): # noqa: E301
"""
Sets gelf console logger. Handler for console output is not included by default,
for testing in non-gelf environments use stdout=True.
Args:
log_level: logging level, default: 'logging.INFO'
transport_layer: 'TCP' or 'UDP', default:'UDP
stdout: if set to True, Stout handler is also included
include_extra_fields:
Include extra GELF fields in the log messages.
e.g. logging.warning('Some warning',
extra={"additional_info": "Extra info to be displayed in the detail"}
Returns: Logger object
"""
# remove existing handlers
for h in logging.getLogger().handlers:
logging.getLogger().removeHandler(h)
if stdout:
CommonInterface.set_default_logger(log_level)
# gelf handler setup
gelf_kwargs['include_extra_fields'] = include_extra_fields
host = os.getenv('KBC_LOGGER_ADDR', 'localhost')
port = os.getenv('KBC_LOGGER_PORT', 12201)
if transport_layer == 'TCP':
gelf = GelfTcpHandler(host=host, port=port, **gelf_kwargs)
elif transport_layer == 'UDP':
gelf = GelfUdpHandler(host=host, port=port, **gelf_kwargs)
else:
raise ValueError(
F'Unsupported gelf transport layer: {transport_layer}. Choose TCP or UDP'
)
logging.getLogger().setLevel(log_level)
logging.getLogger().addHandler(gelf)
logger = logging.getLogger()
return logger
def __get_graylog_handler(self, config):
handler = None
host = config["host"]
port = config["port"]
if config["type"].lower() == "udp":
from pygelf import GelfUdpHandler
handler = GelfUdpHandler(host=host,
port=port,
include_extra_fields=True)
elif config["type"].lower() == "tcp":
from pygelf import GelfTcpHandler
handler = GelfTcpHandler(host=host,
port=port,
include_extra_fields=True)
elif config["type"].lower() == "http":
from pygelf import GelfHttpHandler
handler = GelfHttpHandler(host=host,
port=port,
include_extra_fields=True)
return handler
def __init__(self,
stream,
host='localhost',
port=12201,
protocol='udp',
_filters=None,
environment=None,
json_only=False):
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
self.buffer = deque()
self.environment = str(environment)
if _filters is not None:
self.filters = _filters.split(',')
else:
self.filters = None
self.json_only = bool(json_only)
self.protocol = str(protocol)
self.stream = stream
self.handleropts = {
'host': str(host),
'port': int(port),
'protocol': str(protocol)
}
if str(protocol) == 'udp':
self.gelfhandler = GelfUdpHandler(host=str(host),
port=int(port),
include_extra_fields=True,
compress=True)
elif str(protocol) == 'tcp':
self.gelfhandler = GelfTcpHandler(host=str(host),
port=int(port),
include_extra_fields=True)
elif str(protocol) == 'tls':
self.gelfhandler = GelfTlsHandler(host=str(host),
port=int(port),
include_extra_fields=True)
def handler():
return GelfTcpHandler(host='127.0.0.1', port=12000)
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfTlsHandler
import logging
import json
import pytest
import mock
def log_and_decode(_logger, _send, text):
_logger.warning(text)
message = _send.call_args[0][0].replace(b'\x00', b'').decode('utf-8')
return json.loads(message)
@pytest.mark.parametrize('handler,version', [
(GelfTcpHandler(host='127.0.0.1', port=12000, version='10.7'), '10.7'),
(GelfUdpHandler(
host='127.0.0.1', port=12000, compress=False, version='11.5'), '11.5'),
(GelfTlsHandler(host='127.0.0.1', port=12000, version='12.3'), '12.3')
])
def test_custom_version(handler, version):
logger = logging.getLogger('test')
logger.addHandler(handler)
with mock.patch.object(handler, 'send') as send:
message = log_and_decode(logger, send, 'custom version')
assert message['version'] == version
logger.removeHandler(handler)
chan = {}
current_chan = {}
startchanindex = -1
stack = Queue.Queue()
class ContextFilter(logging.Filter):
def filter(self, record):
record.mac = gray_mac
record.vendor = gray_vendor
record.ssid = gray_ssid
return True
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
logger.addFilter(ContextFilter())
handler = GelfTcpHandler(host=graylog_server, port=graylog_port, include_extra_fields=True)
logger.addHandler(handler)
def checkroot():
if os.getuid() == 0:
return True
else:
return False
def checktool(name):
try:
devnull = open(os.devnull)
subprocess.Popen([name], stdout=devnull, stderr=devnull).communicate()
except OSError, e:
if e.errno == os.errno.ENOENT:
return False
import socket
import pytest
import mock
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfHttpHandler, GelfTlsHandler
from tests.helper import logger, get_unique_message, log_warning, log_exception
SYSLOG_LEVEL_ERROR = 3
SYSLOG_LEVEL_WARNING = 4
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12201),
GelfUdpHandler(host='127.0.0.1', port=12202),
GelfUdpHandler(host='127.0.0.1', port=12202, compress=False),
GelfHttpHandler(host='127.0.0.1', port=12203),
GelfHttpHandler(host='127.0.0.1', port=12203, compress=False),
GelfTlsHandler(host='127.0.0.1', port=12204),
# GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem'),
])
def handler(request):
return request.param
def test_simple_message(logger):
message = get_unique_message()
graylog_response = log_warning(logger, message)
assert graylog_response['message'] == message
assert graylog_response['level'] == SYSLOG_LEVEL_WARNING
assert 'full_message' not in graylog_response
assert 'file' not in graylog_response
assert 'module' not in graylog_response
import pytest
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfHttpHandler, GelfTlsHandler
from tests.helper import logger, get_unique_message, log_warning
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12201, debug=True),
GelfUdpHandler(host='127.0.0.1', port=12202, debug=True),
GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, debug=True),
GelfHttpHandler(host='127.0.0.1', port=12203, debug=True),
GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, debug=True),
GelfTlsHandler(host='127.0.0.1', port=12204, debug=True),
GelfTlsHandler(host='127.0.0.1',
port=12204,
debug=True,
validate=True,
ca_certs='tests/config/cert.pem'),
])
def handler(request):
return request.param
def test_debug_mode(logger):
message = get_unique_message()
graylog_response = log_warning(logger, message)
assert graylog_response['message'] == message
assert graylog_response['file'] == 'helper.py'
assert graylog_response['module'] == 'helper'
assert graylog_response['func'] == 'log_warning'
assert graylog_response['logger_name'] == 'test'
assert 'line' in graylog_response
# mongo.py
from flask import Flask, render_template
from flask import jsonify
from flask import request
from flask_pymongo import PyMongo
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfTlsHandler, GelfHttpHandler
import logging
app = Flask(__name__)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
logger.addHandler(GelfTcpHandler(host='127.0.0.1', port=12201))
logger.addHandler(GelfUdpHandler(host='127.0.0.1', port=12201))
logger.addHandler(GelfTlsHandler(host='127.0.0.1', port=12201))
logger.addHandler(GelfHttpHandler(host='127.0.0.1', port=12201))
logger.info('hello gelf')
app.config['MONGO_DBNAME'] = 'db_teste_itau1'
app.config['MONGO_URI'] = 'mongodb://*****:*****@app.route('/tweets', methods=['GET'])
def get_all_tweets():
teste4 = mongo.db.teste4
output = []
for s in teste4.find():
tweets_list = []
for tweet in results:
try:
user = tweet.user.name
followers = tweet.user.followers_count
posts = tweet.user.statuses_count
text = tweet.text
location = tweet.user.location
created = tweet.created_at.strftime('%d %B %Y - %H:%M:%S')
tweets_list.append({
'Hashtag': hashtag,
'Usuario': user,
'Seguidores': followers,
'Posts': posts,
'Tweet': text,
'Localizacao': location,
'Data': created
})
except:
return {
'message': 'An internal error ocurred trying to get {}'
}.format(hashtag), 500
return jsonify(tweets_list), 200
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()
logger.addHandler(GelfTcpHandler(host='0.0.0.0', port=12201))
if __name__ == '__main__':
app.run(host='0.0.0.0')
token = config.get('token')
limit = config.get('limit', None)
timespan = int(config.get('timespan', 3600)) # in seconds
end_ts = time.time() * 1000
start_ts = end_ts - (timespan * 1000)
_dg_debug = config.getboolean('debug', False)
# Setup GELF logger
h = None
gelf_host = config.get('host', '127.0.0.1')
gelf_port = int(config.get('port', 11201))
gelf_protocol = config.get('protocol', 'udp') # (tcp|udp|tls)
gelf_source = config.get('source_label', 'Dropbox-audit')
if gelf_protocol == 'udp':
h = GelfUdpHandler(host=gelf_host, port=gelf_port, source=gelf_source)
elif gelf_protocol == 'tcp':
h = GelfTcpHandler(host=gelf_host, port=gelf_port, source=gelf_source)
elif gelf_protocol == 'tls':
tls_cafile = config.get('tls_cafile',
'/etc/ssl/certs/ca-certificates.crt')
h = GelfTlsHandler(host=gelf_host,
port=gelf_port,
source=gelf_source,
validate=True,
ca_certs=tls_cafile)
else:
sys.exit('Unknown protocol')
logger = logging.getLogger()
logger.addHandler(h)
# Import audit entries
n = dropbox_to_graylog(token, start_ts, end_ts, limit=limit, handler=h)
h.flush()
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfTlsHandler
import logging
import json
import pytest
import mock
@pytest.yield_fixture
def send(extra_fields_handler):
with mock.patch.object(extra_fields_handler, 'send') as mock_send:
yield mock_send
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12000, include_extra_fields=True),
GelfUdpHandler(host='127.0.0.1', port=12000, compress=False, include_extra_fields=True),
GelfTlsHandler(host='127.0.0.1', port=12000, include_extra_fields=True)
])
def extra_fields_handler(request):
return request.param
class ContextFilter(logging.Filter):
def filter(self, record):
record.ozzy = 'diary of a madman'
record.van_halen = 1984
record.id = 123
return True
@pytest.yield_fixture
from client import update_thr
from rfid_lib import rfid_thr
from loguru import logger
from device_manager import manager
from pygelf import GelfTcpHandler
from dotenv import load_dotenv
load_dotenv()
logger.add('logs/log_{time}.log',
rotation=datetime.timedelta(days=1),
retention=30,
level='INFO')
# Send logs to a remote server
if 'REMOTE_SYSLOG_IP' in os.environ:
logger.add(GelfTcpHandler(host=os.environ['REMOTE_SYSLOG_IP'],
port=os.environ['REMOTE_SYSLOG_PORT']),
level='INFO')
else:
logger.warning(
'no REMOTE_SYSLOG_IP configured, will write logs only to STDOUT and logs/log_*.log'
)
logger.info('setting up GPIO mode')
GPIO.setmode(GPIO.BCM)
logger.info('(DONE) setting up GPIO mode')
try:
DEBUG_ON_ = False
RUN_SERVER = True
logger.info("initializing threads")
manager.door_magnet.close()
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfTlsHandler
import logging
import json
import pytest
import mock
import socket
ADDITIONAL_FIELDS = {
'_ozzy': 'diary of a madman',
'_van_halen': 1984,
'_id': '123'
}
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12000, **ADDITIONAL_FIELDS),
GelfUdpHandler(host='127.0.0.1',
port=12000,
compress=False,
**ADDITIONAL_FIELDS),
GelfTlsHandler(host='127.0.0.1', port=12000, **ADDITIONAL_FIELDS)
])
def handler(request):
return request.param
@pytest.yield_fixture
def send(handler):
with mock.patch.object(handler, 'send') as mock_send:
yield mock_send
import pytest
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfHttpHandler, GelfTlsHandler
from tests.helper import logger, get_unique_message, log_warning
STATIC_FIELDS = {
'_ozzy': 'diary of a madman',
'_van_halen': 1984,
'_id': 42
}
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12201, **STATIC_FIELDS),
GelfUdpHandler(host='127.0.0.1', port=12202, **STATIC_FIELDS),
GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, **STATIC_FIELDS),
GelfHttpHandler(host='127.0.0.1', port=12203, **STATIC_FIELDS),
GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, **STATIC_FIELDS),
GelfTlsHandler(host='127.0.0.1', port=12204, **STATIC_FIELDS),
# GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem', **STATIC_FIELDS),
GelfTcpHandler(host='127.0.0.1', port=12201, static_fields=STATIC_FIELDS, _ozzy='billie jean'),
GelfUdpHandler(host='127.0.0.1', port=12202, static_fields=STATIC_FIELDS, _ozzy='billie jean'),
GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, static_fields=STATIC_FIELDS, _ozzy='billie jean'),
GelfHttpHandler(host='127.0.0.1', port=12203, static_fields=STATIC_FIELDS, _ozzy='billie jean'),
GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, static_fields=STATIC_FIELDS, _ozzy='billie jean'),
GelfTlsHandler(host='127.0.0.1', port=12204, static_fields=STATIC_FIELDS),
# GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem', static_fields=STATIC_FIELDS, _ozzy='billie jean'),
])
def handler(request):
return request.param