def __init__(self,
host,
port,
base_tag,
additional_fields={},
additional_fields_key="additionalContext",
message_field="message",
include_time=False):
self.host = host
self.port = port
self.base_tag = base_tag
self.message_field = message_field
self.include_time = include_time
self.additional_fields = {}
self.additional_fields_key = additional_fields_key
if bool(additional_fields):
self.additional_fields = {}
for key, val in additional_fields.items():
self.additional_fields[key] = val
if host and port:
self.fluentSender = sender.FluentSender(base_tag,
host=host,
port=port)
else:
self.fluentSender = sender.FluentSender(base_tag)
def getLogger():
if os.getenv('LOG_SERVER') is not None:
log_server = os.getenv('LOG_SERVER')
else:
try:
with open("log_server", "r") as f:
log_server = f.read()
except IOError:
log_server = ""
if os.getenv('LOG_SERVER_PORT') is not None:
log_server_port = int(os.getenv('LOG_SERVER_PORT'))
else:
log_server_port = 24224
if os.getenv('LOG_TAG') is not None:
log_tag = os.getenv('LOG_TAG')
else:
log_tag = "audit"
try:
logger = sender.FluentSender( log_tag, host=log_server, port=log_server_port )
print("Connected to Fluentd")
return logger
except:
quit("ERROR: could not open connection to Fluentd server.");
def __init__(self, host, port, app, tag):
self.__sender = sender.FluentSender(app, host=host, port=int(port))
self.__msg_queue = queue.Queue()
self.__tag = tag
self._start = False
self._child_thread = None
self._to_fluent()
def start(self):
host = CONFIG.get('output_stingarfluentd', 'fluent_host')
port = CONFIG.getint('output_stingarfluentd', 'fluent_port')
app = CONFIG.get('output_stingarfluentd', 'app')
self.sender = sender.FluentSender(app, host=host, port=port)
self.meta = {}
def get_prediction():
loaded_model = joblib.load('data/decision_tree/model.pkl')
date_string = request.args.get('date')
date = datetime.strptime(date_string, '%Y-%m-%d')
product = products[request.args.get("item_nbr")]
data = {
"date": date_string,
"item_nbr": request.args.get("item_nbr"),
"family": product['family'],
"class": product['class'],
"perishable": product['perishable'],
"transactions": 1000,
"year": date.year,
"month": date.month,
"day": date.day,
"dayofweek": date.weekday(),
"days_til_end_of_data": 0,
"dayoff": date.weekday() >= 5
}
df = pd.DataFrame(data=data, index=['row1'])
df = decision_tree.encode_categorical_columns(df)
pred = loaded_model.predict(df)t
if FLUENTD_HOST:
logger = sender.FluentSender(TENANT, host=FLUENTD_HOST, port=int(FLUENTD_PORT))
log_payload = {'prediction': pred[0], **data}
print('logging {}'.format(log_payload))
if not logger.emit('prediction', log_payload):
print(logger.last_error)
logger.clear_last_error()
return "%d" % pred[0]
def send_message():
log.debug('Sending Message')
if json_mode == 'object':
out = json.loads(
message.format(unixs=int(time.time()),
unixms=int(time.time() * 1000)))
elif json_mode == 'string':
out = json.dumps(json.loads(
message.format(unixs=int(time.time()),
unixms=int(time.time() * 1000))),
separators=(',', ':'))
else:
out = message.format(unixs=int(time.time()),
unixms=int(time.time() * 1000))
fluent = sender.FluentSender(
fluentd_tag,
host=fluentd_host,
port=fluentd_port,
timeout=1.0,
verbose=False,
nanosecond_precision=True,
)
if not fluent.emit(None, ({message_key: out} if message_key else out)):
print(fluent.last_error)
fluent.clear_last_error()
fluent.close()
def connect_fluentd(hostname, tag):
try:
client = sender.FluentSender(tag, hostname)
logger.info(f"Connected to fluentd at {hostname}")
return client
except Exception:
logger.error(f"Failed to connect to fluentd at {hostname}")
raise
def __init__(self, fluentd_configs):
logging.Handler.__init__(self)
self.log_senders = []
self.result_senders = []
self.runlogdate = "1970-01-01T00:00:00.000000+0000"
self.formatter = logging.Formatter('%(message)s')
for fluentd_config in fluentd_configs:
if fluentd_config.logging_tag:
self.log_senders.append(
sender.FluentSender(fluentd_config.logging_tag, host=fluentd_config.ip,
port=fluentd_config.port))
if fluentd_config.result_tag:
self.result_senders.append(
sender.FluentSender(fluentd_config.result_tag, host=fluentd_config.ip,
port=fluentd_config.port))
self.__warning_counter = 0
self.__error_counter = 0
def cli(fluentd_host):
logger = sender.FluentSender('arctool', host=fluentd_host, port=24224)
message = {
'api-version': __version__,
'command_line': sys.argv,
'unix_username': getpass.getuser()
}
logger.emit('cli_command', message)
def __init__(self, tag, path, log_line):
OutputWriter.__init__(self, 'in_forward', tag, path, log_line)
self.protocol = 'tcp'
self.host, port = self.path.split(':')
self.port = int(port)
self.fluent_sender = sender.FluentSender(self.tag,
host=self.host,
port=self.port)
self.msgpack_buffer = msgpack.packb(
(self.tag, int(time.time()), self.buffer), **{})
def __init__(self,
host,
port,
base_tag,
identifier,
message_field="message",
include_time=False):
self.host = host
self.port = port
self.base_tag = base_tag
self.identifier = identifier
self.message_field = message_field
self.include_time = include_time
if host and port:
self.fluentSender = sender.FluentSender(base_tag,
host=host,
port=port)
else:
self.fluentSender = sender.FluentSender(base_tag)
def __init__(self, fluent_tag_prefix, inifile_path):
self.fluent_logger = sender.FluentSender(fluent_tag_prefix)
self.inifile_path = inifile_path
self.dd = DatadogClient(app_key=os.environ['DD_APP_KEY'],
api_key=os.environ['DD_API_KEY'])
# key: config's section_name
# value: { query: (query string), cf: (ChangeFinder instance) }
self.dd_sections = {}
self.load_dd_config()
def __init__(self, app=None):
self.app = app
if app is not None:
self.init_app(app)
# Send events after every request finishes
app.after_request(self.send_events)
# Unbounded queue for sent events
self.queue = Queue()
tag_label = app.config.get('EVENT_TAG_PREFIX', 'flask.fluentd')
self._sender = sender.FluentSender(tag_label)
def run(self):
fluentdLogger = sender.FluentSender(self._tag,
host=self._ip,
port=self._port)
while True:
message = {'from': 'userA', 'to': 'userB'}
fluentdLogger.emit('follow', message)
print "Send Message %s message Error %s" % (
message, fluentdLogger.last_error)
time.sleep(1)
def __init__(self, tag, fluentd_ip='127.0.0.1', fluentd_port=24224):
logging.Handler.__init__(self)
self.tag = tag
self.formatter = logging.Formatter('%(message)s')
self.sender = sender.FluentSender(self.tag,
host=fluentd_ip,
port=fluentd_port)
self.runlogdate = datetime.utcnow().replace(
tzinfo=pytz.utc).strftime("%Y-%m-%dT%H:%M:%S.%f%z")
self.__warning_counter = 0
self.__error_counter = 0
def main():
args = parse_args()
logger.info("Args: %s", args)
connection_id = get_connection_id(args.connection_id)
logger.info("Using connection ID: %s", connection_id)
with sender.FluentSender("speedtest",
host=args.fluent_host,
port=args.fluent_port) as fs:
loop(fs, connection_id, args.interval, args.verbose)
def __init__(self,
fluentd_utils=Fluentd(
sender.FluentSender('estuary',
host='127.0.0.1',
port=24224)),
poll_interval=1200,
env_expire_in=1440):
"""K8s deployments env expire scheduler."""
super().__init__(fluentd_utils=fluentd_utils,
method=KubectlUtils.env_clean_up,
poll_interval=poll_interval,
args=[fluentd_utils, env_expire_in])
def __init__(self,
tag,
host='localhost',
port=24224,
timeout=3.0,
verbose=False):
self.tag = tag
self.sender = sender.FluentSender(tag,
host=host, port=port,
timeout=timeout, verbose=verbose)
logging.Handler.__init__(self)
def __init__(self,
fluentd_utils=Fluentd(
sender.FluentSender('estuary',
host='127.0.0.1',
port=24224)),
path=EnvInit.init.get(EnvConstants.DEPLOY_PATH),
poll_interval=1200,
env_expire_in=1440):
"""Docker env expire scheduler."""
super().__init__(fluentd_utils=fluentd_utils,
method=DockerUtils.env_clean_up,
poll_interval=poll_interval,
args=[fluentd_utils, path, env_expire_in])
async def open(self) -> None:
"""Open module."""
from fluent import sender
await Module.open(self)
# get handler
self._fluent = sender.FluentSender("pyobs",
host=self._hostname,
port=self._port)
# listen to log events
await self.comm.register_event(LogEvent, self._process_log_entry)
def init_app(self, app):
tag_prefix = app.config.get("FLUENTD_EVENT_TAG_PREFIX",
"flask.fluentd_event")
host = app.config.get("FLUENTD_EVENT_HOST", "localhost")
port = int(app.config.get("FLUENTD_EVENT_PORT", 24224))
self._sender = sender.FluentSender(tag_prefix, host=host, port=port)
# Use the newstyle teardown_appcontext if it's available,
# otherwise fall back to the request context
if hasattr(app, "teardown_appcontext"):
app.teardown_appcontext(self.send_events)
else:
app.teardown_request(self.send_events)
def __init__(self,
tag,
host='localhost',
port=24220,
timeout=3.0,
verbose=False):
Handler.__init__(self)
self.tag = tag
self.sender = sender.FluentSender(tag,
host=host,
port=port,
timeout=timeout,
verbose=verbose)
self.hostname = socket.gethostname()
def send(level):
logger = sender.FluentSender('app', host='fluentd', port=24224)
result = logger.emit('flask.' + level, {
'level': level,
'from': 'heart',
'to': 'sun'
})
if not result:
error = logger.last_error
logger.clear_last_error()
else:
error = 'OK'
logger.close()
return error
def __init__(self):
self.logger = logging.getLogger(__name__)
tenant = os.getenv('TENANT', 'model')
fluentd_host = os.getenv('FLUENTD_HOST', '')
fluentd_port = os.getenv('FLUENTD_PORT', '0')
if fluentd_host == '' or fluentd_port == '0':
self.logger.info(
"Fluentd logger is not configured, check your FLUENTD_HOST and FLUENTD_PORT environment "
"variables, not logging to fluentd")
self.fluentd_logger = None
else:
self.fluentd_logger = sender.FluentSender(tenant,
host=fluentd_host,
port=int(fluentd_port))
def __init__(self,
logger_name: str = 'rekcurd.service',
log_level: int = None,
config: RekcurdConfig = RekcurdConfig()):
"""
Constructor
:param logger_name:
:param log_level:
:param config: RekcurdConfig
"""
self.logger_name = logger_name
self.log_level = int(log_level or logging.DEBUG)
self.config = config
self.ml_service = config.APPLICATION_NAME
self.service_level = config.SERVICE_LEVEL
self.logger = sender.FluentSender(logger_name)
def __init__(self, config):
self.config = config
fluent_host = self.config.get('stingar', 'fluent_host')
fluent_port = self.config.getint('stingar', 'fluent_port')
app = self.config.get('stingar', 'app')
self.hostname = self.config.get('stingar', 'hostname')
self.identifier = self.config.get('stingar', 'identifier')
self.tags = self.config.get('stingar', 'tags')
self.asn = self.config.get('stingar', 'asn')
self.log_get = self.config.getboolean('stingar',
'log_get',
fallback=False)
self.sender = sender.FluentSender(app,
host=fluent_host,
port=fluent_port)
def cli(tag, label, file, fluentd):
env = EnvironmentSingleton.get_instance()
fluentd = fluentd if fluentd is not None else env.get_env_and_virtual_env(
).get(EnvConstants.FLUENTD_IP_PORT)
app_label = label if label is not None else env.get_env_and_virtual_env(
).get(EnvConstants.LABEL)
tag = tag if tag is not None else env.get_env_and_virtual_env().get(
EnvConstants.TAG)
if app_label is None:
raise Exception(
"Fluentd label was not set."
"Please set option '--label' using this CLI or set the 'LABEL' environment variable \n"
)
if tag is None:
raise Exception(
"Fluentd tag was not set."
"Please set option '--tag' using this CLI or set the 'TAG' environment variable \n"
)
if fluentd is None:
raise Exception(
"Fluentd ip:port location was not detected in command and neither environment variable "
"" + EnvConstants.FLUENTD_IP_PORT + "\n" +
"Please set option '--fluentd' using this CLI or set the 'FLUENTD_IP_PORT' environment variable \n"
)
logger = sender.FluentSender(tag=tag,
host=fluentd.split(":")[0],
port=int(fluentd.split(":")[1]))
service = Logger(logger)
try:
messages = IOUtils.read_dict_from_file(file=file)
if isinstance(messages, list):
for message in messages:
click.echo(service.emit(app_label=app_label, msg=message))
elif isinstance(messages, dict):
click.echo(service.emit(app_label=app_label, msg=messages))
else:
raise Exception("Could not deserialize to a List of Dicts / Dict")
except Exception as e:
click.echo(f"Exception {e.__str__()}")
exit(CLIConstants.FAILURE.value)
exit(CLIConstants.SUCCESS.value)
def write(self, eps, override_buffer=None):
import msgpack
from fluent import sender
if self.fluent_sender is None:
self.fluent_sender = sender.FluentSender(self.tag,
host=self.host,
port=self.port)
self.msgpack_msg = msgpack.packb(
(self.tag, int(time.time()), self.msg), **{})
if override_buffer is not None:
self.msg = override_buffer
self.msgpack_msg = msgpack.packb(
(self.tag, int(time.time()), self.msg), **{})
for i in range(eps):
self.fluent_sender._send_internal(self.msgpack_msg)
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
if 'lc_wrapper_fluentd_host' in os.environ:
fluentd_host = os.environ['lc_wrapper_fluentd_host']
fluentd_port = int(os.environ.get('lc_wrapper_fluentd_port', '24224'))
fluentd_tag = os.environ.get('lc_wrapper_fluentd_tag', 'lc_wrapper')
self.sender = sender.FluentSender(fluentd_tag,
host=fluentd_host,
port=fluentd_port)
self.log.info('lc_wrapper: Enabled fluent logger: host=%s, port=%s, tag=%s',
fluentd_host, fluentd_port, fluentd_tag)
else:
self.sender = None
self._init_message_handler()
self.start_ipython_kernel()
def predict():
request_payload = request.json
input_features = pd.DataFrame([], columns=column_order)
input_features = input_features.append(request_payload, ignore_index=True)
input_features = input_features.fillna(0)
prediction = model.predict(input_features.values.tolist()).tolist()[0]
logger = sender.FluentSender('app', host='host.docker.internal', port=24224)
feature_names = column_order.tolist()
feature_values = input_features.values.tolist()[0]
log_payload = {'prediction': prediction, **dict(zip(feature_names, feature_values))}
if not logger.emit('prediction', log_payload):
print('logger error')
print(logger.last_error)
logger.clear_last_error() # clear stored error after handled errors
return jsonify({'predicted price (thousands)': prediction})
