class FluentTraceLogger( object ): def __init__( self, name, host='localhost', port=24224 ): if FluentSender is None: raise Exception("Attempted to use FluentTraceLogger with not Fluent dependency available.") self.lock = threading.Lock() self.thread_local = threading.local() self.name = name self.sender = FluentSender( self.name, host=host, port=port ) def context_set( self, key, value ): self.lock.acquire() if not hasattr( self.thread_local, 'context' ): self.thread_local.context = {} self.thread_local.context[key] = value self.lock.release() def context_remove( self, key ): self.lock.acquire() del self.thread_local.context[key] self.lock.release() def log( self, label, time=None, **kwargs ): self.lock.acquire() if hasattr( self.thread_local, 'context' ): kwargs.update( self.thread_local.context ) self.lock.release() if time is None: time = int( time.time() ) self.sender.emit_with_time( label, time, kwargs )
class FluentTraceLogger(object): def __init__(self, name, host='localhost', port=24224): if FluentSender is None: raise Exception( "Attempted to use FluentTraceLogger with not Fluent dependency available." ) self.lock = threading.Lock() self.thread_local = threading.local() self.name = name self.sender = FluentSender(self.name, host=host, port=port) def context_set(self, key, value): self.lock.acquire() if not hasattr(self.thread_local, 'context'): self.thread_local.context = {} self.thread_local.context[key] = value self.lock.release() def context_remove(self, key): self.lock.acquire() del self.thread_local.context[key] self.lock.release() def log(self, label, time=None, **kwargs): self.lock.acquire() if hasattr(self.thread_local, 'context'): kwargs.update(self.thread_local.context) self.lock.release() if time is None: time = int(time.time()) self.sender.emit_with_time(label, time, kwargs)
class FluentTraceLogger( object ): def __init__( self, name, host='localhost', port=24224 ): assert FluentSender is not None, FLUENT_IMPORT_MESSAGE self.lock = threading.Lock() self.thread_local = threading.local() self.name = name self.sender = FluentSender( self.name, host=host, port=port ) def context_set( self, key, value ): self.lock.acquire() if not hasattr( self.thread_local, 'context' ): self.thread_local.context = {} self.thread_local.context[key] = value self.lock.release() def context_remove( self, key ): self.lock.acquire() del self.thread_local.context[key] self.lock.release() def log( self, label, event_time=None, **kwargs ): self.lock.acquire() if hasattr( self.thread_local, 'context' ): kwargs.update( self.thread_local.context ) self.lock.release() event_time = event_time or time.time() self.sender.emit_with_time( label, int(event_time), kwargs )
async def test_fluentd_forward_source_events(elasticsearch, http_client, docker_ip): """FluentD forwards "native" records to ElasticSearch.""" # Grab the current date (we'll need to use it several times and we don't # want to get flakey results when we execute the tests around midnight). today = datetime.date.today() # Clear the index. url = urljoin(elasticsearch, 'events-%s' % (today.isoformat(), )) async with http_client.delete(url) as resp: # May get 404 if we're the first test to run, but we'll get 200 if we # successfully delete the index. assert resp.status in (200, 404) # Post an event with a tag that matches `events.**` rule in `fluent.conf`. fluent = FluentSender('events.test', host=docker_ip, port=24224) fluent.emit('an-event', { 'some-field': 'some-value', }) # Wait until the record shows up in search results. await asyncio.sleep(datetime.timedelta(seconds=3).total_seconds()) # Grab the record. url = urljoin( elasticsearch, 'events-%04d-%02d-%02d/events/_search' % ( today.year, today.month, today.day, )) async with http_client.get(url) as resp: assert resp.status == 200 body = await resp.json() assert body['hits']['total'] == 1 assert body['hits']['hits'][0]['_source'] == { 'service': 'test', 'event': 'an-event', 'some-field': 'some-value', '@timestamp': mock.ANY, } # Grab index stats, check that index exists and that we have our data. url = urljoin(elasticsearch, '_cat/indices?v') async with http_client.get(url) as resp: assert resp.status == 200 body = await resp.text() print('STATS:') print(body) print('------') lines = body.split('\n') lines = [line.split() for line in lines if line.strip()] assert len(lines) >= 2 stats = [dict(zip(lines[0], line)) for line in lines[1:]] stats = {index['index']: index for index in stats} index = stats.pop('events-%04d-%02d-%02d' % (today.year, today.month, today.day)) assert int(index['docs.count']) >= 1
def __init__(self, tag, pdir, stream_cfg, send_term, max_send_fail, echo, encoding, lines_on_start, max_between_data): """ Trailer common class initialization Args: tag: Classification tag for Fluentd pdir: Position file directory stream_cfg: Log streaming service config (Fluentd / Kinesis) strem_cfg: Log transmission time interval max_send_fail: Maximum number of retries in case of transmission failure echo: Whether to save sent messages encoding: Original message encoding lines_on_start: How many lines of existing log will be resent at startup (for debugging) max_between_data: When the service is restarted, unsent logs smaller than this amount are sent. """ super(BaseTailer, self).__init__() self.fsender = self.kclient = None self.ksent_seqn = self.ksent_shid = None self.linfo("__init__", "max_send_fail: '{}'".format(max_send_fail)) self.last_get_hinfo = 0 self.sname, self.saddr = self.get_host_info() tag = "{}.{}".format(self.sname.lower() if self.sname is not None else None, tag) self.linfo(1, "tag: '{}'".format(tag)) self.tag = tag self.send_term = send_term self.last_send_try = 0 self.last_update = 0 self.kpk_cnt = 0 # count for kinesis partition key self.pdir = pdir max_send_fail = max_send_fail if max_send_fail else MAX_SEND_FAIL tstc = type(stream_cfg) if tstc == FluentCfg: host, port = stream_cfg self.fsender = FluentSender(tag, host, port, max_send_fail=max_send_fail) elif tstc == KinesisCfg: stream_name, region, access_key, secret_key = stream_cfg self.kstream_name = stream_name self.ldebug('query_aws_client kinesis {}'.format(region)) self.kclient = query_aws_client('kinesis', region, access_key, secret_key) self.kagg = aggregator.RecordAggregator() self.send_retry = 0 self.echo_file = StringIO() if echo else None self.cache_sent_pos = {} self.encoding = encoding self.lines_on_start = lines_on_start if lines_on_start else 0 self.max_between_data = max_between_data if max_between_data else\ MAX_BETWEEN_DATA
def __init__(self, name, host='localhost', port=24224): if FluentSender is None: raise Exception( "Attempted to use FluentTraceLogger with not Fluent dependency available." ) self.lock = threading.Lock() self.thread_local = threading.local() self.name = name self.sender = FluentSender(self.name, host=host, port=port)
def loop(fs: sender.FluentSender, connection_id: str, interval: int, verbose: bool): while True: results, timestamp = run_speedtest() if results: if not fs.emit_with_time(connection_id, timestamp, clean_results(results, verbose)): logger.error("fluent error: %s", fs.last_error) fs.clear_last_error() logger.info("Sleeping for %s", interval) time.sleep(interval)
def connect(self, tag=None): """ Connect to a fluentd logger """ if not tag: tag = self.tag timeout = float(self.timeout) port = int(self.port) buffer_max = int(self.buffer_max) self.sender = FluentSender(tag, self.host, port, buffer_max, timeout)
class FluentdConnector(StoqConnectorPlugin): def __init__(self): super().__init__() def activate(self, stoq): self.stoq = stoq super().activate() def save(self, payload, archive=False, **kwargs): """ Save results to fluentd logger :param bytes payload: Content to be sent to fluentd :param **kwargs index: Index name to save content to """ # Define the index name, if available. Will default to the plugin name index = kwargs.get('index', self.parentname) for save_attempt in range(3): try: self.sender.emit(index, payload) except AttributeError: self.connect() super().save() def connect(self, tag=None): """ Connect to a fluentd logger """ if not tag: tag = self.tag timeout = float(self.timeout) port = int(self.port) buffer_max = int(self.buffer_max) self.sender = FluentSender(tag, self.host, port, buffer_max, timeout) def disconnect(self): """ Disconnect to a fluentd logger """ self.sender._close()
def init_global_fsender(tag, host, port): from fluent.sender import FluentSender global fsender if fsender is None: fsender = FluentSender(tag, host, port) linfo("init_global_fsender")
def __init__( self, name, host='localhost', port=24224 ): if FluentSender is None: raise Exception("Attempted to use FluentTraceLogger with not Fluent dependency available.") self.lock = threading.Lock() self.thread_local = threading.local() self.name = name self.sender = FluentSender( self.name, host=host, port=port )
def __init__( self, mongos, mongor_ssl=False, cursor_timeout=600, #10 Minutes fluentd="localhost:24224", logtag="rabidmongoose"): self.connections = {} self.cursor_timeout = cursor_timeout self.mongor_uri = "".join(mongos) self.mongor_host = self.mongor_uri[10:].split(":")[0] self.mongor_port = int(self.mongor_uri[10:].split(":")[1]) self.mongor_ssl = mongor_ssl LOGGER.debug('INIT - URI: %s, Host: %s, Port: %s' % (self.mongor_uri, self.mongor_host, self.mongor_port)) if HAS_FLUENT: self.sender = FluentSender(host=fluentd.split(":")[0], port=int(fluentd.split(":")[1]), tag=logtag) LOGGER.addHandler( FluentHandler("%s.debug" % (logtag), host=fluentd.split(":")[0], port=int(fluentd.split(":")[1]))) else: self.sender = FakeFluentSender()
class FluentdConnector(StoqConnectorPlugin): def __init__(self): super().__init__() def activate(self, stoq): self.stoq = stoq super().activate() def save(self, payload, archive=False, **kwargs): """ Save results to fluentd logger :param bytes payload: Content to be sent to fluentd :param **kwargs kwargs: Additional attributes (unused) """ for save_attempt in range(3): try: self.sender.emit(self.parentname, payload) except AttributeError: self.connect() super().save() def connect(self, tag=None): """ Connect to a fluentd logger """ if not tag: tag = self.tag timeout = float(self.timeout) port = int(self.port) buffer_max = int(self.buffer_max) self.sender = FluentSender(tag, self.host, port, buffer_max, timeout) def disconnect(self): """ Disconnect to a fluentd logger """ self.sender._close()
def main(): parser = argparse.ArgumentParser() parser.add_argument("-i", "--ident", help="Unique identifier of the honeypot.") parser.add_argument("-n", "--hostname", help="Hostname of the honeypot.") parser.add_argument("-t", "--type", help="Honeypot type.") parser.add_argument("-a", "--address", help="IP address of the honeypot.") parser.add_argument("--fluent-host", default="fluentbit", help="Hostname of Fluent Bit server") parser.add_argument("--fluent-port", type=int, default=24284, help="Port of Fluent Bit server") parser.add_argument("--fluent-app", default="stingar", help="Application name for Fluent Bit server") parser.add_argument("--tags", help="Comma separated tags for honeypot.") args = parser.parse_args() dt = datetime.datetime.now() data = { "identifier": args.ident, "hostname": args.hostname, "honeypot": args.type, "ip": args.address, "created": dt.strftime("%Y/%m/%d %H:%M:%S"), "updated": dt.strftime("%Y/%m/%d %H:%M:%S"), "tags": args.tags } sender = FluentSender(args.fluent_app, host=args.fluent_host, port=args.fluent_port) sender.emit(SENSOR_TOPIC, data) return 0
def __init__(self, logsense_token=None, tag='python', meta={}, logsense_host=None, logsense_port=None, verbose=False, nanosecond_precision=False): internal_logger = logging.getLogger('logsense.sender') self._logsense_token = logsense_token if logsense_host: self._logsense_host = logsense_host else: self._logsense_host = getenv('LOGSENSE_HOST', 'logs.logsense.com') if logsense_port: self._logsense_port = logsense_port else: self._logsense_port = int(getenv('LOGSENSE_PORT', '32714')) if self._logsense_token is None: self._logger = None print("LOGSENSE_TOKEN not set - skipping handler") else: self._verbose = verbose self._logger = FluentSender( tag, host=self._logsense_host, ssl_server_hostname=self._logsense_host, port=self._logsense_port, use_ssl=True, verbose=self._verbose) self._base_dict = self.update_meta(meta) self.nanosecond_precision = nanosecond_precision
def _get_sender(self, tag, host, port, bufmax, timeout): """ Get the Fluentd sender for the given arguments. Arguments: tag -- The base tag of the log event, representing the application. host -- The host name or IP address of the fluentd server. port -- The port number the fluentd server is listeneing on. bufmax -- The size of the buffer. timeout -- The timeout Returns: A FluentSender with the given configuration. """ sender = None key = "%s%s%d%d%f" % (tag, host, port, bufmax, timeout) if key in self._senders: sender = self._senders[key] else: sender = FluentSender(tag, host, port, bufmax, timeout) self._senders[key] = sender return sender
def __init__( self, name, host='localhost', port=24224 ): self.lock = threading.Lock() self.thread_local = threading.local() self.name = name self.sender = FluentSender( self.name, host=host, port=port )
class LogSenseSender: def __init__(self, logsense_token=None, tag='python', meta={}, logsense_host=None, logsense_port=None, verbose=False, nanosecond_precision=False): internal_logger = logging.getLogger('logsense.sender') self._logsense_token = logsense_token if logsense_host: self._logsense_host = logsense_host else: self._logsense_host = getenv('LOGSENSE_HOST', 'logs.logsense.com') if logsense_port: self._logsense_port = logsense_port else: self._logsense_port = int(getenv('LOGSENSE_PORT', '32714')) if self._logsense_token is None: self._logger = None print("LOGSENSE_TOKEN not set - skipping handler") else: self._verbose = verbose self._logger = FluentSender( tag, host=self._logsense_host, ssl_server_hostname=self._logsense_host, port=self._logsense_port, use_ssl=True, verbose=self._verbose) self._base_dict = self.update_meta(meta) self.nanosecond_precision = nanosecond_precision def update_meta(self, new_meta): self._base_dict = { **new_meta, **{ 'cs_customer_token': self._logsense_token, 'cs_hostname': socket.gethostname() } } return self._base_dict def _convert_value_to_known_type(self, value): if isinstance(value, str): return value elif isinstance(value, int): return value elif isinstance(value, float): return value elif isinstance(value, dict): return value elif isinstance(value, list): return value else: # This fixes issues with e.g. NumPy serialization return str(value) def emit(self, data={}): if self._logger: if isinstance(data, dict): converted_data = { key: self._convert_value_to_known_type(value) for key, value in data.items() } else: converted_data = {'message': str(data)} self._logger.emit('tag', {**converted_data, **self._base_dict}) def emit_with_time(self, label, timestamp, data): event_timestamp = EventTime(timestamp) if self._logger: if isinstance(data, dict): converted_data = { key: self._convert_value_to_known_type(value) for key, value in data.items() } else: converted_data = {'message': str(data)} self._logger.emit_with_time(label, event_timestamp, { **converted_data, **self._base_dict }) @property def last_error(self): if self._logger: return self._logger.last_error() else: return None @last_error.setter def last_error(self, err): if self._logger: self._logger.last_error(err) def clear_last_error(self, _thread_id=None): if self._logger: self._logger.clear_last_error(_thread_id) def close(self): if self._logger: self._logger.close()
from fluent.sender import FluentSender logger = FluentSender("app", host="localhost", port=24224) logger.emit("follow", {"from": "userA", "to": "userB"}) logger.emit("bof", dict(beuha="aussi", age=42)) logger.close()
class BaseTailer(object): def __init__(self, tag, pdir, stream_cfg, send_term, max_send_fail, echo, encoding, lines_on_start, max_between_data): """ Trailer common class initialization Args: tag: Classification tag for Fluentd pdir: Position file directory stream_cfg: Log streaming service config (Fluentd / Kinesis) strem_cfg: Log transmission time interval max_send_fail: Maximum number of retries in case of transmission failure echo: Whether to save sent messages encoding: Original message encoding lines_on_start: How many lines of existing log will be resent at startup (for debugging) max_between_data: When the service is restarted, unsent logs smaller than this amount are sent. """ super(BaseTailer, self).__init__() self.fsender = self.kclient = None self.ksent_seqn = self.ksent_shid = None self.linfo("__init__", "max_send_fail: '{}'".format(max_send_fail)) self.last_get_hinfo = 0 self.sname, self.saddr = self.get_host_info() tag = "{}.{}".format(self.sname.lower() if self.sname is not None else None, tag) self.linfo(1, "tag: '{}'".format(tag)) self.tag = tag self.send_term = send_term self.last_send_try = 0 self.last_update = 0 self.kpk_cnt = 0 # count for kinesis partition key self.pdir = pdir max_send_fail = max_send_fail if max_send_fail else MAX_SEND_FAIL tstc = type(stream_cfg) if tstc == FluentCfg: host, port = stream_cfg self.fsender = FluentSender(tag, host, port, max_send_fail=max_send_fail) elif tstc == KinesisCfg: stream_name, region, access_key, secret_key = stream_cfg self.kstream_name = stream_name self.ldebug('query_aws_client kinesis {}'.format(region)) self.kclient = query_aws_client('kinesis', region, access_key, secret_key) self.kagg = aggregator.RecordAggregator() self.send_retry = 0 self.echo_file = StringIO() if echo else None self.cache_sent_pos = {} self.encoding = encoding self.lines_on_start = lines_on_start if lines_on_start else 0 self.max_between_data = max_between_data if max_between_data else\ MAX_BETWEEN_DATA def get_host_info(self): sname = saddr = None try: sname = socket.gethostname() self.linfo(" host name: {}".format(sname)) saddr = socket.gethostbyname(sname) self.linfo(" host addr: {}".format(saddr)) except Exception as e: self.lerror("Fail to get host info: {}".format(e)) return sname, saddr def query_host_info(self): invalid = (self.sname is None) or (self.saddr is None) if invalid: if time.time() - self.last_get_hinfo > GET_HINFO_TERM: self.sname, self.saddr = self.get_host_info() self.last_get_hinfo = time.time() self.linfo(" self.sname {}, self.saddr {}".format(self.sname, self.saddr)) return self.sname, self.saddr def ldebug(self, tabfunc, msg=""): _log(self, 'debug', tabfunc, msg) def linfo(self, tabfunc, msg=""): _log(self, 'info', tabfunc, msg) def lwarning(self, tabfunc, msg=""): _log(self, 'warning', tabfunc, msg) def lerror(self, tabfunc, msg=""): _log(self, 'error', tabfunc, msg) def tmain(self): cur = time.time() self.ldebug("tmain {}".format(cur)) return cur def read_sent_pos(self, target, con): """Update the sent position of the target so far. Arguments: target: file path for FileTailer, table name for DBTailer con(DBConnector): DB Connection Returns: (position type): Parsed position type `int` for FileTailer. `datetime` for TableTailer. """ self.linfo("read_sent_pos", "updating for '{}'..".format(target)) tname = escape_path(target) ppath = os.path.join(self.pdir, tname + '.pos') pos = None if os.path.isfile(ppath): with open(ppath, 'r') as f: pos = f.readline() self.linfo(1, "found pos file - {}: {}".format(ppath, pos)) parsed_pos = self.parse_sent_pos(pos) if parsed_pos is None: self.lerror("Invalid pos file: '{}'".format(pos)) pos = None else: pos = parsed_pos if pos is None: pos = self.get_initial_pos(con) self.linfo(1, "can't find valid pos for {}, save as " "initial value {}".format(target, pos)) self._save_sent_pos(target, pos) pos = self.parse_sent_pos(pos) return pos def _save_sent_pos(self, target, pos): """Save sent position for a target flie. Args: target: A target file for which position will be saved. pos: Sent position to save. """ self.linfo(1, "_save_sent_pos for {} - {}".format(target, pos)) tname = escape_path(target) path = os.path.join(self.pdir, tname + '.pos') try: with open(path, 'w') as f: f.write("{}\n".format(pos)) except Exception as e: self.lerror("Fail to write pos file: {} {}".format(e, path)) self.cache_sent_pos[target] = pos def _send_newline(self, msg, msgs): """Send new lines This does not send right away, but waits for a certain number of messages to send for efficiency. Args: msg: A message to send msgs: Bulk message buffer """ # self.ldebug("_send_newline {}".format(msg)) ts = int(time.time()) self.may_echo(msg) msgs.append((ts, msg)) if len(msgs) >= BULK_SEND_SIZE: if self.fsender: bytes_ = self._make_fluent_bulk(msgs) self.fsender._send(bytes_) elif self.kclient: self._kinesis_put(msgs) msgs[:] = [] def _send_remain_msgs(self, msgs): """Send bulk remain messages.""" if len(msgs) > 0: if self.fsender: bytes_ = self._make_fluent_bulk(msgs) self.fsender._send(bytes_) elif self.kclient: self._kinesis_put(msgs) def _handle_send_fail(self, e, rbytes): """Handle send exception. Args: e: Exception instance rbytes: Size of send message in bytes. Raises: Re-raise send exception """ self.lwarning(1, "send fail '{}'".format(e)) self.send_retry += 1 if self.send_retry < MAX_SEND_RETRY: self.lerror(1, "Not exceed max retry({} < {}), will try " "again".format(self.send_retry, MAX_SEND_RETRY)) raise else: self.lerror(1, "Exceed max retry, Giving up this change({}" " Bytes)!!".format(rbytes)) def _make_fluent_bulk(self, msgs): """Make bulk payload for fluentd""" tag = '.'.join((self.tag, "data")) bulk = [msgpack.packb((tag, ts, data)) for ts, data in msgs] return ''.join(bulk) def may_echo(self, line): """Echo sent message for debugging Args: line: Sent message """ if self.echo_file: self.echo_file.write('{}\n'.format(line)) self.echo_file.flush() def _kinesis_put(self, msgs): """Send to AWS Kinesis Make aggregated message and send it. Args: msgs: Messages to send """ self.linfo('_kinesis_put {} messages'.format(len(msgs))) self.kpk_cnt += 1 # round robin shards for aggd in self._iter_kinesis_aggrec(msgs): pk, ehk, data = aggd.get_contents() self.linfo(" kinesis aggregated put_record: {} " "bytes".format(len(data))) st = time.time() ret = self.kclient.put_record( StreamName=self.kstream_name, Data=data, PartitionKey=pk, ExplicitHashKey=ehk ) stat = ret['ResponseMetadata']['HTTPStatusCode'] shid = ret['ShardId'] seqn = ret['SequenceNumber'] self.ksent_seqn = seqn self.ksent_shid = shid elp = time.time() - st if stat == 200: self.linfo("Kinesis put success in {}: ShardId: {}, " "SequenceNumber: {}".format(elp, shid, seqn)) else: self.error("Kineis put failed in {}!: " "{}".format(elp, ret['ResponseMetadata'])) def _iter_kinesis_aggrec(self, msgs): for msg in msgs: data = {'tag_': self.tag + '.data', 'ts_': msg[0]} if isinstance(msg[1], dict): data.update(msg[1]) else: data['value_'] = msg[1] pk = str(uuid.uuid4()) res = self.kagg.add_user_record(pk, str(data)) # if payload fits max send size, send it if res: yield self.kagg.clear_and_get() # send remain payload yield self.kagg.clear_and_get()
from fluent.sender import FluentSender sender = FluentSender('tag') sender.emit('label', dict(a='data', b=dict(c='d')))
def __init__( self, name, host='localhost', port=24224 ): assert FluentSender is not None, FLUENT_IMPORT_MESSAGE self.lock = threading.Lock() self.thread_local = threading.local() self.name = name self.sender = FluentSender( self.name, host=host, port=port )
def main(): parser = argparse.ArgumentParser( add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( '--help', action='help', default=argparse.SUPPRESS, help=argparse._('show this help message and exit') ) parser.add_argument( '-v', '--verbose', action='store_true', help='Display debug messages' ) parser.add_argument('-h', '--host', type=str, default='127.0.0.1', help='Listen host') parser.add_argument('-p', '--port', type=int, default=8080, help='Listen port') parser.add_argument('-w', '--workers', type=int, default=max(1, multiprocessing.cpu_count() - 1), help='Number of workers') parser.add_argument('--sentry-dsn', type=str, help='Sentry DSN') parser.add_argument('--redis-host', type=str, default='127.0.0.1', help='Redis host') parser.add_argument('--redis-port', type=str, default=6379, help='Redis port') parser.add_argument('--redis-password', type=str, default=None, help='Redis password') parser.add_argument('--disable-fluent', action='store_true', default=False, help='If set, do not send logs to fluent') parser.add_argument('--fluent-host', type=str, default='127.0.0.1', help='Fluentd host') parser.add_argument('--fluent-port', type=int, default=24224, help='Fluentd port') parser.add_argument('--auth-enabled', action='store_true', default=False, help='Enable authentication') parser.add_argument('--api-url', type=str, default='http://127.0.0.1:5000', help='APITaxi URL, used when authentication is enabled to retrieve users') args = parser.parse_args() if args.sentry_dsn: sentry_sdk.init(args.sentry_dsn, traces_sample_rate=1.0) loglevel = logging.DEBUG if args.verbose else logging.INFO logging.config.dictConfig({ 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'default': { '()': FormatWithPID, 'format': '%(asctime)s (pid %(pid)s) %(message)s' } }, 'handlers': { 'console': { 'level': loglevel, 'class': 'logging.StreamHandler', 'formatter': 'default', } }, 'loggers': { '': { 'handlers': ['console'], 'level': loglevel, } } }) if not args.auth_enabled: logger.warning('Authentication is not enabled') api_key = os.getenv('API_KEY') if args.auth_enabled and not api_key: parser.error('--enable-auth is set but API_KEY environment variable is not set') if args.disable_fluent: fluent = None else: fluent = FluentSender('geotaxi', host=args.fluent_host, port=args.fluent_port) redis = Redis( host=args.redis_host, port=args.redis_port, password=args.redis_password, socket_keepalive=True, ) worker = Worker( redis, fluent=fluent, auth_enabled=args.auth_enabled, api_url=args.api_url, api_key=api_key ) run_server(args.workers, args.host, args.port, worker)
def __init__(self, name, host='localhost', port=24224): self.lock = threading.Lock() self.thread_local = threading.local() self.name = name self.sender = FluentSender(self.name, host=host, port=port)