def __init__(self):
set_service(self)
sys.excepthook = excepthook
self.ioloop = None
self.logger = None
self.service_id = str(uuid.uuid4())
self.executors = {}
self.start_time = perf_counter()
self.pid = os.getpid()
self.nsq_readers = {} # handler -> Reader
self.nsq_writer = None
self.startup_ts = None
# channel, fields -> data
self._metrics = defaultdict(list)
self.metrics_lock = threading.Lock()
self.metrics_callback = None
self.dcs = None
# Effective address and port
self.server = None
self.address = None
self.port = None
self.is_active = False
self.close_callbacks = []
# Can be initialized in subclasses
self.scheduler = None
# Depends on config
topo = config.get_ch_topology_type()
if topo == CH_UNCLUSTERED:
self.register_metrics = self.register_unclustered_metrics
elif topo == CH_REPLICATED:
self.register_metrics = self.register_replicated_metrics
elif topo == CH_SHARDED:
self.register_metrics = self.register_sharded_metrics
self.total_weight = 0
self.get_shard = self.get_sharding_function()
else:
self.die("Invalid ClickHouse cluster topology")
# NSQ Topics
self.topic_queues = {} # name -> TopicQueue()
self.topic_queue_lock = threading.Lock()
self.topic_shutdown = {} # name -> event
DEFAULT_SHARDING_KEY = "managed_object"
SHARDING_KEYS = {"span": "ctx"}
def __init__(self, table=None, chunk=None):
super(ShardingSharder, self).__init__(table="raw_%s" % table, chunk=chunk)
self.get_shards = ShardingFunction()
def feed(self, records):
"""
Shard and replicate records
:param records:
:return:
"""
for m in records:
if not m:
continue
for ch in self.get_shards(self.table, m):
self.records[ch] += [m]
# Initialize
topo = config.get_ch_topology_type()
if topo == CH_UNCLUSTERED:
Sharder = BaseSharder
elif topo == CH_REPLICATED:
Sharder = ReplicatedSharder
elif topo == CH_SHARDED:
Sharder = ShardingSharder