def __init__(self, clust):
self.clust = clust
self.addr = '0.0.0.0'
self.statsd_port = 8125
self.graphite_port = 2003
self.last_write = time.time()
self.nb_data = 0
self.stats_interval = 10
# Do not step on your own foot...
self.stats_lock = threading.RLock()
# our main data structs
self.gauges = {}
self.timers = {}
self.histograms = {}
self.counters = {}
# our helper objects
#self.its = IdxTSDatabase(self.clust)
self.usender = UDPSender(self.clust)
self.tsb = TSBackend(self.usender, self.clust)
# Now start our threads
# STATSD
threader.create_and_launch(self.launch_statsd_udp_listener, name='TSL_statsd_thread')
threader.create_and_launch(self.launch_compute_stats_thread, name='TSC_thread')
# GRAPHITE
threader.create_and_launch(self.launch_graphite_udp_listener, name='TSL_graphite_udp_thread')
threader.create_and_launch(self.launch_graphite_tcp_listener, name='TSL_graphite_tcp_thread')
self.graphite_queue = []
threader.create_and_launch(self.graphite_reaper, name='graphite-reaper-thread')
def __init__(self, kv, ttldb_dir):
self.lock = threading.RLock()
self.dbs = {}
self.db_cache_size = 100
self.kv = kv
self.ttldb_dir = ttldb_dir
if not os.path.exists(self.ttldb_dir):
os.mkdir(self.ttldb_dir)
# Launch a thread that will look once a minute the old entries
threader.create_and_launch(self.ttl_cleaning_thread, name='ttl-cleaning-thread')
def do_collector_thread(self):
logger.log('COLLECTOR thread launched', part='check')
cur_launchs = {}
while not stopper.interrupted:
now = int(time.time())
for (colname, e) in self.collectors.iteritems():
colname = e['name']
inst = e['inst']
# maybe a collection is already running
if colname in cur_launchs:
continue
if now >= e['next_check']:
logger.debug('COLLECTOR: launching collector %s' % colname, part='check')
t = threader.create_and_launch(inst.main, name='collector-%s' % colname)
cur_launchs[colname] = t
e['next_check'] += 10
e['last_check'] = now
to_del = []
for (colname, t) in cur_launchs.iteritems():
if not t.is_alive():
t.join()
to_del.append(colname)
for colname in to_del:
del cur_launchs[colname]
time.sleep(1)
def set_leave(self, leaved):
addr = leaved['addr']
port = leaved['port']
name = leaved['name']
incarnation = leaved['incarnation']
uuid = leaved['uuid']
tags = leaved.get('tags', [])
services = leaved.get('services', {})
state = 'leave'
print "SET_LEAVE::", leaved
# Maybe we didn't even have this nodes in our list?
if not uuid in self.nodes:
return
node = self.nodes.get(uuid, None)
# The node can vanish by another thread delete
if node is None:
return
# Maybe we already know it's leaved, so don't update it
if node['state'] == 'leave':
return
print "SET LEAVE %s and inner node %s" % (leaved, node)
# If for me it must be with my own incarnation number so we are sure it's really us that should leave
# and not
if uuid == self.uuid:
if incarnation != node['incarnation']:
print "LEAVE INCARNATION NOT THE SAME FOR MYSELF"
return
else:
# If not for me, use the classic 'not already known' rule
if incarnation < node['incarnation']:
print "LEAVE, NOT FOR ME, THE INCARNATION NUMBER IS TOO OLD"
return
print "SET LEAVE UUID and SELF.UUID", uuid, self.uuid
# Maybe it's us?? If so we must send our broadcast and exit in few seconds
if uuid == self.uuid:
logger.log('LEAVE: someone is asking me for leaving.', part='gossip')
self.incarnation += 1
node['incarnation'] = self.incarnation
self.stack_leave_broadcast(node)
def bailout_after_leave(self):
logger.log('Bailing out in few seconds. I was put in leave state')
time.sleep(10)
logger.log('Exiting from a self leave message')
# Will set self.interrupted = True to eavery thread that loop
pubsub.pub('interrupt')
threader.create_and_launch(bailout_after_leave, args=(self,))
return
logger.log('LEAVING: The node %s is leaving' % node['name'], part='gossip')
# Ok it's definitivly someone else that is now suspected, update this, and update it :)
node['incarnation'] = incarnation
node['state'] = state
node['leave_time'] = int(time.time())
node['tags'] = tags
node['services'] = services
self.stack_leave_broadcast(node)
def launch_reaper_thread(self):
threader.create_and_launch(self.do_reaper_thread, name='TS-reaper-thread')
def launch(self):
threader.create_and_launch(self.do_loop, name='docker-loop')
threader.create_and_launch(self.do_stats_loop, name='docker-stats-loop')
def launch(self):
threader.create_and_launch(self.do_thread, name='now-thread')
def launch(self):
if not Client:
logger.warning('Missing docker lib')
return
t = threader.create_and_launch(self.do_loop, name='docker-loop')
