def send_metrics_plaintext(self, metrics, ts):
try:
self.ensure_connected(2003)
self.all_stats = ""
def append(k, v):
line = "%s %f %d\n" % (k, v, ts)
self.all_stats += line
self.forEachPrefixedMetric(metrics, append)
totalsent = 0
l = len(self.all_stats)
iterations = 0
while totalsent < l:
iterations += 1
sent = self.sock.send(self.all_stats[totalsent:])
if sent == 0:
raise RuntimeError("socket connection broken")
totalsent += sent
if iterations != 1:
log("INFO: Send took %s iterations" % iterations)
log("%s out of %s (%s%s) characters sent successfully in %s iteration(s)."
% (totalsent, l, (l / totalsent) * 100, "%", iterations))
finally:
self.close()
def connect(self, port):
if self.sock is not None:
raise Exception("Attempt to connect an already connected socket.")
self.port = port
self.sock = socket.socket()
self.sock.settimeout(self.timeout)
log('Connecting to {}:{}'.format(self.host, self.port))
self.sock.connect((self.host, self.port))
def connect(self, port):
if self.sock is not None:
raise Exception("Attempt to connect an already connected socket.")
self.port = port
self.sock = socket.socket()
self.sock.settimeout(self.timeout)
log('Connecting to {}:{}'.format(self.host, self.port))
self.sock.connect((self.host, self.port))
def send_metrics_plaintext(self, metrics_list):
log('Sending {} metrics via Plaintext'.format(len(metrics_list)))
self.ensure_connected(self.port)
data = []
data = "\n".join(metrics_list) + '\n'
try:
sent = self.sock.sendall(data.encode())
except socket.error as e:
log('ERROR: Socket error during send')
raise RuntimeError("socket connection broken")
return sent
def send_metrics_pickle(self, metrics_list):
log('Send metrics via Pickle')
self.ensure_connected(self.pickle_port)
assert (isinstance(metrics_list[0], tuple))
payload = pickle.dumps(metrics_list, protocol=2)
header = struct.pack("!L", len(payload))
message = header + payload
try:
sent = self.sock.send(message)
except socket.error as e:
log('ERROR: Socket error during send')
raise RuntimeError("socket connection broken")
return sent
def send_metrics(self, metrics, timeout):
self.timeout = timeout
iterations = 1
total = 0
while True:
chunk = self._get_chunk_from_queue(metrics, CHUNK_SIZE)
if not self.dry_run:
if self.pickle:
self.send_metrics_pickle(chunk)
else:
self.send_metrics_plaintext(chunk)
iterations += 1
total += len(chunk)
if len(chunk) < CHUNK_SIZE:
break
if iterations != 1:
log("INFO: Send took %s iterations" % iterations)
log('Sent {} datapoints to Carbon'.format(total))
self.close()
def send_metrics(self, metrics, timeout):
self.timeout = timeout
iterations = 1
total = 0
while True:
chunk = self._get_chunk_from_queue(metrics, CHUNK_SIZE)
if not self.dry_run:
if self.pickle:
self.send_metrics_pickle(chunk)
else:
self.send_metrics_plaintext(chunk)
iterations += 1
total += len(chunk)
if len(chunk) < CHUNK_SIZE:
break
if iterations != 1:
log("INFO: Send took %s iterations" % iterations)
log('Sent {} datapoints to Carbon'.format(total))
self.close()
def send_metrics_plaintext(self, metrics, ts):
try:
self.ensure_connected(2003)
self.all_stats = ""
def append(k, v):
line = "%s %f %d\n" % (k, v, ts)
self.all_stats += line
self.forEachPrefixedMetric(metrics, append)
totalsent = 0
l = len(self.all_stats)
iterations = 0
while totalsent < l:
iterations += 1
sent = self.sock.send(self.all_stats[totalsent:])
if sent == 0:
raise RuntimeError("socket connection broken")
totalsent += sent
if iterations != 1:
log("INFO: Send took %s iterations" % iterations)
log("%s out of %s (%s%s) characters sent successfully in %s iteration(s)." % (totalsent, l, (l/totalsent)*100, "%", iterations))
finally:
self.close()
def send_metrics_pickle(self, metrics_list):
log('Send metrics via Pickle')
self.ensure_connected(self.pickle_port)
assert(isinstance(metrics_list[0], tuple))
payload = pickle.dumps(metrics_list, protocol=2)
header = struct.pack("!L", len(payload))
message = header + payload
try:
sent = self.sock.send(message)
except BrokenPipeError as e:
log('ERROR: Broken Pipe Error during send')
raise RuntimeError("BrokenPipe Error")
except socket.error as e:
log('ERROR: Socket error during send')
raise RuntimeError("socket connection broken")
return sent
def send_metrics_plaintext(self, metrics_list):
log('Sending {} metrics via Plaintext'.format(len(metrics_list)))
self.ensure_connected(self.port)
data = []
data = "\n".join(metrics_list) + '\n'
try:
sent = self.sock.sendall(data.encode())
except BrokenPipeError as e:
log('ERROR: Broken Pipe Error during send')
raise RuntimeError("BrokenPipe Error")
except socket.error as e:
# This will kill the task via a runtime error
# every time we get a socket error
log('ERROR: Socket error during send')
raise RuntimeError("socket connection broken")
return sent
def main_loop(mesos, carbon, singularity, pickle):
should_exit = False
# self-monitoring
assert all([mesos, carbon]) # Mesos and Carbon is mandatory
metrics_queue = queue.Queue()
if singularity:
singularity_carbon = SingularityCarbon(singularity, metrics_queue,
pickle)
mesos_carbon = MesosCarbon(mesos, metrics_queue, singularity, pickle)
else:
mesos_carbon = MesosCarbon(mesos, metrics_queue, pickle=pickle)
while True:
try:
wait_until_beginning_of_clock_minute()
with Timer("Entire collect and send cycle"):
timestamp = time.time()
now = datetime.fromtimestamp(timestamp)
log("Timestamp: %s (%s)" % (now, timestamp))
cycle_timeout = timestamp + 59.0
if singularity:
with Timer("Singularity metrics collection"):
singularity.reset()
singularity.update()
singularity_carbon.flush_all()
if mesos:
with Timer("Mesos metrics collection"):
mesos.reset()
mesos.update()
mesos_carbon.flush_all()
if not metrics_queue:
log("No stats this time; sleeping")
continue
send_timeout = cycle_timeout - time.time()
log("Sending stats (timeout %ss)" % send_timeout)
with Timer("Sending stats to graphite"):
carbon.send_metrics(metrics_queue, send_timeout)
except MesosStatsException as e:
log("%s" % e)
except RuntimeError as e:
log("%s" % e)
should_exit = True
sys.exit(1)
break
except (KeyboardInterrupt, SystemExit):
print("Bye!")
should_exit = True
sys.exit(0)
break
except Exception as e:
traceback.print_exc()
log("Unhandled exception: %s" % e)
except object as o:
traceback.print_exc()
log("Unhandled exception object: %s" % o)
except:
traceback.print_exc()
log("Unhandled unknown exception.")
else:
log("Metrics sent successfully.")
finally:
if should_exit:
return
def wait_until_beginning_of_clock_minute():
iter_start = time.time()
now = datetime.fromtimestamp(iter_start)
sleep_time = 60.0 - ((now.microsecond/1000000.0) + now.second)
log("Sleeping for %ss" % sleep_time)
time.sleep(sleep_time)
def main_loop(mesos, carbon, singularity, pickle):
should_exit = False
# self-monitoring
assert all([mesos, carbon]) # Mesos and Carbon is mandatory
metrics_queue = queue.Queue()
if singularity:
singularity_carbon = SingularityCarbon(singularity, metrics_queue,
pickle)
mesos_carbon = MesosCarbon(mesos, metrics_queue, singularity, pickle)
else:
mesos_carbon = MesosCarbon(mesos, metrics_queue, pickle=pickle)
while True:
try:
wait_until_beginning_of_clock_minute()
with Timer("Entire collect and send cycle"):
timestamp = time.time()
now = datetime.fromtimestamp(timestamp)
log("Timestamp: %s (%s)" % (now, timestamp))
cycle_timeout = timestamp + 59.0
if singularity:
with Timer("Singularity metrics collection"):
singularity.reset()
singularity.update()
singularity_carbon.flush_all()
if mesos:
with Timer("Mesos metrics collection"):
mesos.reset()
mesos.update()
mesos_carbon.flush_all()
if not metrics_queue:
log("No stats this time; sleeping")
continue
send_timeout = cycle_timeout - time.time()
log("Sending stats (timeout %ss)" % send_timeout)
with Timer("Sending stats to graphite"):
carbon.send_metrics(metrics_queue, send_timeout)
except MesosStatsException as e:
log("%s" % e)
except RuntimeError as e:
log("%s" % e)
should_exit = True
sys.exit(1)
break
except (KeyboardInterrupt, SystemExit):
print("Bye!")
should_exit = True
sys.exit(0)
break
except Exception as e:
traceback.print_exc()
log("Unhandled exception: %s" % e)
except object as o:
traceback.print_exc()
log("Unhandled exception object: %s" % o)
except:
traceback.print_exc()
log("Unhandled unknown exception.")
else:
log("Metrics sent successfully.")
finally:
if should_exit:
return
def wait_until_beginning_of_clock_minute():
iter_start = time.time()
now = datetime.fromtimestamp(iter_start)
sleep_time = 60.0 - ((now.microsecond / 1000000.0) + now.second)
log("Sleeping for %ss" % sleep_time)
time.sleep(sleep_time)