class CarbonSender(Service):
host = "127.0.0.1" # type: str
port = 2003 # type: int
send_interval = 5 # type: int
protocol = "udp" # type: str
namespace = "" # type: List[str]
storage = TotalStorage
_handle = None # type: PeriodicCallback
async def start(self):
namespace = ".".join(strip_carbon_ns(item) for item in self.namespace)
client = PROTOCOLS[self.protocol](
self.host,
self.port,
namespace=namespace,
storage=self.storage(),
)
set_client(client)
self._handle = PeriodicCallback(client.send)
self._handle.start(self.send_interval, loop=self.loop)
log.info(
"Periodic carbon metrics sender started. Send to %s://%s:%d with "
"interval %rs",
self.protocol,
self.host,
self.port,
self.send_interval,
)
async def stop(self, *_):
self._handle.stop()
async def test_restart(loop):
counter = 0
def task():
nonlocal counter
counter += 1
periodic = PeriodicCallback(task)
periodic.start(0.1, loop)
await asyncio.sleep(0.5)
periodic.stop()
assert 4 < counter < 7
await asyncio.sleep(0.5)
assert 4 < counter < 7
periodic.start(0.1, loop)
await asyncio.sleep(0.5)
periodic.stop()
assert 8 < counter < 14
await asyncio.sleep(0.5)
assert 8 < counter < 14
class PeriodicService(Service):
__required__ = ("interval", )
interval = None # type: float # in seconds
delay = 0 # type: float # in seconds
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.periodic = PeriodicCallback(self.callback)
async def start(self):
self.periodic.start(self.interval, delay=self.delay, loop=self.loop)
log.info("Periodic service %s started", self)
async def stop(self, err):
if self.periodic.task:
await self.periodic.task
self.periodic.stop()
log.info("Periodic service %s is stopped", self)
async def callback(self):
raise NotImplementedError
def __str__(self):
return "{}(interval={},delay={})".format(
self.__class__.__name__,
self.interval,
self.delay,
)
async def start(self):
self.logger = self.log.getChild(str(id(self)))
self.profiler = cProfile.Profile()
self.periodic = PeriodicCallback(self.save_stats)
self.profiler.enable()
self.periodic.start(self.interval)
async def start(self):
log.warning("Start memory tracer")
tracemalloc.start()
# 创建周期任务，计算内存差异
self._tracer = PeriodicCallback(self.show_stats)
self._tracer.start(self.interval)
self._log = log.getChild(str(id(self)))
self._snapshot_on_start = self.take_snapshot()
class Profiler(Service):
profiler = None # type: cProfile.Profile
periodic = None # type: PeriodicCallback
order = "cumulative" # type: str
path = None # type: str
logger = None # type: logging.Logger
interval = 10 # type: int
top_results = 10 # type: int
log = logging.getLogger(__name__) # type: logging.Logger
async def start(self):
self.logger = self.log.getChild(str(id(self)))
self.profiler = cProfile.Profile()
self.periodic = PeriodicCallback(self.save_stats)
self.profiler.enable()
self.periodic.start(self.interval)
def save_stats(self):
with io.StringIO() as stream:
stats = Stats(
self.profiler,
stream=stream,
).strip_dirs().sort_stats(self.order)
stats.print_stats(self.top_results)
self.logger.info(stream.getvalue())
try:
if self.path is not None:
stats.dump_stats(self.path)
finally:
self.profiler.enable()
async def stop(self, exception: Exception = None):
self.logger.info("Stop profiler")
task = self.periodic.stop()
with suppress(CancelledError):
await task
self.profiler.disable()
async def start(self):
namespace = ".".join(strip_carbon_ns(item) for item in self.namespace)
client = PROTOCOLS[self.protocol](
self.host,
self.port,
namespace=namespace,
storage=self.storage(),
)
set_client(client)
self._handle = PeriodicCallback(client.send)
self._handle.start(self.send_interval, loop=self.loop)
log.info(
"Periodic carbon metrics sender started. Send to %s://%s:%d with "
"interval %rs",
self.protocol,
self.host,
self.port,
self.send_interval,
)
async def test_shield(loop):
counter = 0
async def task():
nonlocal counter
await asyncio.sleep(0.1)
counter += 1
periodic = PeriodicCallback(task)
periodic.start(0.2, loop, shield=True)
# Wait for periodic callback to start
await asyncio.sleep(0.01)
with pytest.raises(asyncio.CancelledError):
await periodic.stop()
# Wait for counter to increment
await asyncio.sleep(0.1)
# Shielded
assert counter == 1
# No shield
counter = 0
periodic = PeriodicCallback(task)
periodic.start(0.2, loop, shield=False)
# Wait for periodic callback to start
await asyncio.sleep(0.01)
with pytest.raises(asyncio.CancelledError):
await periodic.stop()
# Wait for counter to increment
await asyncio.sleep(0.1)
# Cancelled
assert counter == 0
async def test_long_func(loop):
counter = 0
async def task():
nonlocal counter
counter += 1
await asyncio.sleep(0.5)
periodic = PeriodicCallback(task)
periodic.start(0.1, loop)
await asyncio.sleep(1)
periodic.stop()
assert 1 < counter < 3
async def test_delay(loop):
counter = 0
def task():
nonlocal counter
counter += 1
periodic = PeriodicCallback(task)
periodic.start(0.1, loop, delay=0.5)
await asyncio.sleep(0.25)
assert not counter
await asyncio.sleep(0.5)
periodic.stop()
assert 1 < counter < 4
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.periodic = PeriodicCallback(self.callback)
class MemoryTracer(Service):
_tracer = None # type: PeriodicCallback
_log = None # type: logging.Logger
_snapshot_on_start = None
logger = log.info
interval = 5 # type: int
top_results = 20 # type: int
group_by = GroupBy.lineno # type: GroupBy
STAT_FORMAT = (
"%(count)8s | "
"%(count_diff)8s | "
"%(size)8s | "
"%(size_diff)8s | "
"%(traceback)s\n"
)
async def start(self):
log.warning("Start memory tracer")
tracemalloc.start()
# 创建周期任务，计算内存差异
self._tracer = PeriodicCallback(self.show_stats)
self._tracer.start(self.interval)
self._log = log.getChild(str(id(self)))
self._snapshot_on_start = self.take_snapshot()
@staticmethod
def humanize(num, suffix="B"):
for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"):
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, "Yi", suffix)
@staticmethod
def take_snapshot() -> tracemalloc.Snapshot:
return tracemalloc.take_snapshot()
def compare_snapshot(
self, snapshot_from: tracemalloc.Snapshot,
snapshot_to: tracemalloc.Snapshot,
):
return snapshot_to.compare_to(snapshot_from, self.group_by.value)
def log_diff(self, diff):
results = self.STAT_FORMAT % {
"count": "Objects",
"count_diff": "Obj.Diff",
"size": "Memory",
"size_diff": "Mem.Diff",
"traceback": "Traceback",
}
for stat in diff[:self.top_results]:
results += self.STAT_FORMAT % {
"count": stat.count,
"count_diff": stat.count_diff,
"size": self.humanize(stat.size),
"size_diff": self.humanize(stat.size_diff),
"traceback": stat.traceback,
}
self.logger("Top memory usage:\n%s", results)
@threaded
def show_stats(self):
differences = self.compare_snapshot(
self._snapshot_on_start,
self.take_snapshot(),
)
return self.log_diff(differences)
async def stop(self, exception: Exception = None):
tracemalloc.stop()