def perform_example(example_option=False, examples=None): """Perform some example task""" proc = Process(os.getpid()) memory = proc.get_memory_info()[0] timer_then = datetime.now() # ... memory = proc.get_memory_info()[0] if proc.get_memory_info()[0] > memory else memory timer_now = datetime.now() print "Performed examples in %s seconds using %dMB memory" % ((timer_now - timer_then).total_seconds(), int(memory/1000000))
def memory_usage(self): p = Process(self._process.pid) try: mem = p.memory_info() except AttributeError: mem = p.get_memory_info() return mem.rss
def run(self): self._isRunning = True process = Process(os.getpid()) while self._isRunning: self.cpuUsage = process.get_cpu_percent(self._pollInterval) self.memUsage = process.get_memory_info()[0] if self.delegate is not None: self.delegate.processCpuUsage(self.cpuUsage) self.delegate.processMemUsage(self.memUsage)
def _periodically_log_statistics(self): statistics = self._dispersy.statistics process = Process(getpid()) if Process else None while True: statistics.update() # CPU if cpu_percent: self.log("scenario-cpu", percentage=cpu_percent(interval=0, percpu=True)) # memory if process: rss, vms = process.get_memory_info() self.log("scenario-memory", rss=rss, vms=vms) # bandwidth self.log( "scenario-bandwidth", up=self._dispersy.endpoint.total_up, down=self._dispersy.endpoint.total_down, drop_count=self._dispersy.statistics.drop_count, delay_count=statistics.delay_count, delay_send=statistics.delay_send, delay_success=statistics.delay_success, delay_timeout=statistics.delay_timeout, success_count=statistics.success_count, received_count=statistics.received_count, ) # dispersy statistics self.log( "scenario-connection", connection_type=statistics.connection_type, lan_address=statistics.lan_address, wan_address=statistics.wan_address, ) # communities for community in statistics.communities: self.log( "scenario-community", hex_cid=community.hex_cid, classification=community.classification, global_time=community.global_time, sync_bloom_new=community.sync_bloom_new, sync_bloom_reuse=community.sync_bloom_reuse, candidates=[ dict(zip(["lan_address", "wan_address", "global_time"], tup)) for tup in community.candidates ], ) # wait yield self.enable_statistics
def _periodically_log_statistics(self): statistics = self._dispersy.statistics process = Process(getpid()) if Process else None while True: statistics.update() # CPU if cpu_percent: self.log("scenario-cpu", percentage=cpu_percent(interval=0, percpu=True)) # memory if process: rss, vms = process.get_memory_info() self.log("scenario-memory", rss=rss, vms=vms) # bandwidth self.log("scenario-bandwidth", up=self._dispersy.endpoint.total_up, down=self._dispersy.endpoint.total_down, drop_count=self._dispersy.statistics.drop_count, delay_count=statistics.delay_count, delay_send=statistics.delay_send, delay_success=statistics.delay_success, delay_timeout=statistics.delay_timeout, success_count=statistics.success_count, received_count=statistics.received_count) # dispersy statistics self.log("scenario-connection", connection_type=statistics.connection_type, lan_address=statistics.lan_address, wan_address=statistics.wan_address) # communities for community in statistics.communities: self.log( "scenario-community", hex_cid=community.hex_cid, classification=community.classification, global_time=community.global_time, sync_bloom_new=community.sync_bloom_new, sync_bloom_reuse=community.sync_bloom_reuse, candidates=[ dict( zip(["lan_address", "wan_address", "global_time"], tup)) for tup in community.candidates ]) # wait yield self.enable_statistics
def get_mem_used_by_tree(proc = this_proc): """ Gets the memory used by a process and all its children (RSS). If the process is not provided, this process is used. The argument must be a pid or a psutils.Process object. Return value is in bytes. """ # This would be nice, but it turns out it crashes the whole program if the process finished between creating the list of children and getting the memory usage # Adding "if p.is_running()" would help but still have a window for the process to finish before getting the memory usage #return sum((p.get_memory_info()[0] for p in proc.get_children(True)), proc.get_memory_info()[0]) if isinstance(proc, int): proc = Process(proc) # was given a PID mem = proc.get_memory_info()[0] for p in proc.get_children(True): try: if p.is_running(): mem += p.get_memory_info()[0] except: pass return mem
def get_mem_used_by_tree(proc=this_proc): """ Gets the memory used by a process and all its children (RSS). If the process is not provided, this process is used. The argument must be a pid or a psutils.Process object. Return value is in bytes. """ # This would be nice, but it turns out it crashes the whole program if the process finished between creating the list of children and getting the memory usage # Adding "if p.is_running()" would help but still have a window for the process to finish before getting the memory usage #return sum((p.get_memory_info()[0] for p in proc.get_children(True)), proc.get_memory_info()[0]) if isinstance(proc, (int, long)): proc = Process(proc) # was given a PID mem = proc.get_memory_info()[0] for p in proc.get_children(True): try: if p.is_running(): mem += p.get_memory_info()[0] except: pass return mem
def memory_usage(self): self._ensure_initialized() usage = 0 agents = [] for name, container in self._map_container_by_name().iteritems(): info = self._docker.inspect_container(container) pid = info['State']['Pid'] process = Process(pid) try: mem = process.memory_info() except AttributeError: mem = process.get_memory_info() usage = usage + mem.rss agents.append({'name': name, 'memory_usage': mem.rss}) avg = usage / len(agents) if len(agents) > 0 else 0 return {'total_usage': usage, 'average_usage': avg, 'agents': agents}
class Tracer: def __init__(self, pid): self.process = Process(pid) self.gdb = Gdb(pid) self.io_read_bytes = 0 self.io_write_bytes = 0 self.about = (self.process.name, pid, datetime.now()) def snapshot(self): cpu = self.process.get_cpu_percent() mem = self.process.get_memory_info()[0] # only RSS for now io = self.get_io_bytes() stack = self.gdb.get_stack() return (cpu, mem, io, stack) def get_io_bytes(self): _, _, read_bytes, write_bytes = self.process.get_io_counters() io = (read_bytes - self.io_read_bytes, write_bytes - self.io_write_bytes) self.io_read_bytes, self.io_write_bytes = read_bytes, write_bytes return io
def get_info(process=None, interval=0): """Return information about a process. If process is None, will return the information about the current process """ if process is None: process = Process(os.getpid()) info = {} try: mem_info = process.get_memory_info() info['mem_info1'] = bytes2human(mem_info[0]) info['mem_info2'] = bytes2human(mem_info[1]) except AccessDenied: info['mem_info1'] = info['mem_info2'] = "N/A" try: info['cpu'] = process.get_cpu_percent(interval=interval) except AccessDenied: info['cpu'] = "N/A" try: info['mem'] = round(process.get_memory_percent(), 1) except AccessDenied: info['mem'] = "N/A" try: cpu_times = process.get_cpu_times() ctime = timedelta(seconds=sum(cpu_times)) ctime = "%s:%s.%s" % (ctime.seconds // 60 % 60, str((ctime.seconds % 60)).zfill(2), str(ctime.microseconds)[:2]) except AccessDenied: ctime = "N/A" info['ctime'] = ctime try: info['pid'] = process.pid except AccessDenied: info['pid'] = 'N/A' try: info['username'] = process.username except AccessDenied: info['username'] = '******' try: info['nice'] = process.nice except AccessDenied: info['nice'] = 'N/A' except NoSuchProcess: info['nice'] = 'Zombie' try: cmdline = os.path.basename(shlex.split(process.cmdline[0])[0]) except (AccessDenied, IndexError): cmdline = "N/A" info['cmdline'] = cmdline info['children'] = [] for child in process.get_children(): info['children'].append(get_info(child, interval=interval)) return info
class ScenarioScript(ScriptBase): def __init__(self, *args, **kargs): super(ScenarioScript, self).__init__(*args, **kargs) self._my_member = None self._master_member = None self._cid = sha1(self.master_member_public_key).digest() self._is_joined = False self.log("scenario-init", peernumber=int(self._kargs["peernumber"]), hostname=uname()[1]) if self.enable_statistics and Process and cpu_percent: self._process = Process(getpid()) if self.enable_statistics or self.enable_statistics else None self._dispersy.callback.register(self._periodically_log_statistics) @property def enable_wait_for_wan_address(self): return False @property def enable_statistics(self): return 30.0 def run(self): self.add_testcase(self._run_scenario) def _run_scenario(self): for deadline, _, call, args in self.parse_scenario(): yield max(0.0, deadline - time()) if __debug__: dprint(call.__name__) if call(*args) == "END": return @property def my_member_security(self): return u"low" @property def master_member_public_key(self): raise NotImplementedError("must return an experiment specific master member public key") # if False: # # when crypto.py is disabled a public key is slightly # # different... # master_public_key = ";".join(("60", master_public_key[:60].encode("HEX"), "")) # return "3081a7301006072a8648ce3d020106052b81040027038192000404668ed626c6d6bf4a280cf4824c8cd31fe4c7c46767afb127129abfccdf8be3c38d4b1cb8792f66ccb603bfed395e908786049cb64bacab198ef07d49358da490fbc41f43ade33e05c9991a1bb7ef122cda5359d908514b3c935fe17a3679b6626161ca8d8d934d372dec23cc30ff576bfcd9c292f188af4142594ccc5f6376e2986e1521dc874819f7bcb7ae3ce400".decode("HEX") @property def community_class(self): raise NotImplementedError("must return an experiment community class") @property def community_args(self): return () @property def community_kargs(self): return {} def log(self, _message, **kargs): pass def _periodically_log_statistics(self): while True: # CPU self.log("scenario-cpu", percentage=cpu_percent(interval=0, percpu=True)) # memory rss, vms = self._process.get_memory_info() self.log("scenario-memory", rss=rss, vms=vms) # bandwidth self.log("scenario-bandwidth", up=self._dispersy.endpoint.total_up, down=self._dispersy.endpoint.total_down, drop=self._dispersy.statistics.drop_count, success=self._dispersy.statistics.success_count) # wait yield self.enable_statistics def parse_scenario(self): """ Returns a list with (TIMESTAMP, FUNC, ARGS) tuples, where TIMESTAMP is the time when FUNC must be called. [@+][H:]M:S[-[H:]M:S] METHOD [ARG1 [ARG2 ..]] [{PEERNR1 [, PEERNR2, ...] [, PEERNR3-PEERNR6, ...]}] ^^^^ use @ to schedule events based on experiment startstamp use + to schedule events based on peer startstamp ^^^^^^^^^^^^^^^^^ schedule event hours:minutes:seconds after @ or + or add another hours:minutes:seconds pair to schedule uniformly chosen between the two ^^^^^^^^^^^^^^^^^^^^^^^ calls script.schedule_METHOD(ARG1, ARG2) the arguments are passed as strings ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ apply event only to peer 1 and 2, and peers in range 3-6 (including both 3 and 6) """ scenario = [] re_line = re_compile("".join(("^", "(?P<origin>[@+])", "\s*", "(?:(?P<beginH>\d+):)?(?P<beginM>\d+):(?P<beginS>\d+)", "(?:\s*-\s*", "(?:(?P<endH>\d+):)?(?P<endM>\d+):(?P<endS>\d+)", ")?", "\s+", "(?P<method>\w+)(?P<args>\s+(.+?))??", "(?:\s*{(?P<peers>\s*!?\d+(?:-\d+)?(?:\s*,\s*!?\d+(?:-\d+)?)*\s*)})?", "\s*(?:\n)?$"))) peernumber = int(self._kargs["peernumber"]) filename = self._kargs["scenario"] origin = {"@":float(self._kargs["startstamp"]) if "startstamp" in self._kargs else time(), "+":time()} for lineno, line in enumerate(open(filename, "r")): match = re_line.match(line) if match: # remove all entries that are None (allows us to get default per key) dic = dict((key, value) for key, value in match.groupdict().iteritems() if not value is None) # get the peers, if any, for which this line applies yes_peers = set() no_peers = set() for peer in dic.get("peers", "").split(","): peer = peer.strip() if peer: # if the peer number (or peer number pair) is preceded by '!' it negates the result if peer.startswith("!"): peer = peer[1:] peers = no_peers else: peers = yes_peers # parse the peer number (or peer number pair) if "-" in peer: low, high = peer.split("-") peers.update(xrange(int(low), int(high)+1)) else: peers.add(int(peer)) if not (yes_peers or no_peers) or (yes_peers and peernumber in yes_peers) or (no_peers and not peernumber in no_peers): begin = int(dic.get("beginH", 0)) * 3600.0 + int(dic.get("beginM", 0)) * 60.0 + int(dic.get("beginS", 0)) end = int(dic.get("endH", 0)) * 3600.0 + int(dic.get("endM", 0)) * 60.0 + int(dic.get("endS", 0)) assert end == 0.0 or begin <= end, "when end time is given it must be at or after the start time" scenario.append((origin[dic.get("origin", "@")] + begin + (random() * (end - begin) if end else 0.0), lineno, getattr(self, "scenario_" + dic.get("method", "print")), tuple(dic.get("args", "").split()))) assert scenario, "scenario is empty" assert any(func.__name__ == "scenario_end" for _, _, func, _ in scenario), "scenario end is not defined" assert any(func.__name__ == "scenario_start" for _, _, func, _ in scenario), "scenario start is not defined" scenario.sort() for deadline, _, func, args in scenario: if __debug__: dprint("scenario: @", int(deadline - origin["@"]), "s ", func.__name__) self.log("scenario-schedule", deadline=int(deadline - origin["@"]), func=func.__name__, args=args) return scenario def has_community(self, load=False, auto_load=False): try: return self._dispersy.get_community(self._cid, load=load, auto_load=auto_load) except KeyError: return None def scenario_start(self): assert self.has_community() == None ec = ec_generate_key(self.my_member_security) self._my_member = Member(ec_to_public_bin(ec), ec_to_private_bin(ec)) self._master_member = Member(self.master_member_public_key) self.log("scenario-start", my_member=self._my_member.mid, master_member=self._master_member.mid, classification=self.community_class.get_classification()) def scenario_end(self): if __debug__: dprint("END") self.log("scenario-end") return "END" def scenario_print(self, *args): dprint(*args, glue=" ", force=True) def scenario_churn(self, state, duration=None): assert isinstance(state, str), type(state) assert state in ("online", "offline"), state assert duration is None or isinstance(duration, (str, float)), type(duration) duration = None if duration == None else float(duration) community = self.has_community() if state == "online": if community is None: if __debug__: dprint("online for the next ", duration, " seconds") self.log("scenario-churn", state="online", duration=duration) if self._is_joined: self.community_class.load_community(self._master_member, *self.community_args, **self.community_kargs) else: if __debug__: dprint("join community ", self._master_member.mid.encode("HEX"), " as ", self._my_member.mid.encode("HEX")) community = self.community_class.join_community(self._master_member, self._my_member, *self.community_args, **self.community_kargs) community.auto_load = False self._is_joined = True else: if __debug__: dprint("online for the next ", duration, " seconds (we are already online)") self.log("scenario-churn", state="stay-online", duration=duration) elif state == "offline": if community is None: if __debug__: dprint("offline (we are already offline)") self.log("scenario-churn", state="stay-offline") else: if __debug__: dprint("offline") self.log("scenario-churn", state="offline") community.unload_community() else: raise ValueError("state must be either 'online' or 'offline'")
def get_memory_usage(): pid = getpid() proc = Process(pid) mem = proc.get_memory_info() return (mem.rss, mem.vms)