def _create_cgroup(self, path): """ Create the specified cgroup. :param path: The path of the cgroup to create. E.g. cpu/mygroup/mysubgroup :return: the Node associated with the created cgroup. :rtype: cgroupspy.nodes.Node """ node = trees.Tree().root path_split = path.split(os.sep) for path_element in path_split: name_to_node = {x.name: x for x in node.children} if path_element not in name_to_node: self.logger.debug("Creating cgroup {} in {}" .format(path_element, node.path)) subprocess.check_output("sudo mkdir -p {}".format(path_element)) subprocess.check_output("sudo chown -R {} {}".format( self._cur_user, path_element)) else: self.logger.debug("Not creating cgroup {} in {} " "since it already exists" .format(path_element, node.path)) node = name_to_node[path_element] return node
def _create_cgroup(self, path): """ Create the specified cgroup. :param path: The path of the cgroup to create. E.g. cpu/mygroup/mysubgroup :return: the Node associated with the created cgroup. :rtype: cgroupspy.nodes.Node """ node = trees.Tree().root path_split = path.split(os.sep) for path_element in path_split: # node.name is encoded to bytes: # https://github.com/cloudsigma/cgroupspy/blob/e705ac4ccdfe33d8ecc700e9a35a9556084449ca/cgroupspy/nodes.py#L64 name_to_node = {x.name.decode(): x for x in node.children} if path_element not in name_to_node: self.log.debug("Creating cgroup %s in %s", path_element, node.path.decode()) node = node.create_cgroup(path_element) else: self.log.debug( "Not creating cgroup %s in %s since it already exists", path_element, node.path.decode() ) node = name_to_node[path_element] return node
def cpu_cgroup(args, id): t = trees.Tree() cset = t.get_node_by_path('/cpuset/') cpu_cgroup = cset.create_cgroup('cpuset_cgroup_' + id) cpu_cgroup.controller.clone_children = 1 cpu_cgroup.controller.cpus = [args.cpu_num] cpu_cgroup.controller.mems = [0] cpu_cgroup.controller.tasks = os.getpid()
def mem_cgroup(args, id): t = trees.Tree() mem = t.get_node_by_path('/memory/') mem_cgroup = mem.create_cgroup('mem_cgroup_' + id) mem_cgroup.controller.clone_children = 1 mem_cgroup.controller.limit_in_bytes = str(args.mem_size * 1024 * 1024) mem_cgroup.controller.swapiness = 0 mem_cgroup.controller.tasks = os.getpid()
def __init__(self, context): self.logger = logging.getLogger(__name__) self.appID = 1 # App ID counter # Map of apps self.appMap = {} # Context self.context = context # Cgroups self.cgroupTree = trees.Tree() if self.cgroupTree.get_node_by_path('/cpu/riaps/') == None: self.logger.info("attempting to create cgroup 'riaps'") _res = riaps_sudo('user_cgroups %s' % (Config.TARGET_USER)) self.cgroupTree = trees.Tree() self.riapsCPU = self.cgroupTree.get_node_by_path( '/cpu/riaps/') # CPU scheduling self.riapsMem = self.cgroupTree.get_node_by_path( '/memory/riaps/') # Memory usage self.riapsNet = self.cgroupTree.get_node_by_path( '/net_cls/riaps/') # Net usage self.hasCGroup = (self.riapsCPU is not None) and \ (self.riapsMem is not None) and \ (self.riapsNet is not None) if not self.hasCGroup: self.logger.warning( "cgroup 'riaps' is incomplete - limited resource management.") # File space is handled through the quota system self.spcMonitor = SpcMonitorThread(self) self.spcMonitor.start() # Network limits are handled through tc self.cleanupNet() riaps_user = Config.TARGET_USER pw_record = pwd.getpwnam(riaps_user) self.riaps_uid = str(pw_record.pw_uid) _res = riaps_sudo( 'tc qdisc add dev %s root handle %s: htb' # Root qdisc = htb % (Config.NIC_NAME, self.riaps_uid)) _res = riaps_sudo('tc class add dev %s parent %s: ' # Root class 'classid %s:%s htb rate %s ceil %s' % (Config.NIC_NAME, self.riaps_uid, self.riaps_uid, self.riaps_uid, Config.NIC_RATE, Config.NIC_CEIL)) self.netMonitor = NetMonitorThread(self) self.netMonitor.start() self.logger.info("__init__ed")
def cgroup_test(): t = trees.Tree() cpu = t.get_node_by_path("/cpu/") new_cpu = cpu.create_cgroup("testcg") cmd("echo $$ > {}/tasks".format(new_cpu.full_path.decode()) + "; cat {}/tasks".format(new_cpu.full_path.decode())) print(new_cpu.name) print(new_cpu.full_path.decode()) print(new_cpu.controller.shares) cpu.delete_cgroup("testcg")
def print_memory_stats(container_id): cgroup_tree = trees.Tree() cgroup_set = cgroup_tree.get_node_by_path("/memory/pylibcontainer/" + container_id) controler = cgroup_set.controller max_usage_in_bytes = controler.max_usage_in_bytes memsw_max_usage_in_bytes = controler.memsw_max_usage_in_bytes info_list = { "mem used": "{0:.2S}".format(HumanSize(max_usage_in_bytes)), "mem+swap used": "{0:.2S}".format(HumanSize(memsw_max_usage_in_bytes)), } if max_usage_in_bytes > 0: print_list("Max Mem Info", info_list)
def setup_memory_cgroup(container_id, pid): """ Create cgroup for the container id """ cgroup_tree = trees.Tree() cgroup_set = cgroup_tree.get_node_by_path("/memory/pylibcontainer") if not cgroup_set: cgroup_set = cgroup_tree.get_node_by_path("/memory/") pylibcontainer_cgroup = cgroup_set.create_cgroup("pylibcontainer") cgroup_set = pylibcontainer_cgroup # container_mem_cgroup new_group = cgroup_set.create_cgroup(container_id) new_group.controller.tasks = pid new_group.controller.limit_in_bytes = DEFAULT_limit_in_bytes
def _delete_cgroup(self, path): """ Delete the specified cgroup. :param path: The path of the cgroup to delete. E.g. cpu/mygroup/mysubgroup """ node = trees.Tree().root path_split = path.split("/") for path_element in path_split: name_to_node = {x.name.decode(): x for x in node.children} if path_element not in name_to_node: self.log.warning("Cgroup does not exist: %s", path) return else: node = name_to_node[path_element] # node is now the leaf node parent = node.parent self.log.debug("Deleting cgroup %s/%s", parent, node.name) parent.delete_cgroup(node.name.decode())
def __init__(self, pps_client: PpsClient, pfs_client: PfsClient): """ PachydermClient constructor. Args: pps_client (PpsClient): Pachyderm Pipeline System client pfs_client (PfsClient): Pachyderm File System client """ self.pps_client = pps_client self.pfs_client = pfs_client self.pool = PoolManager() # TODO: expose self.max_workers = 20 try: self.memory_limit = trees.Tree().get_node_by_path( "/memory/").controller.limit_in_bytes except FileNotFoundError: # cgroups not found, probably running on local machine self.memory_limit = virtual_memory().available self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
def set_dockerd_docker_cpu_shares(dockerd_shares, docker_shares): assert (dockerd_shares >= 0) and (docker_shares >= 0), "given dockerd: %d; docker: %d" % ( dockerd_shares, docker_shares) if dockerd_shares == 0: dockerd_shares = 2 if docker_shares == 0: docker_shares = 2 t = trees.Tree() docker_cpu_cg = t.get_node_by_path("/cpu/docker/") system_cpu_cg = t.get_node_by_path("/cpu/system.slice/") dockerd_cpu_cg = t.get_node_by_path("/cpu/system.slice/docker.service/") # set up the cpu.shares s.t. c(docker) + c(system/dockerd) == c(total) docker_cpu_cg.controller.shares = docker_shares system_cpu_cg.controller.shares = dockerd_shares # TODO: decide how to set dockerd cpu share within system.slice # dockerd_cpu_cg.controller.shares = dockerd_shares print("cpu shares set; dockerd, docker:") print(system_cpu_cg.controller.shares, docker_cpu_cg.controller.shares)
def _create_cgroup(self, path): """ Create the specified cgroup. :param path: The path of the cgroup to create. E.g. cpu/mygroup/mysubgroup :return: the Node associated with the created cgroup. :rtype: cgroupspy.nodes.Node """ node = trees.Tree().root path_split = path.split(os.sep) for path_element in path_split: name_to_node = {x.name: x for x in node.children} if path_element not in name_to_node: self.log.debug("Creating cgroup %s in %s", path_element, node.path) node = node.create_cgroup(path_element) else: self.log.debug( "Not creating cgroup %s in %s since it already exists", path_element, node.path) node = name_to_node[path_element] return node
def get_cgroups(): cgrp_tree = trees.Tree() cgrp_nodes = cgrp_tree.root.children cgroups = [Cgroup(name=node.name) for node in cgrp_nodes] nix_cgrps = [] return nix_cgrps
from cgroupspy import trees import os import subprocess t=trees.Tree() user=os.getlogin() #name of the user class Login(): def __init__(self): name_cg=input("Enter the name of the cgroup you want to create: ") #name of the cgroup try: #Creating cgroups for cpuset cset=t.get_node_by_path('/cpuset/') cset_cg=cset.create_cgroup(name_cg) #creating cgroups for cpu cpu=t.get_node_by_path('/cpu/') cpu_cg=cpu.create_cgroup(name_cg) #creating cgroups for memory memory=t.get_node_by_path('/memory/') memory_cg=memory.create_cgroup(name_cg) print ("Cgroups created by the name of",name_cg) except: print ("Cgroup",name_cg,"already exists")
def __init__(self, parent, rootPath=""): ScrolledPanel.__init__(self, parent=parent) self.SetupScrolling() self.parent = parent self.rootPath = rootPath parent.Bind(wx.EVT_CLOSE, self.closeWindow) filemenu = wx.Menu() menuAbout = filemenu.Append(wx.ID_ABOUT, "&About", "Information about this project") parent.Bind(wx.EVT_MENU, self.aboutClicked, menuAbout) filemenu.AppendSeparator() if (rootPath == "") or (rootPath[:8] == "/cpuacct") or (rootPath[:7] == "/memory"): menuStopTracking = filemenu.Append(-1, "&Stop tracking") parent.Bind(wx.EVT_MENU, self.stopTrackingClicked, menuStopTracking) menuExit = filemenu.Append(wx.ID_EXIT, "E&xit", "Terminate the program") parent.Bind(wx.EVT_MENU, self.exitClicked, menuExit) menuBar = wx.MenuBar() menuBar.Append(filemenu, "&Options") parent.SetMenuBar(menuBar) #logic connected with cgroups self.centerSizer = wx.BoxSizer(wx.HORIZONTAL) #left sizer -it will show cgroups tree with appropriate intendation # all nodes has buttons - clicking one of them starts tracking cgroup and provide # the right sizer with recorded data (for a chart) leftSizer = wx.BoxSizer(wx.VERTICAL) ln = wx.StaticLine(self, id=-1) leftSizer.Add(ln, 0, wx.EXPAND) label = wx.StaticText(self, label="Cgroup tree") leftSizer.Add(label) leftSizer.AddSpacer(20) self.tree = trees.Tree() rootNode = self.tree.root if rootPath != "": rootNode = self.tree.get_node_by_path(rootPath) print rootPath self.children = rootNode.children self.childrenLen = len(self.children) for i in range(self.childrenLen): c = self.children[i] childTitleSizer = wx.BoxSizer(wx.HORIZONTAL) child = wx.StaticText(self, label="\t" + str(c)) oneNode = self.tree.get_node_by_path(str(c)[6:-1]) childrenBtn = wx.Button(self, id=i, label="Children") paramsBtn = wx.Button(self, id=self.childrenLen + i, label="Params") tasksBtn = wx.Button(self, id=self.childrenLen * 2 + i, label="Tasks") addBtn = wx.Button(self, id=self.childrenLen * 3 + i, label="Add cgroup") deleteBtn = wx.Button(self, id=self.childrenLen * 4 + i, label="Delete cgroup") PIDBtn = wx.Button(self, id=self.childrenLen * 5 + i, label="Move PID here") self.Bind(wx.EVT_BUTTON, self.paramsClicked, paramsBtn) self.Bind(wx.EVT_BUTTON, self.childrenClicked, childrenBtn) self.Bind(wx.EVT_BUTTON, self.tasksClicked, tasksBtn) self.Bind(wx.EVT_BUTTON, self.clickAdd, addBtn) self.Bind(wx.EVT_BUTTON, self.clickDelete, deleteBtn) self.Bind(wx.EVT_BUTTON, self.clickPID, PIDBtn) childTitleSizer.Add(child, 0, wx.CENTER) childTitleSizer.AddSpacer(2) childTitleSizer.Add(childrenBtn, 0, wx.CENTER) childTitleSizer.AddSpacer(2) childTitleSizer.Add(paramsBtn, 0, wx.CENTER) childTitleSizer.AddSpacer(2) childTitleSizer.Add(tasksBtn, 0, wx.CENTER) childTitleSizer.AddSpacer(2) childTitleSizer.Add(addBtn, 0, wx.CENTER) childTitleSizer.AddSpacer(2) childTitleSizer.Add(deleteBtn, 0, wx.CENTER) childTitleSizer.AddSpacer(2) childTitleSizer.Add(PIDBtn, 0, wx.CENTER) childTitleSizer.AddSpacer(2) leftSizer.AddSpacer(8) leftSizer.Add(childTitleSizer) rightSizer = wx.BoxSizer(wx.VERTICAL) if (rootPath == "") or (rootPath[:8] == "/cpuacct") or (rootPath[:7] == "/memory"): ln = wx.StaticLine(self, id=-1) rightButtonSizer = wx.BoxSizer(wx.HORIZONTAL) if (rootPath == ""): # or (rootPath[:8] == "/cpuacct"): cpuBtn = wx.Button(self, id=-1, label="Cpu accounting") self.Bind(wx.EVT_BUTTON, self.cpuChosen, cpuBtn) rightButtonSizer.Add(cpuBtn, 0, wx.CENTER) rightButtonSizer.AddSpacer(2) #if (rootPath == "") or (rootPath[:7] == "/memory"): memoryBtn = wx.Button(self, id=-1, label="Memory") self.Bind(wx.EVT_BUTTON, self.memoryChosen, memoryBtn) rightButtonSizer.Add(memoryBtn, 0, wx.CENTER) rightButtonSizer.AddSpacer(2) rightSizer.Add(rightButtonSizer) self.figure = Figure() self.axes = self.figure.add_subplot(111) self.canvas = FigureCanvas(self, -1, self.figure) self.x = [] self.y = [] rightSizer.Add(ln, 0, wx.EXPAND) rightSizer.Add(self.canvas, 1, wx.LEFT | wx.TOP | wx.GROW) ln = wx.StaticLine(self, id=-1) rightSizer.Add(ln, 0, wx.EXPAND) self.dataInput = wx.StaticText(self, label="") rightSizer.Add(self.dataInput, 0, wx.CENTER) self.thread = ChartThread() self.thread.parent = self if (rootPath == ""): self.thread.trackID = 0 self.thread.rootPath = "/cpuacct" elif (rootPath[:8] == "/cpuacct"): self.thread.trackID = 0 self.thread.rootPath = rootPath else: self.thread.trackID = 1 self.thread.rootPath = rootPath self.alive = False self.startThread() self.centerSizer.Add(leftSizer) self.centerSizer.Add(rightSizer) self.SetSizer(self.centerSizer) self.Show(True)
def __init__(self): while (True): print( "Press s for setup\nPress d for delete\nPress r for running\nPress x to exit\n" ) inp = input("Please Press a command: ") if inp == "s": inp_user = input("Please enter your user name: ") if os.getlogin() == inp_user: cmd1 = "sudo" + " " + self.path_env + " " + self.path_setup subprocess.run(cmd1, shell=True) else: print("You are not the root user, you cannot setup cgroup") elif inp == "d": inp_user = input("Please enter your user name: ") if os.getlogin() == inp_user: cmd2 = "sudo" + " " + self.path_env + " " + self.path_delete subprocess.run(cmd2, shell=True) else: print( "You are not the root user, you cannot delete cgroup") elif inp == "r": t = trees.Tree() name_cg = "test" #name of the cgroup nodepath_cpuset = '/cpuset/' + name_cg nodepath_cpu = '/cpu/' + name_cg nodepath_memory = '/memory/' + name_cg nodepath_cpuacct = '/cpuacct/' + name_cg #assigning cpucores cset = t.get_node_by_path(nodepath_cpuset) cset.controller.cpus = [3] #assigning hard limits on cpu bandwidth #CPU time is divided by means of specifying how much time can be spent running in a given period cpu = t.get_node_by_path(nodepath_cpu) cpu.controller.cfs_quota_us = 50000 #the total available run-time within a period, minimum=1ms cpu.controller.cfs_period_us = 50000 #the length of a period, minimum=1ms, maximum=1s #assigning soft limits on cpu bandwidth #provide tasks in a cgroup with a relative amount of CPU time, providing an opportunity for the tasks to run cpu.controller.shares = 60 #assigning memory limit memory = t.get_node_by_path(nodepath_memory) memory.controller.limit_in_bytes = 10000000 #node for cpuacct cpuacct = t.get_node_by_path(nodepath_cpuacct) #adding PID pid = None path_cpuset = "/sys/fs/cgroup/cpuset/" + name_cg + "/tasks" #print(path_cpuset) path_cpu = "/sys/fs/cgroup/cpu/" + name_cg + "/tasks" path_memory = "/sys/fs/cgroup/memory/" + name_cg + "/tasks" path_cpuset_mems = "/sys/fs/cgroup/cpuset/" + name_cg + "/cpuset.mems" tasklist = ['gnome-mines'] for proc in psutil.process_iter(): if any(task in proc.name() for task in tasklist): print(proc.pid) print(proc) #print(proc) #print(proc.cpu_times()) pid = str(proc.pid) #print(psutil.cpu_freq()) cmd3 = "echo 0 > " + path_cpuset_mems subprocess.run(cmd3, shell=True) cmd4 = 'echo ' + pid + ' > ' + path_cpuset print("Added PID", pid, "to cpuset") cmd5 = 'echo ' + pid + ' > ' + path_cpu print("Added PID", pid, "to cpu") cmd6 = 'echo ' + pid + ' > ' + path_memory print("Added PID", pid, "to memory") subprocess.run(cmd4, shell=True) subprocess.run(cmd5, shell=True) subprocess.run(cmd6, shell=True) print("Press x to exit to the main menu\n") f = open("out_cg.txt", "w") x = 0 while (x <= 15): print("After", x, "seconds") #bandwidth statistics #nr_periods: No. of enforcement intervals that have elapsed #nr_throttled: No. of times the group has been throttled/limited #throttled_time: The total time duration for which entities of the group have been throttled. f.write("Bandwidth stats:" + str(cpu.controller.stat) + "\n") #print("Bandwidth stats:",cpu.controller.stat) #reports the total CPU time (in nanoseconds) spent in user and system mode by all the tasks in the cgroup #user: Time spent by tasks of the cgroup in user mode #system: Time spent by tasks of the cgroup in kernel mode f.write("User and System times:" + str(cpuacct.controller.acct_stat) + "\n") #print("User and System times:",cpuacct.controller.acct_stat) #reports the total CPU time (in nanoseconds) for all tasks in the cgroup f.write("Total CPU time:" + str(cpuacct.controller.usage) + "\n") #print("Total CPU time:",cpuacct.controller.usage) #reports the total CPU time (in nanoseconds) on each CPU core for all tasks in the cgroup f.write("Total CPU time per core:" + str(cpuacct.controller.usage_percpu) + "\n") #print("Total CPU time per core:",cpuacct.controller.usage_percpu) #wall clock time or real time elapsed cmd_time = 'command ps -p ' + pid + ' --no-headers -o etime' proc = subprocess.run(cmd_time, shell=True, capture_output=True, universal_newlines=True) f.write("Wall clock time:" + proc.stdout.strip() + "\n") #print("Wall clock time:",proc.stdout.strip()) #shows the usage of memory f.write("Memory used:" + str(memory.controller.usage_in_bytes) + "\n") #print("Memory used:",memory.controller.usage_in_bytes) #shows the maximum memory usage f.write("Maximum Memory used:" + str(memory.controller.usage_in_bytes) + "\n") #print("Maximum Memory used:",memory.controller.usage_in_bytes) #shows the number of memory usage hits limits f.write("No. of memory usage hits:" + str(memory.controller.failcnt) + "\n") #print("No. of memory usage hits:",memory.controller.failcnt) #shows various statistics f.write("Stats:" + str(memory.controller.stat) + "\n") #print("Stats:",memory.controller.stat) #shows the amount of Physical Memory used f.write("Physical Memory used:" + str(memory.controller.stat['rss'] + memory.controller.stat['mapped_file']) + "\n") #print("Physical Memory used:",memory.controller.stat['rss']+memory.controller.stat['mapped_file']) #show the OOM controls #print(memory.controller.oom_control) f.write( "-------------------------------------------------------------" ) subprocess.run('sleep 5', shell=True) x = x + 5 f.close() elif inp == "x": break
def delete_memory_cgroup(container_id): cgroup_tree = trees.Tree() cgroup_set = cgroup_tree.get_node_by_path("/memory/pylibcontainer") cgroup_set.delete_cgroup(container_id)