def load_modules_from_zip(f): "Given a zip file, import all modules found therin." new_modules = {} z = zipfile.ZipFile(f) for i in z.namelist(): #get just the folder, ie the module p = unurl(i.split('/')[0]) #Remove the.json by getting rid of last 5 chars n = unurl((i.split('/'))[1][:-5]) if p not in new_modules: new_modules[p] = {} f = z.open(i) new_modules[p][n] = json.loads(f.read().decode()) f.close() with modulesLock: for i in new_modules: if i in ActiveModules: raise cherrypy.HTTPRedirect("/errors/alreadyexists") for i in new_modules: ActiveModules[i] = new_modules[i] messagebus.postMessage("/system/notifications","User "+ pages.getAcessingUser() + " uploaded module" + i + " from a zip file") bookkeeponemodule(i) z.close()
def load_modules_from_zip(f): "Given a zip file, import all modules found therin." new_modules = {} z = zipfile.ZipFile(f) for i in z.namelist(): #get just the folder, ie the module p = unurl(i.split('/')[0]) #Remove the.json by getting rid of last 5 chars n = unurl((i.split('/'))[1][:-5]) if p not in new_modules: new_modules[p] = {} f = z.open(i) new_modules[p][n] = json.loads(f.read().decode()) f.close() with modulesLock: for i in new_modules: if i in ActiveModules: raise cherrypy.HTTPRedirect("/errors/alreadyexists") for i in new_modules: ActiveModules[i] = new_modules[i] messagebus.postMessage( "/system/notifications", "User " + pages.getAcessingUser() + " uploaded module" + i + " from a zip file") bookkeeponemodule(i) z.close()
def newmoduletarget(self, **kwargs): global scopes pages.require("/admin/modules.edit") global moduleschanged moduleschanged = True #If there is no module by that name, create a blank template and the scope obj with modulesLock: if kwargs['name'] not in ActiveModules: ActiveModules[kwargs['name']] = { "__description": { "resource-type": "module-description", "text": "Module info here" } } #Create the scope that code in the module will run in scopes[kwargs['name']] = obj() #Go directly to the newly created module messagebus.postMessage( "/system/notifications", "User " + pages.getAcessingUser() + " Created Module " + kwargs['name']) raise cherrypy.HTTPRedirect("/modules/module/" + util.url(kwargs['name'])) else: return pages.get_template("error.html").render( info=" A module already exists by that name,")
def newusertarget(self,**kwargs): #THIS IS A HACK TO PREVENT UNICODE STRINGS IN PY2.XX FROM GETTING THROUGH #BECAUSE QUOTE() IS USUALLY WHERE THEY CRASH. #AWFULHACK quote(kwargs['username']) pages.require("/admin/users.edit") #create the new user auth.addUser(kwargs['username'],kwargs['password']) #Take the user back to the users page messagebus.postMessage('/system/notifications','New user "'+kwargs['username']+'" added') raise cherrypy.HTTPRedirect("/auth/")
def resourceUpdateTarget(module, resource, kwargs): pages.require("/admin/modules.edit") global moduleschanged moduleschanged = True with modulesLock: t = ActiveModules[module][resource]['resource-type'] resourceobj = ActiveModules[module][resource] if t == 'permission': resourceobj['description'] = kwargs['description'] #has its own lock auth.importPermissionsFromModules( ) #sync auth's list of permissions if t == 'event': e = newevt.Event(kwargs['trigger'], kwargs['action'], {}) #Test compile, throw error on fail. resourceobj['trigger'] = kwargs['trigger'] resourceobj['action'] = kwargs['action'] resourceobj['setup'] = kwargs['setup'] resourceobj['priority'] = max([int(kwargs['priority']), 0]) resourceobj['continual'] = 'continual' in kwargs resourceobj['rate-limit'] = float(kwargs['ratelimit']) #I really need to do something about this possibly brittle bookkeeping system #But anyway, when the active modules thing changes we must update the newevt cache thing. newevt.updateOneEvent(resource, module) if t == 'page': resourceobj['body'] = kwargs['body'] resourceobj['no-navheader'] = 'no-navheader' in kwargs resourceobj['no-header'] = 'no-header' in kwargs resourceobj['dont-show-in-index'] = 'dont-show-in-index' in kwargs #Method checkboxes resourceobj['require-method'] = [] if 'allow-GET' in kwargs: resourceobj['require-method'].append('GET') if 'allow-POST' in kwargs: resourceobj['require-method'].append('POST') #permission checkboxes resourceobj['require-permissions'] = [] for i in kwargs: #Since HTTP args don't have namespaces we prefix all the permission checkboxes with permission if i[:10] == 'Permission': if kwargs[i] == 'true': resourceobj['require-permissions'].append(i[10:]) usrpages.updateOnePage(resource, module) messagebus.postMessage( "/system/notifications", "User " + pages.getAcessingUser() + " modified resource " + resource + " of module " + module) #Return user to the module page raise cherrypy.HTTPRedirect( "/modules/module/" + util.url(module)) #+'/resource/'+util.url(resource))
def newusertarget(self, **kwargs): #THIS IS A HACK TO PREVENT UNICODE STRINGS IN PY2.XX FROM GETTING THROUGH #BECAUSE QUOTE() IS USUALLY WHERE THEY CRASH. #AWFULHACK quote(kwargs['username']) pages.require("/admin/users.edit") #create the new user auth.addUser(kwargs['username'], kwargs['password']) #Take the user back to the users page messagebus.postMessage('/system/notifications', 'New user "' + kwargs['username'] + '" added') raise cherrypy.HTTPRedirect("/auth/")
def deletemoduletarget(self,**kwargs): pages.require("/admin/modules.edit") global moduleschanged moduleschanged = True with modulesLock: ActiveModules.pop(kwargs['name']) #Get rid of any lingering cached events newevt.removeModuleEvents(kwargs['name']) #Get rid of any permissions defined in the modules. auth.importPermissionsFromModules() usrpages.removeModulePages(kwargs['name']) messagebus.postMessage("/system/notifications","User "+ pages.getAcessingUser() + " Deleted module " + kwargs['name']) raise cherrypy.HTTPRedirect("/modules")
def addResourceTarget(module, type, name, kwargs): pages.require("/admin/modules.edit") global moduleschanged moduleschanged = True def insertResource(r): ActiveModules[module][kwargs['name']] = r with modulesLock: #Check if a resource by that name is already there if kwargs['name'] in ActiveModules[module]: raise cherrypy.HTTPRedirect("/errors/alreadyexists") #Create a permission if type == 'permission': insertResource({ "resource-type": "permission", "description": kwargs['description'] }) #has its own lock auth.importPermissionsFromModules( ) #sync auth's list of permissions if type == 'event': insertResource({ "resource-type": "event", "trigger": "False", "action": "pass", "once": True }) #newevt maintains a cache of precompiled events that must be kept in sync with #the modules newevt.updateOneEvent(kwargs['name'], module) if type == 'page': insertResource({ "resource-type": "page", "body": "Content here", 'no-navheader': True }) usrpages.updateOnePage(kwargs['name'], module) messagebus.postMessage( "/system/notifications", "User " + pages.getAcessingUser() + " added resource " + kwargs['name'] + " of type " + type + " to module " + module) #Take the user straight to the resource page raise cherrypy.HTTPRedirect("/modules/module/" + util.url(module) + '/resource/' + util.url(name))
def resourceUpdateTarget(module,resource,kwargs): pages.require("/admin/modules.edit") global moduleschanged moduleschanged = True with modulesLock: t = ActiveModules[module][resource]['resource-type'] resourceobj = ActiveModules[module][resource] if t == 'permission': resourceobj['description'] = kwargs['description'] #has its own lock auth.importPermissionsFromModules() #sync auth's list of permissions if t == 'event': e = newevt.Event(kwargs['trigger'],kwargs['action'],{})#Test compile, throw error on fail. resourceobj['trigger'] = kwargs['trigger'] resourceobj['action'] = kwargs['action'] resourceobj['setup'] = kwargs['setup'] resourceobj['priority'] = max([int(kwargs['priority']),0]) resourceobj['continual'] = 'continual' in kwargs resourceobj['rate-limit'] = float(kwargs['ratelimit']) #I really need to do something about this possibly brittle bookkeeping system #But anyway, when the active modules thing changes we must update the newevt cache thing. newevt.updateOneEvent(resource,module) if t == 'page': resourceobj['body'] = kwargs['body'] resourceobj['no-navheader'] = 'no-navheader' in kwargs resourceobj['no-header'] = 'no-header' in kwargs resourceobj['dont-show-in-index'] = 'dont-show-in-index' in kwargs #Method checkboxes resourceobj['require-method'] = [] if 'allow-GET' in kwargs: resourceobj['require-method'].append('GET') if 'allow-POST' in kwargs: resourceobj['require-method'].append('POST') #permission checkboxes resourceobj['require-permissions'] = [] for i in kwargs: #Since HTTP args don't have namespaces we prefix all the permission checkboxes with permission if i[:10] == 'Permission': if kwargs[i] == 'true': resourceobj['require-permissions'].append(i[10:]) usrpages.updateOnePage(resource,module) messagebus.postMessage("/system/notifications", "User "+ pages.getAcessingUser() + " modified resource " + resource + " of module " + module) #Return user to the module page raise cherrypy.HTTPRedirect("/modules/module/"+util.url(module))#+'/resource/'+util.url(resource))
def deletemoduletarget(self, **kwargs): pages.require("/admin/modules.edit") global moduleschanged moduleschanged = True with modulesLock: ActiveModules.pop(kwargs['name']) #Get rid of any lingering cached events newevt.removeModuleEvents(kwargs['name']) #Get rid of any permissions defined in the modules. auth.importPermissionsFromModules() usrpages.removeModulePages(kwargs['name']) messagebus.postMessage( "/system/notifications", "User " + pages.getAcessingUser() + " Deleted module " + kwargs['name']) raise cherrypy.HTTPRedirect("/modules")
def __workerloop(): while(run): try: #We need the timeout othewise it could block forever #and thus not notice if run was False f=__queue.get(timeout = 5) f() except Exception as e: try: import messagebus messagebus.postMessage('system/errors/workers'+ {"function":f.__name__, "module":f.__module__, "traceback":traceback.format_exc()}) except: pass
def newmoduletarget(self,**kwargs): global scopes pages.require("/admin/modules.edit") global moduleschanged moduleschanged = True #If there is no module by that name, create a blank template and the scope obj with modulesLock: if kwargs['name'] not in ActiveModules: ActiveModules[kwargs['name']] = {"__description": {"resource-type":"module-description", "text":"Module info here"}} #Create the scope that code in the module will run in scopes[kwargs['name']] = obj() #Go directly to the newly created module messagebus.postMessage("/system/notifications","User "+ pages.getAcessingUser() + " Created Module " + kwargs['name']) raise cherrypy.HTTPRedirect("/modules/module/"+util.url(kwargs['name'])) else: return pages.get_template("error.html").render(info = " A module already exists by that name,")
def default(self,module,pagename,*args,**kwargs): global _page_list_lock with _page_list_lock: page = _Pages[module][pagename] page.lastaccessed = time.time() #Check user permissions for i in page.permissions: pages.require(i) #Check HTTP Method if cherrypy.request.method not in page.methods: #Raise a redirect the the wrongmethod error page raise cherrypy.HTTPRedirect('/errors/wrongmethod') try: return page.template.render( kaithem = kaithemobj.kaithem, request = cherrypy.request, module = modules.scopes[module], path = args, kwargs = kwargs ) except Exception as e: #The HTTPRedirect is NOT an error, and should not be handled like one. #So we just reraise it unchanged if isinstance(e,cherrypy.HTTPRedirect): raise e #The way we let users securely serve static files is to simply #Give them a function that raises this special exception if isinstance(e,kaithemobj.ServeFileInsteadOfRenderingPageException): return cherrypy.lib.static.serve_file(e.f_filepath,e.f_MIME,e.f_name) #When an error happens, log it and save the time #Note that we are logging to the compiled event object page.errors.append([time.strftime(config['time-format']),e]) #Keep oly the most recent 25 errors #If this is the first error(high level: transition from ok to not ok) #send a global system messsage that will go to the front page. if len(page.errors)==1: messagebus.postMessage('/system/notifications', "Page "+pagename+" of module "+module+ " may need attention") raise (e)
def everyminute(): global pageviewsthisminute global lastsaved, lastdumpedlogs if not config['autosave-state'] == 'never': if (time.time() -lastsaved) > saveinterval: lastsaved = time.time() #This does not dump the log files. The log files change often. #It would suck to have tons of tiny log files so we let the user configure #Them separately util.SaveAllStateExceptLogs() if not config['autosave-logs'] == 'never': if (time.time() -lastdumpedlogs) > dumplogsinterval: lastdumpedlogs = time.time() messagelogging.dumpLogFile() messagebus.postMessage("/system/perf/FPS" , round(newevt.averageFramesPerSecond,2)) pageviewcountsmoother.sample(pageviewsthisminute) pageviewsthisminute = 0
def _handle_exception(self, e): tb = traceback.format_exc() #When an error happens, log it and save the time #Note that we are logging to the compiled event object self.errors.append([time.strftime(config['time-format']),tb]) #Keep only the most recent errors self.errors = self.errors[-(config['errors-to-keep']):] #The mesagebus is largely untested and we don't want to kill the thread. try: messagebus.postMessage('system/errors/events/'+ self.module+'/'+ self.resource,str(tb)) except Exception as e: print (e) #If this is the first error since th module was last saved raise a notification if len(self.errors)==1: messagebus.postMessage('/system/notifications',"Event "+self.resource+" of module "+self.module+ " may need attention")
def addResourceTarget(module,type,name,kwargs): pages.require("/admin/modules.edit") global moduleschanged moduleschanged = True def insertResource(r): ActiveModules[module][kwargs['name']] = r with modulesLock: #Check if a resource by that name is already there if kwargs['name'] in ActiveModules[module]: raise cherrypy.HTTPRedirect("/errors/alreadyexists") #Create a permission if type == 'permission': insertResource({ "resource-type":"permission", "description":kwargs['description']}) #has its own lock auth.importPermissionsFromModules() #sync auth's list of permissions if type == 'event': insertResource({ "resource-type":"event", "trigger":"False", "action":"pass", "once":True}) #newevt maintains a cache of precompiled events that must be kept in sync with #the modules newevt.updateOneEvent(kwargs['name'],module) if type == 'page': insertResource({ "resource-type":"page", "body":"Content here", 'no-navheader':True}) usrpages.updateOnePage(kwargs['name'],module) messagebus.postMessage("/system/notifications", "User "+ pages.getAcessingUser() + " added resource " + kwargs['name'] + " of type " + type+" to module " + module) #Take the user straight to the resource page raise cherrypy.HTTPRedirect("/modules/module/"+util.url(module)+'/resource/'+util.url(name))
def everyminute(): global pageviewsthisminute global lastsaved, lastdumpedlogs if not config['autosave-state'] == 'never': if (time.time() - lastsaved) > saveinterval: lastsaved = time.time() #This does not dump the log files. The log files change often. #It would suck to have tons of tiny log files so we let the user configure #Them separately util.SaveAllStateExceptLogs() if not config['autosave-logs'] == 'never': if (time.time() - lastdumpedlogs) > dumplogsinterval: lastdumpedlogs = time.time() messagelogging.dumpLogFile() messagebus.postMessage("/system/perf/FPS", round(newevt.averageFramesPerSecond, 2)) pageviewcountsmoother.sample(pageviewsthisminute) pageviewsthisminute = 0
def _handle_exception(self, e): tb = traceback.format_exc() #When an error happens, log it and save the time #Note that we are logging to the compiled event object self.errors.append([time.strftime(config['time-format']), tb]) #Keep only the most recent errors self.errors = self.errors[-(config['errors-to-keep']):] #The mesagebus is largely untested and we don't want to kill the thread. try: messagebus.postMessage( 'system/errors/events/' + self.module + '/' + self.resource, str(tb)) except Exception as e: print(e) #If this is the first error since th module was last saved raise a notification if len(self.errors) == 1: messagebus.postMessage( '/system/notifications', "Event " + self.resource + " of module " + self.module + " may need attention")
def module(self,module,*path,**kwargs): global moduleschanged #If we are not performing an action on a module just going to its page if not path: pages.require("/admin/modules.view") return pages.get_template("modules/module.html").render(module = ActiveModules[module],name = module) else: #This gets the interface to add a page if path[0] == 'addresource': #path[1] tells what type of resource is being created and addResourceDispatcher returns the appropriate crud screen return addResourceDispatcher(module,path[1]) #This case handles the POST request from the new resource target if path[0] == 'addresourcetarget': return addResourceTarget(module,path[1],kwargs['name'],kwargs) #This case shows the information and editing page for one resource if path[0] == 'resource': return resourceEditPage(module,path[1]) #This goes to a dispatcher that takes into account the type of resource and updates everything about the resource. if path[0] == 'updateresource': return resourceUpdateTarget(module,path[1],kwargs) #This returns a page to delete any resource by name if path[0] == 'deleteresource': pages.require("/admin/modules.edit") return pages.get_template("modules/deleteresource.html").render(module=module) #This handles the POST request to actually do the deletion if path[0] == 'deleteresourcetarget': pages.require("/admin/modules.edit") moduleschanged = True with modulesLock: r = ActiveModules[module].pop(kwargs['name']) if r['resource-type'] == 'page': usrpages.removeOnePage(module,kwargs['name']) #Annoying bookkeeping crap to get rid of the cached crap if r['resource-type'] == 'event': newevt.removeOneEvent(module,kwargs['name']) if r['resource-type'] == 'permission': auth.importPermissionsFromModules() #sync auth's list of permissions messagebus.postMessage("/system/notifications","User "+ pages.getAcessingUser() + " deleted resource " + kwargs['name'] + " from module " + module) raise cherrypy.HTTPRedirect('/modules') #This is the target used to change the name and description(basic info) of a module if path[0] == 'update': pages.require("/admin/modules.edit") moduleschanged = True with modulesLock: ActiveModules[module]['__description']['text'] = kwargs['description'] ActiveModules[kwargs['name']] = ActiveModules.pop(module) #UHHG. So very much code tht just syncs data structures. #This gets rid of the cache under the old name newevt.removeModuleEvents(module) usrpages.removeModulePages(module) #And calls this function the generate the new cache bookkeeponemodule(kwargs['name']) #Just for fun, we should probably also sync the permissions auth.importPermissionsFromModules() raise cherrypy.HTTPRedirect('/modules/module/'+util.url(kwargs['name']))
conf = { '/static': {'tools.staticdir.on': True, 'tools.staticdir.dir':os.path.join(dn,'data/static'), "tools.sessions.on": False, 'tools.caching.on' : True, 'tools.caching.delay' : 3600, "tools.caching.expires": 3600, "tools.addheader.on": True } } #Let the user create additional static directories for i in config['serve-static']: if i not in conf: conf["/usr/static/"+i]= {'tools.staticdir.on': True, 'tools.staticdir.dir': config['serve-static'][i], 'tools.expires.on' : True, 'tools.expires.secs' : 1000, "tools.sessions.on": False, 'tools.caching.on' : True, 'tools.caching.delay' : 3600, "tools.caching.expires": 3600, "tools.addheader.on": True } cherrypy.config.update(server_config) messagebus.postMessage('/system/startup','System Initialized') messagebus.postMessage('/system/notifications','System Initialized at ' + time.strftime(config['time-format'])) cherrypy.quickstart(root,'/',config=conf)
if y == 1: common.fail("Onchange Event did nothing") #Unregister event and make sure it really goes away x.unregister() y = 1 if not y == 1: common.fail("Onchange Event did not go away when unregistered") #Now we test the message bus event x = newevt.Event("!onmsg /test", "y='test'", locals()) #Register event with polling. x.register() #Give it a value to change from messagebus.postMessage("/test", 'poo') #Let it notice the old value time.sleep(0.25) #y should immediately be set back to 0 at the next polling cycle if not y == 'test': common.fail("Message Event did nothing") #Unregister event and make sure it really goes away x.unregister() y = 1 messagebus.postMessage('poo', "/test") if y == 'test': common.fail("Message Event did not go away when unregistered")
def postMessage(topic, message): print(message) messagebus.postMessage(topic, message)
def module(self, module, *path, **kwargs): global moduleschanged #If we are not performing an action on a module just going to its page if not path: pages.require("/admin/modules.view") return pages.get_template("modules/module.html").render( module=ActiveModules[module], name=module) else: #This gets the interface to add a page if path[0] == 'addresource': #path[1] tells what type of resource is being created and addResourceDispatcher returns the appropriate crud screen return addResourceDispatcher(module, path[1]) #This case handles the POST request from the new resource target if path[0] == 'addresourcetarget': return addResourceTarget(module, path[1], kwargs['name'], kwargs) #This case shows the information and editing page for one resource if path[0] == 'resource': return resourceEditPage(module, path[1]) #This goes to a dispatcher that takes into account the type of resource and updates everything about the resource. if path[0] == 'updateresource': return resourceUpdateTarget(module, path[1], kwargs) #This returns a page to delete any resource by name if path[0] == 'deleteresource': pages.require("/admin/modules.edit") return pages.get_template( "modules/deleteresource.html").render(module=module) #This handles the POST request to actually do the deletion if path[0] == 'deleteresourcetarget': pages.require("/admin/modules.edit") moduleschanged = True with modulesLock: r = ActiveModules[module].pop(kwargs['name']) if r['resource-type'] == 'page': usrpages.removeOnePage(module, kwargs['name']) #Annoying bookkeeping crap to get rid of the cached crap if r['resource-type'] == 'event': newevt.removeOneEvent(module, kwargs['name']) if r['resource-type'] == 'permission': auth.importPermissionsFromModules( ) #sync auth's list of permissions messagebus.postMessage( "/system/notifications", "User " + pages.getAcessingUser() + " deleted resource " + kwargs['name'] + " from module " + module) raise cherrypy.HTTPRedirect('/modules') #This is the target used to change the name and description(basic info) of a module if path[0] == 'update': pages.require("/admin/modules.edit") moduleschanged = True with modulesLock: ActiveModules[module]['__description']['text'] = kwargs[ 'description'] ActiveModules[kwargs['name']] = ActiveModules.pop(module) #UHHG. So very much code tht just syncs data structures. #This gets rid of the cache under the old name newevt.removeModuleEvents(module) usrpages.removeModulePages(module) #And calls this function the generate the new cache bookkeeponemodule(kwargs['name']) #Just for fun, we should probably also sync the permissions auth.importPermissionsFromModules() raise cherrypy.HTTPRedirect('/modules/module/' + util.url(kwargs['name']))
x.unregister() y =1 if not y == 1: common.fail("Onchange Event did not go away when unregistered") #Now we test the message bus event x = newevt.Event("!onmsg /test","y='test'",locals()) #Register event with polling. x.register() #Give it a value to change from messagebus.postMessage("/test",'poo') #Let it notice the old value time.sleep(0.25) #y should immediately be set back to 0 at the next polling cycle if not y == 'test': common.fail("Message Event did nothing") #Unregister event and make sure it really goes away x.unregister() y =1 messagebus.postMessage('poo',"/test") if y == 'test': common.fail("Message Event did not go away when unregistered")
def postMessage(topic,message): print(message) messagebus.postMessage(topic,message)
def savetarget(self): pages.require("/admin/settings.edit") util.SaveAllState() messagebus.postMessage("/system/notifications","Global server state was saved to disk") raise cherrypy.HTTPRedirect('/')
def dump(j,f): f.write(json.dumps(j,sort_keys=True,indent=1).encode()) messagebus.postMessage("system/notifications","Invalid config option for 'log-format' so defaulting to normal")
def dumpLogFile(): if config['log-format'] == 'normal': def dump(j,f): f.write(json.dumps(j,sort_keys=True,indent=1).encode()) elif config['log-format'] == 'tiny': def dump(j,f): f.write(json.dumps(j,sort_keys=True,separators=(',',':')).encode()) elif config['log-format'] == 'pretty': def dump(j,f): f.write(json.dumps(j,sort_keys=True,indent=4, separators=(',', ': ')).encode()) else: def dump(j,f): f.write(json.dumps(j,sort_keys=True,indent=1).encode()) messagebus.postMessage("system/notifications","Invalid config option for 'log-format' so defaulting to normal") if config['log-compress'] == 'bz2': openlog= bz2.BZ2File ext = '.json.bz2' elif config['log-compress'] == 'gzip': openlog = gzip.GzipFile ext = '.json.gz' elif config['log-compress'] == 'none': openlog = open ext = '.json' else: openlog = open messagebus.postMessage("system/notifications","Invalid config option for 'log-compress' so defaulting to no compression") global log,loglistchanged global approxtotallogentries with savelock: temp = log log = defaultdict(list) approxtotallogentries = 0 if loglistchanged: #Save the list of things to dump with open(os.path.join(directories.logdir,"whattosave.txt"),'w') as f: for i in toSave: f.write(i+'\n') loglistchanged = False #Get rid of anything that is not in the list of things to dump to the log temp2 = {} saveset = set(toSave) for i in temp: #Parsetopic is a function that returns all subscriptions that would match a topic if not set(messagebus.MessageBus.parseTopic(i)).isdisjoint(saveset): temp2[i] = temp[i] temp = temp2 #If there is no log entries to save, don't dump an empty file. if not temp: return where =os.path.join(directories.logdir,'dumps') #Actually dump the log. with openlog(os.path.join(where,str(time.time())+ext),'wb') as f: print() dump(temp,f) f.close() asnumbers = {} for i in util.get_files(where): try: #Remove extensions if i.endswith(".json"): asnumbers[float(i[:-5])] = i elif i.endswith(".json.gz"): asnumbers[float(i[:-8])] = i elif i.endswith(".json.bz2"): asnumbers[float(i[:-9])] = i except ValueError: pass maxsize = unitsofmeasure.strToIntWithSIMultipliers(config['keep-log-files']) size = 0 #Loop over all the old log dumps and add up the sizes for i in util.get_files(where): size = size + os.path.getsize(os.path.join(where,i)) #Get rid of oldest log dumps until the total size is within the limit for i in sorted(asnumbers.keys()): if size <= maxsize: break size = size - os.path.getsize(os.path.join(where,i)) os.remove(os.path.join(where,asnumbers[i]))