def load_modules_from_zip(f):
"Given a zip file, import all modules found therin."
new_modules = {}
z = zipfile.ZipFile(f)
for i in z.namelist():
#get just the folder, ie the module
p = unurl(i.split('/')[0])
#Remove the.json by getting rid of last 5 chars
n = unurl((i.split('/'))[1][:-5])
if p not in new_modules:
new_modules[p] = {}
f = z.open(i)
new_modules[p][n] = json.loads(f.read().decode())
f.close()
with modulesLock:
for i in new_modules:
if i in ActiveModules:
raise cherrypy.HTTPRedirect("/errors/alreadyexists")
for i in new_modules:
ActiveModules[i] = new_modules[i]
messagebus.postMessage("/system/notifications","User "+ pages.getAcessingUser() + " uploaded module" + i + " from a zip file")
bookkeeponemodule(i)
z.close()
def load_modules_from_zip(f):
"Given a zip file, import all modules found therin."
new_modules = {}
z = zipfile.ZipFile(f)
for i in z.namelist():
#get just the folder, ie the module
p = unurl(i.split('/')[0])
#Remove the.json by getting rid of last 5 chars
n = unurl((i.split('/'))[1][:-5])
if p not in new_modules:
new_modules[p] = {}
f = z.open(i)
new_modules[p][n] = json.loads(f.read().decode())
f.close()
with modulesLock:
for i in new_modules:
if i in ActiveModules:
raise cherrypy.HTTPRedirect("/errors/alreadyexists")
for i in new_modules:
ActiveModules[i] = new_modules[i]
messagebus.postMessage(
"/system/notifications", "User " + pages.getAcessingUser() +
" uploaded module" + i + " from a zip file")
bookkeeponemodule(i)
z.close()
def newmoduletarget(self, **kwargs):
global scopes
pages.require("/admin/modules.edit")
global moduleschanged
moduleschanged = True
#If there is no module by that name, create a blank template and the scope obj
with modulesLock:
if kwargs['name'] not in ActiveModules:
ActiveModules[kwargs['name']] = {
"__description": {
"resource-type": "module-description",
"text": "Module info here"
}
}
#Create the scope that code in the module will run in
scopes[kwargs['name']] = obj()
#Go directly to the newly created module
messagebus.postMessage(
"/system/notifications",
"User " + pages.getAcessingUser() + " Created Module " +
kwargs['name'])
raise cherrypy.HTTPRedirect("/modules/module/" +
util.url(kwargs['name']))
else:
return pages.get_template("error.html").render(
info=" A module already exists by that name,")
def newusertarget(self,**kwargs):
#THIS IS A HACK TO PREVENT UNICODE STRINGS IN PY2.XX FROM GETTING THROUGH
#BECAUSE QUOTE() IS USUALLY WHERE THEY CRASH. #AWFULHACK
quote(kwargs['username'])
pages.require("/admin/users.edit")
#create the new user
auth.addUser(kwargs['username'],kwargs['password'])
#Take the user back to the users page
messagebus.postMessage('/system/notifications','New user "'+kwargs['username']+'" added')
raise cherrypy.HTTPRedirect("/auth/")
def resourceUpdateTarget(module, resource, kwargs):
pages.require("/admin/modules.edit")
global moduleschanged
moduleschanged = True
with modulesLock:
t = ActiveModules[module][resource]['resource-type']
resourceobj = ActiveModules[module][resource]
if t == 'permission':
resourceobj['description'] = kwargs['description']
#has its own lock
auth.importPermissionsFromModules(
) #sync auth's list of permissions
if t == 'event':
e = newevt.Event(kwargs['trigger'], kwargs['action'],
{}) #Test compile, throw error on fail.
resourceobj['trigger'] = kwargs['trigger']
resourceobj['action'] = kwargs['action']
resourceobj['setup'] = kwargs['setup']
resourceobj['priority'] = max([int(kwargs['priority']), 0])
resourceobj['continual'] = 'continual' in kwargs
resourceobj['rate-limit'] = float(kwargs['ratelimit'])
#I really need to do something about this possibly brittle bookkeeping system
#But anyway, when the active modules thing changes we must update the newevt cache thing.
newevt.updateOneEvent(resource, module)
if t == 'page':
resourceobj['body'] = kwargs['body']
resourceobj['no-navheader'] = 'no-navheader' in kwargs
resourceobj['no-header'] = 'no-header' in kwargs
resourceobj['dont-show-in-index'] = 'dont-show-in-index' in kwargs
#Method checkboxes
resourceobj['require-method'] = []
if 'allow-GET' in kwargs:
resourceobj['require-method'].append('GET')
if 'allow-POST' in kwargs:
resourceobj['require-method'].append('POST')
#permission checkboxes
resourceobj['require-permissions'] = []
for i in kwargs:
#Since HTTP args don't have namespaces we prefix all the permission checkboxes with permission
if i[:10] == 'Permission':
if kwargs[i] == 'true':
resourceobj['require-permissions'].append(i[10:])
usrpages.updateOnePage(resource, module)
messagebus.postMessage(
"/system/notifications", "User " + pages.getAcessingUser() +
" modified resource " + resource + " of module " + module)
#Return user to the module page
raise cherrypy.HTTPRedirect(
"/modules/module/" +
util.url(module)) #+'/resource/'+util.url(resource))
def newusertarget(self, **kwargs):
#THIS IS A HACK TO PREVENT UNICODE STRINGS IN PY2.XX FROM GETTING THROUGH
#BECAUSE QUOTE() IS USUALLY WHERE THEY CRASH. #AWFULHACK
quote(kwargs['username'])
pages.require("/admin/users.edit")
#create the new user
auth.addUser(kwargs['username'], kwargs['password'])
#Take the user back to the users page
messagebus.postMessage('/system/notifications',
'New user "' + kwargs['username'] + '" added')
raise cherrypy.HTTPRedirect("/auth/")
def deletemoduletarget(self,**kwargs):
pages.require("/admin/modules.edit")
global moduleschanged
moduleschanged = True
with modulesLock:
ActiveModules.pop(kwargs['name'])
#Get rid of any lingering cached events
newevt.removeModuleEvents(kwargs['name'])
#Get rid of any permissions defined in the modules.
auth.importPermissionsFromModules()
usrpages.removeModulePages(kwargs['name'])
messagebus.postMessage("/system/notifications","User "+ pages.getAcessingUser() + " Deleted module " + kwargs['name'])
raise cherrypy.HTTPRedirect("/modules")
def addResourceTarget(module, type, name, kwargs):
pages.require("/admin/modules.edit")
global moduleschanged
moduleschanged = True
def insertResource(r):
ActiveModules[module][kwargs['name']] = r
with modulesLock:
#Check if a resource by that name is already there
if kwargs['name'] in ActiveModules[module]:
raise cherrypy.HTTPRedirect("/errors/alreadyexists")
#Create a permission
if type == 'permission':
insertResource({
"resource-type": "permission",
"description": kwargs['description']
})
#has its own lock
auth.importPermissionsFromModules(
) #sync auth's list of permissions
if type == 'event':
insertResource({
"resource-type": "event",
"trigger": "False",
"action": "pass",
"once": True
})
#newevt maintains a cache of precompiled events that must be kept in sync with
#the modules
newevt.updateOneEvent(kwargs['name'], module)
if type == 'page':
insertResource({
"resource-type": "page",
"body": "Content here",
'no-navheader': True
})
usrpages.updateOnePage(kwargs['name'], module)
messagebus.postMessage(
"/system/notifications",
"User " + pages.getAcessingUser() + " added resource " +
kwargs['name'] + " of type " + type + " to module " + module)
#Take the user straight to the resource page
raise cherrypy.HTTPRedirect("/modules/module/" + util.url(module) +
'/resource/' + util.url(name))
def resourceUpdateTarget(module,resource,kwargs):
pages.require("/admin/modules.edit")
global moduleschanged
moduleschanged = True
with modulesLock:
t = ActiveModules[module][resource]['resource-type']
resourceobj = ActiveModules[module][resource]
if t == 'permission':
resourceobj['description'] = kwargs['description']
#has its own lock
auth.importPermissionsFromModules() #sync auth's list of permissions
if t == 'event':
e = newevt.Event(kwargs['trigger'],kwargs['action'],{})#Test compile, throw error on fail.
resourceobj['trigger'] = kwargs['trigger']
resourceobj['action'] = kwargs['action']
resourceobj['setup'] = kwargs['setup']
resourceobj['priority'] = max([int(kwargs['priority']),0])
resourceobj['continual'] = 'continual' in kwargs
resourceobj['rate-limit'] = float(kwargs['ratelimit'])
#I really need to do something about this possibly brittle bookkeeping system
#But anyway, when the active modules thing changes we must update the newevt cache thing.
newevt.updateOneEvent(resource,module)
if t == 'page':
resourceobj['body'] = kwargs['body']
resourceobj['no-navheader'] = 'no-navheader' in kwargs
resourceobj['no-header'] = 'no-header' in kwargs
resourceobj['dont-show-in-index'] = 'dont-show-in-index' in kwargs
#Method checkboxes
resourceobj['require-method'] = []
if 'allow-GET' in kwargs:
resourceobj['require-method'].append('GET')
if 'allow-POST' in kwargs:
resourceobj['require-method'].append('POST')
#permission checkboxes
resourceobj['require-permissions'] = []
for i in kwargs:
#Since HTTP args don't have namespaces we prefix all the permission checkboxes with permission
if i[:10] == 'Permission':
if kwargs[i] == 'true':
resourceobj['require-permissions'].append(i[10:])
usrpages.updateOnePage(resource,module)
messagebus.postMessage("/system/notifications", "User "+ pages.getAcessingUser() + " modified resource " +
resource + " of module " + module)
#Return user to the module page
raise cherrypy.HTTPRedirect("/modules/module/"+util.url(module))#+'/resource/'+util.url(resource))
def deletemoduletarget(self, **kwargs):
pages.require("/admin/modules.edit")
global moduleschanged
moduleschanged = True
with modulesLock:
ActiveModules.pop(kwargs['name'])
#Get rid of any lingering cached events
newevt.removeModuleEvents(kwargs['name'])
#Get rid of any permissions defined in the modules.
auth.importPermissionsFromModules()
usrpages.removeModulePages(kwargs['name'])
messagebus.postMessage(
"/system/notifications", "User " + pages.getAcessingUser() +
" Deleted module " + kwargs['name'])
raise cherrypy.HTTPRedirect("/modules")
def __workerloop():
while(run):
try:
#We need the timeout othewise it could block forever
#and thus not notice if run was False
f=__queue.get(timeout = 5)
f()
except Exception as e:
try:
import messagebus
messagebus.postMessage('system/errors/workers'+
{"function":f.__name__,
"module":f.__module__,
"traceback":traceback.format_exc()})
except:
pass
def newmoduletarget(self,**kwargs):
global scopes
pages.require("/admin/modules.edit")
global moduleschanged
moduleschanged = True
#If there is no module by that name, create a blank template and the scope obj
with modulesLock:
if kwargs['name'] not in ActiveModules:
ActiveModules[kwargs['name']] = {"__description":
{"resource-type":"module-description",
"text":"Module info here"}}
#Create the scope that code in the module will run in
scopes[kwargs['name']] = obj()
#Go directly to the newly created module
messagebus.postMessage("/system/notifications","User "+ pages.getAcessingUser() + " Created Module " + kwargs['name'])
raise cherrypy.HTTPRedirect("/modules/module/"+util.url(kwargs['name']))
else:
return pages.get_template("error.html").render(info = " A module already exists by that name,")
def default(self,module,pagename,*args,**kwargs):
global _page_list_lock
with _page_list_lock:
page = _Pages[module][pagename]
page.lastaccessed = time.time()
#Check user permissions
for i in page.permissions:
pages.require(i)
#Check HTTP Method
if cherrypy.request.method not in page.methods:
#Raise a redirect the the wrongmethod error page
raise cherrypy.HTTPRedirect('/errors/wrongmethod')
try:
return page.template.render(
kaithem = kaithemobj.kaithem,
request = cherrypy.request,
module = modules.scopes[module],
path = args,
kwargs = kwargs
)
except Exception as e:
#The HTTPRedirect is NOT an error, and should not be handled like one.
#So we just reraise it unchanged
if isinstance(e,cherrypy.HTTPRedirect):
raise e
#The way we let users securely serve static files is to simply
#Give them a function that raises this special exception
if isinstance(e,kaithemobj.ServeFileInsteadOfRenderingPageException):
return cherrypy.lib.static.serve_file(e.f_filepath,e.f_MIME,e.f_name)
#When an error happens, log it and save the time
#Note that we are logging to the compiled event object
page.errors.append([time.strftime(config['time-format']),e])
#Keep oly the most recent 25 errors
#If this is the first error(high level: transition from ok to not ok)
#send a global system messsage that will go to the front page.
if len(page.errors)==1:
messagebus.postMessage('/system/notifications',
"Page "+pagename+" of module "+module+
" may need attention")
raise (e)
def everyminute():
global pageviewsthisminute
global lastsaved, lastdumpedlogs
if not config['autosave-state'] == 'never':
if (time.time() -lastsaved) > saveinterval:
lastsaved = time.time()
#This does not dump the log files. The log files change often.
#It would suck to have tons of tiny log files so we let the user configure
#Them separately
util.SaveAllStateExceptLogs()
if not config['autosave-logs'] == 'never':
if (time.time() -lastdumpedlogs) > dumplogsinterval:
lastdumpedlogs = time.time()
messagelogging.dumpLogFile()
messagebus.postMessage("/system/perf/FPS" , round(newevt.averageFramesPerSecond,2))
pageviewcountsmoother.sample(pageviewsthisminute)
pageviewsthisminute = 0
def _handle_exception(self, e):
tb = traceback.format_exc()
#When an error happens, log it and save the time
#Note that we are logging to the compiled event object
self.errors.append([time.strftime(config['time-format']),tb])
#Keep only the most recent errors
self.errors = self.errors[-(config['errors-to-keep']):]
#The mesagebus is largely untested and we don't want to kill the thread.
try:
messagebus.postMessage('system/errors/events/'+
self.module+'/'+
self.resource,str(tb))
except Exception as e:
print (e)
#If this is the first error since th module was last saved raise a notification
if len(self.errors)==1:
messagebus.postMessage('/system/notifications',"Event "+self.resource+" of module "+self.module+ " may need attention")
def addResourceTarget(module,type,name,kwargs):
pages.require("/admin/modules.edit")
global moduleschanged
moduleschanged = True
def insertResource(r):
ActiveModules[module][kwargs['name']] = r
with modulesLock:
#Check if a resource by that name is already there
if kwargs['name'] in ActiveModules[module]:
raise cherrypy.HTTPRedirect("/errors/alreadyexists")
#Create a permission
if type == 'permission':
insertResource({
"resource-type":"permission",
"description":kwargs['description']})
#has its own lock
auth.importPermissionsFromModules() #sync auth's list of permissions
if type == 'event':
insertResource({
"resource-type":"event",
"trigger":"False",
"action":"pass",
"once":True})
#newevt maintains a cache of precompiled events that must be kept in sync with
#the modules
newevt.updateOneEvent(kwargs['name'],module)
if type == 'page':
insertResource({
"resource-type":"page",
"body":"Content here",
'no-navheader':True})
usrpages.updateOnePage(kwargs['name'],module)
messagebus.postMessage("/system/notifications", "User "+ pages.getAcessingUser() + " added resource " +
kwargs['name'] + " of type " + type+" to module " + module)
#Take the user straight to the resource page
raise cherrypy.HTTPRedirect("/modules/module/"+util.url(module)+'/resource/'+util.url(name))
def everyminute():
global pageviewsthisminute
global lastsaved, lastdumpedlogs
if not config['autosave-state'] == 'never':
if (time.time() - lastsaved) > saveinterval:
lastsaved = time.time()
#This does not dump the log files. The log files change often.
#It would suck to have tons of tiny log files so we let the user configure
#Them separately
util.SaveAllStateExceptLogs()
if not config['autosave-logs'] == 'never':
if (time.time() - lastdumpedlogs) > dumplogsinterval:
lastdumpedlogs = time.time()
messagelogging.dumpLogFile()
messagebus.postMessage("/system/perf/FPS",
round(newevt.averageFramesPerSecond, 2))
pageviewcountsmoother.sample(pageviewsthisminute)
pageviewsthisminute = 0
def _handle_exception(self, e):
tb = traceback.format_exc()
#When an error happens, log it and save the time
#Note that we are logging to the compiled event object
self.errors.append([time.strftime(config['time-format']), tb])
#Keep only the most recent errors
self.errors = self.errors[-(config['errors-to-keep']):]
#The mesagebus is largely untested and we don't want to kill the thread.
try:
messagebus.postMessage(
'system/errors/events/' + self.module + '/' + self.resource,
str(tb))
except Exception as e:
print(e)
#If this is the first error since th module was last saved raise a notification
if len(self.errors) == 1:
messagebus.postMessage(
'/system/notifications', "Event " + self.resource +
" of module " + self.module + " may need attention")
def module(self,module,*path,**kwargs):
global moduleschanged
#If we are not performing an action on a module just going to its page
if not path:
pages.require("/admin/modules.view")
return pages.get_template("modules/module.html").render(module = ActiveModules[module],name = module)
else:
#This gets the interface to add a page
if path[0] == 'addresource':
#path[1] tells what type of resource is being created and addResourceDispatcher returns the appropriate crud screen
return addResourceDispatcher(module,path[1])
#This case handles the POST request from the new resource target
if path[0] == 'addresourcetarget':
return addResourceTarget(module,path[1],kwargs['name'],kwargs)
#This case shows the information and editing page for one resource
if path[0] == 'resource':
return resourceEditPage(module,path[1])
#This goes to a dispatcher that takes into account the type of resource and updates everything about the resource.
if path[0] == 'updateresource':
return resourceUpdateTarget(module,path[1],kwargs)
#This returns a page to delete any resource by name
if path[0] == 'deleteresource':
pages.require("/admin/modules.edit")
return pages.get_template("modules/deleteresource.html").render(module=module)
#This handles the POST request to actually do the deletion
if path[0] == 'deleteresourcetarget':
pages.require("/admin/modules.edit")
moduleschanged = True
with modulesLock:
r = ActiveModules[module].pop(kwargs['name'])
if r['resource-type'] == 'page':
usrpages.removeOnePage(module,kwargs['name'])
#Annoying bookkeeping crap to get rid of the cached crap
if r['resource-type'] == 'event':
newevt.removeOneEvent(module,kwargs['name'])
if r['resource-type'] == 'permission':
auth.importPermissionsFromModules() #sync auth's list of permissions
messagebus.postMessage("/system/notifications","User "+ pages.getAcessingUser() + " deleted resource " +
kwargs['name'] + " from module " + module)
raise cherrypy.HTTPRedirect('/modules')
#This is the target used to change the name and description(basic info) of a module
if path[0] == 'update':
pages.require("/admin/modules.edit")
moduleschanged = True
with modulesLock:
ActiveModules[module]['__description']['text'] = kwargs['description']
ActiveModules[kwargs['name']] = ActiveModules.pop(module)
#UHHG. So very much code tht just syncs data structures.
#This gets rid of the cache under the old name
newevt.removeModuleEvents(module)
usrpages.removeModulePages(module)
#And calls this function the generate the new cache
bookkeeponemodule(kwargs['name'])
#Just for fun, we should probably also sync the permissions
auth.importPermissionsFromModules()
raise cherrypy.HTTPRedirect('/modules/module/'+util.url(kwargs['name']))
conf = {
'/static':
{'tools.staticdir.on': True,
'tools.staticdir.dir':os.path.join(dn,'data/static'),
"tools.sessions.on": False,
'tools.caching.on' : True,
'tools.caching.delay' : 3600,
"tools.caching.expires": 3600,
"tools.addheader.on": True
}
}
#Let the user create additional static directories
for i in config['serve-static']:
if i not in conf:
conf["/usr/static/"+i]= {'tools.staticdir.on': True,
'tools.staticdir.dir': config['serve-static'][i],
'tools.expires.on' : True,
'tools.expires.secs' : 1000,
"tools.sessions.on": False,
'tools.caching.on' : True,
'tools.caching.delay' : 3600,
"tools.caching.expires": 3600,
"tools.addheader.on": True
}
cherrypy.config.update(server_config)
messagebus.postMessage('/system/startup','System Initialized')
messagebus.postMessage('/system/notifications','System Initialized at ' + time.strftime(config['time-format']))
cherrypy.quickstart(root,'/',config=conf)
if y == 1:
common.fail("Onchange Event did nothing")
#Unregister event and make sure it really goes away
x.unregister()
y = 1
if not y == 1:
common.fail("Onchange Event did not go away when unregistered")
#Now we test the message bus event
x = newevt.Event("!onmsg /test", "y='test'", locals())
#Register event with polling.
x.register()
#Give it a value to change from
messagebus.postMessage("/test", 'poo')
#Let it notice the old value
time.sleep(0.25)
#y should immediately be set back to 0 at the next polling cycle
if not y == 'test':
common.fail("Message Event did nothing")
#Unregister event and make sure it really goes away
x.unregister()
y = 1
messagebus.postMessage('poo', "/test")
if y == 'test':
common.fail("Message Event did not go away when unregistered")
def postMessage(topic, message):
print(message)
messagebus.postMessage(topic, message)
def module(self, module, *path, **kwargs):
global moduleschanged
#If we are not performing an action on a module just going to its page
if not path:
pages.require("/admin/modules.view")
return pages.get_template("modules/module.html").render(
module=ActiveModules[module], name=module)
else:
#This gets the interface to add a page
if path[0] == 'addresource':
#path[1] tells what type of resource is being created and addResourceDispatcher returns the appropriate crud screen
return addResourceDispatcher(module, path[1])
#This case handles the POST request from the new resource target
if path[0] == 'addresourcetarget':
return addResourceTarget(module, path[1], kwargs['name'],
kwargs)
#This case shows the information and editing page for one resource
if path[0] == 'resource':
return resourceEditPage(module, path[1])
#This goes to a dispatcher that takes into account the type of resource and updates everything about the resource.
if path[0] == 'updateresource':
return resourceUpdateTarget(module, path[1], kwargs)
#This returns a page to delete any resource by name
if path[0] == 'deleteresource':
pages.require("/admin/modules.edit")
return pages.get_template(
"modules/deleteresource.html").render(module=module)
#This handles the POST request to actually do the deletion
if path[0] == 'deleteresourcetarget':
pages.require("/admin/modules.edit")
moduleschanged = True
with modulesLock:
r = ActiveModules[module].pop(kwargs['name'])
if r['resource-type'] == 'page':
usrpages.removeOnePage(module, kwargs['name'])
#Annoying bookkeeping crap to get rid of the cached crap
if r['resource-type'] == 'event':
newevt.removeOneEvent(module, kwargs['name'])
if r['resource-type'] == 'permission':
auth.importPermissionsFromModules(
) #sync auth's list of permissions
messagebus.postMessage(
"/system/notifications",
"User " + pages.getAcessingUser() + " deleted resource " +
kwargs['name'] + " from module " + module)
raise cherrypy.HTTPRedirect('/modules')
#This is the target used to change the name and description(basic info) of a module
if path[0] == 'update':
pages.require("/admin/modules.edit")
moduleschanged = True
with modulesLock:
ActiveModules[module]['__description']['text'] = kwargs[
'description']
ActiveModules[kwargs['name']] = ActiveModules.pop(module)
#UHHG. So very much code tht just syncs data structures.
#This gets rid of the cache under the old name
newevt.removeModuleEvents(module)
usrpages.removeModulePages(module)
#And calls this function the generate the new cache
bookkeeponemodule(kwargs['name'])
#Just for fun, we should probably also sync the permissions
auth.importPermissionsFromModules()
raise cherrypy.HTTPRedirect('/modules/module/' +
util.url(kwargs['name']))
x.unregister()
y =1
if not y == 1:
common.fail("Onchange Event did not go away when unregistered")
#Now we test the message bus event
x = newevt.Event("!onmsg /test","y='test'",locals())
#Register event with polling.
x.register()
#Give it a value to change from
messagebus.postMessage("/test",'poo')
#Let it notice the old value
time.sleep(0.25)
#y should immediately be set back to 0 at the next polling cycle
if not y == 'test':
common.fail("Message Event did nothing")
#Unregister event and make sure it really goes away
x.unregister()
y =1
messagebus.postMessage('poo',"/test")
if y == 'test':
common.fail("Message Event did not go away when unregistered")
def postMessage(topic,message):
print(message)
messagebus.postMessage(topic,message)
def savetarget(self):
pages.require("/admin/settings.edit")
util.SaveAllState()
messagebus.postMessage("/system/notifications","Global server state was saved to disk")
raise cherrypy.HTTPRedirect('/')
def dump(j,f):
f.write(json.dumps(j,sort_keys=True,indent=1).encode())
messagebus.postMessage("system/notifications","Invalid config option for 'log-format' so defaulting to normal")
def dumpLogFile():
if config['log-format'] == 'normal':
def dump(j,f):
f.write(json.dumps(j,sort_keys=True,indent=1).encode())
elif config['log-format'] == 'tiny':
def dump(j,f):
f.write(json.dumps(j,sort_keys=True,separators=(',',':')).encode())
elif config['log-format'] == 'pretty':
def dump(j,f):
f.write(json.dumps(j,sort_keys=True,indent=4, separators=(',', ': ')).encode())
else:
def dump(j,f):
f.write(json.dumps(j,sort_keys=True,indent=1).encode())
messagebus.postMessage("system/notifications","Invalid config option for 'log-format' so defaulting to normal")
if config['log-compress'] == 'bz2':
openlog= bz2.BZ2File
ext = '.json.bz2'
elif config['log-compress'] == 'gzip':
openlog = gzip.GzipFile
ext = '.json.gz'
elif config['log-compress'] == 'none':
openlog = open
ext = '.json'
else:
openlog = open
messagebus.postMessage("system/notifications","Invalid config option for 'log-compress' so defaulting to no compression")
global log,loglistchanged
global approxtotallogentries
with savelock:
temp = log
log = defaultdict(list)
approxtotallogentries = 0
if loglistchanged:
#Save the list of things to dump
with open(os.path.join(directories.logdir,"whattosave.txt"),'w') as f:
for i in toSave:
f.write(i+'\n')
loglistchanged = False
#Get rid of anything that is not in the list of things to dump to the log
temp2 = {}
saveset = set(toSave)
for i in temp:
#Parsetopic is a function that returns all subscriptions that would match a topic
if not set(messagebus.MessageBus.parseTopic(i)).isdisjoint(saveset):
temp2[i] = temp[i]
temp = temp2
#If there is no log entries to save, don't dump an empty file.
if not temp:
return
where =os.path.join(directories.logdir,'dumps')
#Actually dump the log.
with openlog(os.path.join(where,str(time.time())+ext),'wb') as f:
print()
dump(temp,f)
f.close()
asnumbers = {}
for i in util.get_files(where):
try:
#Remove extensions
if i.endswith(".json"):
asnumbers[float(i[:-5])] = i
elif i.endswith(".json.gz"):
asnumbers[float(i[:-8])] = i
elif i.endswith(".json.bz2"):
asnumbers[float(i[:-9])] = i
except ValueError:
pass
maxsize = unitsofmeasure.strToIntWithSIMultipliers(config['keep-log-files'])
size = 0
#Loop over all the old log dumps and add up the sizes
for i in util.get_files(where):
size = size + os.path.getsize(os.path.join(where,i))
#Get rid of oldest log dumps until the total size is within the limit
for i in sorted(asnumbers.keys()):
if size <= maxsize:
break
size = size - os.path.getsize(os.path.join(where,i))
os.remove(os.path.join(where,asnumbers[i]))
