class ProcessTerminationMonitor(plugins.Observable): def __init__(self): plugins.Observable.__init__(self) self.processesForKill = OrderedDict() self.exitHandlers = OrderedDict() def listRunningProcesses(self): processesToCheck = guiConfig.getCompositeValue("query_kill_processes", "", modeDependent=True) if "all" in processesToCheck: processesToCheck = [".*"] if len(processesToCheck) == 0: return [] running = [] triggerGroup = plugins.TextTriggerGroup(processesToCheck) for process, description in self.processesForKill.values(): if triggerGroup.stringContainsText(description): running.append("PID " + str(process.pid) + " : " + description) return running def getProcessIdentifier(self, process): # Unfortunately the child_watch_add method needs different ways to # identify the process on different platforms... if os.name == "posix": return process.pid else: return process._handle def startProcess( self, cmdArgs, description="", killOnTermination=True, exitHandler=None, exitHandlerArgs=(), **kwargs ): process = subprocess.Popen(cmdArgs, stdin=open(os.devnull), **kwargs) pidOrHandle = self.getProcessIdentifier(process) self.exitHandlers[int(pidOrHandle)] = (exitHandler, exitHandlerArgs) if killOnTermination: self.processesForKill[int(pidOrHandle)] = (process, description) gobject.child_watch_add(pidOrHandle, self.processExited) def processExited(self, pid, *args): if self.processesForKill.has_key(pid): del self.processesForKill[pid] if self.exitHandlers.has_key(pid): exitHandler, exitHandlerArgs = self.exitHandlers.pop(pid) if exitHandler: exitHandler(*exitHandlerArgs) def notifyKillProcesses(self, sig=None): # Don't leak processes if len(self.processesForKill) == 0: return diag = logging.getLogger("kill processes") self.notify("Status", "Terminating all external viewers ...") for pid, (process, description) in self.processesForKill.items(): if self.exitHandlers.has_key(pid): self.exitHandlers.pop(pid) # don't call exit handlers in this case, we're terminating self.notify("ActionProgress") diag.info("Killing '" + description + "' interactive process") killSubProcessAndChildren(process, sig)
def test_pop(self): pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) od = OrderedDict(pairs) shuffle(pairs) while pairs: k, v = pairs.pop() self.assertEqual(od.pop(k), v) self.assertRaises(KeyError, od.pop, 'xyz') self.assertEqual(len(od), 0) self.assertEqual(od.pop(k, 12345), 12345)
class DictCache(AbstractCache): """Implementation of a cache in a memory dictionary """ def __init__(self, ttl=60): """Creates a new instance params: ``ttl`` Time to live of the data. """ super(DictCache, self).__init__(ttl=ttl) try: self._ich = collections.OrderedDict() self._ttds = collections.OrderedDict() except AttributeError: #This version of python does not support OrderedDict from ordereddict import OrderedDict self._ich = OrderedDict() self._ttds = OrderedDict() def store_data(self, k, ttl, v): self._ich[k] = v self._ttds[k] = ( time.time() + ttl if ttl != None else None ) def retrieve_data(self, k): ttd = self._ttds.get(k, 0) if ttd == None or time.time() < ttd: return self._ich[k] elif ttd: self._ttds.pop(k) self._ich.pop(k) def clear_expired(self): for k, ttd in self._ttds.items(): if ttd != None and ttd < time.time(): self._ttds.pop(k) self._ich.pop(k) else: break def clear(self): self._ich.clear() self._ttds.clear()
def _build(self, dirinfo): if self._belongs_to_exclude(dirinfo): return BinaryItem(dirinfo.name, filename=dirinfo.filename) files = OrderedDict((x.name, x) for x in unique_sorted_listing(dirinfo.filename, extension_priority=self.extension_priority)) if self.directory_item_name in files \ and text_file(files[self.directory_item_name].filename): info = files.pop(self.directory_item_name) factory = query_item_factory(info.extension, default=ContentItem) item = lambda children: factory( dirinfo.name, info.filename, info.extension, children=children) else: item = lambda children: Item(dirinfo.name, children=children) children = [self._build(x) for x in files.values()] return item(children)
def get_bibliography_entries(self, docname, id_, warn): """Return filtered bibliography entries, sorted by citation order.""" # get entries, ordered by bib file occurrence entries = OrderedDict((entry.key, entry) for entry in self._get_bibliography_entries( docname=docname, id_=id_, warn=warn)) # order entries according to which were cited first # first, we add all keys that were cited # then, we add all remaining keys sorted_entries = [] for key in self.get_all_cited_keys(): try: entry = entries.pop(key) except KeyError: pass else: sorted_entries.append(entry) sorted_entries += six.itervalues(entries) return sorted_entries
def get_bibliography_entries(self, docname, id_, warn): """Return filtered bibliography entries, sorted by citation order.""" # get entries, ordered by bib file occurrence entries = OrderedDict( (entry.key, entry) for entry in self._get_bibliography_entries( docname=docname, id_=id_, warn=warn)) # order entries according to which were cited first # first, we add all keys that were cited # then, we add all remaining keys sorted_entries = [] for key in self.get_all_cited_keys(): try: entry = entries.pop(key) except KeyError: pass else: sorted_entries.append(entry) sorted_entries += six.itervalues(entries) return sorted_entries
class HeaderParser(object): """Parses a file with family info and creates a family object with individuals.""" def __init__(self, infile): super(HeaderParser, self).__init__() self.metadata=OrderedDict() self.header=[] self.line_counter = 0 self.individuals = [] self.metadata_pattern = re.compile(r'''\#\#COLUMNNAME="(?P<colname>[^"]*)" (?P<info>.*)''', re.VERBOSE) with open(infile, 'rb') as f: for line in f: self.line_counter += 1 line = line.rstrip() if line.startswith('##'): match = self.metadata_pattern.match(line) if not match: raise SyntaxError("One of the metadata lines is malformed: %s" % line) matches = [match.group('colname'), match.group('info')] self.metadata[match.group('colname')] = line elif line.startswith('#'): self.header = line[1:].split('\t') for entry in self.header: if entry[:3] == 'IDN': self.individuals.append(entry.split(':')[1]) else: self.check_header(entry) else: break def add_metadata(self, column_name, data_type=None, version=None, description=None, dbname=None, delimiter='\t'): """Add metadata info to the header.""" data_line = '##COLUMNNAME='+'"'+ column_name +'"' if column_name not in self.metadata: if data_type: if data_type not in ['Float', 'String', 'Integer']: raise SyntaxError("Type must be 'Float', 'String' or 'Integer'. You tried: %s" % data_type) data_line += delimiter + 'TYPE="' + data_type + '"' if version: data_line += delimiter + 'VERSION="' + version + '"' if description: data_line += delimiter + 'DESCRIPTION="' + description + '"' if dbname: data_line += delimiter + 'SCOUTHEADER="' + dbname + '"' self.metadata.pop(column_name, 0) self.metadata[column_name] = data_line return def add_header(self, header_name): """Add an entry to the header line. The entry must be specified among the metadata lines first.""" self.check_header(header_name) if header_name not in self.header: self.header.append(header_name) return def get_headers_for_print(self): """Returns a list with the metadata lines on correct format.""" lines_for_print = [] for header in self.metadata: lines_for_print.append(self.metadata[header]) lines_for_print.append('\t'.join(self.header)) lines_for_print[-1] = '#' + lines_for_print[-1] return lines_for_print def check_header(self, entry): """All entrys in the header must be specified in the metadata lines.""" if entry not in self.metadata: raise SyntaxError("Header entry must be described in the metadata lines. Entry: %s is not in metadata." % entry)
class HybridView(newcls, JSONResponseMixin): """Middleware class which add the JSONResponseMixin in the view to handle ajax requests""" def __init__(self, *args, **kwargs): """Our newcls can be an instance of several mixins working with the get and post functions. e.g : CreateView is instance of BaseCreateView and ProcessFormView Let's add our custom mixins that implement get and post without returning a render_to_response, and call """ newcls.__init__(self, **kwargs) # The order matters for the get/post calls. self.mixins = OrderedDict() self.mixins[BaseListView] = BaseListViewMixin self.mixins[BaseCreateView] = BaseCreateViewMixin self.mixins[BaseUpdateView] = BaseUpdateViewMixin self.mixins[BaseDetailView] = BaseDetailViewMixin self.mixins[BaseDeleteView] = BaseDeleteViewMixin self.mixins[ProcessFormView] = ProcessFormViewMixin [self.mixins.pop(baseView) for baseView in self.mixins.iterkeys() if not isinstance(self, baseView)] @property def is_ajax(self): """Check the META HTTP_X_REQUESTED_WITH and CONTENT_TYPE""" meta = self.request.META return meta.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest'\ or "json" in meta.get("CONTENT_TYPE") def render_to_response(self, context): cls = type(self).__bases__[self.is_ajax] return cls.render_to_response(self, context) def get(self, request, **kwargs): if not self.is_ajax: "If it's not ajax, return the inherited get" return super(HybridView, self).get(self, **kwargs) for mixin in self.mixins.itervalues(): self, kwargs = mixin().get(self, **kwargs) context = kwargs context.update(self.get_json_context(**kwargs)) context.pop("form", None) context.pop("object", None) context.pop("object_list", None) return self.render_to_response(context) def post(self, request, **kwargs): """Hybrid post to handle all parents post actions""" if not self.is_ajax: "If it's not ajax, return the inherited get" return super(HybridView, self).post(self, **kwargs) for mixin in self.mixins.itervalues(): try: self, kwargs = mixin().post(self, **kwargs) except AttributeError: """The inherited view doesn't handle post""" pass context = kwargs context.update(self.get_json_context(**kwargs)) context.pop("form", None) return self.render_to_response(context)
class CnCGraph(object): def __init__(self, name, g): verifyCollectionDecls("item", g.itemColls) steps = [ x.step for x in g.stepRels ] verifyCollectionDecls("step", steps) self.name = name # items self.itemDeclarations = OrderedDict((i.collName, makeItemDecl(i)) for i in g.itemColls) self.concreteItems = [ i for i in self.itemDeclarations.values() if not i.isVirtual ] # item virtual mappings self.vms = [ i for i in self.itemDeclarations.values() if i.isVirtual ] self.inlineVms = [ i for i in self.vms if i.isInline ] self.externVms = [ i for i in self.vms if not i.isInline ] # steps / pseudo-steps self.stepFunctions = OrderedDict((x.step.collName, StepFunction(x)) for x in g.stepRels) verifyEnv(self.stepFunctions) self.initFunction = self.stepFunctions.pop(initNameRaw) self.initFunction.collName = "cncInitialize" self.finalizeFunction = self.stepFunctions.pop(finalizeNameRaw) self.finalizeFunction.collName = "cncFinalize" self.finalAndSteps = [self.finalizeFunction] + self.stepFunctions.values() # set up step attribute lookup dict self.stepLikes = OrderedDict(self.stepFunctions) self.stepLikes[self.initFunction.collName] = self.initFunction self.stepLikes[self.finalizeFunction.collName] = self.finalizeFunction # attribute tracking self.allAttrNames = set() # context self.ctxParams = filter(bool, map(strip, g.ctx.splitlines())) if g.ctx else [] def hasTuning(self, name): return name in self.allAttrNames def hasCustomDist(self): return self.hasTuning('distfn') or self.hasTuning('placeWith') def lookupType(self, item): return self.itemDeclarations[item.collName].type def itemDistFn(self, collName, ranksExpr): return getDistFn(self.itemDeclarations, collName, ranksExpr) def stepDistFn(self, collName, ranksExpr): return getDistFn(self.stepLikes, collName, ranksExpr) def itemTuningFn(self, collName, name, ranksExpr, default): return getTuningFn(self.itemDeclarations, collName, name, ranksExpr, default) def stepTuningFn(self, collName, name, ranksExpr, default): return getTuningFn(self.stepLikes, collName, name, ranksExpr, default) def priorityFn(self, collName, ranksExpr): return self.stepTuningFn(collName, 'priority', ranksExpr, "0") def addTunings(self, tuningSpec): for t in tuningSpec.itemTunings: x = self.itemDeclarations.get(t.collName) assert x, "Unknown item in tuning: {0}".format(t.collName) x.attrs.update(t.attrs) self.allAttrNames.update(t.attrs.keys()) for t in tuningSpec.stepTunings: x = self.stepLikes.get(t.collName) assert x, "Unknown step in tuning: {0}".format(t.collName) if t.inputName: i = x.inputsDict.get(t.inputName) assert i, "Unknown input in tuning: {0} <- {1}".format(t.collName, t.inputName) i.attrs.update(t.attrs) self.allAttrNames.update(t.attrs.keys()) else: x.attrs.update(t.attrs) self.allAttrNames.update(t.attrs.keys())
class CnCGraph(object): def __init__(self, name, g): verifyCollectionDecls("item", g.itemColls) steps = [x.step for x in g.stepRels] verifyCollectionDecls("step", steps) self.name = name # items self.itemDeclarations = OrderedDict( (i.collName, makeItemDecl(i)) for i in g.itemColls) self.concreteItems = [ i for i in self.itemDeclarations.values() if not i.isVirtual ] # item virtual mappings self.vms = [i for i in self.itemDeclarations.values() if i.isVirtual] self.inlineVms = [i for i in self.vms if i.isInline] self.externVms = [i for i in self.vms if not i.isInline] # steps / pseudo-steps self.stepFunctions = OrderedDict( (x.step.collName, StepFunction(x)) for x in g.stepRels) verifyEnv(self.stepFunctions) self.initFunction = self.stepFunctions.pop(initNameRaw) self.initFunction.collName = "cncInitialize" self.finalizeFunction = self.stepFunctions.pop(finalizeNameRaw) self.finalizeFunction.collName = "cncFinalize" self.finalAndSteps = [self.finalizeFunction ] + self.stepFunctions.values() # set up step attribute lookup dict self.stepLikes = OrderedDict(self.stepFunctions) self.stepLikes[self.initFunction.collName] = self.initFunction self.stepLikes[self.finalizeFunction.collName] = self.finalizeFunction # attribute tracking self.allAttrNames = set() # context self.ctxParams = filter(bool, map( strip, g.ctx.splitlines())) if g.ctx else [] def hasTuning(self, name): return name in self.allAttrNames def hasCustomDist(self): return self.hasTuning('distfn') or self.hasTuning('placeWith') def lookupType(self, item): return self.itemDeclarations[item.collName].type def itemDistFn(self, collName, ranksExpr): return getDistFn(self.itemDeclarations, collName, ranksExpr) def stepDistFn(self, collName, ranksExpr): return getDistFn(self.stepLikes, collName, ranksExpr) def itemTuningFn(self, collName, name, ranksExpr, default): return getTuningFn(self.itemDeclarations, collName, name, ranksExpr, default) def stepTuningFn(self, collName, name, ranksExpr, default): return getTuningFn(self.stepLikes, collName, name, ranksExpr, default) def priorityFn(self, collName, ranksExpr): return self.stepTuningFn(collName, 'priority', ranksExpr, "0") def addTunings(self, tuningSpec): for t in tuningSpec.itemTunings: x = self.itemDeclarations.get(t.collName) assert x, "Unknown item in tuning: {0}".format(t.collName) x.attrs.update(t.attrs) self.allAttrNames.update(t.attrs.keys()) for t in tuningSpec.stepTunings: x = self.stepLikes.get(t.collName) assert x, "Unknown step in tuning: {0}".format(t.collName) if t.inputName: i = x.inputsDict.get(t.inputName) assert i, "Unknown input in tuning: {0} <- {1}".format( t.collName, t.inputName) i.attrs.update(t.attrs) self.allAttrNames.update(t.attrs.keys()) else: x.attrs.update(t.attrs) self.allAttrNames.update(t.attrs.keys())
class RenderableContainer(object): is_group = True def __init__(self): self._components = [] self._componentmap = OrderedDict() self._bindmap = OrderedDict() def __json__(self, request): return {"components": self._componentmap} def rmRenderable(self, renderable_id): renderable = self._componentmap.pop(renderable_id) self._components.remove(renderable) def addRenderable(self, renderable, pos=None): """ Add renderable. If pos is given, insert into that position, otherwise just append""" if pos is None: self._components.append(renderable) else: self._components.insert(pos, renderable) # todo: see if we can use the pos parameter for the insertion # into the _componentmap ordereddict self._componentmap[renderable.id] = renderable if getattr(renderable, 'bind', None): self._bindmap[renderable.bind] = renderable def getRenderables(self, recursive=False): """ retrieve all renderables. If recursive is true, then also return all renderables from the children recursively """ if recursive: result = self._componentmap.values() for r in self._componentmap.values(): try: result += r.getRenderables(recursive) except: pass return result else: return self._components def getRenderable(self, id): """ find renderable by id in the complete (recursive) tree """ found = self._componentmap.get(id, None) if not found: # search children for r in self.getRenderables(False): try: found = r.getRenderable(id) if found: break except: pass return found def getRenderableByBind(self, bind): found = self._bindmap.get(bind, None) if not found: # search children for r in self.getRenderables(False): try: found = r.getRenderableByBind(bind) if found: break except: pass return found
class SymbolLRUCacheManager(RequiredConfig): """for cleaning up the symbols cache""" required_config = Namespace() required_config.add_option( 'symbol_cache_path', doc="the cache directory to automatically remove files from", default=os.path.join(tempfile.gettempdir(), 'symbols')) required_config.add_option('symbol_cache_size', doc="the maximum size of the symbols cache", default='1G', from_string_converter=from_string_to_parse_size) required_config.add_option( 'verbosity', doc="how chatty should this be? 1 - writes to stdout," " 2 - uses the logger", default=0, from_string_converter=int) #-------------------------------------------------------------------------- def __init__(self, config, quit_check_callback=None): """constructor for a registration object that runs an LRU cache cleaner""" self.config = config self.directory = os.path.abspath(config.symbol_cache_path) self.max_size = config.symbol_cache_size self.verbosity = config.verbosity # Cache state self.total_size = 0 self._lru = OrderedDict() # pyinotify bits self._wm = pyinotify.WatchManager() self._handler = EventHandler(self, verbosity=config.verbosity) self._notifier = pyinotify.ThreadedNotifier(self._wm, self._handler) mask = pyinotify.IN_DELETE | pyinotify.IN_CREATE \ | pyinotify.IN_OPEN | pyinotify.IN_MOVED_FROM \ | pyinotify.IN_MOVED_TO | pyinotify.IN_MODIFY self._wdd = self._wm.add_watch(self.directory, mask, rec=True, auto_add=True) # Load existing files into the cache. self._get_existing_files(self.directory) self._notifier.start() #-------------------------------------------------------------------------- @property def num_files(self): return len(self._lru) #-------------------------------------------------------------------------- def _rm_empty_dirs(self, path): ''' Attempt to remove any empty directories that are parents of path and children of self.directory. ''' path = os.path.dirname(path) while not os.path.samefile(path, self.directory): if not os.listdir(path): os.rmdir(path) path = os.path.dirname(path) #-------------------------------------------------------------------------- def _update_cache(self, path, update_size=False): if path in self._lru: size = self._lru.pop(path) if update_size: self.total_size -= size else: update_size = True if update_size: try: size = os.stat(path).st_size except OSError: self.config.logger.warning( 'file was not found while cleaning cache: %s', path) return self.total_size += size # If we're out of space, remove items from the cache until # we fit again. while self.total_size > self.max_size and self._lru: rm_path, rm_size = self._lru.popitem(last=False) self.total_size -= rm_size os.unlink(rm_path) self._rm_empty_dirs(rm_path) if self.verbosity >= 2: self.config.logger.debug('RM %s', rm_path) self._lru[path] = size #-------------------------------------------------------------------------- def _remove_cached(self, path): # We might have already removed this file in _update_cache. if path in self._lru: size = self._lru.pop(path) self.total_size -= size #-------------------------------------------------------------------------- def _get_existing_files(self, path): for base, dirs, files in os.walk(path): for f in files: f = os.path.join(base, f) self._update_cache(f) #-------------------------------------------------------------------------- def close(self): self._notifier.stop()
class Application(object): """Poor WSGI application which is called by WSGI server. Working of is describe in PEP 0333. This object store route dispatch table, and have methods for it's using and of course __call__ method for use as WSGI application. """ __instances = [] def __init__(self, name="__main__"): """Application class is per name singleton. That means, there could be exist only one instance with same name. """ if Application.__instances.count(name): raise RuntimeError('Application with name %s exist yet.' % name) Application.__instances.append(name) # Application name self.__name = name # list of pre and post process handlers self.__pre = [] self.__post = [] # dhandlers table for default handers on methods {METHOD_GET: handler} self.__dhandlers = {} # handlers table of simple paths: {'/path': {METHOD_GET: handler}} self.__handlers = {} self.__filters = { ':int': (r'-?\d+', int), ':float': (r'-?\d+(\.\d+)?', float), ':word': (r'\w+', uni), ':hex': (r'[0-9a-fA-F]+', str), ':re:': (None, uni), 'none': (r'[^/]+', uni) } # handlers of regex paths: {r'/user/([a-z]?)': {METHOD_GET: handler}} self.__rhandlers = OrderedDict() # http state handlers: {HTTP_NOT_FOUND: {METHOD_GET: my_404_handler}} self.__shandlers = {} # -- Application variable self.__config = { 'auto_args': True, 'auto_form': True, 'auto_json': True, 'keep_blank_values': 0, 'strict_parsing': 0, 'json_content_types': [ 'application/json', 'application/javascript', 'application/merge-patch+json'], 'auto_cookies': True, 'debug': 'Off', 'document_root': '', 'document_index': 'Off', 'secret_key': '%s%s%s%s' % (__version__, version, getcwd(), ''.join(str(x) for x in uname())) } try: self.__log_level = levels[environ.get('poor_LogLevel', 'warn').lower()] except: self.__log_level = LOG_WARNING self.log_error('Bad poor_LogLevel, default is warn.', LOG_WARNING) # endtry # enddef def __regex(self, match): groups = match.groups() _filter = str(groups[1]).lower() if _filter in self.__filters: regex = self.__filters[_filter][0] elif _filter[:4] == ':re:': # :re: filter have user defined regex regex = _filter[4:] else: try: regex = self.__filters[_filter][0] except KeyError: raise RuntimeError("Undefined route group filter '%s'" % _filter) return "(?P<%s>%s)" % (groups[0], regex) # enddef def __convertor(self, _filter): _filter = str(_filter).lower() _filter = ':re:' if _filter[:4] == ':re:' else _filter try: return self.__filters[_filter][1] except KeyError: raise RuntimeError("Undefined route group filter '%s'" % _filter) @property def name(self): """Return application name.""" return self.__name @property def filters(self): """Copy of filter table. Filter table contains regular expressions and convert functions, see Application.set_filter and Application.route. Default filters are: :int - match number and convert it to int :float - match number and convert it to float :word - match one unicoee word :hex - match hexadecimal value and convert it to str :re: - match user defined regular expression none - match any string withount '/' character For more details see {/debug-info} page of your application, where you see all filters with regular expression definition. """ return self.__filters.copy() @property def pre(self): """Tuple of table with pre-process handlers. See Application.pre_process. """ return tuple(self.__pre) @property def post(self): """Tuple of table with post-process handlers. See Application.post_process. """ return tuple(self.__post) @property def dhandlers(self): """Copy of table with default handlers. See Application.set_default """ return self.__dhandlers.copy() @property def handlers(self): """Copy of table with static handlers. See Application.route. """ return self.__handlers.copy() @property def rhandlers(self): """Copy of table with regular expression handlers. See Application.route and Application.rroute. """ return self.__rhandlers.copy() @property def shandlers(self): """Copy of table with http state aka error handlers. See Application.http_state """ return self.__shandlers.copy() @property def auto_args(self): """Automatic parsing request arguments from uri. If it is True (default), Request object do automatic parsing request uri to its args variable. """ return self.__config['auto_args'] @auto_args.setter def auto_args(self, value): self.__config['auto_args'] = bool(value) @property def auto_form(self): """Automatic parsing arguments from request body. If it is True (default) and method is POST, PUT or PATCH, Request object do automatic parsing request body to its form variable. """ return self.__config['auto_form'] @auto_form.setter def auto_form(self, value): self.__config['auto_form'] = bool(value) @property def auto_json(self): """Automatic parsing JSON from request body. If it is True (default), method is POST, PUT or PATCH and request content type is one of json_content_types, Request object do automatic parsing request body to json variable. """ return self.__config['auto_json'] @auto_json.setter def auto_json(self, value): self.__config['auto_json'] = bool(value) @property def auto_cookies(self): """Automatic parsing cookies from request headers. If it is True (default) and Cookie request header was set, SimpleCookie object was paresed to Request property cookies. """ return self.__config['auto_cookies'] @auto_cookies.setter def auto_cookies(self, value): self.__config['auto_cookies'] = bool(value) @property def debug(self): """Application debug as another way how to set poor_Debug. This setting will be rewrite by poor_Debug environment variable. """ return self.__config['debug'] == 'On' @debug.setter def debug(self, value): self.__config['debug'] = 'On' if bool(value) else 'Off' @property def document_root(self): """Application document_root as another way how to set poor_DocumentRoot. This setting will be rewrite by poor_DocumentRoot environ variable. """ return self.__config['document_root'] @document_root.setter def document_root(self, value): self.__config['document_root'] = value @property def document_index(self): """Application document_root as another way how to set poor_DocumentRoot. This setting will be rewrite by poor_DocumentRoot environ variable. """ return self.__config['document_index'] == 'On' @document_index.setter def document_index(self, value): self.__config['document_index'] = 'On' if bool(value) else 'Off' @property def secret_key(self): """Application secret_key could be replace by poor_SecretKey in request. Secret key is used by PoorSession class. It is generate from some server variables, and the best way is set to your own long key.""" return self.__config['secret_key'] @secret_key.setter def secret_key(self, value): self.__config['secret_key'] = value @property def keep_blank_values(self): """Keep blank values in request arguments. If it is 1 (0 is default), automatic parsing request uri or body keep blank values as empty string. """ return self.__config['keep_blank_values'] @keep_blank_values.setter def keep_blank_values(self, value): self.__config['keep_blank_values'] = int(value) @property def strict_parsing(self): """Strict parse request arguments. If it is 1 (0 is default), automatic parsing request uri or body raise with exception on parsing error. """ return self.__config['strict_parsing'] @strict_parsing.setter def strict_parsing(self, value): self.__config['strict_parsing'] = int(value) @property def json_content_types(self): """Copy of json content type list. Containt list of strings as json content types, which is use for testing, when automatics Json object is create from request body. """ return self.__config['json_content_types'] def set_filter(self, name, regex, convertor=uni): """Create new filter or overwrite builtins. Arguments: name - Name of filter which is used in route or set_route method. regex - regular expression which used for filter convertor - convertor function or class, which gets unicode in input. Default is uni function, which is wrapper to unicode string. app.set_filter('uint', r'\d+', int) """ name = ':'+name if name[0] != ':' else name self.__filters[name] = (regex, convertor) def pre_process(self): """Append pre process hendler. This is decorator for function to call before each request. @app.pre_process() def before_each_request(req): ... """ def wrapper(fn): self.__pre.append(fn) return fn return wrapper # enddef def add_pre_process(self, fn): """Append pre proccess handler. Method adds function to list functions which is call before each request. app.add_pre_process(before_each_request) """ self.__pre.append(fn) # enddef def post_process(self): """Append post process handler. This decorator append function to be called after each request, if you want to use it redefined all outputs. @app.pre_process() def after_each_request(req): ... """ def wrapper(fn): self.__post.append(fn) return fn return wrapper # enddef def add_post_process(self, fn): """Append post process handler. Method for direct append function to list functions which are called after each request. app.add_post_process(after_each_request) """ self.__post.append(fn) # enddef def default(self, method=METHOD_HEAD | METHOD_GET): """Set default handler. This is decorator for default handler for http method (called before error_not_found). @app.default(METHOD_GET_POST) def default_get_post(req): # this function will be called if no uri match in internal # uri table with method. It's similar like not_found error, # but without error ... """ def wrapper(fn): self.set_default(fn, method) return wrapper # enddef def set_default(self, fn, method=METHOD_HEAD | METHOD_GET): """Set default handler. Set fn default handler for http method called befor error_not_found. app.set_default(default_get_post, METHOD_GET_POST) """ for m in methods.values(): if method & m: self.__dhandlers[m] = fn # enddef def pop_default(self, method): """Pop default handler for method.""" return self.__dhandlers(method) def route(self, uri, method=METHOD_HEAD | METHOD_GET): """Wrap function to be handler for uri and specified method. You can define uri as static path or as groups which are hand to handler as next parameters. # static uri @app.route('/user/post', method=METHOD_POST) def user_create(req): ... # group regular expression @app.route('/user/<name>') def user_detail(req, name): ... # group regular expression with filter @app.route('/<surname:word>/<age:int>') def surnames_by_age(req, surname, age): ... # group with own regular expression filter @app.route('/<car:re:\w+>/<color:re:#[\da-fA-F]+>') def car(req, car, color): ... If you can use some name of group which is python keyword, like class, you can use **kwargs syntax: @app.route('/<class>/<len:int>') def classes(req, **kwargs): return "'%s' class is %d lenght." % \ (kwargs['class'], kwargs['len']) Be sure with ordering of call this decorator or set_route function with groups regular expression. Regular expression routes are check with the same ordering, as you create internal table of them. First match stops any other searching. In fact, if groups are detect, they will be transfer to normal regular expression, and will be add to second internal table. """ def wrapper(fn): self.set_route(uri, fn, method) return fn return wrapper # enddef def set_route(self, uri, fn, method=METHOD_HEAD | METHOD_GET): """Set handler for uri and method. Another way to add fn as handler for uri. See Application.route documentation for details. app.set_route('/use/post', user_create, METHOD_POST) """ uri = uni(uri) if re_filter.search(uri): r_uri = re_filter.sub(self.__regex, uri) + '$' convertors = tuple((g[0], self.__convertor(g[1])) for g in (m.groups() for m in re_filter.finditer(uri))) self.set_rroute(r_uri, fn, method, convertors) else: if uri not in self.__handlers: self.__handlers[uri] = {} for m in methods.values(): if method & m: self.__handlers[uri][m] = fn # enddef def pop_route(self, uri, method): """Pop handler for uri and method from handers table. Method must be define unique, so METHOD_GET_POST could not be use. If you want to remove handler for both methods, you must call pop route for each method state. """ uri = uni(uri) if re_filter.search(uri): r_uri = re_filter.sub(self.__regex, uri) + '$' return self.pop_rroute(r_uri, method) else: handlers = self.__handlers.get(uri, {}) rv = handlers.pop(method) if not handlers: # is empty self.__handlers.pop(uri, None) return rv def is_route(self, uri): """Check if uri have any registered record.""" uri = uni(uri) if re_filter.search(uri): r_uri = re_filter.sub(self.__regex, uri) + '$' return self.is_rroute(r_uri) return uri in self.__handlers def rroute(self, ruri, method=METHOD_HEAD | METHOD_GET): """Wrap function to be handler for uri defined by regular expression. Both of function, rroute and set_rroute store routes to special internal table, which is another to table of static routes. @app.rroute(r'/user/\w+') # simple regular expression def any_user(req): ... @app.rroute(r'/user/(?P<user>\w+)') # regular expression with def user_detail(req, user): # groups ... Be sure with ordering of call this decorator or set_rroute function. Regular expression routes are check with the same ordering, as you create internal table of them. First match stops any other searching. """ def wrapper(fn): self.set_rroute(ruri, fn, method) return fn return wrapper # enddef def set_rroute(self, r_uri, fn, method=METHOD_HEAD | METHOD_GET, convertors=()): """Set hanlder for uri defined by regular expression. Another way to add fn as handler for uri defined by regular expression. See Application.rroute documentation for details. app.set_rroute('/use/\w+/post', user_create, METHOD_POST) This method is internally use, when groups are found in static route, adding by route or set_route method. """ r_uri = re.compile(r_uri, re.U) if r_uri not in self.__rhandlers: self.__rhandlers[r_uri] = {} for m in methods.values(): if method & m: self.__rhandlers[r_uri][m] = (fn, convertors) # enddef def pop_rroute(self, r_uri, method): """Pop handler and convertors for uri and method from handlers table. For mor details see Application.pop_route. """ r_uri = re.compile(r_uri, re.U) handlers = self.__rhandlers.get(r_uri, {}) rv = handlers.pop(method) if not handlers: # is empty self.__rhandlers.pop(r_uri, None) return rv def is_rroute(self, r_uri): """Check if regular expression uri have any registered record.""" r_uri = re.compile(r_uri, re.U) return r_uri in self.__rhandlers def http_state(self, code, method=METHOD_HEAD | METHOD_GET | METHOD_POST): """Wrap function to handle http status codes like http errors.""" def wrapper(fn): self.set_http_state(code, fn, method) return wrapper # enddef def set_http_state(self, code, fn, method=METHOD_HEAD | METHOD_GET | METHOD_POST): """Set fn as handler for http state code and method.""" if code not in self.__shandlers: self.__shandlers[code] = {} for m in methods.values(): if method & m: self.__shandlers[code][m] = fn # enddef def pop_http_state(self, code, method): """Pop handerl for http state and method. As Application.pop_route, for pop multimethod handler, you must call pop_http_state for each method. """ handlers = self.__shandlers(code, {}) return handlers.pop(method) def error_from_table(self, req, code): """Internal method, which is called if error was accured. If status code is in Application.shandlers (fill with http_state function), call this handler. """ if code in self.__shandlers \ and req.method_number in self.__shandlers[code]: try: handler = self.__shandlers[code][req.method_number] if 'uri_handler' not in req.__dict__: req.uri_rule = '_%d_error_handler_' % code req.uri_handler = handler self.handler_from_pre(req) # call pre handlers now handler(req) except: internal_server_error(req) elif code in default_shandlers: handler = default_shandlers[code][METHOD_GET] handler(req) else: not_implemented(req, code) # enddef def handler_from_default(self, req): """Internal method, which is called if no handler is found.""" if req.method_number in self.__dhandlers: req.uri_rule = '_default_handler_' req.uri_handler = self.__dhandlers[req.method_number] self.handler_from_pre(req) # call pre handlers now retval = self.__dhandlers[req.method_number](req) if retval != DECLINED: raise SERVER_RETURN(retval) # enddef def handler_from_pre(self, req): """Internal method, which run all pre (pre_proccess) handlers. This method was call before end-point route handler. """ for fn in self.__pre: fn(req) def handler_from_table(self, req): """Call right handler from handlers table (fill with route function). If no handler is fined, try to find directory or file if Document Root, resp. Document Index is set. Then try to call default handler for right method or call handler for status code 404 - not found. """ # static routes if req.uri in self.__handlers: if req.method_number in self.__handlers[req.uri]: handler = self.__handlers[req.uri][req.method_number] req.uri_rule = req.uri # nice variable for pre handlers req.uri_handler = handler self.handler_from_pre(req) # call pre handlers now retval = handler(req) # call right handler now # return text is allowed if isinstance(retval, str) \ or (_unicode_exist and isinstance(retval, unicode)): req.write(retval, 1) # write data and flush retval = DONE if retval != DECLINED: raise SERVER_RETURN(retval or DONE) # could be state.DONE else: raise SERVER_RETURN(HTTP_METHOD_NOT_ALLOWED) # endif # endif # regular expression for ruri in self.__rhandlers.keys(): match = ruri.match(req.uri) if match and req.method_number in self.__rhandlers[ruri]: handler, convertors = self.__rhandlers[ruri][req.method_number] req.uri_rule = ruri.pattern # nice variable for pre handlers req.uri_handler = handler self.handler_from_pre(req) # call pre handlers now if len(convertors): # create OrderedDict from match insead of dict for # convertors applying req.groups = OrderedDict( (g, c(v))for ((g, c), v) in zip(convertors, match.groups())) retval = handler(req, *req.groups.values()) else: req.groups = match.groupdict() retval = handler(req, *match.groups()) # return text is allowed if isinstance(retval, str) \ or (_unicode_exist and isinstance(retval, unicode)): req.write(retval, 1) # write data and flush retval = DONE if retval != DECLINED: raise SERVER_RETURN(retval or DONE) # could be state.DONE # endif - no METHOD_NOT_ALLOWED here # endfor # try file or index if req.document_root(): rfile = "%s%s" % (uni(req.document_root()), path.normpath("%s" % uni(req.uri))) if not path.exists(rfile): if req.debug and req.uri == '/debug-info': # work if debug req.uri_rule = '_debug_info_' req.uri_handler = debug_info self.handler_from_pre(req) # call pre handlers now raise SERVER_RETURN(debug_info(req, self)) self.handler_from_default(req) # try default raise SERVER_RETURN(HTTP_NOT_FOUND) # not found # return file if path.isfile(rfile) and access(rfile, R_OK): req.uri_rule = '_send_file_' req.uri_handler = send_file self.handler_from_pre(req) # call pre handlers now req.log_error("Return file: %s" % req.uri, LOG_INFO) raise SERVER_RETURN(send_file(req, rfile)) # return directory index if req.document_index and path.isdir(rfile) \ and access(rfile, R_OK): req.log_error("Return directory: %s" % req.uri, LOG_INFO) req.uri_rule = '_directory_index_' req.uri_handler = directory_index self.handler_from_pre(req) # call pre handlers now raise SERVER_RETURN(directory_index(req, rfile)) raise SERVER_RETURN(HTTP_FORBIDDEN) # endif if req.debug and req.uri == '/debug-info': req.uri_rule = '_debug_info_' req.uri_handler = debug_info self.handler_from_pre(req) # call pre handlers now raise SERVER_RETURN(debug_info(req, self)) self.handler_from_default(req) req.log_error("404 Not Found: %s" % req.uri, LOG_ERR) raise SERVER_RETURN(HTTP_NOT_FOUND) # enddef def __request__(self, environ, start_response): """Create Request instance and return wsgi response. This method create Request object, call handlers from Application.__pre (Application.handler_from_pre), uri handler (handler_from_table), default handler (Application.handler_from_default) or error handler (Application.error_from_table), and handlers from Application.__post. """ req = Request(environ, start_response, self.__config) try: self.handler_from_table(req) except SERVER_RETURN as e: code = e.args[0] if code in (OK, HTTP_OK, DONE): pass # XXX: elif code in (HTTP_MOVED_PERMANENTLY, # HTTP_MOVED_TEMPORARILY): else: req.status = code self.error_from_table(req, code) except (BrokenClientConnection, SystemExit) as e: req.log_error(str(e), LOG_ERR) req.log_error(' *** You shoud ignore next error ***', LOG_ERR) return () except: self.error_from_table(req, 500) # endtry try: # call post_process handler for fn in self.__post: fn(req) except: self.error_from_table(req, 500) # endtry return req.__end_of_request__() # private call of request # enddef def __call__(self, environ, start_response): """Callable define for Application instance. This method run __request__ method. """ if self.__name == '__poorwsgi__': stderr.write("[W] Using deprecated instance of Application.\n") stderr.write(" Please, create your own instance\n") stderr.flush() return self.__request__(environ, start_response) def __profile_request__(self, environ, start_response): """Profiler version of __request__. This method is used if set_profile is used.""" def wrapper(rv): rv.append(self.__original_request__(environ, start_response)) rv = [] uri_dump = (self._dump + environ.get('PATH_INFO').replace('/', '_') + '.profile') self.log_error('Generate %s' % uri_dump, LOG_INFO) self._runctx('wrapper(rv)', globals(), locals(), filename=uri_dump) return rv[0] # enddef def __repr__(self): return '%s - callable Application class instance' % self.__name def set_profile(self, runctx, dump): """Set profiler for __call__ function. Arguments: runctx - function from profiler module dump - path and prefix for .profile files Typical usage: import cProfile cProfile.runctx('from simple import *', globals(), locals(), filename="log/init.profile") app.set_profile(cProfile.runctx, 'log/req') """ self._runctx = runctx self._dump = dump self.__original_request__ = self.__request__ self.__request__ = self.__profile_request__ # enddef def del_profile(self): """Remove profiler from application.""" self.__request__ = self.__original_request__ def get_options(self): """Returns dictionary with application variables from system environment. Application variables start with {app_} prefix, but in returned dictionary is set without this prefix. #!ini poor_LogLevel = warn # Poor WSGI variable app_db_server = localhost # application variable db_server app_templates = app/templ # application variable templates This method works like Request.get_options, but work with os.environ, so it works only with wsgi servers, which set not only request environ, but os.environ too. Apaches mod_wsgi don't do that, uWsgi and PoorHTTP do that. """ options = {} for key, val in environ.items(): key = key.strip() if key[:4].lower() == 'app_': options[key[4:].lower()] = val.strip() return options # enddef def log_error(self, message, level=LOG_ERR): """Logging method with the same functionality like in Request object. But as get_options read configuration from os.environ which could not work in same wsgi servers like Apaches mod_wsgi. This method write to stderr so messages, could not be found in servers error log! """ if self.__log_level[0] >= level[0]: if _unicode_exist and isinstance(message, unicode): message = message.encode('utf-8') try: stderr.write("<%s> [%s] %s\n" % (level[1], self.__name, message)) except UnicodeEncodeError: if _unicode_exist: message = message.decode('utf-8').encode( 'ascii', 'backslashreplace') else: message = message.encode( 'ascii', 'backslashreplace').decode('ascii') stderr.write("<%s> [%s] %s\n" % (level[1], self.__name, message)) stderr.flush() # enddef def log_info(self, message): """Logging method, which create message as LOG_INFO level.""" self.log_error(message, LOG_INFO) def log_debug(self, message): """Logging method, which create message as LOG_DEBUG level.""" self.log_error(message, LOG_DEBUG) def log_warning(self, message): """Logging method, which create message as LOG_WARNING level.""" self.log_error(message, LOG_WARNING)
class SymbolLRUCacheManager(RequiredConfig): """for cleaning up the symbols cache""" required_config = Namespace() required_config.add_option( 'symbol_cache_path', doc="the cache directory to automatically remove files from", default=os.path.join(tempfile.gettempdir(), 'symbols') ) required_config.add_option( 'symbol_cache_size', doc="the maximum size of the symbols cache", default='1G', from_string_converter=from_string_to_parse_size ) required_config.add_option( 'verbosity', doc="how chatty should this be? 1 - writes to stdout," " 2 - uses the logger", default=0, from_string_converter=int ) #-------------------------------------------------------------------------- def __init__(self, config, quit_check_callback=None): """constructor for a registration object that runs an LRU cache cleaner""" self.config = config self.directory = os.path.abspath(config.symbol_cache_path) self.max_size = config.symbol_cache_size self.verbosity = config.verbosity # Cache state self.total_size = 0 self._lru = OrderedDict() # pyinotify bits self._wm = pyinotify.WatchManager() self._handler = EventHandler(self, verbosity=config.verbosity) self._notifier = pyinotify.ThreadedNotifier(self._wm, self._handler) mask = pyinotify.IN_DELETE | pyinotify.IN_CREATE \ | pyinotify.IN_OPEN | pyinotify.IN_MOVED_FROM \ | pyinotify.IN_MOVED_TO | pyinotify.IN_MODIFY self._wdd = self._wm.add_watch( self.directory, mask, rec=True, auto_add=True ) # Load existing files into the cache. self._get_existing_files(self.directory) self._notifier.start() #-------------------------------------------------------------------------- @property def num_files(self): return len(self._lru) #-------------------------------------------------------------------------- def _rm_empty_dirs(self, path): ''' Attempt to remove any empty directories that are parents of path and children of self.directory. ''' path = os.path.dirname(path) while not os.path.samefile(path, self.directory): if not os.listdir(path): os.rmdir(path) path = os.path.dirname(path) #-------------------------------------------------------------------------- def _update_cache(self, path, update_size=False): if path in self._lru: size = self._lru.pop(path) if update_size: self.total_size -= size else: update_size = True if update_size: try: size = os.stat(path).st_size except OSError: self.config.logger.warning( 'file was not found while cleaning cache: %s', path ) return self.total_size += size # If we're out of space, remove items from the cache until # we fit again. while self.total_size > self.max_size and self._lru: rm_path, rm_size = self._lru.popitem(last=False) self.total_size -= rm_size os.unlink(rm_path) self._rm_empty_dirs(rm_path) if self.verbosity >= 2: self.config.logger.debug('RM %s', rm_path) self._lru[path] = size #-------------------------------------------------------------------------- def _remove_cached(self, path): # We might have already removed this file in _update_cache. if path in self._lru: size = self._lru.pop(path) self.total_size -= size #-------------------------------------------------------------------------- def _get_existing_files(self, path): for base, dirs, files in os.walk(path): for f in files: f = os.path.join(base, f) self._update_cache(f) #-------------------------------------------------------------------------- def close(self): self._notifier.stop()
class Application(object): """Poor WSGI application which is called by WSGI server. Working of is describe in PEP 0333. This object store route dispatch table, and have methods for it's using and of course __call__ method for use as WSGI application. """ __instances = [] def __init__(self, name="__main__"): """Application class is per name singleton. That means, there could be exist only one instance with same name. """ if Application.__instances.count(name): raise RuntimeError('Application with name %s exist yet.' % name) Application.__instances.append(name) # Application name self.__name = name # list of pre and post process handlers self.__pre = [] self.__post = [] # dhandlers table for default handers on methods {METHOD_GET: handler} self.__dhandlers = {} # handlers table of simple paths: {'/path': {METHOD_GET: handler}} self.__handlers = {} self.__filters = { ':int': (r'-?\d+', int), ':float': (r'-?\d+(\.\d+)?', float), ':word': (r'\w+', uni), ':hex': (r'[0-9a-fA-F]+', str), ':re:': (None, uni), 'none': (r'[^/]+', uni) } # handlers of regex paths: {r'/user/([a-z]?)': {METHOD_GET: handler}} self.__rhandlers = OrderedDict() # http state handlers: {HTTP_NOT_FOUND: {METHOD_GET: my_404_handler}} self.__shandlers = {} # -- Application variable self.__config = { 'auto_args': True, 'auto_form': True, 'auto_json': True, 'keep_blank_values': 0, 'strict_parsing': 0, 'json_content_types': [ 'application/json', 'application/javascript', 'application/merge-patch+json'], 'form_content_types': [ 'application/x-www-form-urlencoded', 'multipart/form-data' ], 'auto_cookies': True, 'debug': 'Off', 'document_root': '', 'document_index': 'Off', 'secret_key': '%s%s%s%s' % (__version__, version, getcwd(), ''.join(str(x) for x in uname())) } try: self.__log_level = levels[environ.get('poor_LogLevel', 'warn').lower()] except: self.__log_level = LOG_WARNING self.log_error('Bad poor_LogLevel, default is warn.', LOG_WARNING) # endtry # enddef def __regex(self, match): groups = match.groups() _filter = str(groups[1]).lower() if _filter in self.__filters: regex = self.__filters[_filter][0] elif _filter[:4] == ':re:': # :re: filter have user defined regex regex = _filter[4:] else: try: regex = self.__filters[_filter][0] except KeyError: raise RuntimeError("Undefined route group filter '%s'" % _filter) return "(?P<%s>%s)" % (groups[0], regex) # enddef def __convertor(self, _filter): _filter = str(_filter).lower() _filter = ':re:' if _filter[:4] == ':re:' else _filter try: return self.__filters[_filter][1] except KeyError: raise RuntimeError("Undefined route group filter '%s'" % _filter) @property def name(self): """Return application name.""" return self.__name @property def filters(self): """Copy of filter table. Filter table contains regular expressions and convert functions, see Application.set_filter and Application.route. Default filters are: :int - match number and convert it to int :float - match number and convert it to float :word - match one unicoee word :hex - match hexadecimal value and convert it to str :re: - match user defined regular expression none - match any string withount '/' character For more details see {/debug-info} page of your application, where you see all filters with regular expression definition. """ return self.__filters.copy() @property def pre(self): """Tuple of table with pre-process handlers. See Application.pre_process. """ return tuple(self.__pre) @property def post(self): """Tuple of table with post-process handlers. See Application.post_process. """ return tuple(self.__post) @property def dhandlers(self): """Copy of table with default handlers. See Application.set_default """ return self.__dhandlers.copy() @property def handlers(self): """Copy of table with static handlers. See Application.route. """ return self.__handlers.copy() @property def rhandlers(self): """Copy of table with regular expression handlers. See Application.route and Application.rroute. """ return self.__rhandlers.copy() @property def shandlers(self): """Copy of table with http state aka error handlers. See Application.http_state """ return self.__shandlers.copy() @property def auto_args(self): """Automatic parsing request arguments from uri. If it is True (default), Request object do automatic parsing request uri to its args variable. """ return self.__config['auto_args'] @auto_args.setter def auto_args(self, value): self.__config['auto_args'] = bool(value) @property def auto_form(self): """Automatic parsing arguments from request body. If it is True (default) and method is POST, PUT or PATCH, and request content type is one of form_content_types, Request object do automatic parsing request body to its form variable. """ return self.__config['auto_form'] @auto_form.setter def auto_form(self, value): self.__config['auto_form'] = bool(value) @property def auto_json(self): """Automatic parsing JSON from request body. If it is True (default), method is POST, PUT or PATCH and request content type is one of json_content_types, Request object do automatic parsing request body to json variable. """ return self.__config['auto_json'] @auto_json.setter def auto_json(self, value): self.__config['auto_json'] = bool(value) @property def auto_cookies(self): """Automatic parsing cookies from request headers. If it is True (default) and Cookie request header was set, SimpleCookie object was paresed to Request property cookies. """ return self.__config['auto_cookies'] @auto_cookies.setter def auto_cookies(self, value): self.__config['auto_cookies'] = bool(value) @property def debug(self): """Application debug as another way how to set poor_Debug. This setting will be rewrite by poor_Debug environment variable. """ return self.__config['debug'] == 'On' @debug.setter def debug(self, value): self.__config['debug'] = 'On' if bool(value) else 'Off' @property def document_root(self): """Application document_root as another way how to set poor_DocumentRoot. This setting will be rewrite by poor_DocumentRoot environ variable. """ return self.__config['document_root'] @document_root.setter def document_root(self, value): self.__config['document_root'] = value @property def document_index(self): """Application document_root as another way how to set poor_DocumentRoot. This setting will be rewrite by poor_DocumentRoot environ variable. """ return self.__config['document_index'] == 'On' @document_index.setter def document_index(self, value): self.__config['document_index'] = 'On' if bool(value) else 'Off' @property def secret_key(self): """Application secret_key could be replace by poor_SecretKey in request. Secret key is used by PoorSession class. It is generate from some server variables, and the best way is set to your own long key.""" return self.__config['secret_key'] @secret_key.setter def secret_key(self, value): self.__config['secret_key'] = value @property def keep_blank_values(self): """Keep blank values in request arguments. If it is 1 (0 is default), automatic parsing request uri or body keep blank values as empty string. """ return self.__config['keep_blank_values'] @keep_blank_values.setter def keep_blank_values(self, value): self.__config['keep_blank_values'] = int(value) @property def strict_parsing(self): """Strict parse request arguments. If it is 1 (0 is default), automatic parsing request uri or body raise with exception on parsing error. """ return self.__config['strict_parsing'] @strict_parsing.setter def strict_parsing(self, value): self.__config['strict_parsing'] = int(value) @property def json_content_types(self): """Copy of json content type list. Containt list of strings as json content types, which is use for testing, when automatics Json object is create from request body. """ return self.__config['json_content_types'] @property def form_content_types(self): """Copy of form content type list. Containt list of strings as form content types, which is use for testing, when automatics Form object is create from request body. """ return self.__config['form_content_types'] def set_filter(self, name, regex, convertor=uni): """Create new filter or overwrite builtins. Arguments: name - Name of filter which is used in route or set_route method. regex - regular expression which used for filter convertor - convertor function or class, which gets unicode in input. Default is uni function, which is wrapper to unicode string. app.set_filter('uint', r'\d+', int) """ name = ':'+name if name[0] != ':' else name self.__filters[name] = (regex, convertor) def pre_process(self): """Append pre process hendler. This is decorator for function to call before each request. @app.pre_process() def before_each_request(req): ... """ def wrapper(fn): self.__pre.append(fn) return fn return wrapper # enddef def add_pre_process(self, fn): """Append pre proccess handler. Method adds function to list functions which is call before each request. app.add_pre_process(before_each_request) """ self.__pre.append(fn) # enddef def post_process(self): """Append post process handler. This decorator append function to be called after each request, if you want to use it redefined all outputs. @app.pre_process() def after_each_request(req): ... """ def wrapper(fn): self.__post.append(fn) return fn return wrapper # enddef def add_post_process(self, fn): """Append post process handler. Method for direct append function to list functions which are called after each request. app.add_post_process(after_each_request) """ self.__post.append(fn) # enddef def default(self, method=METHOD_HEAD | METHOD_GET): """Set default handler. This is decorator for default handler for http method (called before error_not_found). @app.default(METHOD_GET_POST) def default_get_post(req): # this function will be called if no uri match in internal # uri table with method. It's similar like not_found error, # but without error ... """ def wrapper(fn): self.set_default(fn, method) return wrapper # enddef def set_default(self, fn, method=METHOD_HEAD | METHOD_GET): """Set default handler. Set fn default handler for http method called befor error_not_found. app.set_default(default_get_post, METHOD_GET_POST) """ for m in methods.values(): if method & m: self.__dhandlers[m] = fn # enddef def pop_default(self, method): """Pop default handler for method.""" return self.__dhandlers(method) def route(self, uri, method=METHOD_HEAD | METHOD_GET): """Wrap function to be handler for uri and specified method. You can define uri as static path or as groups which are hand to handler as next parameters. # static uri @app.route('/user/post', method=METHOD_POST) def user_create(req): ... # group regular expression @app.route('/user/<name>') def user_detail(req, name): ... # group regular expression with filter @app.route('/<surname:word>/<age:int>') def surnames_by_age(req, surname, age): ... # group with own regular expression filter @app.route('/<car:re:\w+>/<color:re:#[\da-fA-F]+>') def car(req, car, color): ... If you can use some name of group which is python keyword, like class, you can use **kwargs syntax: @app.route('/<class>/<len:int>') def classes(req, **kwargs): return "'%s' class is %d lenght." % \ (kwargs['class'], kwargs['len']) Be sure with ordering of call this decorator or set_route function with groups regular expression. Regular expression routes are check with the same ordering, as you create internal table of them. First match stops any other searching. In fact, if groups are detect, they will be transfer to normal regular expression, and will be add to second internal table. """ def wrapper(fn): self.set_route(uri, fn, method) return fn return wrapper # enddef def set_route(self, uri, fn, method=METHOD_HEAD | METHOD_GET): """Set handler for uri and method. Another way to add fn as handler for uri. See Application.route documentation for details. app.set_route('/use/post', user_create, METHOD_POST) """ uri = uni(uri) if re_filter.search(uri): r_uri = re_filter.sub(self.__regex, uri) + '$' convertors = tuple((g[0], self.__convertor(g[1])) for g in (m.groups() for m in re_filter.finditer(uri))) self.set_rroute(r_uri, fn, method, convertors) else: if uri not in self.__handlers: self.__handlers[uri] = {} for m in methods.values(): if method & m: self.__handlers[uri][m] = fn # enddef def pop_route(self, uri, method): """Pop handler for uri and method from handers table. Method must be define unique, so METHOD_GET_POST could not be use. If you want to remove handler for both methods, you must call pop route for each method state. """ uri = uni(uri) if re_filter.search(uri): r_uri = re_filter.sub(self.__regex, uri) + '$' return self.pop_rroute(r_uri, method) else: handlers = self.__handlers.get(uri, {}) rv = handlers.pop(method) if not handlers: # is empty self.__handlers.pop(uri, None) return rv def is_route(self, uri): """Check if uri have any registered record.""" uri = uni(uri) if re_filter.search(uri): r_uri = re_filter.sub(self.__regex, uri) + '$' return self.is_rroute(r_uri) return uri in self.__handlers def rroute(self, ruri, method=METHOD_HEAD | METHOD_GET): """Wrap function to be handler for uri defined by regular expression. Both of function, rroute and set_rroute store routes to special internal table, which is another to table of static routes. @app.rroute(r'/user/\w+') # simple regular expression def any_user(req): ... @app.rroute(r'/user/(?P<user>\w+)') # regular expression with def user_detail(req, user): # groups ... Be sure with ordering of call this decorator or set_rroute function. Regular expression routes are check with the same ordering, as you create internal table of them. First match stops any other searching. """ def wrapper(fn): self.set_rroute(ruri, fn, method) return fn return wrapper # enddef def set_rroute(self, r_uri, fn, method=METHOD_HEAD | METHOD_GET, convertors=()): """Set hanlder for uri defined by regular expression. Another way to add fn as handler for uri defined by regular expression. See Application.rroute documentation for details. app.set_rroute('/use/\w+/post', user_create, METHOD_POST) This method is internally use, when groups are found in static route, adding by route or set_route method. """ r_uri = re.compile(r_uri, re.U) if r_uri not in self.__rhandlers: self.__rhandlers[r_uri] = {} for m in methods.values(): if method & m: self.__rhandlers[r_uri][m] = (fn, convertors) # enddef def pop_rroute(self, r_uri, method): """Pop handler and convertors for uri and method from handlers table. For mor details see Application.pop_route. """ r_uri = re.compile(r_uri, re.U) handlers = self.__rhandlers.get(r_uri, {}) rv = handlers.pop(method) if not handlers: # is empty self.__rhandlers.pop(r_uri, None) return rv def is_rroute(self, r_uri): """Check if regular expression uri have any registered record.""" r_uri = re.compile(r_uri, re.U) return r_uri in self.__rhandlers def http_state(self, code, method=METHOD_HEAD | METHOD_GET | METHOD_POST): """Wrap function to handle http status codes like http errors.""" def wrapper(fn): self.set_http_state(code, fn, method) return wrapper # enddef def set_http_state(self, code, fn, method=METHOD_HEAD | METHOD_GET | METHOD_POST): """Set fn as handler for http state code and method.""" if code not in self.__shandlers: self.__shandlers[code] = {} for m in methods.values(): if method & m: self.__shandlers[code][m] = fn # enddef def pop_http_state(self, code, method): """Pop handerl for http state and method. As Application.pop_route, for pop multimethod handler, you must call pop_http_state for each method. """ handlers = self.__shandlers(code, {}) return handlers.pop(method) def error_from_table(self, req, code): """Internal method, which is called if error was accured. If status code is in Application.shandlers (fill with http_state function), call this handler. """ if code in self.__shandlers \ and req.method_number in self.__shandlers[code]: try: handler = self.__shandlers[code][req.method_number] if 'uri_handler' not in req.__dict__: req.uri_rule = '_%d_error_handler_' % code req.uri_handler = handler self.handler_from_pre(req) # call pre handlers now handler(req) except: internal_server_error(req) elif code in default_shandlers: handler = default_shandlers[code][METHOD_GET] handler(req) else: not_implemented(req, code) # enddef def handler_from_default(self, req): """Internal method, which is called if no handler is found.""" if req.method_number in self.__dhandlers: req.uri_rule = '_default_handler_' req.uri_handler = self.__dhandlers[req.method_number] self.handler_from_pre(req) # call pre handlers now retval = self.__dhandlers[req.method_number](req) if retval != DECLINED: raise SERVER_RETURN(retval) # enddef def handler_from_pre(self, req): """Internal method, which run all pre (pre_proccess) handlers. This method was call before end-point route handler. """ for fn in self.__pre: fn(req) def handler_from_table(self, req): """Call right handler from handlers table (fill with route function). If no handler is fined, try to find directory or file if Document Root, resp. Document Index is set. Then try to call default handler for right method or call handler for status code 404 - not found. """ # static routes if req.uri in self.__handlers: if req.method_number in self.__handlers[req.uri]: handler = self.__handlers[req.uri][req.method_number] req.uri_rule = req.uri # nice variable for pre handlers req.uri_handler = handler self.handler_from_pre(req) # call pre handlers now retval = handler(req) # call right handler now # return text is allowed if isinstance(retval, str) \ or (_unicode_exist and isinstance(retval, unicode)): req.write(retval, 1) # write data and flush retval = DONE if retval != DECLINED: raise SERVER_RETURN(retval or DONE) # could be state.DONE else: raise SERVER_RETURN(HTTP_METHOD_NOT_ALLOWED) # endif # endif # regular expression for ruri in self.__rhandlers.keys(): match = ruri.match(req.uri) if match and req.method_number in self.__rhandlers[ruri]: handler, convertors = self.__rhandlers[ruri][req.method_number] req.uri_rule = ruri.pattern # nice variable for pre handlers req.uri_handler = handler self.handler_from_pre(req) # call pre handlers now if len(convertors): # create OrderedDict from match insead of dict for # convertors applying req.groups = OrderedDict( (g, c(v))for ((g, c), v) in zip(convertors, match.groups())) retval = handler(req, *req.groups.values()) else: req.groups = match.groupdict() retval = handler(req, *match.groups()) # return text is allowed if isinstance(retval, str) \ or (_unicode_exist and isinstance(retval, unicode)): req.write(retval, 1) # write data and flush retval = DONE if retval != DECLINED: raise SERVER_RETURN(retval or DONE) # could be state.DONE # endif - no METHOD_NOT_ALLOWED here # endfor # try file or index if req.document_root(): rfile = "%s%s" % (uni(req.document_root()), path.normpath("%s" % uni(req.uri))) if not path.exists(rfile): if req.debug and req.uri == '/debug-info': # work if debug req.uri_rule = '_debug_info_' req.uri_handler = debug_info self.handler_from_pre(req) # call pre handlers now raise SERVER_RETURN(debug_info(req, self)) self.handler_from_default(req) # try default raise SERVER_RETURN(HTTP_NOT_FOUND) # not found # return file if path.isfile(rfile) and access(rfile, R_OK): req.uri_rule = '_send_file_' req.uri_handler = send_file self.handler_from_pre(req) # call pre handlers now req.log_error("Return file: %s" % req.uri, LOG_INFO) raise SERVER_RETURN(send_file(req, rfile)) # return directory index if req.document_index and path.isdir(rfile) \ and access(rfile, R_OK): req.log_error("Return directory: %s" % req.uri, LOG_INFO) req.uri_rule = '_directory_index_' req.uri_handler = directory_index self.handler_from_pre(req) # call pre handlers now raise SERVER_RETURN(directory_index(req, rfile)) raise SERVER_RETURN(HTTP_FORBIDDEN) # endif if req.debug and req.uri == '/debug-info': req.uri_rule = '_debug_info_' req.uri_handler = debug_info self.handler_from_pre(req) # call pre handlers now raise SERVER_RETURN(debug_info(req, self)) self.handler_from_default(req) req.log_error("404 Not Found: %s" % req.uri, LOG_ERR) raise SERVER_RETURN(HTTP_NOT_FOUND) # enddef def __request__(self, environ, start_response): """Create Request instance and return wsgi response. This method create Request object, call handlers from Application.__pre (Application.handler_from_pre), uri handler (handler_from_table), default handler (Application.handler_from_default) or error handler (Application.error_from_table), and handlers from Application.__post. """ req = Request(environ, start_response, self.__config) try: self.handler_from_table(req) except SERVER_RETURN as e: code = e.args[0] if code in (OK, HTTP_OK, DONE): pass # XXX: elif code in (HTTP_MOVED_PERMANENTLY, # HTTP_MOVED_TEMPORARILY): else: req.status = code self.error_from_table(req, code) except (BrokenClientConnection, SystemExit) as e: req.log_error(str(e), LOG_ERR) req.log_error(' *** You shoud ignore next error ***', LOG_ERR) return () except: self.error_from_table(req, 500) # endtry try: # call post_process handler for fn in self.__post: fn(req) except: self.error_from_table(req, 500) # endtry return req.__end_of_request__() # private call of request # enddef def __call__(self, environ, start_response): """Callable define for Application instance. This method run __request__ method. """ if self.__name == '__poorwsgi__': stderr.write("[W] Using deprecated instance of Application.\n") stderr.write(" Please, create your own instance\n") stderr.flush() return self.__request__(environ, start_response) def __profile_request__(self, environ, start_response): """Profiler version of __request__. This method is used if set_profile is used.""" def wrapper(rv): rv.append(self.__original_request__(environ, start_response)) rv = [] uri_dump = (self._dump + environ.get('PATH_INFO').replace('/', '_') + '.profile') self.log_error('Generate %s' % uri_dump, LOG_INFO) self._runctx('wrapper(rv)', globals(), locals(), filename=uri_dump) return rv[0] # enddef def __repr__(self): return '%s - callable Application class instance' % self.__name def set_profile(self, runctx, dump): """Set profiler for __call__ function. Arguments: runctx - function from profiler module dump - path and prefix for .profile files Typical usage: import cProfile cProfile.runctx('from simple import *', globals(), locals(), filename="log/init.profile") app.set_profile(cProfile.runctx, 'log/req') """ self._runctx = runctx self._dump = dump self.__original_request__ = self.__request__ self.__request__ = self.__profile_request__ # enddef def del_profile(self): """Remove profiler from application.""" self.__request__ = self.__original_request__ def get_options(self): """Returns dictionary with application variables from system environment. Application variables start with {app_} prefix, but in returned dictionary is set without this prefix. #!ini poor_LogLevel = warn # Poor WSGI variable app_db_server = localhost # application variable db_server app_templates = app/templ # application variable templates This method works like Request.get_options, but work with os.environ, so it works only with wsgi servers, which set not only request environ, but os.environ too. Apaches mod_wsgi don't do that, uWsgi and PoorHTTP do that. """ options = {} for key, val in environ.items(): key = key.strip() if key[:4].lower() == 'app_': options[key[4:].lower()] = val.strip() return options # enddef def log_error(self, message, level=LOG_ERR): """Logging method with the same functionality like in Request object. But as get_options read configuration from os.environ which could not work in same wsgi servers like Apaches mod_wsgi. This method write to stderr so messages, could not be found in servers error log! """ if self.__log_level[0] >= level[0]: if _unicode_exist and isinstance(message, unicode): message = message.encode('utf-8') try: stderr.write("<%s> [%s] %s\n" % (level[1], self.__name, message)) except UnicodeEncodeError: if _unicode_exist: message = message.decode('utf-8').encode( 'ascii', 'backslashreplace') else: message = message.encode( 'ascii', 'backslashreplace').decode('ascii') stderr.write("<%s> [%s] %s\n" % (level[1], self.__name, message)) stderr.flush() # enddef def log_info(self, message): """Logging method, which create message as LOG_INFO level.""" self.log_error(message, LOG_INFO) def log_debug(self, message): """Logging method, which create message as LOG_DEBUG level.""" self.log_error(message, LOG_DEBUG) def log_warning(self, message): """Logging method, which create message as LOG_WARNING level.""" self.log_error(message, LOG_WARNING)
class RenderableContainer(object): is_group = True def __init__(self): self._components = [] self._componentmap = OrderedDict() self._bindmap = OrderedDict() def __json__(self, request): return {"components": self._componentmap} def rmRenderable(self, renderable_id): renderable = self._componentmap.pop(renderable_id) self._components.remove(renderable) def addRenderable(self, renderable, pos=None): """ Add renderable. If pos is given, insert into that position, otherwise just append""" if pos is None: self._components.append(renderable) else: self._components.insert(pos, renderable) # todo: see if we can use the pos parameter for the insertion # into the _componentmap ordereddict self._componentmap[renderable.id] = renderable if getattr(renderable, "bind", None): self._bindmap[renderable.bind] = renderable def getRenderables(self, recursive=False): """ retrieve all renderables. If recursive is true, then also return all renderables from the children recursively """ if recursive: result = self._componentmap.values() for r in self._componentmap.values(): try: result += r.getRenderables(recursive) except: pass return result else: return self._components def getRenderable(self, id): """ find renderable by id in the complete (recursive) tree """ found = self._componentmap.get(id, None) if not found: # search children for r in self.getRenderables(False): try: found = r.getRenderable(id) if found: break except: pass return found def getRenderableByBind(self, bind): found = self._bindmap.get(bind, None) if not found: # search children for r in self.getRenderables(False): try: found = r.getRenderableByBind(bind) if found: break except: pass return found
def apply(self): """Transform each :class:`~sphinxcontrib.bibtex.nodes.bibliography` node into a list of citations. """ env = self.document.settings.env for bibnode in self.document.traverse(bibliography): # get the information of this bibliography node # by looking up its id in the bibliography cache id_ = bibnode['ids'][0] infos = [info for other_id, info in env.bibtex_cache.bibliographies.items() if other_id == id_ and info.docname == env.docname] if not infos: raise RuntimeError( "document %s has no bibliography nodes with id '%s'" % (env.docname, id_)) elif len(infos) >= 2: raise RuntimeError( "document %s has multiple bibliography nodes with id '%s'" % (env.docname, id_)) info = infos[0] # generate entries entries = OrderedDict() for bibfile in info.bibfiles: # XXX entries are modified below in an unpickable way # XXX so fetch a deep copy data = env.bibtex_cache.bibfiles[bibfile].data if info.cite == "all": bibfile_entries = data.entries.values() elif info.cite == "cited": bibfile_entries = ( entry for entry in data.entries.values() if env.bibtex_cache.is_cited(entry.key)) elif info.cite == "notcited": bibfile_entries = ( entry for entry in data.entries.values() if not env.bibtex_cache.is_cited(entry.key)) else: raise RuntimeError("invalid cite option (%s)" % info.cite) for entry in bibfile_entries: entries[entry.key] = copy.deepcopy(entry) # order entries according to which were cited first # first, we add all keys that were cited # then, we add all remaining keys sorted_entries = [] for key in env.bibtex_cache.get_all_cited_keys(): try: entry = entries.pop(key) except KeyError: pass else: sorted_entries.append(entry) sorted_entries += entries.values() # locate and instantiate style plugin style_cls = find_plugin( 'pybtex.style.formatting', info.style) style = style_cls() # create citation nodes for all references backend = output_backend() if info.list_ == "enumerated": nodes = docutils.nodes.enumerated_list() nodes['enumtype'] = info.enumtype if info.start >= 1: nodes['start'] = info.start env.bibtex_cache.set_enum_count(env.docname, info.start) else: nodes['start'] = env.bibtex_cache.get_enum_count(env.docname) elif info.list_ == "bullet": nodes = docutils.nodes.bullet_list() else: # "citation" nodes = docutils.nodes.paragraph() # XXX style.format_entries modifies entries in unpickable way for entry in style.format_entries(sorted_entries): if info.list_ == "enumerated" or info.list_ == "bullet": citation = docutils.nodes.list_item() citation += entry.text.render(backend) else: # "citation" citation = backend.citation(entry, self.document) # backend.citation(...) uses entry.key as citation label # we change it to entry.label later onwards # but we must note the entry.label now; # at this point, we also already prefix the label key = citation[0].astext() info.labels[key] = info.labelprefix + entry.label node_text_transform(citation, transform_url_command) if info.curly_bracket_strip: node_text_transform(citation, transform_curly_bracket_strip) nodes += citation if info.list_ == "enumerated": env.bibtex_cache.inc_enum_count(env.docname) bibnode.replace_self(nodes)
def apply(self): """Transform each :class:`~sphinxcontrib.bibtex.nodes.bibliography` node into a list of citations. """ env = self.document.settings.env for bibnode in self.document.traverse(bibliography): # get the information of this bibliography node # by looking up its id in the bibliography cache id_ = bibnode['ids'][0] infos = [info for other_id, info in env.bibtex_cache.bibliographies.iteritems() if other_id == id_ and info.docname == env.docname] if not infos: raise RuntimeError( "document %s has no bibliography nodes with id '%s'" % (env.docname, id_)) elif len(infos) >= 2: raise RuntimeError( "document %s has multiple bibliography nodes with id '%s'" % (env.docname, id_)) info = infos[0] # generate entries entries = OrderedDict() for bibfile in info.bibfiles: # XXX entries are modified below in an unpickable way # XXX so fetch a deep copy data = env.bibtex_cache.bibfiles[bibfile].data if info.cite == "all": bibfile_entries = data.entries.itervalues() elif info.cite == "cited": bibfile_entries = ( entry for entry in data.entries.itervalues() if env.bibtex_cache.is_cited(entry.key)) elif info.cite == "notcited": bibfile_entries = ( entry for entry in data.entries.itervalues() if not env.bibtex_cache.is_cited(entry.key)) else: raise RuntimeError("invalid cite option (%s)" % info.cite) for entry in bibfile_entries: entries[entry.key] = copy.deepcopy(entry) # order entries according to which were cited first # first, we add all keys that were cited # then, we add all remaining keys sorted_entries = [] for key in env.bibtex_cache.get_all_cited_keys(): try: entry = entries.pop(key) except KeyError: pass else: sorted_entries.append(entry) sorted_entries += entries.itervalues() # locate and instantiate style plugin style_cls = find_plugin( 'pybtex.style.formatting', info.style) style = style_cls() # create citation nodes for all references backend = output_backend() if info.list_ == "enumerated": nodes = docutils.nodes.enumerated_list() nodes['enumtype'] = info.enumtype if info.start >= 1: nodes['start'] = info.start env.bibtex_cache.set_enum_count(env.docname, info.start) else: nodes['start'] = env.bibtex_cache.get_enum_count(env.docname) elif info.list_ == "bullet": nodes = docutils.nodes.bullet_list() else: # "citation" nodes = docutils.nodes.paragraph() # XXX style.format_entries modifies entries in unpickable way for entry in style.format_entries(sorted_entries): if info.list_ == "enumerated" or info.list_ == "bullet": citation = docutils.nodes.list_item() citation += entry.text.render(backend) else: # "citation" citation = backend.citation(entry, self.document) # backend.citation(...) uses entry.key as citation label # we change it to entry.label later onwards # but we must note the entry.label now; # at this point, we also already prefix the label key = citation[0].astext() info.labels[key] = info.labelprefix + entry.label node_text_transform(citation, transform_url_command) if info.curly_bracket_strip: node_text_transform(citation, transform_curly_bracket_strip) nodes += citation if info.list_ == "enumerated": env.bibtex_cache.inc_enum_count(env.docname) bibnode.replace_self(nodes)
def apply(self): """Transform each :class:`~sphinxcontrib.bibtex.nodes.bibliography` node into a list of citations. """ env = self.document.settings.env for bibnode in self.document.traverse(bibliography): # get the information of this bibliography node # by looking up its id in the bibliography cache id_ = bibnode['ids'][0] infos = [info for other_id, info in env.bibtex_cache.bibliographies.iteritems() if other_id == id_ and info.docname == env.docname] assert infos, "no bibliography id '%s' in %s" % ( id_, env.docname) assert len(infos) == 1, "duplicate bibliography ids '%s' in %s" % ( id_, env.docname) info = infos[0] # generate entries entries = OrderedDict() for bibfile in info.bibfiles: # XXX entries are modified below in an unpickable way # XXX so fetch a deep copy data = env.bibtex_cache.bibfiles[bibfile].data for entry in data.entries.itervalues(): visitor = FilterVisitor( entry=entry, is_cited=env.bibtex_cache.is_cited(entry.key)) try: ok = visitor.visit(info.filter_) except ValueError as e: env.app.warn( "syntax error in :filter: expression; %s" % e) # recover by falling back to the default ok = env.bibtex_cache.is_cited(entry.key) if ok: entries[entry.key] = copy.deepcopy(entry) # order entries according to which were cited first # first, we add all keys that were cited # then, we add all remaining keys sorted_entries = [] for key in env.bibtex_cache.get_all_cited_keys(): try: entry = entries.pop(key) except KeyError: pass else: sorted_entries.append(entry) sorted_entries += entries.itervalues() # locate and instantiate style and backend plugins style = find_plugin('pybtex.style.formatting', info.style)() backend = find_plugin('pybtex.backends', 'docutils')() # create citation nodes for all references if info.list_ == "enumerated": nodes = docutils.nodes.enumerated_list() nodes['enumtype'] = info.enumtype if info.start >= 1: nodes['start'] = info.start env.bibtex_cache.set_enum_count(env.docname, info.start) else: nodes['start'] = env.bibtex_cache.get_enum_count( env.docname) elif info.list_ == "bullet": nodes = docutils.nodes.bullet_list() else: # "citation" nodes = docutils.nodes.paragraph() # XXX style.format_entries modifies entries in unpickable way for entry in style.format_entries(sorted_entries): if info.list_ == "enumerated" or info.list_ == "bullet": citation = docutils.nodes.list_item() citation += entry.text.render(backend) else: # "citation" citation = backend.citation(entry, self.document) # backend.citation(...) uses entry.key as citation label # we change it to entry.label later onwards # but we must note the entry.label now; # at this point, we also already prefix the label key = citation[0].astext() info.labels[key] = info.labelprefix + entry.label node_text_transform(citation, transform_url_command) if info.curly_bracket_strip: node_text_transform( citation, transform_curly_bracket_strip) nodes += citation if info.list_ == "enumerated": env.bibtex_cache.inc_enum_count(env.docname) bibnode.replace_self(nodes)
class HeaderParser(object): """Class for holding and manipulating vcf headers Attributes: info_dict=OrderedDict() extra_info = {} filter_lines=[] filter_dict=OrderedDict() contig_lines=[] contig_dict=OrderedDict() format_lines=[] format_dict=OrderedDict() alt_lines=[] alt_dict=OrderedDict() other_lines=[] other_dict=OrderedDict() header=['CHROM','POS','ID','REF','ALT','QUAL','FILTER','INFO'] header_keys={'info' : ['ID', 'Number', 'Type', 'Description'], 'form' : ['ID', 'Number', 'Type', 'Description'], 'filt' : ['ID', 'Description'], 'alt' : ['ID', 'Description'], 'contig' : ['ID']} fileformat = None filedate = None reference = None phasing = None source = None line_counter = 0 individuals = [] vep_columns = [] snpeff_columns = [] info_pattern filter_pattern contig_pattern format_pattern alt_pattern meta_pattern Methods: parse_meta_data(line) parse_header_line(line) add_fileformat(fileformat) print_header(): Return a list with the header lines add_meta_linekey, value) add_info(info_id, number, entry_type, description) add_filter(filter_id, description) add_format(format_id, number, entry_type, description) add_alt(alt_id, description) add_contig(contig_id, length) add_version_tracking(info_id, version, date, command_line='') """ def __init__(self): super(HeaderParser, self).__init__() self.info_lines = [] self.info_dict = OrderedDict() # This is a dictionary cantaining specific information about the info fields # It will have info name as key and then another dictionary with ID, Number, Type and Description self.extra_info = {} self.filter_lines = [] self.filter_dict = OrderedDict() self.contig_lines = [] self.contig_dict = OrderedDict() self.format_lines = [] self.format_dict = OrderedDict() self.alt_lines = [] self.alt_dict = OrderedDict() self.other_lines = [] self.other_dict = OrderedDict() self.header = ["CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO"] self.header_keys = { "info": ["ID", "Number", "Type", "Description"], "form": ["ID", "Number", "Type", "Description"], "filt": ["ID", "Description"], "alt": ["ID", "Description"], "contig": ["ID", "length"], } self.fileformat = None self.filedate = None self.reference = None self.phasing = None self.source = None self.line_counter = 0 self.individuals = [] self.vep_columns = [] self.snpeff_columns = [] self.info_pattern = re.compile( r"""\#\#INFO=< ID=(?P<id>[^,]+), Number=(?P<number>-?\d+|\.|[AGR]), Type=(?P<type>Integer|Float|Flag|Character|String), Description="(?P<desc>[^"]*)" >""", re.VERBOSE, ) self.filter_pattern = re.compile( r"""\#\#FILTER=< ID=(?P<id>[^,]+), Description="(?P<desc>[^"]*)" >""", re.VERBOSE, ) self.contig_pattern = re.compile( r"""\#\#contig=< ID=(?P<id>[^>,]+) (,.*length=(?P<length>-?\d+))? .* >""", re.VERBOSE, ) self.format_pattern = re.compile( r"""\#\#FORMAT=< ID=(?P<id>.+), Number=(?P<number>-?\d+|\.|[AGR]), Type=(?P<type>.+), Description="(?P<desc>.*)" >""", re.VERBOSE, ) self.alt_pattern = re.compile( r"""\#\#ALT=< ID=(?P<id>[^,]+), Description="(?P<desc>[^"]*)" >""", re.VERBOSE, ) self.meta_pattern = re.compile(r"""##(?P<key>.+?)=(?P<val>.+)""") def parse_meta_data(self, line): """Parse a vcf metadataline""" line = line.rstrip() logger.debug("Parsing metadata line:{0}".format(line)) line_info = line[2:].split("=") match = False if line_info[0] == "fileformat": logger.debug("Parsing fileformat") try: self.fileformat = line_info[1] logger.debug("Found fileformat {0}".format(self.fileformat)) except IndexError: raise SyntaxError("fileformat must have a value") elif line_info[0] == "INFO": match = self.info_pattern.match(line) if not match: raise SyntaxError("One of the INFO lines is malformed:{0}".format(line)) matches = [match.group("id"), match.group("number"), match.group("type"), match.group("desc")] # extra_info is a dictionary to check the metadata about the INFO values: self.extra_info[matches[0]] = dict(zip(self.header_keys["info"][1:], matches[1:])) info_line = dict(list(zip(self.header_keys["info"], matches))) if len(info_line["Description"].split("Format:")) > 1: info_line["Format"] = [info.strip() for info in info_line["Description"].split("Format:")][-1] self.info_lines.append(info_line) # Store the vep columns: if info_line["ID"] == "CSQ": self.vep_columns = info_line.get("Format", "").split("|") if info_line["ID"] == "ANN": self.snpeff_columns = [ annotation.strip("' ") for annotation in info_line.get("Description", "").split(":")[-1].split("|") ] self.info_dict[match.group("id")] = line elif line_info[0] == "FILTER": match = self.filter_pattern.match(line) if not match: raise SyntaxError("One of the FILTER lines is malformed: {0}".format(line)) matches = [match.group("id"), match.group("desc")] self.filter_lines.append(dict(list(zip(self.header_keys["filt"], matches)))) self.filter_dict[match.group("id")] = line elif line_info[0] == "contig": match = self.contig_pattern.match(line) if not match: print() raise SyntaxError("One of the contig lines is malformed: {0}".format(line)) matches = [match.group("id"), match.group("length")] self.contig_lines.append(dict(list(zip(self.header_keys["contig"], matches)))) self.contig_dict[match.group("id")] = line elif line_info[0] == "FORMAT": match = self.format_pattern.match(line) if not match: raise SyntaxError("One of the FORMAT lines is malformed: {0}".format(line)) matches = [match.group("id"), match.group("number"), match.group("type"), match.group("desc")] self.format_lines.append(dict(list(zip(self.header_keys["form"], matches)))) self.format_dict[match.group("id")] = line elif line_info[0] == "ALT": match = self.alt_pattern.match(line) if not match: raise SyntaxError("One of the ALT lines is malformed: {0}".format(line)) matches = [match.group("id"), match.group("desc")] self.alt_lines.append(dict(list(zip(self.header_keys["alt"], matches)))) self.alt_dict[match.group("id")] = line else: match = self.meta_pattern.match(line) if not match: raise SyntaxError("One of the meta data lines is malformed: {0}".format(line)) self.other_lines.append({match.group("key"): match.group("val")}) self.other_dict[match.group("key")] = line def parse_header_line(self, line): """docstring for parse_header_line""" self.header = line[1:].rstrip().split("\t") if len(self.header) < 9: self.header = line[1:].rstrip().split() self.individuals = self.header[9:] def remove_header(self, name): """Remove a field from the header""" if name in self.info_dict: self.info_dict.pop(name) logger.info("Removed '{0}' from INFO".format(name)) if name in self.filter_dict: self.filter_dict.pop(name) logger.info("Removed '{0}' from FILTER".format(name)) if name in self.format_dict: self.format_dict.pop(name) logger.info("Removed '{0}' from FORMAT".format(name)) if name in self.contig_dict: self.contig_dict.pop(name) logger.info("Removed '{0}' from CONTIG".format(name)) if name in self.alt_dict: self.alt_dict.pop(name) logger.info("Removed '{0}' from ALT".format(name)) if name in self.other_dict: self.other_dict.pop(name) logger.info("Removed '{0}' from OTHER".format(name)) return def print_header(self): """Returns a list with the header lines if proper format""" lines_to_print = [] lines_to_print.append("##fileformat=" + self.fileformat) if self.filedate: lines_to_print.append("##fileformat=" + self.fileformat) for filt in self.filter_dict: lines_to_print.append(self.filter_dict[filt]) for form in self.format_dict: lines_to_print.append(self.format_dict[form]) for info in self.info_dict: lines_to_print.append(self.info_dict[info]) for contig in self.contig_dict: lines_to_print.append(self.contig_dict[contig]) for alt in self.alt_dict: lines_to_print.append(self.alt_dict[alt]) for other in self.other_dict: lines_to_print.append(self.other_dict[other]) lines_to_print.append("#" + "\t".join(self.header)) return lines_to_print def add_fileformat(self, fileformat): """ Add fileformat line to the header. Arguments: fileformat (str): The id of the info line """ self.fileformat = fileformat logger.info("Adding fileformat to vcf: {0}".format(fileformat)) return def add_meta_line(self, key, value): """ Adds an arbitrary metadata line to the header. This must be a key value pair Arguments: key (str): The key of the metadata line value (str): The value of the metadata line """ meta_line = "##{0}={1}".format(key, value) logger.info("Adding meta line to vcf: {0}".format(meta_line)) self.parse_meta_data(meta_line) return def add_info(self, info_id, number, entry_type, description): """ Add an info line to the header. Arguments: info_id (str): The id of the info line number (str): Integer or any of [A,R,G,.] entry_type (str): Any of [Integer,Float,Flag,Character,String] description (str): A description of the info line """ info_line = '##INFO=<ID={0},Number={1},Type={2},Description="{3}">'.format( info_id, number, entry_type, description ) logger.info("Adding info line to vcf: {0}".format(info_line)) self.parse_meta_data(info_line) return def add_filter(self, filter_id, description): """ Add a filter line to the header. Arguments: filter_id (str): The id of the filter line description (str): A description of the info line """ filter_line = '##FILTER=<ID={0},Description="{1}">'.format(filter_id, description) logger.info("Adding filter line to vcf: {0}".format(filter_line)) self.parse_meta_data(filter_line) return def add_format(self, format_id, number, entry_type, description): """ Add a format line to the header. Arguments: format_id (str): The id of the format line number (str): Integer or any of [A,R,G,.] entry_type (str): Any of [Integer,Float,Flag,Character,String] description (str): A description of the info line """ format_line = '##FORMAT=<ID={0},Number={1},Type={2},Description="{3}">'.format( format_id, number, entry_type, description ) logger.info("Adding format line to vcf: {0}".format(format_line)) self.parse_meta_data(format_line) return def add_alt(self, alt_id, description): """ Add a alternative allele format field line to the header. Arguments: alt_id (str): The id of the alternative line description (str): A description of the info line """ alt_line = '##ALT=<ID={0},Description="{1}">'.format(alt_id, description) logger.info("Adding alternative allele line to vcf: {0}".format(alt_line)) self.parse_meta_data(alt_line) return def add_contig(self, contig_id, length): """ Add a contig line to the header. Arguments: contig_id (str): The id of the alternative line length (str): A description of the info line """ contig_line = "##contig=<ID={0},length={1}>".format(contig_id, length) logger.info("Adding contig line to vcf: {0}".format(contig_line)) self.parse_meta_data(contig_line) return def add_version_tracking(self, info_id, version, date, command_line=""): """ Add a line with information about which software that was run and when to the header. Arguments: info_id (str): The id of the info line version (str): The version of the software used date (str): Date when software was run command_line (str): The command line that was used for run """ other_line = '##Software=<ID={0},Version={1},Date="{2}",CommandLineOptions="{3}">'.format( info_id, version, date, command_line ) self.other_dict[info_id] = other_line return
class HeaderParser(object): """Class for holding and manipulating vcf headers Attributes: info_dict=OrderedDict() extra_info = {} filter_lines=[] filter_dict=OrderedDict() contig_lines=[] contig_dict=OrderedDict() format_lines=[] format_dict=OrderedDict() alt_lines=[] alt_dict=OrderedDict() other_lines=[] other_dict=OrderedDict() header=['CHROM','POS','ID','REF','ALT','QUAL','FILTER','INFO'] header_keys={'info' : ['ID', 'Number', 'Type', 'Description'], 'form' : ['ID', 'Number', 'Type', 'Description'], 'filt' : ['ID', 'Description'], 'alt' : ['ID', 'Description'], 'contig' : ['ID']} fileformat = None filedate = None reference = None phasing = None source = None line_counter = 0 individuals = [] vep_columns = [] snpeff_columns = [] info_pattern filter_pattern contig_pattern format_pattern alt_pattern meta_pattern Methods: parse_meta_data(line) parse_header_line(line) add_fileformat(fileformat) print_header(): Return a list with the header lines add_meta_linekey, value) add_info(info_id, number, entry_type, description) add_filter(filter_id, description) add_format(format_id, number, entry_type, description) add_alt(alt_id, description) add_contig(contig_id, length) add_version_tracking(info_id, version, date, command_line='') """ def __init__(self): super(HeaderParser, self).__init__() self.info_lines = [] self.info_dict = OrderedDict() #This is a dictionary cantaining specific information about the info fields #It will have info name as key and then another dictionary with ID, Number, Type and Description self.extra_info = {} self.filter_lines = [] self.filter_dict = OrderedDict() self.contig_lines = [] self.contig_dict = OrderedDict() self.format_lines = [] self.format_dict = OrderedDict() self.alt_lines = [] self.alt_dict = OrderedDict() self.other_lines = [] self.other_dict = OrderedDict() self.header = [ 'CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO' ] self.header_keys = { 'info': ['ID', 'Number', 'Type', 'Description'], 'form': ['ID', 'Number', 'Type', 'Description'], 'filt': ['ID', 'Description'], 'alt': ['ID', 'Description'], 'contig': ['ID', 'length'] } self.fileformat = None self.filedate = None self.reference = None self.phasing = None self.source = None self.line_counter = 0 self.individuals = [] self.vep_columns = [] self.snpeff_columns = [] self.info_pattern = re.compile( r'''\#\#INFO=< ID=(?P<id>[^,]+), Number=(?P<number>-?\d+|\.|[AGR]), Type=(?P<type>Integer|Float|Flag|Character|String), Description="(?P<desc>[^"]*)" >''', re.VERBOSE) self.filter_pattern = re.compile( r'''\#\#FILTER=< ID=(?P<id>[^,]+), Description="(?P<desc>[^"]*)" >''', re.VERBOSE) self.contig_pattern = re.compile( r'''\#\#contig=< ID=(?P<id>[^>,]+) (,.*length=(?P<length>-?\d+))? .* >''', re.VERBOSE) self.format_pattern = re.compile( r'''\#\#FORMAT=< ID=(?P<id>.+), Number=(?P<number>-?\d+|\.|[AGR]), Type=(?P<type>.+), Description="(?P<desc>.*)" >''', re.VERBOSE) self.alt_pattern = re.compile( r'''\#\#ALT=< ID=(?P<id>[^,]+), Description="(?P<desc>[^"]*)" >''', re.VERBOSE) self.meta_pattern = re.compile(r'''##(?P<key>.+?)=(?P<val>.+)''') def parse_meta_data(self, line): """Parse a vcf metadataline""" line = line.rstrip() logger.debug("Parsing metadata line:{0}".format(line)) line_info = line[2:].split('=') match = False if line_info[0] == 'fileformat': logger.debug("Parsing fileformat") try: self.fileformat = line_info[1] logger.debug("Found fileformat {0}".format(self.fileformat)) except IndexError: raise SyntaxError("fileformat must have a value") elif line_info[0] == 'INFO': match = self.info_pattern.match(line) if not match: raise SyntaxError( "One of the INFO lines is malformed:{0}".format(line)) matches = [ match.group('id'), match.group('number'), match.group('type'), match.group('desc') ] # extra_info is a dictionary to check the metadata about the INFO values: self.extra_info[matches[0]] = dict( zip(self.header_keys['info'][1:], matches[1:])) info_line = dict(list(zip(self.header_keys['info'], matches))) if len(info_line['Description'].split('Format:')) > 1: info_line['Format'] = [ info.strip() for info in info_line['Description'].split('Format:') ][-1] self.info_lines.append(info_line) # Store the vep columns: if info_line['ID'] == 'CSQ': self.vep_columns = info_line.get('Format', '').split('|') if info_line['ID'] == 'ANN': self.snpeff_columns = [ annotation.strip("' ") for annotation in info_line.get( 'Description', '').split(':')[-1].split('|') ] self.info_dict[match.group('id')] = line elif line_info[0] == 'FILTER': match = self.filter_pattern.match(line) if not match: raise SyntaxError( "One of the FILTER lines is malformed: {0}".format(line)) matches = [match.group('id'), match.group('desc')] self.filter_lines.append( dict(list(zip(self.header_keys['filt'], matches)))) self.filter_dict[match.group('id')] = line elif line_info[0] == 'contig': match = self.contig_pattern.match(line) if not match: print() raise SyntaxError( "One of the contig lines is malformed: {0}".format(line)) matches = [match.group('id'), match.group('length')] self.contig_lines.append( dict(list(zip(self.header_keys['contig'], matches)))) self.contig_dict[match.group('id')] = line elif line_info[0] == 'FORMAT': match = self.format_pattern.match(line) if not match: raise SyntaxError( "One of the FORMAT lines is malformed: {0}".format(line)) matches = [ match.group('id'), match.group('number'), match.group('type'), match.group('desc') ] self.format_lines.append( dict(list(zip(self.header_keys['form'], matches)))) self.format_dict[match.group('id')] = line elif line_info[0] == 'ALT': match = self.alt_pattern.match(line) if not match: raise SyntaxError( "One of the ALT lines is malformed: {0}".format(line)) matches = [match.group('id'), match.group('desc')] self.alt_lines.append( dict(list(zip(self.header_keys['alt'], matches)))) self.alt_dict[match.group('id')] = line else: match = self.meta_pattern.match(line) if not match: raise SyntaxError( "One of the meta data lines is malformed: {0}".format( line)) self.other_lines.append({match.group('key'): match.group('val')}) self.other_dict[match.group('key')] = line def parse_header_line(self, line): """docstring for parse_header_line""" self.header = line[1:].rstrip().split('\t') if len(self.header) < 9: self.header = line[1:].rstrip().split() self.individuals = self.header[9:] def remove_header(self, name): """Remove a field from the header""" if name in self.info_dict: self.info_dict.pop(name) logger.info("Removed '{0}' from INFO".format(name)) if name in self.filter_dict: self.filter_dict.pop(name) logger.info("Removed '{0}' from FILTER".format(name)) if name in self.format_dict: self.format_dict.pop(name) logger.info("Removed '{0}' from FORMAT".format(name)) if name in self.contig_dict: self.contig_dict.pop(name) logger.info("Removed '{0}' from CONTIG".format(name)) if name in self.alt_dict: self.alt_dict.pop(name) logger.info("Removed '{0}' from ALT".format(name)) if name in self.other_dict: self.other_dict.pop(name) logger.info("Removed '{0}' from OTHER".format(name)) return def print_header(self): """Returns a list with the header lines if proper format""" lines_to_print = [] lines_to_print.append('##fileformat=' + self.fileformat) if self.filedate: lines_to_print.append('##fileformat=' + self.fileformat) for filt in self.filter_dict: lines_to_print.append(self.filter_dict[filt]) for form in self.format_dict: lines_to_print.append(self.format_dict[form]) for info in self.info_dict: lines_to_print.append(self.info_dict[info]) for contig in self.contig_dict: lines_to_print.append(self.contig_dict[contig]) for alt in self.alt_dict: lines_to_print.append(self.alt_dict[alt]) for other in self.other_dict: lines_to_print.append(self.other_dict[other]) lines_to_print.append('#' + '\t'.join(self.header)) return lines_to_print def add_fileformat(self, fileformat): """ Add fileformat line to the header. Arguments: fileformat (str): The id of the info line """ self.fileformat = fileformat logger.info("Adding fileformat to vcf: {0}".format(fileformat)) return def add_meta_line(self, key, value): """ Adds an arbitrary metadata line to the header. This must be a key value pair Arguments: key (str): The key of the metadata line value (str): The value of the metadata line """ meta_line = '##{0}={1}'.format(key, value) logger.info("Adding meta line to vcf: {0}".format(meta_line)) self.parse_meta_data(meta_line) return def add_info(self, info_id, number, entry_type, description): """ Add an info line to the header. Arguments: info_id (str): The id of the info line number (str): Integer or any of [A,R,G,.] entry_type (str): Any of [Integer,Float,Flag,Character,String] description (str): A description of the info line """ info_line = '##INFO=<ID={0},Number={1},Type={2},Description="{3}">'.format( info_id, number, entry_type, description) logger.info("Adding info line to vcf: {0}".format(info_line)) self.parse_meta_data(info_line) return def add_filter(self, filter_id, description): """ Add a filter line to the header. Arguments: filter_id (str): The id of the filter line description (str): A description of the info line """ filter_line = '##FILTER=<ID={0},Description="{1}">'.format( filter_id, description) logger.info("Adding filter line to vcf: {0}".format(filter_line)) self.parse_meta_data(filter_line) return def add_format(self, format_id, number, entry_type, description): """ Add a format line to the header. Arguments: format_id (str): The id of the format line number (str): Integer or any of [A,R,G,.] entry_type (str): Any of [Integer,Float,Flag,Character,String] description (str): A description of the info line """ format_line = '##FORMAT=<ID={0},Number={1},Type={2},Description="{3}">'.format( format_id, number, entry_type, description) logger.info("Adding format line to vcf: {0}".format(format_line)) self.parse_meta_data(format_line) return def add_alt(self, alt_id, description): """ Add a alternative allele format field line to the header. Arguments: alt_id (str): The id of the alternative line description (str): A description of the info line """ alt_line = '##ALT=<ID={0},Description="{1}">'.format( alt_id, description) logger.info( "Adding alternative allele line to vcf: {0}".format(alt_line)) self.parse_meta_data(alt_line) return def add_contig(self, contig_id, length): """ Add a contig line to the header. Arguments: contig_id (str): The id of the alternative line length (str): A description of the info line """ contig_line = '##contig=<ID={0},length={1}>'.format(contig_id, length) logger.info("Adding contig line to vcf: {0}".format(contig_line)) self.parse_meta_data(contig_line) return def add_version_tracking(self, info_id, version, date, command_line=''): """ Add a line with information about which software that was run and when to the header. Arguments: info_id (str): The id of the info line version (str): The version of the software used date (str): Date when software was run command_line (str): The command line that was used for run """ other_line = '##Software=<ID={0},Version={1},Date="{2}",CommandLineOptions="{3}">'.format( info_id, version, date, command_line) self.other_dict[info_id] = other_line return
class Cache(object): ''' Cache to hold cached contents ''' index_filename = "index.pickle" # active requests are not counted. def __init__(self, path, entry_limit): self.path = path self.on_memory_entry_limit = entry_limit self.on_memory = OrderedDict() #FastAVLTree() self.index = {} self.load_index() def __contains__(self, key): return key in self.index def __len__(self): return len(self.index) def __iter__(self): return self.index.values() def _make_path(self, fname): return os.path.join(self.path, fname) def make_entry(self, key): ''' reserve ''' assert key not in self.index entry = CacheEntry(key, self.path, 0.0, self.on_notify) #FIXME because cache entry is write once. read many. self.index[key] = entry return entry def on_notify(self, entry): print 'cache entries: ', len(self.on_memory) if entry.key in self.on_memory: self.touch(entry) else: self.push_to_memory(entry) def push_to_memory(self, entry): if len(self.on_memory) >= self.on_memory_entry_limit: key, purged = self.on_memory.popitem(False) #popping first item print 'purged cache life=%f s since %f for %s' % (time.time() - purged.last_touch, purged.last_touch, purged.key) purged.move_to_disk() entry.touch() print "putting entry %s" % (entry.key) assert entry.datafile assert entry.last_touch > 1.0 self.on_memory[entry.key] = entry def touch(self, entry): #revoke x = self.on_memory.pop(entry.key) assert x == entry # activate it as a new entry entry.touch() self.on_memory[entry.key] = entry def get(self, key): e = self.index.get(key, None) return e def pop(self, key): return self.index.pop(key) def load_index(self): p = self._make_path(self.index_filename) no_index = False try: f = open(p) except: f = None pass if f: try: self.index = pickle.load(f) except: no_index = True finally: f.close() else: no_index = True if no_index: self.index = {} self.save_index() def save_entries(self): print 'Cache.save_entries' for entry in self.index.itervalues(): if entry.datafile: entry.move_to_disk() def save_index(self): print 'Cache.save_index' p = self._make_path(self.index_filename) for entry in self.index.itervalues(): entry.abort() with open(p, 'w') as f: pickle.dump(self.index, f) def fix(self): to_delete = [] for k, v in self.index.items(): p = self._make_path(v) if not os.access(p, os.F_OK | os.R_OK | os.W_OK): to_delete.append[k] for k in to_delete: del self.index[k] self.save_index() def scan(self): ''' wrong idea. cant generate url from file name...''' #return os.path.join(self.path, fname) for fname in os.listdir(self.path): if fname == self.index_filename: continue def html_index(self): count = len(self.index) x = [] for key, ce in self.on_memory.iteritems(): x.append('<li>[%s ]: %s</li>\n'%(key, ce.status())) frag_mem = '<ol>%s</ol>'%(''.join(x)) y = [] for key, ce in self.index.iteritems(): y.append('<li>[%s ]: %s</li>\n'%(key, ce.status())) frag_index = '<ol>%s</ol>'%(''.join(y)) html = '''<html><body> <p>count:%s</p> %s <hr /> %s </body></html>''' return html%(count, frag_mem, frag_index)