class ProcessTerminationMonitor(plugins.Observable):
    def __init__(self):
        plugins.Observable.__init__(self)
        self.processesForKill = OrderedDict()
        self.exitHandlers = OrderedDict()

    def listRunningProcesses(self):
        processesToCheck = guiConfig.getCompositeValue("query_kill_processes", "", modeDependent=True)
        if "all" in processesToCheck:
            processesToCheck = [".*"]
        if len(processesToCheck) == 0:
            return []

        running = []
        triggerGroup = plugins.TextTriggerGroup(processesToCheck)
        for process, description in self.processesForKill.values():
            if triggerGroup.stringContainsText(description):
                running.append("PID " + str(process.pid) + " : " + description)

        return running

    def getProcessIdentifier(self, process):
        # Unfortunately the child_watch_add method needs different ways to
        # identify the process on different platforms...
        if os.name == "posix":
            return process.pid
        else:
            return process._handle

    def startProcess(
        self, cmdArgs, description="", killOnTermination=True, exitHandler=None, exitHandlerArgs=(), **kwargs
    ):
        process = subprocess.Popen(cmdArgs, stdin=open(os.devnull), **kwargs)
        pidOrHandle = self.getProcessIdentifier(process)
        self.exitHandlers[int(pidOrHandle)] = (exitHandler, exitHandlerArgs)
        if killOnTermination:
            self.processesForKill[int(pidOrHandle)] = (process, description)
        gobject.child_watch_add(pidOrHandle, self.processExited)

    def processExited(self, pid, *args):
        if self.processesForKill.has_key(pid):
            del self.processesForKill[pid]

        if self.exitHandlers.has_key(pid):
            exitHandler, exitHandlerArgs = self.exitHandlers.pop(pid)
            if exitHandler:
                exitHandler(*exitHandlerArgs)

    def notifyKillProcesses(self, sig=None):
        # Don't leak processes
        if len(self.processesForKill) == 0:
            return
        diag = logging.getLogger("kill processes")
        self.notify("Status", "Terminating all external viewers ...")
        for pid, (process, description) in self.processesForKill.items():
            if self.exitHandlers.has_key(pid):
                self.exitHandlers.pop(pid)  # don't call exit handlers in this case, we're terminating
            self.notify("ActionProgress")
            diag.info("Killing '" + description + "' interactive process")
            killSubProcessAndChildren(process, sig)
 def test_pop(self):
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     shuffle(pairs)
     od = OrderedDict(pairs)
     shuffle(pairs)
     while pairs:
         k, v = pairs.pop()
         self.assertEqual(od.pop(k), v)
     self.assertRaises(KeyError, od.pop, 'xyz')
     self.assertEqual(len(od), 0)
     self.assertEqual(od.pop(k, 12345), 12345)
 def test_pop(self):
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     shuffle(pairs)
     od = OrderedDict(pairs)
     shuffle(pairs)
     while pairs:
         k, v = pairs.pop()
         self.assertEqual(od.pop(k), v)
     self.assertRaises(KeyError, od.pop, 'xyz')
     self.assertEqual(len(od), 0)
     self.assertEqual(od.pop(k, 12345), 12345)
Exemple #4
0
class DictCache(AbstractCache):
    """Implementation of a cache in a memory dictionary

    """
    def __init__(self, ttl=60):
        """Creates a new instance

        params:

            ``ttl``
                Time to live of the data.

        """
        super(DictCache, self).__init__(ttl=ttl)
        try:
            self._ich = collections.OrderedDict()
            self._ttds = collections.OrderedDict()
        except AttributeError:
            #This version of python does not support OrderedDict
            from ordereddict import OrderedDict
            self._ich = OrderedDict()
            self._ttds = OrderedDict()

    def store_data(self, k, ttl, v):
        self._ich[k] = v
        self._ttds[k] = (
            time.time() + ttl if ttl != None else None
        )

    def retrieve_data(self, k):
        ttd = self._ttds.get(k, 0)
        if ttd == None or time.time() < ttd:
            return self._ich[k]
        elif ttd:
            self._ttds.pop(k)
            self._ich.pop(k)

    def clear_expired(self):
        for k, ttd in self._ttds.items():
            if ttd != None and ttd < time.time():
                self._ttds.pop(k)
                self._ich.pop(k)
            else:
                break

    def clear(self):
        self._ich.clear()
        self._ttds.clear()
Exemple #5
0
    def _build(self, dirinfo):
        if self._belongs_to_exclude(dirinfo):
            return BinaryItem(dirinfo.name, filename=dirinfo.filename)
        files = OrderedDict((x.name, x) for x
            in unique_sorted_listing(dirinfo.filename,
                extension_priority=self.extension_priority))

        if self.directory_item_name in files \
          and text_file(files[self.directory_item_name].filename):
            info = files.pop(self.directory_item_name)
            factory = query_item_factory(info.extension, default=ContentItem)
            item = lambda children: factory(
                dirinfo.name, info.filename, info.extension, children=children)
        else:
            item = lambda children: Item(dirinfo.name, children=children)
        children = [self._build(x) for x in files.values()]
        return item(children)
Exemple #6
0
 def get_bibliography_entries(self, docname, id_, warn):
     """Return filtered bibliography entries, sorted by citation order."""
     # get entries, ordered by bib file occurrence
     entries = OrderedDict((entry.key, entry)
                           for entry in self._get_bibliography_entries(
                               docname=docname, id_=id_, warn=warn))
     # order entries according to which were cited first
     # first, we add all keys that were cited
     # then, we add all remaining keys
     sorted_entries = []
     for key in self.get_all_cited_keys():
         try:
             entry = entries.pop(key)
         except KeyError:
             pass
         else:
             sorted_entries.append(entry)
     sorted_entries += six.itervalues(entries)
     return sorted_entries
Exemple #7
0
 def get_bibliography_entries(self, docname, id_, warn):
     """Return filtered bibliography entries, sorted by citation order."""
     # get entries, ordered by bib file occurrence
     entries = OrderedDict(
         (entry.key, entry) for entry in
         self._get_bibliography_entries(
             docname=docname, id_=id_, warn=warn))
     # order entries according to which were cited first
     # first, we add all keys that were cited
     # then, we add all remaining keys
     sorted_entries = []
     for key in self.get_all_cited_keys():
         try:
             entry = entries.pop(key)
         except KeyError:
             pass
         else:
             sorted_entries.append(entry)
     sorted_entries += six.itervalues(entries)
     return sorted_entries
Exemple #8
0
class HeaderParser(object):
    """Parses a file with family info and creates a family object with individuals."""
    def __init__(self, infile):
        super(HeaderParser, self).__init__()
        self.metadata=OrderedDict()
        self.header=[]
        self.line_counter = 0
        self.individuals = []
        self.metadata_pattern = re.compile(r'''\#\#COLUMNNAME="(?P<colname>[^"]*)"
            (?P<info>.*)''', re.VERBOSE)
        with open(infile, 'rb') as f:
            for line in f:
                self.line_counter += 1
                line = line.rstrip()
                if line.startswith('##'):
                    match = self.metadata_pattern.match(line)
                    if not match:
                        raise SyntaxError("One of the metadata lines is malformed: %s" % line)
                    matches = [match.group('colname'), match.group('info')]
                    self.metadata[match.group('colname')] = line
                elif line.startswith('#'):
                    self.header = line[1:].split('\t')
                    for entry in self.header:
                        if entry[:3] == 'IDN':
                            self.individuals.append(entry.split(':')[1])
                        else:
                            self.check_header(entry)
                else:
                    break
        
    def add_metadata(self, column_name, data_type=None, version=None, description=None, dbname=None, delimiter='\t'):
        """Add metadata info to the header."""
        data_line = '##COLUMNNAME='+'"'+ column_name +'"'
        if column_name not in self.metadata:
            if data_type:
                if data_type not in ['Float', 'String', 'Integer']:
                    raise SyntaxError("Type must be 'Float', 'String' or 'Integer'. You tried: %s" % data_type)
                data_line += delimiter + 'TYPE="' + data_type + '"'
            if version:
                data_line += delimiter + 'VERSION="' + version + '"'
            if description:
                data_line += delimiter + 'DESCRIPTION="' + description + '"'
            if dbname:
                data_line += delimiter + 'SCOUTHEADER="' + dbname + '"'
            self.metadata.pop(column_name, 0)
            self.metadata[column_name] = data_line
        return
    
    def add_header(self, header_name):
        """Add an entry to the header line. The entry must be specified among the metadata lines first."""
        self.check_header(header_name)
        if header_name not in self.header:
            self.header.append(header_name)
        return
    
    def get_headers_for_print(self):
        """Returns a list with the metadata lines on correct format."""
        lines_for_print = []
        for header in self.metadata:
            lines_for_print.append(self.metadata[header])
        lines_for_print.append('\t'.join(self.header))
        lines_for_print[-1] = '#' + lines_for_print[-1]
        return lines_for_print
    
    def check_header(self, entry):
        """All entrys in the header must be specified in the metadata lines."""
        if entry not in self.metadata:
            raise SyntaxError("Header entry must be described in the metadata lines. Entry: %s is not in metadata." % entry)
    class HybridView(newcls, JSONResponseMixin):
        """Middleware class which add the JSONResponseMixin in the view to
        handle ajax requests"""

        def __init__(self, *args, **kwargs):
            """Our newcls can be an instance of several mixins working with the
            get and post functions.
            e.g : CreateView is instance of BaseCreateView and ProcessFormView
            Let's add our custom mixins that implement get and post without
            returning a render_to_response, and call """

            newcls.__init__(self, **kwargs)
            # The order matters for the get/post calls.
            self.mixins = OrderedDict()
            self.mixins[BaseListView] = BaseListViewMixin
            self.mixins[BaseCreateView] = BaseCreateViewMixin
            self.mixins[BaseUpdateView] = BaseUpdateViewMixin
            self.mixins[BaseDetailView] = BaseDetailViewMixin
            self.mixins[BaseDeleteView] = BaseDeleteViewMixin
            self.mixins[ProcessFormView] = ProcessFormViewMixin
            [self.mixins.pop(baseView) for baseView in self.mixins.iterkeys()
                if not isinstance(self, baseView)]

        @property
        def is_ajax(self):
            """Check the META HTTP_X_REQUESTED_WITH and CONTENT_TYPE"""
            meta = self.request.META
            return meta.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest'\
                or "json" in meta.get("CONTENT_TYPE")

        def render_to_response(self, context):
            cls = type(self).__bases__[self.is_ajax]
            return cls.render_to_response(self, context)

        def get(self, request, **kwargs):
            if not self.is_ajax:
                "If it's not ajax, return the inherited get"
                return super(HybridView, self).get(self, **kwargs)

            for mixin in self.mixins.itervalues():
                self, kwargs = mixin().get(self, **kwargs)

            context = kwargs
            context.update(self.get_json_context(**kwargs))
            context.pop("form", None)
            context.pop("object", None)
            context.pop("object_list", None)
            return self.render_to_response(context)

        def post(self, request, **kwargs):
            """Hybrid post to handle all parents post actions"""
            if not self.is_ajax:
                "If it's not ajax, return the inherited get"
                return super(HybridView, self).post(self, **kwargs)

            for mixin in self.mixins.itervalues():
                try:
                    self, kwargs = mixin().post(self, **kwargs)
                except AttributeError:
                    """The inherited view doesn't handle post"""
                    pass

            context = kwargs
            context.update(self.get_json_context(**kwargs))
            context.pop("form", None)
            return self.render_to_response(context)
Exemple #10
0
class CnCGraph(object):
    def __init__(self, name, g):
        verifyCollectionDecls("item", g.itemColls)
        steps = [ x.step for x in g.stepRels ]
        verifyCollectionDecls("step", steps)
        self.name = name
        # items
        self.itemDeclarations = OrderedDict((i.collName, makeItemDecl(i)) for i in g.itemColls)
        self.concreteItems = [ i for i in self.itemDeclarations.values() if not i.isVirtual ]
        # item virtual mappings
        self.vms = [ i for i in self.itemDeclarations.values() if i.isVirtual ]
        self.inlineVms = [ i for i in self.vms if i.isInline ]
        self.externVms = [ i for i in self.vms if not i.isInline ]
        # steps / pseudo-steps
        self.stepFunctions = OrderedDict((x.step.collName, StepFunction(x)) for x in g.stepRels)
        verifyEnv(self.stepFunctions)
        self.initFunction = self.stepFunctions.pop(initNameRaw)
        self.initFunction.collName = "cncInitialize"
        self.finalizeFunction = self.stepFunctions.pop(finalizeNameRaw)
        self.finalizeFunction.collName = "cncFinalize"
        self.finalAndSteps = [self.finalizeFunction] + self.stepFunctions.values()
        # set up step attribute lookup dict
        self.stepLikes = OrderedDict(self.stepFunctions)
        self.stepLikes[self.initFunction.collName] = self.initFunction
        self.stepLikes[self.finalizeFunction.collName] = self.finalizeFunction
        # attribute tracking
        self.allAttrNames = set()
        # context
        self.ctxParams = filter(bool, map(strip, g.ctx.splitlines())) if g.ctx else []

    def hasTuning(self, name):
        return name in self.allAttrNames

    def hasCustomDist(self):
        return self.hasTuning('distfn') or self.hasTuning('placeWith')

    def lookupType(self, item):
        return self.itemDeclarations[item.collName].type

    def itemDistFn(self, collName, ranksExpr):
        return getDistFn(self.itemDeclarations, collName, ranksExpr)

    def stepDistFn(self, collName, ranksExpr):
        return getDistFn(self.stepLikes, collName, ranksExpr)

    def itemTuningFn(self, collName, name, ranksExpr, default):
        return getTuningFn(self.itemDeclarations, collName, name, ranksExpr, default)

    def stepTuningFn(self, collName, name, ranksExpr, default):
        return getTuningFn(self.stepLikes, collName, name, ranksExpr, default)

    def priorityFn(self, collName, ranksExpr):
        return self.stepTuningFn(collName, 'priority', ranksExpr, "0")

    def addTunings(self, tuningSpec):
        for t in tuningSpec.itemTunings:
            x = self.itemDeclarations.get(t.collName)
            assert x, "Unknown item in tuning: {0}".format(t.collName)
            x.attrs.update(t.attrs)
            self.allAttrNames.update(t.attrs.keys())
        for t in tuningSpec.stepTunings:
            x = self.stepLikes.get(t.collName)
            assert x, "Unknown step in tuning: {0}".format(t.collName)
            if t.inputName:
                i = x.inputsDict.get(t.inputName)
                assert i, "Unknown input in tuning: {0} <- {1}".format(t.collName, t.inputName)
                i.attrs.update(t.attrs)
                self.allAttrNames.update(t.attrs.keys())
            else:
                x.attrs.update(t.attrs)
                self.allAttrNames.update(t.attrs.keys())
Exemple #11
0
class CnCGraph(object):
    def __init__(self, name, g):
        verifyCollectionDecls("item", g.itemColls)
        steps = [x.step for x in g.stepRels]
        verifyCollectionDecls("step", steps)
        self.name = name
        # items
        self.itemDeclarations = OrderedDict(
            (i.collName, makeItemDecl(i)) for i in g.itemColls)
        self.concreteItems = [
            i for i in self.itemDeclarations.values() if not i.isVirtual
        ]
        # item virtual mappings
        self.vms = [i for i in self.itemDeclarations.values() if i.isVirtual]
        self.inlineVms = [i for i in self.vms if i.isInline]
        self.externVms = [i for i in self.vms if not i.isInline]
        # steps / pseudo-steps
        self.stepFunctions = OrderedDict(
            (x.step.collName, StepFunction(x)) for x in g.stepRels)
        verifyEnv(self.stepFunctions)
        self.initFunction = self.stepFunctions.pop(initNameRaw)
        self.initFunction.collName = "cncInitialize"
        self.finalizeFunction = self.stepFunctions.pop(finalizeNameRaw)
        self.finalizeFunction.collName = "cncFinalize"
        self.finalAndSteps = [self.finalizeFunction
                              ] + self.stepFunctions.values()
        # set up step attribute lookup dict
        self.stepLikes = OrderedDict(self.stepFunctions)
        self.stepLikes[self.initFunction.collName] = self.initFunction
        self.stepLikes[self.finalizeFunction.collName] = self.finalizeFunction
        # attribute tracking
        self.allAttrNames = set()
        # context
        self.ctxParams = filter(bool, map(
            strip, g.ctx.splitlines())) if g.ctx else []

    def hasTuning(self, name):
        return name in self.allAttrNames

    def hasCustomDist(self):
        return self.hasTuning('distfn') or self.hasTuning('placeWith')

    def lookupType(self, item):
        return self.itemDeclarations[item.collName].type

    def itemDistFn(self, collName, ranksExpr):
        return getDistFn(self.itemDeclarations, collName, ranksExpr)

    def stepDistFn(self, collName, ranksExpr):
        return getDistFn(self.stepLikes, collName, ranksExpr)

    def itemTuningFn(self, collName, name, ranksExpr, default):
        return getTuningFn(self.itemDeclarations, collName, name, ranksExpr,
                           default)

    def stepTuningFn(self, collName, name, ranksExpr, default):
        return getTuningFn(self.stepLikes, collName, name, ranksExpr, default)

    def priorityFn(self, collName, ranksExpr):
        return self.stepTuningFn(collName, 'priority', ranksExpr, "0")

    def addTunings(self, tuningSpec):
        for t in tuningSpec.itemTunings:
            x = self.itemDeclarations.get(t.collName)
            assert x, "Unknown item in tuning: {0}".format(t.collName)
            x.attrs.update(t.attrs)
            self.allAttrNames.update(t.attrs.keys())
        for t in tuningSpec.stepTunings:
            x = self.stepLikes.get(t.collName)
            assert x, "Unknown step in tuning: {0}".format(t.collName)
            if t.inputName:
                i = x.inputsDict.get(t.inputName)
                assert i, "Unknown input in tuning: {0} <- {1}".format(
                    t.collName, t.inputName)
                i.attrs.update(t.attrs)
                self.allAttrNames.update(t.attrs.keys())
            else:
                x.attrs.update(t.attrs)
                self.allAttrNames.update(t.attrs.keys())
Exemple #12
0
class RenderableContainer(object):
    is_group = True

    def __init__(self):

        self._components = []
        self._componentmap = OrderedDict()
        self._bindmap = OrderedDict()

    def __json__(self, request):
        return {"components": self._componentmap}

    def rmRenderable(self, renderable_id):

        renderable = self._componentmap.pop(renderable_id)
        self._components.remove(renderable)

    def addRenderable(self, renderable, pos=None):
        """ Add renderable. If pos is given, insert into that
        position, otherwise just append"""

        if pos is None:
            self._components.append(renderable)
        else:
            self._components.insert(pos, renderable)

        # todo: see if we can use the pos parameter for the insertion
        # into the _componentmap ordereddict
        self._componentmap[renderable.id] = renderable

        if getattr(renderable, 'bind', None):
            self._bindmap[renderable.bind] = renderable

    def getRenderables(self, recursive=False):
        """ retrieve all renderables. If recursive is true, then
        also return all renderables from the children recursively """

        if recursive:
            result = self._componentmap.values()
            for r in self._componentmap.values():
                try:
                    result += r.getRenderables(recursive)
                except:
                    pass
            return result

        else:
            return self._components

    def getRenderable(self, id):
        """ find renderable by id in the complete (recursive) tree """

        found = self._componentmap.get(id, None)
        if not found:
            # search children
            for r in self.getRenderables(False):
                try:
                    found = r.getRenderable(id)
                    if found:
                        break
                except:
                    pass
        return found

    def getRenderableByBind(self, bind):

        found = self._bindmap.get(bind, None)
        if not found:
            # search children
            for r in self.getRenderables(False):
                try:
                    found = r.getRenderableByBind(bind)
                    if found:
                        break
                except:
                    pass
        return found
class HeaderParser(object):
    """Parses a file with family info and creates a family object with individuals."""
    def __init__(self, infile):
        super(HeaderParser, self).__init__()
        self.metadata=OrderedDict()
        self.header=[]
        self.line_counter = 0
        self.individuals = []
        self.metadata_pattern = re.compile(r'''\#\#COLUMNNAME="(?P<colname>[^"]*)"
            (?P<info>.*)''', re.VERBOSE)
        with open(infile, 'rb') as f:
            for line in f:
                self.line_counter += 1
                line = line.rstrip()
                if line.startswith('##'):
                    match = self.metadata_pattern.match(line)
                    if not match:
                        raise SyntaxError("One of the metadata lines is malformed: %s" % line)
                    matches = [match.group('colname'), match.group('info')]
                    self.metadata[match.group('colname')] = line
                elif line.startswith('#'):
                    self.header = line[1:].split('\t')
                    for entry in self.header:
                        if entry[:3] == 'IDN':
                            self.individuals.append(entry.split(':')[1])
                        else:
                            self.check_header(entry)
                else:
                    break
        
    def add_metadata(self, column_name, data_type=None, version=None, description=None, dbname=None, delimiter='\t'):
        """Add metadata info to the header."""
        data_line = '##COLUMNNAME='+'"'+ column_name +'"'
        if column_name not in self.metadata:
            if data_type:
                if data_type not in ['Float', 'String', 'Integer']:
                    raise SyntaxError("Type must be 'Float', 'String' or 'Integer'. You tried: %s" % data_type)
                data_line += delimiter + 'TYPE="' + data_type + '"'
            if version:
                data_line += delimiter + 'VERSION="' + version + '"'
            if description:
                data_line += delimiter + 'DESCRIPTION="' + description + '"'
            if dbname:
                data_line += delimiter + 'SCOUTHEADER="' + dbname + '"'
            self.metadata.pop(column_name, 0)
            self.metadata[column_name] = data_line
        return
    
    def add_header(self, header_name):
        """Add an entry to the header line. The entry must be specified among the metadata lines first."""
        self.check_header(header_name)
        if header_name not in self.header:
            self.header.append(header_name)
        return
    
    def get_headers_for_print(self):
        """Returns a list with the metadata lines on correct format."""
        lines_for_print = []
        for header in self.metadata:
            lines_for_print.append(self.metadata[header])
        lines_for_print.append('\t'.join(self.header))
        lines_for_print[-1] = '#' + lines_for_print[-1]
        return lines_for_print
    
    def check_header(self, entry):
        """All entrys in the header must be specified in the metadata lines."""
        if entry not in self.metadata:
            raise SyntaxError("Header entry must be described in the metadata lines. Entry: %s is not in metadata." % entry)
Exemple #14
0
class SymbolLRUCacheManager(RequiredConfig):
    """for cleaning up the symbols cache"""
    required_config = Namespace()
    required_config.add_option(
        'symbol_cache_path',
        doc="the cache directory to automatically remove files from",
        default=os.path.join(tempfile.gettempdir(), 'symbols'))
    required_config.add_option('symbol_cache_size',
                               doc="the maximum size of the symbols cache",
                               default='1G',
                               from_string_converter=from_string_to_parse_size)
    required_config.add_option(
        'verbosity',
        doc="how chatty should this be? 1 - writes to stdout,"
        " 2 - uses the logger",
        default=0,
        from_string_converter=int)

    #--------------------------------------------------------------------------
    def __init__(self, config, quit_check_callback=None):
        """constructor for a registration object that runs an LRU cache
       cleaner"""
        self.config = config

        self.directory = os.path.abspath(config.symbol_cache_path)
        self.max_size = config.symbol_cache_size
        self.verbosity = config.verbosity
        # Cache state
        self.total_size = 0
        self._lru = OrderedDict()
        # pyinotify bits
        self._wm = pyinotify.WatchManager()
        self._handler = EventHandler(self, verbosity=config.verbosity)
        self._notifier = pyinotify.ThreadedNotifier(self._wm, self._handler)
        mask = pyinotify.IN_DELETE | pyinotify.IN_CREATE \
            | pyinotify.IN_OPEN | pyinotify.IN_MOVED_FROM \
            | pyinotify.IN_MOVED_TO | pyinotify.IN_MODIFY
        self._wdd = self._wm.add_watch(self.directory,
                                       mask,
                                       rec=True,
                                       auto_add=True)
        # Load existing files into the cache.
        self._get_existing_files(self.directory)
        self._notifier.start()

    #--------------------------------------------------------------------------
    @property
    def num_files(self):
        return len(self._lru)

    #--------------------------------------------------------------------------
    def _rm_empty_dirs(self, path):
        '''
       Attempt to remove any empty directories that are parents of path
       and children of self.directory.
       '''
        path = os.path.dirname(path)
        while not os.path.samefile(path, self.directory):
            if not os.listdir(path):
                os.rmdir(path)
            path = os.path.dirname(path)

    #--------------------------------------------------------------------------
    def _update_cache(self, path, update_size=False):
        if path in self._lru:
            size = self._lru.pop(path)
            if update_size:
                self.total_size -= size
        else:
            update_size = True

        if update_size:
            try:
                size = os.stat(path).st_size
            except OSError:
                self.config.logger.warning(
                    'file was not found while cleaning cache: %s', path)
                return

            self.total_size += size
            # If we're out of space, remove items from the cache until
            # we fit again.
            while self.total_size > self.max_size and self._lru:
                rm_path, rm_size = self._lru.popitem(last=False)
                self.total_size -= rm_size
                os.unlink(rm_path)
                self._rm_empty_dirs(rm_path)
                if self.verbosity >= 2:
                    self.config.logger.debug('RM %s', rm_path)
        self._lru[path] = size

    #--------------------------------------------------------------------------
    def _remove_cached(self, path):
        # We might have already removed this file in _update_cache.
        if path in self._lru:
            size = self._lru.pop(path)
            self.total_size -= size

    #--------------------------------------------------------------------------
    def _get_existing_files(self, path):
        for base, dirs, files in os.walk(path):
            for f in files:
                f = os.path.join(base, f)
                self._update_cache(f)

    #--------------------------------------------------------------------------
    def close(self):
        self._notifier.stop()
Exemple #15
0
class Application(object):
    """Poor WSGI application which is called by WSGI server.

    Working of is describe in PEP 0333. This object store route dispatch table,
    and have methods for it's using and of course __call__ method for use
    as WSGI application.
    """

    __instances = []

    def __init__(self, name="__main__"):
        """Application class is per name singleton.

        That means, there could be exist only one instance with same name.
        """

        if Application.__instances.count(name):
            raise RuntimeError('Application with name %s exist yet.' % name)
        Application.__instances.append(name)

        # Application name
        self.__name = name

        # list of pre and post process handlers
        self.__pre = []
        self.__post = []

        # dhandlers table for default handers on methods {METHOD_GET: handler}
        self.__dhandlers = {}

        # handlers table of simple paths: {'/path': {METHOD_GET: handler}}
        self.__handlers = {}

        self.__filters = {
            ':int': (r'-?\d+', int),
            ':float': (r'-?\d+(\.\d+)?', float),
            ':word': (r'\w+', uni),
            ':hex': (r'[0-9a-fA-F]+', str),
            ':re:': (None, uni),
            'none': (r'[^/]+', uni)
        }

        # handlers of regex paths: {r'/user/([a-z]?)': {METHOD_GET: handler}}
        self.__rhandlers = OrderedDict()

        # http state handlers: {HTTP_NOT_FOUND: {METHOD_GET: my_404_handler}}
        self.__shandlers = {}

        # -- Application variable
        self.__config = {
            'auto_args': True,
            'auto_form': True,
            'auto_json': True,
            'keep_blank_values': 0,
            'strict_parsing': 0,
            'json_content_types': [
                'application/json',
                'application/javascript',
                'application/merge-patch+json'],
            'auto_cookies': True,
            'debug': 'Off',
            'document_root': '',
            'document_index': 'Off',
            'secret_key': '%s%s%s%s' %
                          (__version__, version, getcwd(),
                           ''.join(str(x) for x in uname()))
        }

        try:
            self.__log_level = levels[environ.get('poor_LogLevel',
                                                  'warn').lower()]
        except:
            self.__log_level = LOG_WARNING
            self.log_error('Bad poor_LogLevel, default is warn.', LOG_WARNING)
        # endtry
    # enddef

    def __regex(self, match):
        groups = match.groups()
        _filter = str(groups[1]).lower()

        if _filter in self.__filters:
            regex = self.__filters[_filter][0]
        elif _filter[:4] == ':re:':     # :re: filter have user defined regex
            regex = _filter[4:]
        else:
            try:
                regex = self.__filters[_filter][0]
            except KeyError:
                raise RuntimeError("Undefined route group filter '%s'" %
                                   _filter)

        return "(?P<%s>%s)" % (groups[0], regex)
    # enddef

    def __convertor(self, _filter):
        _filter = str(_filter).lower()
        _filter = ':re:' if _filter[:4] == ':re:' else _filter
        try:
            return self.__filters[_filter][1]
        except KeyError:
            raise RuntimeError("Undefined route group filter '%s'" % _filter)

    @property
    def name(self):
        """Return application name."""
        return self.__name

    @property
    def filters(self):
        """Copy of filter table.

        Filter table contains regular expressions and convert functions,
        see Application.set_filter and Application.route.

        Default filters are:
            :int - match number and convert it to int
            :float - match number and convert it to float
            :word - match one unicoee word
            :hex - match hexadecimal value and convert it to str
            :re: - match user defined regular expression
            none - match any string withount '/' character

        For more details see {/debug-info} page of your application, where
        you see all filters with regular expression definition.
        """
        return self.__filters.copy()

    @property
    def pre(self):
        """Tuple of table with pre-process handlers.

        See Application.pre_process.
        """
        return tuple(self.__pre)

    @property
    def post(self):
        """Tuple of table with post-process handlers.

        See Application.post_process.
        """
        return tuple(self.__post)

    @property
    def dhandlers(self):
        """Copy of table with default handlers.

        See Application.set_default
        """
        return self.__dhandlers.copy()

    @property
    def handlers(self):
        """Copy of table with static handlers.

        See Application.route.
        """
        return self.__handlers.copy()

    @property
    def rhandlers(self):
        """Copy of table with regular expression handlers.

        See Application.route and Application.rroute.
        """
        return self.__rhandlers.copy()

    @property
    def shandlers(self):
        """Copy of table with http state aka error handlers.

        See Application.http_state
        """
        return self.__shandlers.copy()

    @property
    def auto_args(self):
        """Automatic parsing request arguments from uri.

        If it is True (default), Request object do automatic parsing request
        uri to its args variable.
        """
        return self.__config['auto_args']

    @auto_args.setter
    def auto_args(self, value):
        self.__config['auto_args'] = bool(value)

    @property
    def auto_form(self):
        """Automatic parsing arguments from request body.

        If it is True (default) and method is POST, PUT or PATCH, Request
        object do automatic parsing request body to its form variable.
        """
        return self.__config['auto_form']

    @auto_form.setter
    def auto_form(self, value):
        self.__config['auto_form'] = bool(value)

    @property
    def auto_json(self):
        """Automatic parsing JSON from request body.

        If it is True (default), method is POST, PUT or PATCH and request
        content type is one of json_content_types, Request object do
        automatic parsing request body to json variable.
        """
        return self.__config['auto_json']

    @auto_json.setter
    def auto_json(self, value):
        self.__config['auto_json'] = bool(value)

    @property
    def auto_cookies(self):
        """Automatic parsing cookies from request headers.

        If it is True (default) and Cookie request header was set,
        SimpleCookie object was paresed to Request property cookies.
        """
        return self.__config['auto_cookies']

    @auto_cookies.setter
    def auto_cookies(self, value):
        self.__config['auto_cookies'] = bool(value)

    @property
    def debug(self):
        """Application debug as another way how to set poor_Debug.

        This setting will be rewrite by poor_Debug environment variable.
        """
        return self.__config['debug'] == 'On'

    @debug.setter
    def debug(self, value):
        self.__config['debug'] = 'On' if bool(value) else 'Off'

    @property
    def document_root(self):
        """Application document_root as another way how to set poor_DocumentRoot.

        This setting will be rewrite by poor_DocumentRoot environ variable.
        """
        return self.__config['document_root']

    @document_root.setter
    def document_root(self, value):
        self.__config['document_root'] = value

    @property
    def document_index(self):
        """Application document_root as another way how to set poor_DocumentRoot.

        This setting will be rewrite by poor_DocumentRoot environ variable.
        """
        return self.__config['document_index'] == 'On'

    @document_index.setter
    def document_index(self, value):
        self.__config['document_index'] = 'On' if bool(value) else 'Off'

    @property
    def secret_key(self):
        """Application secret_key could be replace by poor_SecretKey in request.

        Secret key is used by PoorSession class. It is generate from
        some server variables, and the best way is set to your own long
        key."""
        return self.__config['secret_key']

    @secret_key.setter
    def secret_key(self, value):
        self.__config['secret_key'] = value

    @property
    def keep_blank_values(self):
        """Keep blank values in request arguments.

        If it is 1 (0 is default), automatic parsing request uri or body
        keep blank values as empty string.
        """
        return self.__config['keep_blank_values']

    @keep_blank_values.setter
    def keep_blank_values(self, value):
        self.__config['keep_blank_values'] = int(value)

    @property
    def strict_parsing(self):
        """Strict parse request arguments.

        If it is 1 (0 is default), automatic parsing request uri or body
        raise with exception on parsing error.
        """
        return self.__config['strict_parsing']

    @strict_parsing.setter
    def strict_parsing(self, value):
        self.__config['strict_parsing'] = int(value)

    @property
    def json_content_types(self):
        """Copy of json content type list.

        Containt list of strings as json content types, which is use for
        testing, when automatics Json object is create from request body.
        """
        return self.__config['json_content_types']

    def set_filter(self, name, regex, convertor=uni):
        """Create new filter or overwrite builtins.

        Arguments:
            name      - Name of filter which is used in route or set_route
                        method.
            regex     - regular expression which used for filter
            convertor - convertor function or class, which gets unicode in
                        input. Default is uni function, which is wrapper
                        to unicode string.

            app.set_filter('uint', r'\d+', int)
        """
        name = ':'+name if name[0] != ':' else name
        self.__filters[name] = (regex, convertor)

    def pre_process(self):
        """Append pre process hendler.

        This is decorator for function to call before each request.

            @app.pre_process()
            def before_each_request(req):
                ...
        """
        def wrapper(fn):
            self.__pre.append(fn)
            return fn
        return wrapper
    # enddef

    def add_pre_process(self, fn):
        """Append pre proccess handler.

        Method adds function to list functions which is call before each
        request.

            app.add_pre_process(before_each_request)
        """
        self.__pre.append(fn)
    # enddef

    def post_process(self):
        """Append post process handler.

        This decorator append function to be called after each request,
        if you want to use it redefined all outputs.

            @app.pre_process()
            def after_each_request(req):
                ...
        """
        def wrapper(fn):
            self.__post.append(fn)
            return fn
        return wrapper
    # enddef

    def add_post_process(self, fn):
        """Append post process handler.

        Method for direct append function to list functions which are called
        after each request.

            app.add_post_process(after_each_request)
        """
        self.__post.append(fn)
    # enddef

    def default(self, method=METHOD_HEAD | METHOD_GET):
        """Set default handler.

        This is decorator for default handler for http method (called before
        error_not_found).

            @app.default(METHOD_GET_POST)
            def default_get_post(req):
                # this function will be called if no uri match in internal
                # uri table with method. It's similar like not_found error,
                # but without error
                ...
        """
        def wrapper(fn):
            self.set_default(fn, method)
        return wrapper
    # enddef

    def set_default(self, fn, method=METHOD_HEAD | METHOD_GET):
        """Set default handler.

        Set fn default handler for http method called befor error_not_found.

            app.set_default(default_get_post, METHOD_GET_POST)
        """
        for m in methods.values():
            if method & m:
                self.__dhandlers[m] = fn
    # enddef

    def pop_default(self, method):
        """Pop default handler for method."""
        return self.__dhandlers(method)

    def route(self, uri, method=METHOD_HEAD | METHOD_GET):
        """Wrap function to be handler for uri and specified method.

        You can define uri as static path or as groups which are hand
        to handler as next parameters.

            # static uri
            @app.route('/user/post', method=METHOD_POST)
            def user_create(req):
                ...

            # group regular expression
            @app.route('/user/<name>')
            def user_detail(req, name):
                ...

            # group regular expression with filter
            @app.route('/<surname:word>/<age:int>')
            def surnames_by_age(req, surname, age):
                ...

            # group with own regular expression filter
            @app.route('/<car:re:\w+>/<color:re:#[\da-fA-F]+>')
            def car(req, car, color):
                ...

        If you can use some name of group which is python keyword, like class,
        you can use **kwargs syntax:

            @app.route('/<class>/<len:int>')
            def classes(req, **kwargs):
                return "'%s' class is %d lenght." % \
                    (kwargs['class'], kwargs['len'])

        Be sure with ordering of call this decorator or set_route function with
        groups regular expression. Regular expression routes are check with the
        same ordering, as you create internal table of them. First match stops
        any other searching. In fact, if groups are detect, they will be
        transfer to normal regular expression, and will be add to second
        internal table.
        """
        def wrapper(fn):
            self.set_route(uri, fn, method)
            return fn
        return wrapper
    # enddef

    def set_route(self, uri, fn, method=METHOD_HEAD | METHOD_GET):
        """Set handler for uri and method.

        Another way to add fn as handler for uri. See Application.route
        documentation for details.

            app.set_route('/use/post', user_create, METHOD_POST)
        """
        uri = uni(uri)

        if re_filter.search(uri):
            r_uri = re_filter.sub(self.__regex, uri) + '$'
            convertors = tuple((g[0], self.__convertor(g[1]))
                               for g in (m.groups()
                               for m in re_filter.finditer(uri)))
            self.set_rroute(r_uri, fn, method, convertors)
        else:
            if uri not in self.__handlers:
                self.__handlers[uri] = {}
            for m in methods.values():
                if method & m:
                    self.__handlers[uri][m] = fn
    # enddef

    def pop_route(self, uri, method):
        """Pop handler for uri and method from handers table.

        Method must be define unique, so METHOD_GET_POST could not be use.
        If you want to remove handler for both methods, you must call pop route
        for each method state.
        """
        uri = uni(uri)

        if re_filter.search(uri):
            r_uri = re_filter.sub(self.__regex, uri) + '$'
            return self.pop_rroute(r_uri, method)
        else:
            handlers = self.__handlers.get(uri, {})
            rv = handlers.pop(method)
            if not handlers:    # is empty
                self.__handlers.pop(uri, None)
            return rv

    def is_route(self, uri):
        """Check if uri have any registered record."""
        uri = uni(uri)
        if re_filter.search(uri):
            r_uri = re_filter.sub(self.__regex, uri) + '$'
            return self.is_rroute(r_uri)
        return uri in self.__handlers

    def rroute(self, ruri, method=METHOD_HEAD | METHOD_GET):
        """Wrap function to be handler for uri defined by regular expression.

        Both of function, rroute and set_rroute store routes to special
        internal table, which is another to table of static routes.

            @app.rroute(r'/user/\w+')               # simple regular expression
            def any_user(req):
                ...

            @app.rroute(r'/user/(?P<user>\w+)')     # regular expression with
            def user_detail(req, user):             # groups
                ...

        Be sure with ordering of call this decorator or set_rroute function.
        Regular expression routes are check with the same ordering, as you
        create internal table of them. First match stops any other searching.
        """
        def wrapper(fn):
            self.set_rroute(ruri, fn, method)
            return fn
        return wrapper
    # enddef

    def set_rroute(self, r_uri, fn, method=METHOD_HEAD | METHOD_GET,
                   convertors=()):
        """Set hanlder for uri defined by regular expression.

        Another way to add fn as handler for uri defined by regular expression.
        See Application.rroute documentation for details.

            app.set_rroute('/use/\w+/post', user_create, METHOD_POST)

        This method is internally use, when groups are found in static route,
        adding by route or set_route method.
        """
        r_uri = re.compile(r_uri, re.U)
        if r_uri not in self.__rhandlers:
            self.__rhandlers[r_uri] = {}
        for m in methods.values():
            if method & m:
                self.__rhandlers[r_uri][m] = (fn, convertors)
    # enddef

    def pop_rroute(self, r_uri, method):
        """Pop handler and convertors for uri and method from handlers table.

        For mor details see Application.pop_route.
        """
        r_uri = re.compile(r_uri, re.U)
        handlers = self.__rhandlers.get(r_uri, {})
        rv = handlers.pop(method)
        if not handlers:    # is empty
            self.__rhandlers.pop(r_uri, None)
        return rv

    def is_rroute(self, r_uri):
        """Check if regular expression uri have any registered record."""
        r_uri = re.compile(r_uri, re.U)
        return r_uri in self.__rhandlers

    def http_state(self, code, method=METHOD_HEAD | METHOD_GET | METHOD_POST):
        """Wrap function to handle http status codes like http errors."""
        def wrapper(fn):
            self.set_http_state(code, fn, method)
        return wrapper
    # enddef

    def set_http_state(self, code, fn,
                       method=METHOD_HEAD | METHOD_GET | METHOD_POST):
        """Set fn as handler for http state code and method."""
        if code not in self.__shandlers:
            self.__shandlers[code] = {}
        for m in methods.values():
            if method & m:
                self.__shandlers[code][m] = fn
    # enddef

    def pop_http_state(self, code, method):
        """Pop handerl for http state and method.

        As Application.pop_route, for pop multimethod handler, you must call
        pop_http_state for each method.
        """
        handlers = self.__shandlers(code, {})
        return handlers.pop(method)

    def error_from_table(self, req, code):
        """Internal method, which is called if error was accured.

        If status code is in Application.shandlers (fill with http_state
        function), call this handler.
        """
        if code in self.__shandlers \
                and req.method_number in self.__shandlers[code]:
            try:
                handler = self.__shandlers[code][req.method_number]
                if 'uri_handler' not in req.__dict__:
                    req.uri_rule = '_%d_error_handler_' % code
                    req.uri_handler = handler
                self.handler_from_pre(req)       # call pre handlers now
                handler(req)
            except:
                internal_server_error(req)
        elif code in default_shandlers:
            handler = default_shandlers[code][METHOD_GET]
            handler(req)
        else:
            not_implemented(req, code)
    # enddef

    def handler_from_default(self, req):
        """Internal method, which is called if no handler is found."""
        if req.method_number in self.__dhandlers:
            req.uri_rule = '_default_handler_'
            req.uri_handler = self.__dhandlers[req.method_number]
            self.handler_from_pre(req)       # call pre handlers now
            retval = self.__dhandlers[req.method_number](req)
            if retval != DECLINED:
                raise SERVER_RETURN(retval)
    # enddef

    def handler_from_pre(self, req):
        """Internal method, which run all pre (pre_proccess) handlers.

        This method was call before end-point route handler.
        """
        for fn in self.__pre:
            fn(req)

    def handler_from_table(self, req):
        """Call right handler from handlers table (fill with route function).

        If no handler is fined, try to find directory or file if Document Root,
        resp. Document Index is set. Then try to call default handler for right
        method or call handler for status code 404 - not found.
        """

        # static routes
        if req.uri in self.__handlers:
            if req.method_number in self.__handlers[req.uri]:
                handler = self.__handlers[req.uri][req.method_number]
                req.uri_rule = req.uri      # nice variable for pre handlers
                req.uri_handler = handler
                self.handler_from_pre(req)  # call pre handlers now
                retval = handler(req)       # call right handler now
                # return text is allowed
                if isinstance(retval, str) \
                        or (_unicode_exist and isinstance(retval, unicode)):
                    req.write(retval, 1)    # write data and flush
                    retval = DONE
                if retval != DECLINED:
                    raise SERVER_RETURN(retval or DONE)  # could be state.DONE
            else:
                raise SERVER_RETURN(HTTP_METHOD_NOT_ALLOWED)
            # endif
        # endif

        # regular expression
        for ruri in self.__rhandlers.keys():
            match = ruri.match(req.uri)
            if match and req.method_number in self.__rhandlers[ruri]:
                handler, convertors = self.__rhandlers[ruri][req.method_number]
                req.uri_rule = ruri.pattern  # nice variable for pre handlers
                req.uri_handler = handler
                self.handler_from_pre(req)   # call pre handlers now
                if len(convertors):
                    # create OrderedDict from match insead of dict for
                    # convertors applying
                    req.groups = OrderedDict(
                        (g, c(v))for ((g, c), v) in zip(convertors,
                                                        match.groups()))
                    retval = handler(req, *req.groups.values())
                else:
                    req.groups = match.groupdict()
                    retval = handler(req, *match.groups())
                # return text is allowed
                if isinstance(retval, str) \
                        or (_unicode_exist and isinstance(retval, unicode)):
                    req.write(retval, 1)    # write data and flush
                    retval = DONE
                if retval != DECLINED:
                    raise SERVER_RETURN(retval or DONE)  # could be state.DONE
            # endif - no METHOD_NOT_ALLOWED here
        # endfor

        # try file or index
        if req.document_root():
            rfile = "%s%s" % (uni(req.document_root()),
                              path.normpath("%s" % uni(req.uri)))

            if not path.exists(rfile):
                if req.debug and req.uri == '/debug-info':      # work if debug
                    req.uri_rule = '_debug_info_'
                    req.uri_handler = debug_info
                    self.handler_from_pre(req)  # call pre handlers now
                    raise SERVER_RETURN(debug_info(req, self))
                self.handler_from_default(req)                  # try default
                raise SERVER_RETURN(HTTP_NOT_FOUND)             # not found

            # return file
            if path.isfile(rfile) and access(rfile, R_OK):
                req.uri_rule = '_send_file_'
                req.uri_handler = send_file
                self.handler_from_pre(req)      # call pre handlers now
                req.log_error("Return file: %s" % req.uri, LOG_INFO)
                raise SERVER_RETURN(send_file(req, rfile))

            # return directory index
            if req.document_index and path.isdir(rfile) \
                    and access(rfile, R_OK):
                req.log_error("Return directory: %s" % req.uri, LOG_INFO)
                req.uri_rule = '_directory_index_'
                req.uri_handler = directory_index
                self.handler_from_pre(req)      # call pre handlers now
                raise SERVER_RETURN(directory_index(req, rfile))

            raise SERVER_RETURN(HTTP_FORBIDDEN)
        # endif

        if req.debug and req.uri == '/debug-info':
            req.uri_rule = '_debug_info_'
            req.uri_handler = debug_info
            self.handler_from_pre(req)          # call pre handlers now
            raise SERVER_RETURN(debug_info(req, self))

        self.handler_from_default(req)

        req.log_error("404 Not Found: %s" % req.uri, LOG_ERR)
        raise SERVER_RETURN(HTTP_NOT_FOUND)
    # enddef

    def __request__(self, environ, start_response):
        """Create Request instance and return wsgi response.

        This method create Request object, call handlers from
        Application.__pre (Application.handler_from_pre),
        uri handler (handler_from_table), default handler
        (Application.handler_from_default) or error handler
        (Application.error_from_table), and handlers from
        Application.__post.
        """
        req = Request(environ, start_response, self.__config)

        try:
            self.handler_from_table(req)
        except SERVER_RETURN as e:
            code = e.args[0]
            if code in (OK, HTTP_OK, DONE):
                pass
            # XXX: elif code in (HTTP_MOVED_PERMANENTLY,
            #                    HTTP_MOVED_TEMPORARILY):
            else:
                req.status = code
                self.error_from_table(req, code)
        except (BrokenClientConnection, SystemExit) as e:
            req.log_error(str(e), LOG_ERR)
            req.log_error('   ***   You shoud ignore next error   ***',
                          LOG_ERR)
            return ()
        except:
            self.error_from_table(req, 500)
        # endtry

        try:    # call post_process handler
            for fn in self.__post:
                fn(req)
        except:
            self.error_from_table(req, 500)
        # endtry

        return req.__end_of_request__()    # private call of request
    # enddef

    def __call__(self, environ, start_response):
        """Callable define for Application instance.

        This method run __request__ method.
        """
        if self.__name == '__poorwsgi__':
            stderr.write("[W] Using deprecated instance of Application.\n")
            stderr.write("    Please, create your own instance\n")
            stderr.flush()
        return self.__request__(environ, start_response)

    def __profile_request__(self, environ, start_response):
        """Profiler version of __request__.

        This method is used if set_profile is used."""
        def wrapper(rv):
            rv.append(self.__original_request__(environ, start_response))

        rv = []
        uri_dump = (self._dump + environ.get('PATH_INFO').replace('/', '_')
                    + '.profile')
        self.log_error('Generate %s' % uri_dump, LOG_INFO)
        self._runctx('wrapper(rv)', globals(), locals(), filename=uri_dump)
        return rv[0]
    # enddef

    def __repr__(self):
        return '%s - callable Application class instance' % self.__name

    def set_profile(self, runctx, dump):
        """Set profiler for __call__ function.

        Arguments:
            runctx - function from profiler module
            dump - path and prefix for .profile files

        Typical usage:

            import cProfile

            cProfile.runctx('from simple import *', globals(), locals(),
                            filename="log/init.profile")
            app.set_profile(cProfile.runctx, 'log/req')
        """
        self._runctx = runctx
        self._dump = dump

        self.__original_request__ = self.__request__
        self.__request__ = self.__profile_request__
    # enddef

    def del_profile(self):
        """Remove profiler from application."""
        self.__request__ = self.__original_request__

    def get_options(self):
        """Returns dictionary with application variables from system environment.

        Application variables start with {app_} prefix,
        but in returned dictionary is set without this prefix.

            #!ini
            poor_LogLevel = warn        # Poor WSGI variable
            app_db_server = localhost   # application variable db_server
            app_templates = app/templ   # application variable templates

        This method works like Request.get_options, but work with
        os.environ, so it works only with wsgi servers, which set not only
        request environ, but os.environ too. Apaches mod_wsgi don't do that,
        uWsgi and PoorHTTP do that.
        """
        options = {}
        for key, val in environ.items():
            key = key.strip()
            if key[:4].lower() == 'app_':
                options[key[4:].lower()] = val.strip()
        return options
    # enddef

    def log_error(self, message, level=LOG_ERR):
        """Logging method with the same functionality like in Request object.

        But as get_options read configuration from os.environ which could
        not work in same wsgi servers like Apaches mod_wsgi.

        This method write to stderr so messages, could not be found in
        servers error log!
        """
        if self.__log_level[0] >= level[0]:
            if _unicode_exist and isinstance(message, unicode):
                message = message.encode('utf-8')
            try:
                stderr.write("<%s> [%s] %s\n" % (level[1], self.__name,
                                                 message))
            except UnicodeEncodeError:
                if _unicode_exist:
                    message = message.decode('utf-8').encode(
                        'ascii', 'backslashreplace')
                else:
                    message = message.encode(
                        'ascii', 'backslashreplace').decode('ascii')

                stderr.write("<%s> [%s] %s\n" % (level[1], self.__name,
                                                 message))
            stderr.flush()
    # enddef

    def log_info(self, message):
        """Logging method, which create message as LOG_INFO level."""
        self.log_error(message, LOG_INFO)

    def log_debug(self, message):
        """Logging method, which create message as LOG_DEBUG level."""
        self.log_error(message, LOG_DEBUG)

    def log_warning(self, message):
        """Logging method, which create message as LOG_WARNING level."""
        self.log_error(message, LOG_WARNING)
class SymbolLRUCacheManager(RequiredConfig):
    """for cleaning up the symbols cache"""
    required_config = Namespace()
    required_config.add_option(
        'symbol_cache_path',
        doc="the cache directory to automatically remove files from",
        default=os.path.join(tempfile.gettempdir(), 'symbols')
    )
    required_config.add_option(
        'symbol_cache_size',
        doc="the maximum size of the symbols cache",
        default='1G',
        from_string_converter=from_string_to_parse_size
    )
    required_config.add_option(
        'verbosity',
        doc="how chatty should this be? 1 - writes to stdout,"
            " 2 - uses the logger",
        default=0,
        from_string_converter=int
    )

    #--------------------------------------------------------------------------
    def __init__(self, config, quit_check_callback=None):
        """constructor for a registration object that runs an LRU cache
       cleaner"""
        self.config = config

        self.directory = os.path.abspath(config.symbol_cache_path)
        self.max_size = config.symbol_cache_size
        self.verbosity = config.verbosity
        # Cache state
        self.total_size = 0
        self._lru = OrderedDict()
        # pyinotify bits
        self._wm = pyinotify.WatchManager()
        self._handler = EventHandler(self, verbosity=config.verbosity)
        self._notifier = pyinotify.ThreadedNotifier(self._wm, self._handler)
        mask = pyinotify.IN_DELETE | pyinotify.IN_CREATE \
            | pyinotify.IN_OPEN | pyinotify.IN_MOVED_FROM \
            | pyinotify.IN_MOVED_TO | pyinotify.IN_MODIFY
        self._wdd = self._wm.add_watch(
            self.directory,
            mask,
            rec=True,
            auto_add=True
        )
        # Load existing files into the cache.
        self._get_existing_files(self.directory)
        self._notifier.start()

    #--------------------------------------------------------------------------
    @property
    def num_files(self):
        return len(self._lru)

    #--------------------------------------------------------------------------
    def _rm_empty_dirs(self, path):
        '''
       Attempt to remove any empty directories that are parents of path
       and children of self.directory.
       '''
        path = os.path.dirname(path)
        while not os.path.samefile(path, self.directory):
            if not os.listdir(path):
                os.rmdir(path)
            path = os.path.dirname(path)

    #--------------------------------------------------------------------------
    def _update_cache(self, path, update_size=False):
        if path in self._lru:
            size = self._lru.pop(path)
            if update_size:
                self.total_size -= size
        else:
            update_size = True

        if update_size:
            try:
                size = os.stat(path).st_size
            except OSError:
                self.config.logger.warning(
                    'file was not found while cleaning cache: %s', path
                )
                return

            self.total_size += size
            # If we're out of space, remove items from the cache until
            # we fit again.
            while self.total_size > self.max_size and self._lru:
                rm_path, rm_size = self._lru.popitem(last=False)
                self.total_size -= rm_size
                os.unlink(rm_path)
                self._rm_empty_dirs(rm_path)
                if self.verbosity >= 2:
                    self.config.logger.debug('RM %s', rm_path)
        self._lru[path] = size

    #--------------------------------------------------------------------------
    def _remove_cached(self, path):
        # We might have already removed this file in _update_cache.
        if path in self._lru:
            size = self._lru.pop(path)
            self.total_size -= size

    #--------------------------------------------------------------------------
    def _get_existing_files(self, path):
        for base, dirs, files in os.walk(path):
            for f in files:
                f = os.path.join(base, f)
                self._update_cache(f)

    #--------------------------------------------------------------------------
    def close(self):
        self._notifier.stop()
Exemple #17
0
class Application(object):
    """Poor WSGI application which is called by WSGI server.

    Working of is describe in PEP 0333. This object store route dispatch table,
    and have methods for it's using and of course __call__ method for use
    as WSGI application.
    """

    __instances = []

    def __init__(self, name="__main__"):
        """Application class is per name singleton.

        That means, there could be exist only one instance with same name.
        """

        if Application.__instances.count(name):
            raise RuntimeError('Application with name %s exist yet.' % name)
        Application.__instances.append(name)

        # Application name
        self.__name = name

        # list of pre and post process handlers
        self.__pre = []
        self.__post = []

        # dhandlers table for default handers on methods {METHOD_GET: handler}
        self.__dhandlers = {}

        # handlers table of simple paths: {'/path': {METHOD_GET: handler}}
        self.__handlers = {}

        self.__filters = {
            ':int': (r'-?\d+', int),
            ':float': (r'-?\d+(\.\d+)?', float),
            ':word': (r'\w+', uni),
            ':hex': (r'[0-9a-fA-F]+', str),
            ':re:': (None, uni),
            'none': (r'[^/]+', uni)
        }

        # handlers of regex paths: {r'/user/([a-z]?)': {METHOD_GET: handler}}
        self.__rhandlers = OrderedDict()

        # http state handlers: {HTTP_NOT_FOUND: {METHOD_GET: my_404_handler}}
        self.__shandlers = {}

        # -- Application variable
        self.__config = {
            'auto_args': True,
            'auto_form': True,
            'auto_json': True,
            'keep_blank_values': 0,
            'strict_parsing': 0,
            'json_content_types': [
                'application/json',
                'application/javascript',
                'application/merge-patch+json'],
            'form_content_types': [
                'application/x-www-form-urlencoded',
                'multipart/form-data'
            ],
            'auto_cookies': True,
            'debug': 'Off',
            'document_root': '',
            'document_index': 'Off',
            'secret_key': '%s%s%s%s' %
                          (__version__, version, getcwd(),
                           ''.join(str(x) for x in uname()))
        }

        try:
            self.__log_level = levels[environ.get('poor_LogLevel',
                                                  'warn').lower()]
        except:
            self.__log_level = LOG_WARNING
            self.log_error('Bad poor_LogLevel, default is warn.', LOG_WARNING)
        # endtry
    # enddef

    def __regex(self, match):
        groups = match.groups()
        _filter = str(groups[1]).lower()

        if _filter in self.__filters:
            regex = self.__filters[_filter][0]
        elif _filter[:4] == ':re:':     # :re: filter have user defined regex
            regex = _filter[4:]
        else:
            try:
                regex = self.__filters[_filter][0]
            except KeyError:
                raise RuntimeError("Undefined route group filter '%s'" %
                                   _filter)

        return "(?P<%s>%s)" % (groups[0], regex)
    # enddef

    def __convertor(self, _filter):
        _filter = str(_filter).lower()
        _filter = ':re:' if _filter[:4] == ':re:' else _filter
        try:
            return self.__filters[_filter][1]
        except KeyError:
            raise RuntimeError("Undefined route group filter '%s'" % _filter)

    @property
    def name(self):
        """Return application name."""
        return self.__name

    @property
    def filters(self):
        """Copy of filter table.

        Filter table contains regular expressions and convert functions,
        see Application.set_filter and Application.route.

        Default filters are:
            :int - match number and convert it to int
            :float - match number and convert it to float
            :word - match one unicoee word
            :hex - match hexadecimal value and convert it to str
            :re: - match user defined regular expression
            none - match any string withount '/' character

        For more details see {/debug-info} page of your application, where
        you see all filters with regular expression definition.
        """
        return self.__filters.copy()

    @property
    def pre(self):
        """Tuple of table with pre-process handlers.

        See Application.pre_process.
        """
        return tuple(self.__pre)

    @property
    def post(self):
        """Tuple of table with post-process handlers.

        See Application.post_process.
        """
        return tuple(self.__post)

    @property
    def dhandlers(self):
        """Copy of table with default handlers.

        See Application.set_default
        """
        return self.__dhandlers.copy()

    @property
    def handlers(self):
        """Copy of table with static handlers.

        See Application.route.
        """
        return self.__handlers.copy()

    @property
    def rhandlers(self):
        """Copy of table with regular expression handlers.

        See Application.route and Application.rroute.
        """
        return self.__rhandlers.copy()

    @property
    def shandlers(self):
        """Copy of table with http state aka error handlers.

        See Application.http_state
        """
        return self.__shandlers.copy()

    @property
    def auto_args(self):
        """Automatic parsing request arguments from uri.

        If it is True (default), Request object do automatic parsing request
        uri to its args variable.
        """
        return self.__config['auto_args']

    @auto_args.setter
    def auto_args(self, value):
        self.__config['auto_args'] = bool(value)

    @property
    def auto_form(self):
        """Automatic parsing arguments from request body.

        If it is True (default) and method is POST, PUT or PATCH, and
        request content type is one of form_content_types, Request
        object do automatic parsing request body to its form variable.
        """
        return self.__config['auto_form']

    @auto_form.setter
    def auto_form(self, value):
        self.__config['auto_form'] = bool(value)

    @property
    def auto_json(self):
        """Automatic parsing JSON from request body.

        If it is True (default), method is POST, PUT or PATCH and request
        content type is one of json_content_types, Request object do
        automatic parsing request body to json variable.
        """
        return self.__config['auto_json']

    @auto_json.setter
    def auto_json(self, value):
        self.__config['auto_json'] = bool(value)

    @property
    def auto_cookies(self):
        """Automatic parsing cookies from request headers.

        If it is True (default) and Cookie request header was set,
        SimpleCookie object was paresed to Request property cookies.
        """
        return self.__config['auto_cookies']

    @auto_cookies.setter
    def auto_cookies(self, value):
        self.__config['auto_cookies'] = bool(value)

    @property
    def debug(self):
        """Application debug as another way how to set poor_Debug.

        This setting will be rewrite by poor_Debug environment variable.
        """
        return self.__config['debug'] == 'On'

    @debug.setter
    def debug(self, value):
        self.__config['debug'] = 'On' if bool(value) else 'Off'

    @property
    def document_root(self):
        """Application document_root as another way how to set poor_DocumentRoot.

        This setting will be rewrite by poor_DocumentRoot environ variable.
        """
        return self.__config['document_root']

    @document_root.setter
    def document_root(self, value):
        self.__config['document_root'] = value

    @property
    def document_index(self):
        """Application document_root as another way how to set poor_DocumentRoot.

        This setting will be rewrite by poor_DocumentRoot environ variable.
        """
        return self.__config['document_index'] == 'On'

    @document_index.setter
    def document_index(self, value):
        self.__config['document_index'] = 'On' if bool(value) else 'Off'

    @property
    def secret_key(self):
        """Application secret_key could be replace by poor_SecretKey in request.

        Secret key is used by PoorSession class. It is generate from
        some server variables, and the best way is set to your own long
        key."""
        return self.__config['secret_key']

    @secret_key.setter
    def secret_key(self, value):
        self.__config['secret_key'] = value

    @property
    def keep_blank_values(self):
        """Keep blank values in request arguments.

        If it is 1 (0 is default), automatic parsing request uri or body
        keep blank values as empty string.
        """
        return self.__config['keep_blank_values']

    @keep_blank_values.setter
    def keep_blank_values(self, value):
        self.__config['keep_blank_values'] = int(value)

    @property
    def strict_parsing(self):
        """Strict parse request arguments.

        If it is 1 (0 is default), automatic parsing request uri or body
        raise with exception on parsing error.
        """
        return self.__config['strict_parsing']

    @strict_parsing.setter
    def strict_parsing(self, value):
        self.__config['strict_parsing'] = int(value)

    @property
    def json_content_types(self):
        """Copy of json content type list.

        Containt list of strings as json content types, which is use for
        testing, when automatics Json object is create from request body.
        """
        return self.__config['json_content_types']

    @property
    def form_content_types(self):
        """Copy of form content type list.

        Containt list of strings as form content types, which is use for
        testing, when automatics Form object is create from request body.
        """
        return self.__config['form_content_types']

    def set_filter(self, name, regex, convertor=uni):
        """Create new filter or overwrite builtins.

        Arguments:
            name      - Name of filter which is used in route or set_route
                        method.
            regex     - regular expression which used for filter
            convertor - convertor function or class, which gets unicode in
                        input. Default is uni function, which is wrapper
                        to unicode string.

            app.set_filter('uint', r'\d+', int)
        """
        name = ':'+name if name[0] != ':' else name
        self.__filters[name] = (regex, convertor)

    def pre_process(self):
        """Append pre process hendler.

        This is decorator for function to call before each request.

            @app.pre_process()
            def before_each_request(req):
                ...
        """
        def wrapper(fn):
            self.__pre.append(fn)
            return fn
        return wrapper
    # enddef

    def add_pre_process(self, fn):
        """Append pre proccess handler.

        Method adds function to list functions which is call before each
        request.

            app.add_pre_process(before_each_request)
        """
        self.__pre.append(fn)
    # enddef

    def post_process(self):
        """Append post process handler.

        This decorator append function to be called after each request,
        if you want to use it redefined all outputs.

            @app.pre_process()
            def after_each_request(req):
                ...
        """
        def wrapper(fn):
            self.__post.append(fn)
            return fn
        return wrapper
    # enddef

    def add_post_process(self, fn):
        """Append post process handler.

        Method for direct append function to list functions which are called
        after each request.

            app.add_post_process(after_each_request)
        """
        self.__post.append(fn)
    # enddef

    def default(self, method=METHOD_HEAD | METHOD_GET):
        """Set default handler.

        This is decorator for default handler for http method (called before
        error_not_found).

            @app.default(METHOD_GET_POST)
            def default_get_post(req):
                # this function will be called if no uri match in internal
                # uri table with method. It's similar like not_found error,
                # but without error
                ...
        """
        def wrapper(fn):
            self.set_default(fn, method)
        return wrapper
    # enddef

    def set_default(self, fn, method=METHOD_HEAD | METHOD_GET):
        """Set default handler.

        Set fn default handler for http method called befor error_not_found.

            app.set_default(default_get_post, METHOD_GET_POST)
        """
        for m in methods.values():
            if method & m:
                self.__dhandlers[m] = fn
    # enddef

    def pop_default(self, method):
        """Pop default handler for method."""
        return self.__dhandlers(method)

    def route(self, uri, method=METHOD_HEAD | METHOD_GET):
        """Wrap function to be handler for uri and specified method.

        You can define uri as static path or as groups which are hand
        to handler as next parameters.

            # static uri
            @app.route('/user/post', method=METHOD_POST)
            def user_create(req):
                ...

            # group regular expression
            @app.route('/user/<name>')
            def user_detail(req, name):
                ...

            # group regular expression with filter
            @app.route('/<surname:word>/<age:int>')
            def surnames_by_age(req, surname, age):
                ...

            # group with own regular expression filter
            @app.route('/<car:re:\w+>/<color:re:#[\da-fA-F]+>')
            def car(req, car, color):
                ...

        If you can use some name of group which is python keyword, like class,
        you can use **kwargs syntax:

            @app.route('/<class>/<len:int>')
            def classes(req, **kwargs):
                return "'%s' class is %d lenght." % \
                    (kwargs['class'], kwargs['len'])

        Be sure with ordering of call this decorator or set_route function with
        groups regular expression. Regular expression routes are check with the
        same ordering, as you create internal table of them. First match stops
        any other searching. In fact, if groups are detect, they will be
        transfer to normal regular expression, and will be add to second
        internal table.
        """
        def wrapper(fn):
            self.set_route(uri, fn, method)
            return fn
        return wrapper
    # enddef

    def set_route(self, uri, fn, method=METHOD_HEAD | METHOD_GET):
        """Set handler for uri and method.

        Another way to add fn as handler for uri. See Application.route
        documentation for details.

            app.set_route('/use/post', user_create, METHOD_POST)
        """
        uri = uni(uri)

        if re_filter.search(uri):
            r_uri = re_filter.sub(self.__regex, uri) + '$'
            convertors = tuple((g[0], self.__convertor(g[1]))
                               for g in (m.groups()
                               for m in re_filter.finditer(uri)))
            self.set_rroute(r_uri, fn, method, convertors)
        else:
            if uri not in self.__handlers:
                self.__handlers[uri] = {}
            for m in methods.values():
                if method & m:
                    self.__handlers[uri][m] = fn
    # enddef

    def pop_route(self, uri, method):
        """Pop handler for uri and method from handers table.

        Method must be define unique, so METHOD_GET_POST could not be use.
        If you want to remove handler for both methods, you must call pop route
        for each method state.
        """
        uri = uni(uri)

        if re_filter.search(uri):
            r_uri = re_filter.sub(self.__regex, uri) + '$'
            return self.pop_rroute(r_uri, method)
        else:
            handlers = self.__handlers.get(uri, {})
            rv = handlers.pop(method)
            if not handlers:    # is empty
                self.__handlers.pop(uri, None)
            return rv

    def is_route(self, uri):
        """Check if uri have any registered record."""
        uri = uni(uri)
        if re_filter.search(uri):
            r_uri = re_filter.sub(self.__regex, uri) + '$'
            return self.is_rroute(r_uri)
        return uri in self.__handlers

    def rroute(self, ruri, method=METHOD_HEAD | METHOD_GET):
        """Wrap function to be handler for uri defined by regular expression.

        Both of function, rroute and set_rroute store routes to special
        internal table, which is another to table of static routes.

            @app.rroute(r'/user/\w+')               # simple regular expression
            def any_user(req):
                ...

            @app.rroute(r'/user/(?P<user>\w+)')     # regular expression with
            def user_detail(req, user):             # groups
                ...

        Be sure with ordering of call this decorator or set_rroute function.
        Regular expression routes are check with the same ordering, as you
        create internal table of them. First match stops any other searching.
        """
        def wrapper(fn):
            self.set_rroute(ruri, fn, method)
            return fn
        return wrapper
    # enddef

    def set_rroute(self, r_uri, fn, method=METHOD_HEAD | METHOD_GET,
                   convertors=()):
        """Set hanlder for uri defined by regular expression.

        Another way to add fn as handler for uri defined by regular expression.
        See Application.rroute documentation for details.

            app.set_rroute('/use/\w+/post', user_create, METHOD_POST)

        This method is internally use, when groups are found in static route,
        adding by route or set_route method.
        """
        r_uri = re.compile(r_uri, re.U)
        if r_uri not in self.__rhandlers:
            self.__rhandlers[r_uri] = {}
        for m in methods.values():
            if method & m:
                self.__rhandlers[r_uri][m] = (fn, convertors)
    # enddef

    def pop_rroute(self, r_uri, method):
        """Pop handler and convertors for uri and method from handlers table.

        For mor details see Application.pop_route.
        """
        r_uri = re.compile(r_uri, re.U)
        handlers = self.__rhandlers.get(r_uri, {})
        rv = handlers.pop(method)
        if not handlers:    # is empty
            self.__rhandlers.pop(r_uri, None)
        return rv

    def is_rroute(self, r_uri):
        """Check if regular expression uri have any registered record."""
        r_uri = re.compile(r_uri, re.U)
        return r_uri in self.__rhandlers

    def http_state(self, code, method=METHOD_HEAD | METHOD_GET | METHOD_POST):
        """Wrap function to handle http status codes like http errors."""
        def wrapper(fn):
            self.set_http_state(code, fn, method)
        return wrapper
    # enddef

    def set_http_state(self, code, fn,
                       method=METHOD_HEAD | METHOD_GET | METHOD_POST):
        """Set fn as handler for http state code and method."""
        if code not in self.__shandlers:
            self.__shandlers[code] = {}
        for m in methods.values():
            if method & m:
                self.__shandlers[code][m] = fn
    # enddef

    def pop_http_state(self, code, method):
        """Pop handerl for http state and method.

        As Application.pop_route, for pop multimethod handler, you must call
        pop_http_state for each method.
        """
        handlers = self.__shandlers(code, {})
        return handlers.pop(method)

    def error_from_table(self, req, code):
        """Internal method, which is called if error was accured.

        If status code is in Application.shandlers (fill with http_state
        function), call this handler.
        """
        if code in self.__shandlers \
                and req.method_number in self.__shandlers[code]:
            try:
                handler = self.__shandlers[code][req.method_number]
                if 'uri_handler' not in req.__dict__:
                    req.uri_rule = '_%d_error_handler_' % code
                    req.uri_handler = handler
                self.handler_from_pre(req)       # call pre handlers now
                handler(req)
            except:
                internal_server_error(req)
        elif code in default_shandlers:
            handler = default_shandlers[code][METHOD_GET]
            handler(req)
        else:
            not_implemented(req, code)
    # enddef

    def handler_from_default(self, req):
        """Internal method, which is called if no handler is found."""
        if req.method_number in self.__dhandlers:
            req.uri_rule = '_default_handler_'
            req.uri_handler = self.__dhandlers[req.method_number]
            self.handler_from_pre(req)       # call pre handlers now
            retval = self.__dhandlers[req.method_number](req)
            if retval != DECLINED:
                raise SERVER_RETURN(retval)
    # enddef

    def handler_from_pre(self, req):
        """Internal method, which run all pre (pre_proccess) handlers.

        This method was call before end-point route handler.
        """
        for fn in self.__pre:
            fn(req)

    def handler_from_table(self, req):
        """Call right handler from handlers table (fill with route function).

        If no handler is fined, try to find directory or file if Document Root,
        resp. Document Index is set. Then try to call default handler for right
        method or call handler for status code 404 - not found.
        """

        # static routes
        if req.uri in self.__handlers:
            if req.method_number in self.__handlers[req.uri]:
                handler = self.__handlers[req.uri][req.method_number]
                req.uri_rule = req.uri      # nice variable for pre handlers
                req.uri_handler = handler
                self.handler_from_pre(req)  # call pre handlers now
                retval = handler(req)       # call right handler now
                # return text is allowed
                if isinstance(retval, str) \
                        or (_unicode_exist and isinstance(retval, unicode)):
                    req.write(retval, 1)    # write data and flush
                    retval = DONE
                if retval != DECLINED:
                    raise SERVER_RETURN(retval or DONE)  # could be state.DONE
            else:
                raise SERVER_RETURN(HTTP_METHOD_NOT_ALLOWED)
            # endif
        # endif

        # regular expression
        for ruri in self.__rhandlers.keys():
            match = ruri.match(req.uri)
            if match and req.method_number in self.__rhandlers[ruri]:
                handler, convertors = self.__rhandlers[ruri][req.method_number]
                req.uri_rule = ruri.pattern  # nice variable for pre handlers
                req.uri_handler = handler
                self.handler_from_pre(req)   # call pre handlers now
                if len(convertors):
                    # create OrderedDict from match insead of dict for
                    # convertors applying
                    req.groups = OrderedDict(
                        (g, c(v))for ((g, c), v) in zip(convertors,
                                                        match.groups()))
                    retval = handler(req, *req.groups.values())
                else:
                    req.groups = match.groupdict()
                    retval = handler(req, *match.groups())
                # return text is allowed
                if isinstance(retval, str) \
                        or (_unicode_exist and isinstance(retval, unicode)):
                    req.write(retval, 1)    # write data and flush
                    retval = DONE
                if retval != DECLINED:
                    raise SERVER_RETURN(retval or DONE)  # could be state.DONE
            # endif - no METHOD_NOT_ALLOWED here
        # endfor

        # try file or index
        if req.document_root():
            rfile = "%s%s" % (uni(req.document_root()),
                              path.normpath("%s" % uni(req.uri)))

            if not path.exists(rfile):
                if req.debug and req.uri == '/debug-info':      # work if debug
                    req.uri_rule = '_debug_info_'
                    req.uri_handler = debug_info
                    self.handler_from_pre(req)  # call pre handlers now
                    raise SERVER_RETURN(debug_info(req, self))
                self.handler_from_default(req)                  # try default
                raise SERVER_RETURN(HTTP_NOT_FOUND)             # not found

            # return file
            if path.isfile(rfile) and access(rfile, R_OK):
                req.uri_rule = '_send_file_'
                req.uri_handler = send_file
                self.handler_from_pre(req)      # call pre handlers now
                req.log_error("Return file: %s" % req.uri, LOG_INFO)
                raise SERVER_RETURN(send_file(req, rfile))

            # return directory index
            if req.document_index and path.isdir(rfile) \
                    and access(rfile, R_OK):
                req.log_error("Return directory: %s" % req.uri, LOG_INFO)
                req.uri_rule = '_directory_index_'
                req.uri_handler = directory_index
                self.handler_from_pre(req)      # call pre handlers now
                raise SERVER_RETURN(directory_index(req, rfile))

            raise SERVER_RETURN(HTTP_FORBIDDEN)
        # endif

        if req.debug and req.uri == '/debug-info':
            req.uri_rule = '_debug_info_'
            req.uri_handler = debug_info
            self.handler_from_pre(req)          # call pre handlers now
            raise SERVER_RETURN(debug_info(req, self))

        self.handler_from_default(req)

        req.log_error("404 Not Found: %s" % req.uri, LOG_ERR)
        raise SERVER_RETURN(HTTP_NOT_FOUND)
    # enddef

    def __request__(self, environ, start_response):
        """Create Request instance and return wsgi response.

        This method create Request object, call handlers from
        Application.__pre (Application.handler_from_pre),
        uri handler (handler_from_table), default handler
        (Application.handler_from_default) or error handler
        (Application.error_from_table), and handlers from
        Application.__post.
        """
        req = Request(environ, start_response, self.__config)

        try:
            self.handler_from_table(req)
        except SERVER_RETURN as e:
            code = e.args[0]
            if code in (OK, HTTP_OK, DONE):
                pass
            # XXX: elif code in (HTTP_MOVED_PERMANENTLY,
            #                    HTTP_MOVED_TEMPORARILY):
            else:
                req.status = code
                self.error_from_table(req, code)
        except (BrokenClientConnection, SystemExit) as e:
            req.log_error(str(e), LOG_ERR)
            req.log_error('   ***   You shoud ignore next error   ***',
                          LOG_ERR)
            return ()
        except:
            self.error_from_table(req, 500)
        # endtry

        try:    # call post_process handler
            for fn in self.__post:
                fn(req)
        except:
            self.error_from_table(req, 500)
        # endtry

        return req.__end_of_request__()    # private call of request
    # enddef

    def __call__(self, environ, start_response):
        """Callable define for Application instance.

        This method run __request__ method.
        """
        if self.__name == '__poorwsgi__':
            stderr.write("[W] Using deprecated instance of Application.\n")
            stderr.write("    Please, create your own instance\n")
            stderr.flush()
        return self.__request__(environ, start_response)

    def __profile_request__(self, environ, start_response):
        """Profiler version of __request__.

        This method is used if set_profile is used."""
        def wrapper(rv):
            rv.append(self.__original_request__(environ, start_response))

        rv = []
        uri_dump = (self._dump + environ.get('PATH_INFO').replace('/', '_')
                    + '.profile')
        self.log_error('Generate %s' % uri_dump, LOG_INFO)
        self._runctx('wrapper(rv)', globals(), locals(), filename=uri_dump)
        return rv[0]
    # enddef

    def __repr__(self):
        return '%s - callable Application class instance' % self.__name

    def set_profile(self, runctx, dump):
        """Set profiler for __call__ function.

        Arguments:
            runctx - function from profiler module
            dump - path and prefix for .profile files

        Typical usage:

            import cProfile

            cProfile.runctx('from simple import *', globals(), locals(),
                            filename="log/init.profile")
            app.set_profile(cProfile.runctx, 'log/req')
        """
        self._runctx = runctx
        self._dump = dump

        self.__original_request__ = self.__request__
        self.__request__ = self.__profile_request__
    # enddef

    def del_profile(self):
        """Remove profiler from application."""
        self.__request__ = self.__original_request__

    def get_options(self):
        """Returns dictionary with application variables from system environment.

        Application variables start with {app_} prefix,
        but in returned dictionary is set without this prefix.

            #!ini
            poor_LogLevel = warn        # Poor WSGI variable
            app_db_server = localhost   # application variable db_server
            app_templates = app/templ   # application variable templates

        This method works like Request.get_options, but work with
        os.environ, so it works only with wsgi servers, which set not only
        request environ, but os.environ too. Apaches mod_wsgi don't do that,
        uWsgi and PoorHTTP do that.
        """
        options = {}
        for key, val in environ.items():
            key = key.strip()
            if key[:4].lower() == 'app_':
                options[key[4:].lower()] = val.strip()
        return options
    # enddef

    def log_error(self, message, level=LOG_ERR):
        """Logging method with the same functionality like in Request object.

        But as get_options read configuration from os.environ which could
        not work in same wsgi servers like Apaches mod_wsgi.

        This method write to stderr so messages, could not be found in
        servers error log!
        """
        if self.__log_level[0] >= level[0]:
            if _unicode_exist and isinstance(message, unicode):
                message = message.encode('utf-8')
            try:
                stderr.write("<%s> [%s] %s\n" % (level[1], self.__name,
                                                 message))
            except UnicodeEncodeError:
                if _unicode_exist:
                    message = message.decode('utf-8').encode(
                        'ascii', 'backslashreplace')
                else:
                    message = message.encode(
                        'ascii', 'backslashreplace').decode('ascii')

                stderr.write("<%s> [%s] %s\n" % (level[1], self.__name,
                                                 message))
            stderr.flush()
    # enddef

    def log_info(self, message):
        """Logging method, which create message as LOG_INFO level."""
        self.log_error(message, LOG_INFO)

    def log_debug(self, message):
        """Logging method, which create message as LOG_DEBUG level."""
        self.log_error(message, LOG_DEBUG)

    def log_warning(self, message):
        """Logging method, which create message as LOG_WARNING level."""
        self.log_error(message, LOG_WARNING)
class RenderableContainer(object):
    is_group = True

    def __init__(self):

        self._components = []
        self._componentmap = OrderedDict()
        self._bindmap = OrderedDict()

    def __json__(self, request):
        return {"components": self._componentmap}

    def rmRenderable(self, renderable_id):

        renderable = self._componentmap.pop(renderable_id)
        self._components.remove(renderable)

    def addRenderable(self, renderable, pos=None):

        """ Add renderable. If pos is given, insert into that
        position, otherwise just append"""

        if pos is None:
            self._components.append(renderable)
        else:
            self._components.insert(pos, renderable)

        # todo: see if we can use the pos parameter for the insertion
        # into the _componentmap ordereddict
        self._componentmap[renderable.id] = renderable

        if getattr(renderable, "bind", None):
            self._bindmap[renderable.bind] = renderable

    def getRenderables(self, recursive=False):
        """ retrieve all renderables. If recursive is true, then
        also return all renderables from the children recursively """

        if recursive:
            result = self._componentmap.values()
            for r in self._componentmap.values():
                try:
                    result += r.getRenderables(recursive)
                except:
                    pass
            return result

        else:
            return self._components

    def getRenderable(self, id):
        """ find renderable by id in the complete (recursive) tree """

        found = self._componentmap.get(id, None)
        if not found:
            # search children
            for r in self.getRenderables(False):
                try:
                    found = r.getRenderable(id)
                    if found:
                        break
                except:
                    pass
        return found

    def getRenderableByBind(self, bind):

        found = self._bindmap.get(bind, None)
        if not found:
            # search children
            for r in self.getRenderables(False):
                try:
                    found = r.getRenderableByBind(bind)
                    if found:
                        break
                except:
                    pass
        return found
Exemple #19
0
 def apply(self):
     """Transform each
     :class:`~sphinxcontrib.bibtex.nodes.bibliography` node into a
     list of citations.
     """
     env = self.document.settings.env
     for bibnode in self.document.traverse(bibliography):
         # get the information of this bibliography node
         # by looking up its id in the bibliography cache
         id_ = bibnode['ids'][0]
         infos = [info for other_id, info
                  in env.bibtex_cache.bibliographies.items()
                  if other_id == id_ and info.docname == env.docname]
         if not infos:
             raise RuntimeError(
                 "document %s has no bibliography nodes with id '%s'"
                 % (env.docname, id_))
         elif len(infos) >= 2:
             raise RuntimeError(
                 "document %s has multiple bibliography nodes with id '%s'"
                 % (env.docname, id_))
         info = infos[0]
         # generate entries
         entries = OrderedDict()
         for bibfile in info.bibfiles:
             # XXX entries are modified below in an unpickable way
             # XXX so fetch a deep copy
             data = env.bibtex_cache.bibfiles[bibfile].data
             if info.cite == "all":
                 bibfile_entries = data.entries.values()
             elif info.cite == "cited":
                 bibfile_entries = (
                     entry for entry in data.entries.values()
                     if env.bibtex_cache.is_cited(entry.key))
             elif info.cite == "notcited":
                 bibfile_entries = (
                     entry for entry in data.entries.values()
                     if not env.bibtex_cache.is_cited(entry.key))
             else:
                 raise RuntimeError("invalid cite option (%s)" % info.cite)
             for entry in bibfile_entries:
                 entries[entry.key] = copy.deepcopy(entry)
         # order entries according to which were cited first
         # first, we add all keys that were cited
         # then, we add all remaining keys
         sorted_entries = []
         for key in env.bibtex_cache.get_all_cited_keys():
             try:
                 entry = entries.pop(key)
             except KeyError:
                 pass
             else:
                 sorted_entries.append(entry)
         sorted_entries += entries.values()
         # locate and instantiate style plugin
         style_cls = find_plugin(
             'pybtex.style.formatting', info.style)
         style = style_cls()
         # create citation nodes for all references
         backend = output_backend()
         if info.list_ == "enumerated":
             nodes = docutils.nodes.enumerated_list()
             nodes['enumtype'] = info.enumtype
             if info.start >= 1:
                 nodes['start'] = info.start
                 env.bibtex_cache.set_enum_count(env.docname, info.start)
             else:
                 nodes['start'] = env.bibtex_cache.get_enum_count(env.docname)
         elif info.list_ == "bullet":
             nodes = docutils.nodes.bullet_list()
         else: # "citation"
             nodes = docutils.nodes.paragraph()
         # XXX style.format_entries modifies entries in unpickable way
         for entry in style.format_entries(sorted_entries):
             if info.list_ == "enumerated" or info.list_ == "bullet":
                 citation = docutils.nodes.list_item()
                 citation += entry.text.render(backend)
             else: # "citation"
                 citation = backend.citation(entry, self.document)
                 # backend.citation(...) uses entry.key as citation label
                 # we change it to entry.label later onwards
                 # but we must note the entry.label now;
                 # at this point, we also already prefix the label
                 key = citation[0].astext()
                 info.labels[key] = info.labelprefix + entry.label
             node_text_transform(citation, transform_url_command)
             if info.curly_bracket_strip:
                 node_text_transform(citation, transform_curly_bracket_strip)
             nodes += citation
             if info.list_ == "enumerated":
                 env.bibtex_cache.inc_enum_count(env.docname)
         bibnode.replace_self(nodes)
Exemple #20
0
 def apply(self):
     """Transform each
     :class:`~sphinxcontrib.bibtex.nodes.bibliography` node into a
     list of citations.
     """
     env = self.document.settings.env
     for bibnode in self.document.traverse(bibliography):
         # get the information of this bibliography node
         # by looking up its id in the bibliography cache
         id_ = bibnode['ids'][0]
         infos = [info for other_id, info
                  in env.bibtex_cache.bibliographies.iteritems()
                  if other_id == id_ and info.docname == env.docname]
         if not infos:
             raise RuntimeError(
                 "document %s has no bibliography nodes with id '%s'"
                 % (env.docname, id_))
         elif len(infos) >= 2:
             raise RuntimeError(
                 "document %s has multiple bibliography nodes with id '%s'"
                 % (env.docname, id_))
         info = infos[0]
         # generate entries
         entries = OrderedDict()
         for bibfile in info.bibfiles:
             # XXX entries are modified below in an unpickable way
             # XXX so fetch a deep copy
             data = env.bibtex_cache.bibfiles[bibfile].data
             if info.cite == "all":
                 bibfile_entries = data.entries.itervalues()
             elif info.cite == "cited":
                 bibfile_entries = (
                     entry for entry in data.entries.itervalues()
                     if env.bibtex_cache.is_cited(entry.key))
             elif info.cite == "notcited":
                 bibfile_entries = (
                     entry for entry in data.entries.itervalues()
                     if not env.bibtex_cache.is_cited(entry.key))
             else:
                 raise RuntimeError("invalid cite option (%s)" % info.cite)
             for entry in bibfile_entries:
                 entries[entry.key] = copy.deepcopy(entry)
         # order entries according to which were cited first
         # first, we add all keys that were cited
         # then, we add all remaining keys
         sorted_entries = []
         for key in env.bibtex_cache.get_all_cited_keys():
             try:
                 entry = entries.pop(key)
             except KeyError:
                 pass
             else:
                 sorted_entries.append(entry)
         sorted_entries += entries.itervalues()
         # locate and instantiate style plugin
         style_cls = find_plugin(
             'pybtex.style.formatting', info.style)
         style = style_cls()
         # create citation nodes for all references
         backend = output_backend()
         if info.list_ == "enumerated":
             nodes = docutils.nodes.enumerated_list()
             nodes['enumtype'] = info.enumtype
             if info.start >= 1:
                 nodes['start'] = info.start
                 env.bibtex_cache.set_enum_count(env.docname, info.start)
             else:
                 nodes['start'] = env.bibtex_cache.get_enum_count(env.docname)
         elif info.list_ == "bullet":
             nodes = docutils.nodes.bullet_list()
         else: # "citation"
             nodes = docutils.nodes.paragraph()
         # XXX style.format_entries modifies entries in unpickable way
         for entry in style.format_entries(sorted_entries):
             if info.list_ == "enumerated" or info.list_ == "bullet":
                 citation = docutils.nodes.list_item()
                 citation += entry.text.render(backend)
             else: # "citation"
                 citation = backend.citation(entry, self.document)
                 # backend.citation(...) uses entry.key as citation label
                 # we change it to entry.label later onwards
                 # but we must note the entry.label now;
                 # at this point, we also already prefix the label
                 key = citation[0].astext()
                 info.labels[key] = info.labelprefix + entry.label
             node_text_transform(citation, transform_url_command)
             if info.curly_bracket_strip:
                 node_text_transform(citation, transform_curly_bracket_strip)
             nodes += citation
             if info.list_ == "enumerated":
                 env.bibtex_cache.inc_enum_count(env.docname)
         bibnode.replace_self(nodes)
 def apply(self):
     """Transform each
     :class:`~sphinxcontrib.bibtex.nodes.bibliography` node into a
     list of citations.
     """
     env = self.document.settings.env
     for bibnode in self.document.traverse(bibliography):
         # get the information of this bibliography node
         # by looking up its id in the bibliography cache
         id_ = bibnode['ids'][0]
         infos = [info for other_id, info
                  in env.bibtex_cache.bibliographies.iteritems()
                  if other_id == id_ and info.docname == env.docname]
         assert infos, "no bibliography id '%s' in %s" % (
             id_, env.docname)
         assert len(infos) == 1, "duplicate bibliography ids '%s' in %s" % (
             id_, env.docname)
         info = infos[0]
         # generate entries
         entries = OrderedDict()
         for bibfile in info.bibfiles:
             # XXX entries are modified below in an unpickable way
             # XXX so fetch a deep copy
             data = env.bibtex_cache.bibfiles[bibfile].data
             for entry in data.entries.itervalues():
                 visitor = FilterVisitor(
                     entry=entry,
                     is_cited=env.bibtex_cache.is_cited(entry.key))
                 try:
                     ok = visitor.visit(info.filter_)
                 except ValueError as e:
                     env.app.warn(
                         "syntax error in :filter: expression; %s" %
                         e)
                     # recover by falling back to the default
                     ok = env.bibtex_cache.is_cited(entry.key)
                 if ok:
                     entries[entry.key] = copy.deepcopy(entry)
         # order entries according to which were cited first
         # first, we add all keys that were cited
         # then, we add all remaining keys
         sorted_entries = []
         for key in env.bibtex_cache.get_all_cited_keys():
             try:
                 entry = entries.pop(key)
             except KeyError:
                 pass
             else:
                 sorted_entries.append(entry)
         sorted_entries += entries.itervalues()
         # locate and instantiate style and backend plugins
         style = find_plugin('pybtex.style.formatting', info.style)()
         backend = find_plugin('pybtex.backends', 'docutils')()
         # create citation nodes for all references
         if info.list_ == "enumerated":
             nodes = docutils.nodes.enumerated_list()
             nodes['enumtype'] = info.enumtype
             if info.start >= 1:
                 nodes['start'] = info.start
                 env.bibtex_cache.set_enum_count(env.docname, info.start)
             else:
                 nodes['start'] = env.bibtex_cache.get_enum_count(
                     env.docname)
         elif info.list_ == "bullet":
             nodes = docutils.nodes.bullet_list()
         else:  # "citation"
             nodes = docutils.nodes.paragraph()
         # XXX style.format_entries modifies entries in unpickable way
         for entry in style.format_entries(sorted_entries):
             if info.list_ == "enumerated" or info.list_ == "bullet":
                 citation = docutils.nodes.list_item()
                 citation += entry.text.render(backend)
             else:  # "citation"
                 citation = backend.citation(entry, self.document)
                 # backend.citation(...) uses entry.key as citation label
                 # we change it to entry.label later onwards
                 # but we must note the entry.label now;
                 # at this point, we also already prefix the label
                 key = citation[0].astext()
                 info.labels[key] = info.labelprefix + entry.label
             node_text_transform(citation, transform_url_command)
             if info.curly_bracket_strip:
                 node_text_transform(
                     citation,
                     transform_curly_bracket_strip)
             nodes += citation
             if info.list_ == "enumerated":
                 env.bibtex_cache.inc_enum_count(env.docname)
         bibnode.replace_self(nodes)
Exemple #22
0
class HeaderParser(object):
    """Class for holding and manipulating vcf headers

        Attributes:
            info_dict=OrderedDict()
            extra_info = {}

            filter_lines=[]
            filter_dict=OrderedDict()

            contig_lines=[]
            contig_dict=OrderedDict()

            format_lines=[]
            format_dict=OrderedDict()

            alt_lines=[]
            alt_dict=OrderedDict()

            other_lines=[]
            other_dict=OrderedDict()

            header=['CHROM','POS','ID','REF','ALT','QUAL','FILTER','INFO']
            header_keys={'info' : ['ID', 'Number', 'Type', 'Description'],
                           'form' : ['ID', 'Number', 'Type', 'Description'],
                           'filt' : ['ID', 'Description'],
                           'alt' : ['ID', 'Description'],
                           'contig' : ['ID']}
            fileformat = None
            filedate = None
            reference = None
            phasing = None
            source = None
            line_counter = 0
            individuals = []
            vep_columns = []
            snpeff_columns = []
            info_pattern
            filter_pattern
            contig_pattern
            format_pattern
            alt_pattern
            meta_pattern

        Methods:
            parse_meta_data(line)
            parse_header_line(line)
            add_fileformat(fileformat)
            print_header(): Return a list with the header lines
            add_meta_linekey, value)
            add_info(info_id, number, entry_type, description)
            add_filter(filter_id, description)
            add_format(format_id, number, entry_type, description)
            add_alt(alt_id, description)
            add_contig(contig_id, length)
            add_version_tracking(info_id, version, date, command_line='')

    """

    def __init__(self):
        super(HeaderParser, self).__init__()
        self.info_lines = []
        self.info_dict = OrderedDict()
        # This is a dictionary cantaining specific information about the info fields
        # It will have info name as key and then another dictionary with ID, Number, Type and Description
        self.extra_info = {}

        self.filter_lines = []
        self.filter_dict = OrderedDict()

        self.contig_lines = []
        self.contig_dict = OrderedDict()

        self.format_lines = []
        self.format_dict = OrderedDict()

        self.alt_lines = []
        self.alt_dict = OrderedDict()

        self.other_lines = []
        self.other_dict = OrderedDict()

        self.header = ["CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO"]
        self.header_keys = {
            "info": ["ID", "Number", "Type", "Description"],
            "form": ["ID", "Number", "Type", "Description"],
            "filt": ["ID", "Description"],
            "alt": ["ID", "Description"],
            "contig": ["ID", "length"],
        }
        self.fileformat = None
        self.filedate = None
        self.reference = None
        self.phasing = None
        self.source = None
        self.line_counter = 0
        self.individuals = []
        self.vep_columns = []
        self.snpeff_columns = []
        self.info_pattern = re.compile(
            r"""\#\#INFO=<
            ID=(?P<id>[^,]+),
            Number=(?P<number>-?\d+|\.|[AGR]),
            Type=(?P<type>Integer|Float|Flag|Character|String),
            Description="(?P<desc>[^"]*)"
            >""",
            re.VERBOSE,
        )
        self.filter_pattern = re.compile(
            r"""\#\#FILTER=<
            ID=(?P<id>[^,]+),
            Description="(?P<desc>[^"]*)"
            >""",
            re.VERBOSE,
        )
        self.contig_pattern = re.compile(
            r"""\#\#contig=<
            ID=(?P<id>[^>,]+)
            (,.*length=(?P<length>-?\d+))?
            .*
            >""",
            re.VERBOSE,
        )
        self.format_pattern = re.compile(
            r"""\#\#FORMAT=<
            ID=(?P<id>.+),
            Number=(?P<number>-?\d+|\.|[AGR]),
            Type=(?P<type>.+),
            Description="(?P<desc>.*)"
            >""",
            re.VERBOSE,
        )
        self.alt_pattern = re.compile(
            r"""\#\#ALT=<
            ID=(?P<id>[^,]+),
            Description="(?P<desc>[^"]*)"
            >""",
            re.VERBOSE,
        )
        self.meta_pattern = re.compile(r"""##(?P<key>.+?)=(?P<val>.+)""")

    def parse_meta_data(self, line):
        """Parse a vcf metadataline"""
        line = line.rstrip()
        logger.debug("Parsing metadata line:{0}".format(line))
        line_info = line[2:].split("=")
        match = False

        if line_info[0] == "fileformat":
            logger.debug("Parsing fileformat")
            try:
                self.fileformat = line_info[1]
                logger.debug("Found fileformat {0}".format(self.fileformat))
            except IndexError:
                raise SyntaxError("fileformat must have a value")

        elif line_info[0] == "INFO":
            match = self.info_pattern.match(line)
            if not match:
                raise SyntaxError("One of the INFO lines is malformed:{0}".format(line))

            matches = [match.group("id"), match.group("number"), match.group("type"), match.group("desc")]

            # extra_info is a dictionary to check the metadata about the INFO values:
            self.extra_info[matches[0]] = dict(zip(self.header_keys["info"][1:], matches[1:]))

            info_line = dict(list(zip(self.header_keys["info"], matches)))

            if len(info_line["Description"].split("Format:")) > 1:
                info_line["Format"] = [info.strip() for info in info_line["Description"].split("Format:")][-1]
            self.info_lines.append(info_line)

            # Store the vep columns:
            if info_line["ID"] == "CSQ":
                self.vep_columns = info_line.get("Format", "").split("|")

            if info_line["ID"] == "ANN":
                self.snpeff_columns = [
                    annotation.strip("' ") for annotation in info_line.get("Description", "").split(":")[-1].split("|")
                ]

            self.info_dict[match.group("id")] = line

        elif line_info[0] == "FILTER":
            match = self.filter_pattern.match(line)
            if not match:
                raise SyntaxError("One of the FILTER lines is malformed: {0}".format(line))
            matches = [match.group("id"), match.group("desc")]
            self.filter_lines.append(dict(list(zip(self.header_keys["filt"], matches))))
            self.filter_dict[match.group("id")] = line

        elif line_info[0] == "contig":
            match = self.contig_pattern.match(line)
            if not match:
                print()
                raise SyntaxError("One of the contig lines is malformed: {0}".format(line))

            matches = [match.group("id"), match.group("length")]
            self.contig_lines.append(dict(list(zip(self.header_keys["contig"], matches))))
            self.contig_dict[match.group("id")] = line

        elif line_info[0] == "FORMAT":
            match = self.format_pattern.match(line)
            if not match:
                raise SyntaxError("One of the FORMAT lines is malformed: {0}".format(line))

            matches = [match.group("id"), match.group("number"), match.group("type"), match.group("desc")]
            self.format_lines.append(dict(list(zip(self.header_keys["form"], matches))))
            self.format_dict[match.group("id")] = line

        elif line_info[0] == "ALT":
            match = self.alt_pattern.match(line)
            if not match:
                raise SyntaxError("One of the ALT lines is malformed: {0}".format(line))

            matches = [match.group("id"), match.group("desc")]
            self.alt_lines.append(dict(list(zip(self.header_keys["alt"], matches))))
            self.alt_dict[match.group("id")] = line

        else:
            match = self.meta_pattern.match(line)
            if not match:
                raise SyntaxError("One of the meta data lines is malformed: {0}".format(line))

            self.other_lines.append({match.group("key"): match.group("val")})
            self.other_dict[match.group("key")] = line

    def parse_header_line(self, line):
        """docstring for parse_header_line"""
        self.header = line[1:].rstrip().split("\t")
        if len(self.header) < 9:
            self.header = line[1:].rstrip().split()
        self.individuals = self.header[9:]

    def remove_header(self, name):
        """Remove a field from the header"""
        if name in self.info_dict:
            self.info_dict.pop(name)
            logger.info("Removed '{0}' from INFO".format(name))
        if name in self.filter_dict:
            self.filter_dict.pop(name)
            logger.info("Removed '{0}' from FILTER".format(name))
        if name in self.format_dict:
            self.format_dict.pop(name)
            logger.info("Removed '{0}' from FORMAT".format(name))
        if name in self.contig_dict:
            self.contig_dict.pop(name)
            logger.info("Removed '{0}' from CONTIG".format(name))
        if name in self.alt_dict:
            self.alt_dict.pop(name)
            logger.info("Removed '{0}' from ALT".format(name))
        if name in self.other_dict:
            self.other_dict.pop(name)
            logger.info("Removed '{0}' from OTHER".format(name))
        return

    def print_header(self):
        """Returns a list with the header lines if proper format"""
        lines_to_print = []
        lines_to_print.append("##fileformat=" + self.fileformat)
        if self.filedate:
            lines_to_print.append("##fileformat=" + self.fileformat)

        for filt in self.filter_dict:
            lines_to_print.append(self.filter_dict[filt])
        for form in self.format_dict:
            lines_to_print.append(self.format_dict[form])
        for info in self.info_dict:
            lines_to_print.append(self.info_dict[info])
        for contig in self.contig_dict:
            lines_to_print.append(self.contig_dict[contig])
        for alt in self.alt_dict:
            lines_to_print.append(self.alt_dict[alt])
        for other in self.other_dict:
            lines_to_print.append(self.other_dict[other])
        lines_to_print.append("#" + "\t".join(self.header))
        return lines_to_print

    def add_fileformat(self, fileformat):
        """
        Add fileformat line to the header.

        Arguments:
            fileformat (str): The id of the info line

        """
        self.fileformat = fileformat
        logger.info("Adding fileformat to vcf: {0}".format(fileformat))
        return

    def add_meta_line(self, key, value):
        """
        Adds an arbitrary metadata line to the header.

        This must be a key value pair

        Arguments:
            key (str): The key of the metadata line
            value (str): The value of the metadata line

        """
        meta_line = "##{0}={1}".format(key, value)
        logger.info("Adding meta line to vcf: {0}".format(meta_line))
        self.parse_meta_data(meta_line)
        return

    def add_info(self, info_id, number, entry_type, description):
        """
        Add an info line to the header.

        Arguments:
            info_id (str): The id of the info line
            number (str): Integer or any of [A,R,G,.]
            entry_type (str): Any of [Integer,Float,Flag,Character,String]
            description (str): A description of the info line

        """
        info_line = '##INFO=<ID={0},Number={1},Type={2},Description="{3}">'.format(
            info_id, number, entry_type, description
        )
        logger.info("Adding info line to vcf: {0}".format(info_line))
        self.parse_meta_data(info_line)
        return

    def add_filter(self, filter_id, description):
        """
        Add a filter line to the header.

        Arguments:
            filter_id (str): The id of the filter line
            description (str): A description of the info line

        """
        filter_line = '##FILTER=<ID={0},Description="{1}">'.format(filter_id, description)
        logger.info("Adding filter line to vcf: {0}".format(filter_line))
        self.parse_meta_data(filter_line)
        return

    def add_format(self, format_id, number, entry_type, description):
        """
        Add a format line to the header.

        Arguments:
            format_id (str): The id of the format line
            number (str): Integer or any of [A,R,G,.]
            entry_type (str): Any of [Integer,Float,Flag,Character,String]
            description (str): A description of the info line

        """
        format_line = '##FORMAT=<ID={0},Number={1},Type={2},Description="{3}">'.format(
            format_id, number, entry_type, description
        )
        logger.info("Adding format line to vcf: {0}".format(format_line))
        self.parse_meta_data(format_line)
        return

    def add_alt(self, alt_id, description):
        """
        Add a alternative allele format field line to the header.

        Arguments:
            alt_id (str): The id of the alternative line
            description (str): A description of the info line

        """
        alt_line = '##ALT=<ID={0},Description="{1}">'.format(alt_id, description)
        logger.info("Adding alternative allele line to vcf: {0}".format(alt_line))
        self.parse_meta_data(alt_line)
        return

    def add_contig(self, contig_id, length):
        """
        Add a contig line to the header.

        Arguments:
            contig_id (str): The id of the alternative line
            length (str): A description of the info line

        """
        contig_line = "##contig=<ID={0},length={1}>".format(contig_id, length)
        logger.info("Adding contig line to vcf: {0}".format(contig_line))
        self.parse_meta_data(contig_line)
        return

    def add_version_tracking(self, info_id, version, date, command_line=""):
        """
        Add a line with information about which software that was run and when
        to the header.

        Arguments:
            info_id (str): The id of the info line
            version (str): The version of the software used
            date (str): Date when software was run
            command_line (str): The command line that was used for run

        """
        other_line = '##Software=<ID={0},Version={1},Date="{2}",CommandLineOptions="{3}">'.format(
            info_id, version, date, command_line
        )
        self.other_dict[info_id] = other_line
        return
Exemple #23
0
class HeaderParser(object):
    """Class for holding and manipulating vcf headers

        Attributes:
            info_dict=OrderedDict()
            extra_info = {}

            filter_lines=[]
            filter_dict=OrderedDict()

            contig_lines=[]
            contig_dict=OrderedDict()

            format_lines=[]
            format_dict=OrderedDict()

            alt_lines=[]
            alt_dict=OrderedDict()

            other_lines=[]
            other_dict=OrderedDict()

            header=['CHROM','POS','ID','REF','ALT','QUAL','FILTER','INFO']
            header_keys={'info' : ['ID', 'Number', 'Type', 'Description'],
                           'form' : ['ID', 'Number', 'Type', 'Description'],
                           'filt' : ['ID', 'Description'],
                           'alt' : ['ID', 'Description'],
                           'contig' : ['ID']}
            fileformat = None
            filedate = None
            reference = None
            phasing = None
            source = None
            line_counter = 0
            individuals = []
            vep_columns = []
            snpeff_columns = []
            info_pattern
            filter_pattern
            contig_pattern
            format_pattern
            alt_pattern
            meta_pattern

        Methods:
            parse_meta_data(line)
            parse_header_line(line)
            add_fileformat(fileformat)
            print_header(): Return a list with the header lines
            add_meta_linekey, value)
            add_info(info_id, number, entry_type, description)
            add_filter(filter_id, description)
            add_format(format_id, number, entry_type, description)
            add_alt(alt_id, description)
            add_contig(contig_id, length)
            add_version_tracking(info_id, version, date, command_line='')

    """
    def __init__(self):
        super(HeaderParser, self).__init__()
        self.info_lines = []
        self.info_dict = OrderedDict()
        #This is a dictionary cantaining specific information about the info fields
        #It will have info name as key and then another dictionary with ID, Number, Type and Description
        self.extra_info = {}

        self.filter_lines = []
        self.filter_dict = OrderedDict()

        self.contig_lines = []
        self.contig_dict = OrderedDict()

        self.format_lines = []
        self.format_dict = OrderedDict()

        self.alt_lines = []
        self.alt_dict = OrderedDict()

        self.other_lines = []
        self.other_dict = OrderedDict()

        self.header = [
            'CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO'
        ]
        self.header_keys = {
            'info': ['ID', 'Number', 'Type', 'Description'],
            'form': ['ID', 'Number', 'Type', 'Description'],
            'filt': ['ID', 'Description'],
            'alt': ['ID', 'Description'],
            'contig': ['ID', 'length']
        }
        self.fileformat = None
        self.filedate = None
        self.reference = None
        self.phasing = None
        self.source = None
        self.line_counter = 0
        self.individuals = []
        self.vep_columns = []
        self.snpeff_columns = []
        self.info_pattern = re.compile(
            r'''\#\#INFO=<
            ID=(?P<id>[^,]+),
            Number=(?P<number>-?\d+|\.|[AGR]),
            Type=(?P<type>Integer|Float|Flag|Character|String),
            Description="(?P<desc>[^"]*)"
            >''', re.VERBOSE)
        self.filter_pattern = re.compile(
            r'''\#\#FILTER=<
            ID=(?P<id>[^,]+),
            Description="(?P<desc>[^"]*)"
            >''', re.VERBOSE)
        self.contig_pattern = re.compile(
            r'''\#\#contig=<
            ID=(?P<id>[^>,]+)
            (,.*length=(?P<length>-?\d+))?
            .*
            >''', re.VERBOSE)
        self.format_pattern = re.compile(
            r'''\#\#FORMAT=<
            ID=(?P<id>.+),
            Number=(?P<number>-?\d+|\.|[AGR]),
            Type=(?P<type>.+),
            Description="(?P<desc>.*)"
            >''', re.VERBOSE)
        self.alt_pattern = re.compile(
            r'''\#\#ALT=<
            ID=(?P<id>[^,]+),
            Description="(?P<desc>[^"]*)"
            >''', re.VERBOSE)
        self.meta_pattern = re.compile(r'''##(?P<key>.+?)=(?P<val>.+)''')

    def parse_meta_data(self, line):
        """Parse a vcf metadataline"""
        line = line.rstrip()
        logger.debug("Parsing metadata line:{0}".format(line))
        line_info = line[2:].split('=')
        match = False

        if line_info[0] == 'fileformat':
            logger.debug("Parsing fileformat")
            try:
                self.fileformat = line_info[1]
                logger.debug("Found fileformat {0}".format(self.fileformat))
            except IndexError:
                raise SyntaxError("fileformat must have a value")

        elif line_info[0] == 'INFO':
            match = self.info_pattern.match(line)
            if not match:
                raise SyntaxError(
                    "One of the INFO lines is malformed:{0}".format(line))

            matches = [
                match.group('id'),
                match.group('number'),
                match.group('type'),
                match.group('desc')
            ]

            # extra_info is a dictionary to check the metadata about the INFO values:
            self.extra_info[matches[0]] = dict(
                zip(self.header_keys['info'][1:], matches[1:]))

            info_line = dict(list(zip(self.header_keys['info'], matches)))

            if len(info_line['Description'].split('Format:')) > 1:
                info_line['Format'] = [
                    info.strip()
                    for info in info_line['Description'].split('Format:')
                ][-1]
            self.info_lines.append(info_line)

            # Store the vep columns:
            if info_line['ID'] == 'CSQ':
                self.vep_columns = info_line.get('Format', '').split('|')

            if info_line['ID'] == 'ANN':
                self.snpeff_columns = [
                    annotation.strip("' ") for annotation in info_line.get(
                        'Description', '').split(':')[-1].split('|')
                ]

            self.info_dict[match.group('id')] = line

        elif line_info[0] == 'FILTER':
            match = self.filter_pattern.match(line)
            if not match:
                raise SyntaxError(
                    "One of the FILTER lines is malformed: {0}".format(line))
            matches = [match.group('id'), match.group('desc')]
            self.filter_lines.append(
                dict(list(zip(self.header_keys['filt'], matches))))
            self.filter_dict[match.group('id')] = line

        elif line_info[0] == 'contig':
            match = self.contig_pattern.match(line)
            if not match:
                print()
                raise SyntaxError(
                    "One of the contig lines is malformed: {0}".format(line))

            matches = [match.group('id'), match.group('length')]
            self.contig_lines.append(
                dict(list(zip(self.header_keys['contig'], matches))))
            self.contig_dict[match.group('id')] = line

        elif line_info[0] == 'FORMAT':
            match = self.format_pattern.match(line)
            if not match:
                raise SyntaxError(
                    "One of the FORMAT lines is malformed: {0}".format(line))

            matches = [
                match.group('id'),
                match.group('number'),
                match.group('type'),
                match.group('desc')
            ]
            self.format_lines.append(
                dict(list(zip(self.header_keys['form'], matches))))
            self.format_dict[match.group('id')] = line

        elif line_info[0] == 'ALT':
            match = self.alt_pattern.match(line)
            if not match:
                raise SyntaxError(
                    "One of the ALT lines is malformed: {0}".format(line))

            matches = [match.group('id'), match.group('desc')]
            self.alt_lines.append(
                dict(list(zip(self.header_keys['alt'], matches))))
            self.alt_dict[match.group('id')] = line

        else:
            match = self.meta_pattern.match(line)
            if not match:
                raise SyntaxError(
                    "One of the meta data lines is malformed: {0}".format(
                        line))

            self.other_lines.append({match.group('key'): match.group('val')})
            self.other_dict[match.group('key')] = line

    def parse_header_line(self, line):
        """docstring for parse_header_line"""
        self.header = line[1:].rstrip().split('\t')
        if len(self.header) < 9:
            self.header = line[1:].rstrip().split()
        self.individuals = self.header[9:]

    def remove_header(self, name):
        """Remove a field from the header"""
        if name in self.info_dict:
            self.info_dict.pop(name)
            logger.info("Removed '{0}' from INFO".format(name))
        if name in self.filter_dict:
            self.filter_dict.pop(name)
            logger.info("Removed '{0}' from FILTER".format(name))
        if name in self.format_dict:
            self.format_dict.pop(name)
            logger.info("Removed '{0}' from FORMAT".format(name))
        if name in self.contig_dict:
            self.contig_dict.pop(name)
            logger.info("Removed '{0}' from CONTIG".format(name))
        if name in self.alt_dict:
            self.alt_dict.pop(name)
            logger.info("Removed '{0}' from ALT".format(name))
        if name in self.other_dict:
            self.other_dict.pop(name)
            logger.info("Removed '{0}' from OTHER".format(name))
        return

    def print_header(self):
        """Returns a list with the header lines if proper format"""
        lines_to_print = []
        lines_to_print.append('##fileformat=' + self.fileformat)
        if self.filedate:
            lines_to_print.append('##fileformat=' + self.fileformat)

        for filt in self.filter_dict:
            lines_to_print.append(self.filter_dict[filt])
        for form in self.format_dict:
            lines_to_print.append(self.format_dict[form])
        for info in self.info_dict:
            lines_to_print.append(self.info_dict[info])
        for contig in self.contig_dict:
            lines_to_print.append(self.contig_dict[contig])
        for alt in self.alt_dict:
            lines_to_print.append(self.alt_dict[alt])
        for other in self.other_dict:
            lines_to_print.append(self.other_dict[other])
        lines_to_print.append('#' + '\t'.join(self.header))
        return lines_to_print

    def add_fileformat(self, fileformat):
        """
        Add fileformat line to the header.

        Arguments:
            fileformat (str): The id of the info line

        """
        self.fileformat = fileformat
        logger.info("Adding fileformat to vcf: {0}".format(fileformat))
        return

    def add_meta_line(self, key, value):
        """
        Adds an arbitrary metadata line to the header.

        This must be a key value pair

        Arguments:
            key (str): The key of the metadata line
            value (str): The value of the metadata line

        """
        meta_line = '##{0}={1}'.format(key, value)
        logger.info("Adding meta line to vcf: {0}".format(meta_line))
        self.parse_meta_data(meta_line)
        return

    def add_info(self, info_id, number, entry_type, description):
        """
        Add an info line to the header.

        Arguments:
            info_id (str): The id of the info line
            number (str): Integer or any of [A,R,G,.]
            entry_type (str): Any of [Integer,Float,Flag,Character,String]
            description (str): A description of the info line

        """
        info_line = '##INFO=<ID={0},Number={1},Type={2},Description="{3}">'.format(
            info_id, number, entry_type, description)
        logger.info("Adding info line to vcf: {0}".format(info_line))
        self.parse_meta_data(info_line)
        return

    def add_filter(self, filter_id, description):
        """
        Add a filter line to the header.

        Arguments:
            filter_id (str): The id of the filter line
            description (str): A description of the info line

        """
        filter_line = '##FILTER=<ID={0},Description="{1}">'.format(
            filter_id, description)
        logger.info("Adding filter line to vcf: {0}".format(filter_line))
        self.parse_meta_data(filter_line)
        return

    def add_format(self, format_id, number, entry_type, description):
        """
        Add a format line to the header.

        Arguments:
            format_id (str): The id of the format line
            number (str): Integer or any of [A,R,G,.]
            entry_type (str): Any of [Integer,Float,Flag,Character,String]
            description (str): A description of the info line

        """
        format_line = '##FORMAT=<ID={0},Number={1},Type={2},Description="{3}">'.format(
            format_id, number, entry_type, description)
        logger.info("Adding format line to vcf: {0}".format(format_line))
        self.parse_meta_data(format_line)
        return

    def add_alt(self, alt_id, description):
        """
        Add a alternative allele format field line to the header.

        Arguments:
            alt_id (str): The id of the alternative line
            description (str): A description of the info line

        """
        alt_line = '##ALT=<ID={0},Description="{1}">'.format(
            alt_id, description)
        logger.info(
            "Adding alternative allele line to vcf: {0}".format(alt_line))
        self.parse_meta_data(alt_line)
        return

    def add_contig(self, contig_id, length):
        """
        Add a contig line to the header.

        Arguments:
            contig_id (str): The id of the alternative line
            length (str): A description of the info line

        """
        contig_line = '##contig=<ID={0},length={1}>'.format(contig_id, length)
        logger.info("Adding contig line to vcf: {0}".format(contig_line))
        self.parse_meta_data(contig_line)
        return

    def add_version_tracking(self, info_id, version, date, command_line=''):
        """
        Add a line with information about which software that was run and when
        to the header.

        Arguments:
            info_id (str): The id of the info line
            version (str): The version of the software used
            date (str): Date when software was run
            command_line (str): The command line that was used for run

        """
        other_line = '##Software=<ID={0},Version={1},Date="{2}",CommandLineOptions="{3}">'.format(
            info_id, version, date, command_line)
        self.other_dict[info_id] = other_line
        return
Exemple #24
0
class Cache(object):
  '''
    Cache to hold cached contents
  '''
  index_filename = "index.pickle"
  # active requests are not counted.

  def __init__(self, path, entry_limit):
    self.path = path
    self.on_memory_entry_limit = entry_limit
    self.on_memory = OrderedDict() #FastAVLTree()
    self.index = {}
    self.load_index()

  def __contains__(self, key):
    return key in self.index 

  def __len__(self):
    return len(self.index)
  
  def __iter__(self):
    return self.index.values()

  def _make_path(self, fname):
    return os.path.join(self.path, fname)

  def make_entry(self, key):
    '''
      reserve
    '''
    assert key not in self.index
    entry = CacheEntry(key, self.path, 0.0, self.on_notify)
    #FIXME because cache entry is write once. read many.
    self.index[key] = entry
    return entry

  def on_notify(self, entry):
    print 'cache entries: ', len(self.on_memory)
    if entry.key in self.on_memory:
      self.touch(entry)
    else:
      self.push_to_memory(entry)

  def push_to_memory(self, entry):
    if len(self.on_memory) >=  self.on_memory_entry_limit:
      key, purged = self.on_memory.popitem(False) #popping first item
      print 'purged cache life=%f s since %f for %s' % (time.time() - purged.last_touch, purged.last_touch, purged.key)
      purged.move_to_disk()
    entry.touch()
    print "putting entry %s" % (entry.key)
    assert entry.datafile
    assert entry.last_touch > 1.0
    self.on_memory[entry.key] = entry

  def touch(self, entry):
    #revoke
    x = self.on_memory.pop(entry.key)
    assert x == entry
    # activate it as a new entry
    entry.touch()
    self.on_memory[entry.key] = entry

  def get(self, key):
    e = self.index.get(key, None)
    return e

  def pop(self, key):
    return self.index.pop(key)
    
  def load_index(self):
    p = self._make_path(self.index_filename)
    
    no_index = False
    try:
      f = open(p)
    except:
      f = None
      pass
    if f:
      try:
        self.index = pickle.load(f)
      except:
        no_index = True        
      finally:
        f.close()
    else:
      no_index = True

    if no_index: 
      self.index = {}
      self.save_index()

  def save_entries(self):
    print 'Cache.save_entries'
    for entry in self.index.itervalues():
      if entry.datafile:
        entry.move_to_disk()

  def save_index(self):
    print 'Cache.save_index'
    p = self._make_path(self.index_filename)
    for entry in self.index.itervalues():
      entry.abort()
    with open(p, 'w') as f:
      pickle.dump(self.index, f)

  def fix(self):
    to_delete = []
    for k, v in self.index.items():
      p = self._make_path(v)
      if not os.access(p, os.F_OK | os.R_OK | os.W_OK):
        to_delete.append[k]
    for k in to_delete:
      del self.index[k]
    self.save_index()

  def scan(self):
    ''' wrong idea. cant generate url from file name...'''
    #return os.path.join(self.path, fname)
    for fname in os.listdir(self.path):
      if fname == self.index_filename:
        continue

    

  def html_index(self):
    count = len(self.index)
    x = []
    for key, ce in self.on_memory.iteritems():
      x.append('<li>[%s ]: %s</li>\n'%(key, ce.status()))
    frag_mem = '<ol>%s</ol>'%(''.join(x))

    y = []
    for key, ce in self.index.iteritems():
      y.append('<li>[%s ]: %s</li>\n'%(key, ce.status()))
    frag_index = '<ol>%s</ol>'%(''.join(y))
    html = '''<html><body>
      <p>count:%s</p>
      %s
      <hr />
      %s
      </body></html>'''
    return html%(count, frag_mem, frag_index)