Beispiel #1
0
 def fmtInfo(self) -> MultiDict:
     if self._fmtInfo is None:
         self._fmtInfo = MultiDict()
         for prop in self.get_values('logset:logFormatInfo'):
             key, sep, val = prop.partition('=')
             self._fmtInfo.add(key, val)
     return self._fmtInfo
Beispiel #2
0
class LuceneQuery(object):
    MANDATORY = 'mandatory'
    PROHIBITED = 'prohibited'
    OPTIONAL = 'optional'

    required_modifiers = {MANDATORY: '+',
                          PROHIBITED: '-',
                          OPTIONAL: ''}

    def __init__(self, *args, **kw):
        first_arg = args[0]
        if hasattr(first_arg, 'solr_field_name'):
            self.field, args = first_arg, args[1:]
        elif isinstance(first_arg, LuceneQuery):
            # Only pass LuceneQuery if all the other args are LuceneQueries
            self.field = None
            self.components = itertools.chain([query.components for query in args])
        else:
            self.field = None

        self.use_colon = True
        self.components = list(args)
        self.local_params = MultiDict()
        self.required = LuceneQuery.OPTIONAL
        self.boost_factor = None

    def __pow__(self, power):
        if not self.components:
            self.use_colon = False
        self.boost_factor = power
        return self

    def tag(self, tag):
        self.local_params.add('tag', tag)
        return self

    def require(self):
        self.required = LuceneQuery.MANDATORY
        return self

    def negate(self):
        self.required = LuceneQuery.PROHIBITED
        return self

    def fuzzy(self, factor=NotGiven):
        self.components.append('~')
        if factor != NotGiven:
            self.components.append(factor)
        return self

    def __unicode__(self):
        modifier = self.required_modifiers[self.required]
        field_clause = '' if not self.field else unicode(self.field)
        separator = ':' if self.use_colon else ''
        local_params = ''
        if self.local_params:
            local_params = '{!' + ' '.join(['%s=%s' % (key, unicode(val)) for key, val in self.local_params.iteritems() ]) + '}'
        component_clause = ''.join([unicode(component) for component in self.components])
        boost_factor = '^%s' % self.boost_factor if self.boost_factor else ''
        return u''.join([local_params, modifier, field_clause, separator, component_clause, boost_factor])
Beispiel #3
0
 def __init__(self,environ):
   self.environ = environ
   self.path = environ.get('PATH_INFO','').decode('utf8')
   self.method = environ.get('REQUEST_METHOD','')
   self.cookies = Cookie.SimpleCookie(str(environ.get('HTTP_COOKIE','')))
   self.GET = MultiDict(cgi.parse_qsl(urllib2.unquote(environ.get('QUERY_STRING','')).decode('utf8'),keep_blank_values=1))
   self.POST = {}
   self.FILES = {}
   if self.method == 'POST':
     post = cgi.FieldStorage(fp=environ.get('wsgi.input',''),environ=environ,keep_blank_values=1)
     self.POST = MultiDict([(item.name,item.value) for item in post.list if not item.filename])
     self.FILES = MultiDict([(item.name,item) for item in post.list if item.filename])
   self.REQUEST = MultiDict(self.GET.items() + self.POST.items()) # + self.FILES.items())
Beispiel #4
0
 def __init__(self,body='',content_type='text/html',charset='utf8',redirect=None,status_code=None,status_msg=None):
   self.headers = MultiDict()
   self.content_type = content_type
   self.charset = charset
   self.body = body
   self.redirect = redirect
   self.status_code = status_code
   self.status_msg = status_msg
   self.cookies = Cookie.SimpleCookie()
Beispiel #5
0
 def finalize(self):
   self.headers['Content-Type'] = '%s; charset=%s' % (self.content_type, self.charset)
   self.headers['Content-Length'] = len(self.body)
   if self.redirect:
     self.headers['Location'] = self.redirect
     self.status_code = self.status_code or 302
   for key in self.cookies:
     self.cookies[key]['path'] = self.cookies[key]['path'] or '/'
     self.headers['Set-Cookie'] = self.cookies[key].OutputString()
   self.status_code = self.status_code or 200
   self.headers = MultiDict([(safestr(k,self.charset),safestr(v,self.charset)) for (k,v) in self.headers.items()])
Beispiel #6
0
    def __init__(self, properties: MultiDict = None, **kwargs) -> None:
        # lazy getting of uri is going to be common enough to just build it into the base class:
        self._uri: Optional[str] = None
        self._namespace: Optional[str] = None
        self._label: Optional[str] = None

        # eg "dct:title": set(Literal("my title"))
        self.properties = MultiDict(properties)
        for key, val in kwargs.items():
            self.properties.add(key, val)

        if 'uri' in self.properties:
            self._uri = self.properties.one('uri')
            self.properties.remove('uri')

        if 'namespace' in self.properties:
            self._namespace = self.properties.one('namespace')
            self.properties.remove('namespace')

#        if properties is not None and 'uri' in properties:
#            logging.debug("setting uri from properties: {0}".format(properties['uri']))
#            #self._uri = properties.pop('uri')[0]
#            self._uri = properties.one('uri')
#            properties.remove('uri')
#
#        if properties is not None and 'namespace' in properties:
#            logging.debug("setting namespace from properties: {0}".format(properties['namespace']))
#            #self._uri = properties.pop('uri')[0]
#            self._namespace = properties.one('namespace')
#            properties.remove('namespace')

# all attributes we have a getter for should have an
# entry in properties, even if it is empty:
        for predicate in self.getters:
            if predicate not in self.properties:
                self.properties.add(predicate)

        # when adding a node to the graph and recursing into its properties,
        # we want a mechanism to bypass adding nodes that are already in the
        # graph:
        self._in_graph = False
Beispiel #7
0
    def __init__(self, *args, **kw):
        first_arg = args[0]
        if hasattr(first_arg, 'solr_field_name'):
            self.field, args = first_arg, args[1:]
        elif isinstance(first_arg, LuceneQuery):
            # Only pass LuceneQuery if all the other args are LuceneQueries
            self.field = None
            self.components = itertools.chain([query.components for query in args])
        else:
            self.field = None

        self.use_colon = True
        self.components = list(args)
        self.local_params = MultiDict()
        self.required = LuceneQuery.OPTIONAL
        self.boost_factor = None
Beispiel #8
0
 def __init__(self, target_url: str = None, fmtinfo: MultiDict = None):
     #                 properties:MultiDict=None) -> None:
     """ constructor should take properties as a keyword argument
         (to pass to the Node superclass constructor). target_url
         (eg the file path to be opened) should be supported but
         not required (because Node factories might not provide it),
         same for fmtinfo
     """
     #super().__init__(properties=properties)
     self._size = None
     self._t_earliest = None
     self._t_latest = None
     self._actual_file = TextFile.factory(target_url)
     for attr, parserule in self.fmtinfo_parsers.items():
         if attr in fmtinfo:
             value = fmtinfo.one(attr)
             m = parserule.regex.search(value)
             setattr(self, attr, parserule.parser(m))
     self.filters = {}
Beispiel #9
0
    def __init__(self, mods : dict, ties : set, name : str = ""):
        super().__init__(name)

        #build operations
        _ops = dict()
        for mod_name, opcode in mods.items():
            _ops[mod_name] = Operation(mod_name, opcode)

        #gather values
        _ties = MultiDict() # src -> (dst, dst_port)
        for src_name, dst_name, dst_port in ties:
            _ties[_ops[src_name]] = (_ops[dst_name], dst_port)


        #build actual val objects
        _values = set()
        for src in _ties:
            _values.add(Value(src, _ties[src]))

        self._operations = frozenset(_ops.values())
        self._values     = frozenset(_values)
Beispiel #10
0
 def instance(cls, uri):
     """ given the uri of a node, instantiate an object of this class based
         on the properties of that node in the graph
     """
     if not isinstance(uri, URIRef):
         uri = graph.geturi(uri)
     mdict = MultiDict(uri=[uri])
     preds = [k for k in cls.getters.keys()]  # impose an order
     preds_by_uri = {graph.geturi(p): p for p in preds}
     for pred_uri, obj in self.graph.predicate_objects(uri):
         if pred_uri in preds_by_uri:
             mdict.add(preds_by_uri[pred_uri], obj)
         else:
             # its a property we're not prepared for, just include it as an Identifier
             mdict.add(pred_uri, obj)
     return cls(properties=mdict)
    def catalog(self, candidates: Set[FileInfo],
                context: Context) -> Set[FileInfo]:
        filepatterns = context['filepatterns']
        matching = set()
        for regex in filepatterns:
            logging.info("filepattern: {0}: {1}".format(regex, regex.pattern))
            logging.info("{0:d} candidates".format(len(candidates)))
            logging.info("candidates: {0}".format(candidates))
            logging.info(
                "after filtering: " +
                str(filter(lambda x: regex.match(x.filename), candidates)))
            matching |= set(
                filter(lambda x: regex.match(x.filename), candidates))

        # filter by MIMEtype:
        mediatypes = self.properties['dcat:mediaType']
        mimetest = lambda x: self.right_mime_type(x)
        matching -= set(filter(mimetest, matching))

        # args for the LogFormatType that will handle the actual file/source:
        handler_args = {
            'rdf_class': context['logFormatType'],
            'fmtinfo': context['formatinfo']
        }
        #  'properties': self.properties
        #context.push(handler_factory=handlers.factory)
        context.push(handler_args=handler_args)

        # hmm, everything the ConcreteLog infers, we can just pass as properties:
        common_properties = MultiDict()
        common_properties.add('logset:isInstanceOf', context['logseries_uri'])
        common_properties.add('dcat:accessURL', context['dcat:accessURL'])
        #common_properties.add_values('logset:subject', context['subjects'])
        common_properties.add('logset:subject', *context['subjects'])
        common_properties.add('namespace', context['namespace'])
        logging.info("logset for concretelog has: {0}".format(
            context['logset']))

        # in most cases we are looking file-by-file .. FilePerTimepoint
        # can override this default behavior
        for f in matching:
            #context.push(label=f.filename)
            #context.push({'dcat:downloadURL': f.relpath + os.sep + f.filename})
            properties = MultiDict(common_properties)
            properties.add('rdfs:label', Literal(f.filename))
            relpath = (f.relpath + os.sep + f.filename).lstrip(os.sep)
            properties.add('dcat:downloadURL', Literal(relpath))
            logging.info(
                "properties for concretelog has: {0}".format(properties))

            handler_args['target_url'] = os.sep.join(f)  # full local path
            log = ConcreteLog(properties=properties)
            logging.info("adding log to graph: {0}".format(log))
            try:
                log.add_to_graph(context)
            except UnsupportedLogFormatHandler as err:
                logging.warn("logformat {0} not implemented".format(err))

            properties.remove('rdfs:label')
            properties.remove('dcat:downloadURL')
        context.pop(('handler_args', ))
        #context.pop('handler_factory')

        return candidates - matching
Beispiel #12
0
class Response(object):
  def __init__(self,body='',content_type='text/html',charset='utf8',redirect=None,status_code=None,status_msg=None):
    self.headers = MultiDict()
    self.content_type = content_type
    self.charset = charset
    self.body = body
    self.redirect = redirect
    self.status_code = status_code
    self.status_msg = status_msg
    self.cookies = Cookie.SimpleCookie()
  @property
  def status(self):
    status_map = {200:'OK',301:'Moved Permanently',302:'Found',404:'Not Found',500:'Internal Server Error'}
    if not self.status_msg: self.status_msg = status_map.get(self.status_code,'')
    return "%s %s" % (self.status_code,self.status_msg)
  def finalize(self):
    self.headers['Content-Type'] = '%s; charset=%s' % (self.content_type, self.charset)
    self.headers['Content-Length'] = len(self.body)
    if self.redirect:
      self.headers['Location'] = self.redirect
      self.status_code = self.status_code or 302
    for key in self.cookies:
      self.cookies[key]['path'] = self.cookies[key]['path'] or '/'
      self.headers['Set-Cookie'] = self.cookies[key].OutputString()
    self.status_code = self.status_code or 200
    self.headers = MultiDict([(safestr(k,self.charset),safestr(v,self.charset)) for (k,v) in self.headers.items()])
Beispiel #13
0
def factory(node: str):
    cls = classes[node]
    properties = MultiDict(uri=[cls.rdf_node])
    return cls(properties=properties)
Beispiel #14
0
class Node:
    """ A node in the global RDF graph """

    # what RDF class is this? (corresponds to "thing a class" in RDF). Note
    # that this is a string like "logset:ConcreteLog" - we won't know the
    # URI until after the graph has been constructed
    # Should be concrete, ie vcard:Organization not vcard:Kind
    rdf_class: ClassVar[str] = ''  # eg "foaf:Organization"

    # certain node types are created as a specific class (eg foaf:Organization
    # for Agent), but when querying the graph we need to find any subclass of
    # some superclass (eg foaf:Agent for Agent). If that is the case, the
    # Node class should specify the superclass with:
    rdf_superclass: ClassVar[str] = ''  # eg "foaf:Agent"

    # when adding a Node to the graph, triples for a certain set of properties
    # (based on the type of Node) are expected. These might be obtained by
    # querying a file/source or asking the user or infering from context, etc.
    # Each class specifies the expected properties and how to obtain them via a
    # class-variable 'getters', which is a dict mapping a predicate (str) to
    # the method (of self) that obtains it.
    # getter methods are called like:
    #   values:PropertyValues = self.getter(context:Context)
    getters: ClassVar[PropertyGetterDict] = {}
    # some predicates *must* be present, indicate these with:
    required_properties: ClassVar[Set[str]] = set()

    # to support finding known nodes of a given type, Node classes should
    # define a sparql query that returns rows from which a Node can be
    # instantiated. This is done by mapping each row to the finder_fields
    # and passing the result as a PropertyDict to __init__ (see method known())
    finder_query: str = ''
    finder_fields: List[str] = []

    # for 'select' and 'multi_select' getters, what target class is being selected?
    targets: Dict[str, NodeType] = {}  # eg 'dct:publisher': Agent

    # prompts for simple questions system can "ask" to get some properties:
    prompts: Dict[str, str] = {}  # eg 'dct:title': "Give the LogSet a title"

    # which property to use as a label? (this is useful because when defining
    # a new Thing and asking the user questions about it, it is helpful to
    # create the instance with a label that is included in the prompts. The
    # label generally corresponds with one of the properties of the Node, so
    # we require each subclass to indicate which property it will use
    label_property: str = None
    label_alternate = 'this'

    @property
    def graph(self):
        #logging.debug("getting graph: {0}".format(graph.Graph.the_graph))
        return graph.Graph.the_graph

    #def __init__(self, properties:PropertyDict = None) -> None:
    def __init__(self, properties: MultiDict = None, **kwargs) -> None:
        # lazy getting of uri is going to be common enough to just build it into the base class:
        self._uri: Optional[str] = None
        self._namespace: Optional[str] = None
        self._label: Optional[str] = None

        # eg "dct:title": set(Literal("my title"))
        self.properties = MultiDict(properties)
        for key, val in kwargs.items():
            self.properties.add(key, val)

        if 'uri' in self.properties:
            self._uri = self.properties.one('uri')
            self.properties.remove('uri')

        if 'namespace' in self.properties:
            self._namespace = self.properties.one('namespace')
            self.properties.remove('namespace')

#        if properties is not None and 'uri' in properties:
#            logging.debug("setting uri from properties: {0}".format(properties['uri']))
#            #self._uri = properties.pop('uri')[0]
#            self._uri = properties.one('uri')
#            properties.remove('uri')
#
#        if properties is not None and 'namespace' in properties:
#            logging.debug("setting namespace from properties: {0}".format(properties['namespace']))
#            #self._uri = properties.pop('uri')[0]
#            self._namespace = properties.one('namespace')
#            properties.remove('namespace')

# all attributes we have a getter for should have an
# entry in properties, even if it is empty:
        for predicate in self.getters:
            if predicate not in self.properties:
                self.properties.add(predicate)

        # when adding a node to the graph and recursing into its properties,
        # we want a mechanism to bypass adding nodes that are already in the
        # graph:
        self._in_graph = False

    @classmethod
    def instance(cls, uri):
        """ given the uri of a node, instantiate an object of this class based
            on the properties of that node in the graph
        """
        if not isinstance(uri, URIRef):
            uri = graph.geturi(uri)
        mdict = MultiDict(uri=[uri])
        preds = [k for k in cls.getters.keys()]  # impose an order
        preds_by_uri = {graph.geturi(p): p for p in preds}
        for pred_uri, obj in self.graph.predicate_objects(uri):
            if pred_uri in preds_by_uri:
                mdict.add(preds_by_uri[pred_uri], obj)
            else:
                # its a property we're not prepared for, just include it as an Identifier
                mdict.add(pred_uri, obj)
        return cls(properties=mdict)

    @classmethod
    def known(cls, filters: Dict[str, str] = dict()):
        """ generator-constructor over known nodes of a given type: """
        # TODO instead of sampling property values, use "order by" and a douple
        # loop to actually get the full set of properties for a uri
        # (the general finder query is like:
        # select ?uri <other fields> <optional fields where {
        #    ?uri a <self.rdf_class> .
        #    ?uri <other predicate> <other variable> .
        #    optional {
        #       ?uri <other predicate> <other variable> .
        #    } } order by ?uri
        # then make a multi-dict of the properties
        # so:
        #  - first add a class var: required: list of required properties
        #
        #   # list of variables:
        #   nrequired = len(self.required)
        #   optionals = [ key for key in self.getters if key not in self.required ]
        #   ntotal = nrequired + len(optionals)
        #   required = [ '?v{:d}'.format(i) for i in range(nrequired) ]
        #   optional = [ '?v{:d}'.format(i) for i in range(nrequired,ntotal) ]
        #   query  = "SELECT ?uri " + ' '.join(self.required) + ' '.join(optionals)
        #   query += " WHERE { "
        #   if self.rdf_superclass is None:
        #       query += "   ?uri a {0} .".format(self.rdf_class)
        #   else:
        #       query += "   ?uri a ?type ."
        #       query += "   ?type rdfs:subClassOf* {0} .".format(self.rdf_superclass)
        #   for clause in zip(self.required, required):
        #       query += " ?uri {0} {1} . ".format(clause[0], clause[1])
        #   if noptional > 0:
        #       query += " OPTIONAL { "
        #       for clause in zip(optionals, optional):
        #           query += " ?uri {0} {1} . ".format(clause[0], clause[1])
        #       query += " } "
        #   query += " } ORDER BY ?uri "
        #   curr = None
        #   next = None
        #   for row in  Graph.graph.query(query):
        #       next = row[0] # the uri
        #       if next != curr:
        #           mdict = Multidict(next)
        #           if curr is not None:
        #               yield cls(properties=mdict)
        #           curr = next
        #       # add each var to mdict
        #       for key,val in zip(required+optional,row):
        #           mdict.add(key, [val])
        #   yield cls(properties=mdict) # the last one
        #
        # or easier still: since we don't actually enforce that certain properties are
        # required, make everything optional:
        #
        # if filters has uris/rdflib identifiers, then to convert to str they need < > around it
        # but if they are a string like 'ddict:someThing' then they should stay as they are:
        as_str = lambda x: "<{0}>".format(str(x)) if isinstance(x, Identifier
                                                                ) else x
        logging.debug("filters has: {0}".format(filters))
        preds = [k for k in cls.getters.keys()]  # impose an order
        qvars = ['?v{:d}'.format(i) for i in range(len(preds))]
        query = "SELECT ?uri {0} WHERE {{ ".format(' '.join(qvars))
        if cls.rdf_superclass is None:
            query += "?uri a {0} . ".format(cls.rdf_class)
        else:
            query += "?uri a ?type . "
            query += "?type rdfs:subClassOf* {0} . ".format(cls.rdf_superclass)
        for pred, var in zip(preds, qvars):
            if pred in filters:
                #query += "?uri {0} {1} . ".format(pred,str(filters[pred]))
                query += "?uri {0} {1} . ".format(pred, as_str(filters[pred]))
            else:
                query += "OPTIONAL {{ ?uri {0} {1} . }} ".format(pred, var)
        query += "} ORDER BY ?uri "
        logging.debug("query is: {0}".format(query))

        curr_uri = None
        next_uri = None
        mdict: MultiDict = None
        #for row in graph.Graph.the_graph.query(query):
        for row in graph.query(query):
            logging.debug("found {0}".format(str(row)))
            next_uri = row[0]  # the uri
            if next_uri != curr_uri:
                if curr_uri is not None:
                    logging.debug("making a {0} with props {1}".format(
                        cls.__name__, str(mdict)))
                    yield cls(properties=mdict)
                mdict = MultiDict(uri=[next_uri])
                curr_uri = next_uri
            # add each var to mdict
            for key, val in zip(preds, row[1:]):
                mdict.add(key, val)
        if mdict is not None:
            logging.debug("making a {0} with props {1}".format(
                cls.__name__, str(mdict)))
            yield cls(properties=mdict)  # the last one
        else:
            logging.debug("no nodes of type {0} found".format(cls.__name__))

        #
        #if cls.finder_query is None:
        #    return
        #for row in Graph.graph.query(cls.finder_query):
        #    logging.info("found row:" + str(row))
        #    # each think in the row needs to be a list for MultiDict:
        #    lists = [ [r] for r in row ]
        #    props = dict(zip(cls.finder_fields, lists))
        #    logging.debug("making a {0} with props {1}".format(cls.__name__, str(props)))
        #    yield cls(properties=props)

    def get_values(self,
                   predicate: str,
                   context: Optional[Context] = None) -> PropertyValues:
        """ return a set of values for a property """
        if context is None:
            context = Context(predicate=predicate)
        logging.debug("looking for {0} in {1}".format(predicate,
                                                      str(self.properties)))
        props = self.properties.get(predicate)
        if len(props) == 0:
            logging.debug("calling a getter for {0}".format(predicate))
            getter = getattr(self, self.getters[predicate])
            logging.debug("got getter {0}, context {1}".format(
                getter, context))
            generator = (v for v in getter(context))
            self.properties.add(predicate, *generator)
            logging.debug("now {0} has: {1}".format(
                predicate, str(self.properties[predicate])))
        return self.properties[predicate]

    def get_one_value(self,
                      predicate: str,
                      context: Context = None) -> PropertyValue:
        values = self.get_values(predicate, context)
        if values is None or len(values) == 0:
            return None
        else:
            return values.pop()

    #def label(self, context:Context=None) -> str:
    @property
    def label(self) -> str:
        """ when asking questions of the user, hinting at who is asking 
            is helpful. If the label should have more to it than the 
            content of the label property, then the subclass should 
            override this (eg, the subjectttype label is from
            'skos:prefLabel', but the logset label is "this logset {dct:title}"
        """
        #return self.properties.one(self.label_property, 'this')
        #return self.get_one_value(self.label_property, context) or 'this'
        # Note: this should not trigger getters, so we use self.properties
        # not self.get_one_value:
        #self._label = self.get_one_value(self.label_property) or self.label_alternate
        if self._label is None:
            candidates = self.properties[self.label_property]
            if len(candidates) > 0:
                self._label = candidates.pop()
            else:
                self._label = self.label_alternate
        return self._label

    def __str__(self):
        # note that this should not trigger getters, so we use _uri not uri
        return "{0}: {1}".format(self.label, str(self._uri))

    @property
    def uri(self):
        # many subclasses will override this to lazily set a uri based on
        # the value of a property
        return self._uri

    @uri.setter
    def uri(self, value):
        self._uri = value

    def add_to_graph(self, context: Context = None):
        if self._in_graph:
            logging.debug("already in graph, skipping")
            return

        if context is None:
            context = Context()

        # describe my properties first, so subclasses can use them to generate a helpful uri if necessary:
        # I think that to avoid loops we need to do all of the asking (the user)
        # before doing any of the adding to graph:
        triples = []
        context.push(node=self)
        for predicate in self.properties:
            context.push(predicate=predicate)
            # need to convert string eg foaf:name to an actual uri for adding
            # to graph:
            logging.debug("calling Graph.geturi on {0}".format(predicate))
            pred_uri = graph.geturi(predicate)
            logging.debug("calling get_values with {0}, {1}".format(
                str(predicate), str(context)))
            for v in self.get_values(predicate, context):
                if isinstance(v, Identifier):
                    triples.append((self.uri, pred_uri, v))
                elif isinstance(v, Node):
                    triples.append((self.uri, pred_uri, v.uri))
                else:
                    # I'm pretty sure this should never happen
                    raise Exception("oh oh! " + str(v) + " ... " +
                                    str(type(v)))
            context.pop(('predicate', ))
        context.pop(('node', ))
        for triple in triples:
            logging.debug("adding triple {0}".format(triple))
            self.graph.add(triple)

        # finally, describe me:
        rdf = graph.getns('rdf')
        myclass = graph.geturi(self.rdf_class)
        logging.info("adding me to graph: {0}, {1}, {2}".format(
            self.uri, str(rdf.type), str(myclass)))
        self.graph.add((self.uri, rdf.type, myclass))

        self._in_graph = True

    # some common getters:
    def skip(self, context: Context) -> PropertyValues:
        """ if it's not there, don't include it """
        logging.debug("calling skip wiht {0}".format(str(context)))
        return set()

    def abort(self, context: Context) -> None:
        """ if it's not there, something is badly wrong """
        logging.debug("calling abort wiht {0}".format(str(context)))
        predicate = str(context['predicate'])
        msg = "{0} {1} missing predicate {2}".format(self.rdf_class,
                                                     str(self.uri), predicate)
        raise Exception(msg)

    def ask(self, context: Context) -> PropertyValues:
        """ ask user for simple text descriptions (subclasses should override
            if needing something more complicated)
        """
        logging.debug("calling ask wiht {0}".format(str(context)))
        retval = set()
        predicate = context['predicate']
        logging.debug("asking about {0} (label property is {1}".format(
            str(predicate), str(self.label_property)))
        label = self.label_alternate if predicate == self.label_property else self.label
        prompt = self.prompts.get(predicate)
        if prompt is not None:
            insist = predicate in self.required_properties
            value = UI.ask(prompt.format(label), insist=insist)
            if value != '':
                #self._properties.add(pred, Literal(value))
                retval.add(Literal(value))
        return retval

    def truefalse(self, context: Context) -> PropertyValues:
        prompt = self.prompts.get(context['predicate'])
        choice = UI.truefalse(prompt)
        response = 'true' if choice else 'false'
        xsd = graph.getns('xsd')
        logging.debug("truefalse returning " + response)
        retval = set([Literal(response, datatype=xsd.boolean)])
        return retval

    def select(self, context: Context) -> PropertyValues:
        """ get a list of existing nodes of appropriate type (via the target
            class), and ask the user to select one, or create a new one
        """
        logging.debug("calling select with {0}".format(str(context)))
        predicate = context['predicate']
        label = self.label_alternate if predicate == self.label_property else self.label
        return self.select_from_known(context, label, multi=False)

    #@classmethod
    def multi_select(self, context: Context) -> PropertyValues:
        logging.debug("calling multi_select with {0}".format(str(context)))
        predicate = context['predicate']
        label = self.label_alternate if predicate == self.label_property else self.label
        return self.select_from_known(context, label)

    @classmethod
    def select_from_known(
        cls,
        context: Context = Context(),
        label=None,
        multi=True,
        allow_create=True,
        filters: Dict[str, str] = dict()) -> PropertyValues:
        """ classmethod select getter ..."""
        retval = set()
        predicate = context.get('predicate')
        target_cls = cls.targets.get(predicate, cls)
        known = list(target_cls.known(filters))

        if label is None:
            label = target_cls.__name__

        if multi:
            default_prompt = "Please select one or more {0} "
            default_prompt += "(space-separated{1} or empty when done) "
            #ui_method = UI.multi_select
        else:
            default_prompt = "Please select a {0}{1}"
            #ui_method = UI.select

        prompt_new = ", or (n)ew " if allow_create else ""
        additional = ['n'] if allow_create else []

        prompt = (context.get('prompt') or cls.prompts.get(predicate)
                  or default_prompt).format(label, prompt_new)

        if multi:
            selection = UI.multi_select(prompt, known, *additional)
        else:
            selection = [UI.select(prompt, known, *additional)]

        while True:
            # loop so user can create multiple new entries
            if len(selection) == 0:
                break
            # for multi, but with single it is same logic:
            for choice in selection:
                if allow_create and str(choice).lower() == 'n':
                    # now it gets giddyingly recursive, make a new target_cls Node
                    # and add that:
                    classname = target_cls.__name__
                    label = UI.ask(
                        "please give the new {0} a label: ".format(classname))
                    props = {target_cls.label_property: [label]}
                    new = target_cls(properties=props)
                    new.add_to_graph(context)
                    uri = new.uri
                else:
                    logging.info(
                        f"got choice {choice} from selection {selection}")
                    #obj = known[choice]
                    obj = choice
                    uri = obj.uri
                retval.add(uri)
            selection = UI.multi_select("more? ", known, 'n') if multi else []
            #selection = ui_method("more? ", known, 'n') if multi else []
        return retval

    def make_uri(self, prop, namespace):
        """ make a candidate uri that is human-readable if possible based on 
            the value of a selected property. If that property is not set, or
            the uri would clash with one that already exists, add some 
            random characters
        """
        # make an rdf-friendly name:
        try:
            name = re.sub('^[^A-Za-z]+|\W', '', self.properties.one(prop))
        except KeyError:
            name = ran_str(8)

        # does it exist already?
        while existing in self.graph.predicate_objects(namespace[name]):
            # yep, choose a new name by adding some random characters to the end
            name = '{0}_{1}'.format(name, ran_str(4))

#        if len(name)==0:
#            name = ran_str(8)
#        else:
#            # does it exist already?
#            logging.info("makeing a uri in namespace {0}".format(namespace))
#            for existing in self.graph.predicate_objects(namespace[name]):
#                # yep, better choose a new name
#                name = '{0}_{1}'.format(name, ran_str(4))
#                break
#        logging.info("makeuri returning {0}".format(name))
        return namespace[name]
Beispiel #15
0
    def known(cls, filters: Dict[str, str] = dict()):
        """ generator-constructor over known nodes of a given type: """
        # TODO instead of sampling property values, use "order by" and a douple
        # loop to actually get the full set of properties for a uri
        # (the general finder query is like:
        # select ?uri <other fields> <optional fields where {
        #    ?uri a <self.rdf_class> .
        #    ?uri <other predicate> <other variable> .
        #    optional {
        #       ?uri <other predicate> <other variable> .
        #    } } order by ?uri
        # then make a multi-dict of the properties
        # so:
        #  - first add a class var: required: list of required properties
        #
        #   # list of variables:
        #   nrequired = len(self.required)
        #   optionals = [ key for key in self.getters if key not in self.required ]
        #   ntotal = nrequired + len(optionals)
        #   required = [ '?v{:d}'.format(i) for i in range(nrequired) ]
        #   optional = [ '?v{:d}'.format(i) for i in range(nrequired,ntotal) ]
        #   query  = "SELECT ?uri " + ' '.join(self.required) + ' '.join(optionals)
        #   query += " WHERE { "
        #   if self.rdf_superclass is None:
        #       query += "   ?uri a {0} .".format(self.rdf_class)
        #   else:
        #       query += "   ?uri a ?type ."
        #       query += "   ?type rdfs:subClassOf* {0} .".format(self.rdf_superclass)
        #   for clause in zip(self.required, required):
        #       query += " ?uri {0} {1} . ".format(clause[0], clause[1])
        #   if noptional > 0:
        #       query += " OPTIONAL { "
        #       for clause in zip(optionals, optional):
        #           query += " ?uri {0} {1} . ".format(clause[0], clause[1])
        #       query += " } "
        #   query += " } ORDER BY ?uri "
        #   curr = None
        #   next = None
        #   for row in  Graph.graph.query(query):
        #       next = row[0] # the uri
        #       if next != curr:
        #           mdict = Multidict(next)
        #           if curr is not None:
        #               yield cls(properties=mdict)
        #           curr = next
        #       # add each var to mdict
        #       for key,val in zip(required+optional,row):
        #           mdict.add(key, [val])
        #   yield cls(properties=mdict) # the last one
        #
        # or easier still: since we don't actually enforce that certain properties are
        # required, make everything optional:
        #
        # if filters has uris/rdflib identifiers, then to convert to str they need < > around it
        # but if they are a string like 'ddict:someThing' then they should stay as they are:
        as_str = lambda x: "<{0}>".format(str(x)) if isinstance(x, Identifier
                                                                ) else x
        logging.debug("filters has: {0}".format(filters))
        preds = [k for k in cls.getters.keys()]  # impose an order
        qvars = ['?v{:d}'.format(i) for i in range(len(preds))]
        query = "SELECT ?uri {0} WHERE {{ ".format(' '.join(qvars))
        if cls.rdf_superclass is None:
            query += "?uri a {0} . ".format(cls.rdf_class)
        else:
            query += "?uri a ?type . "
            query += "?type rdfs:subClassOf* {0} . ".format(cls.rdf_superclass)
        for pred, var in zip(preds, qvars):
            if pred in filters:
                #query += "?uri {0} {1} . ".format(pred,str(filters[pred]))
                query += "?uri {0} {1} . ".format(pred, as_str(filters[pred]))
            else:
                query += "OPTIONAL {{ ?uri {0} {1} . }} ".format(pred, var)
        query += "} ORDER BY ?uri "
        logging.debug("query is: {0}".format(query))

        curr_uri = None
        next_uri = None
        mdict: MultiDict = None
        #for row in graph.Graph.the_graph.query(query):
        for row in graph.query(query):
            logging.debug("found {0}".format(str(row)))
            next_uri = row[0]  # the uri
            if next_uri != curr_uri:
                if curr_uri is not None:
                    logging.debug("making a {0} with props {1}".format(
                        cls.__name__, str(mdict)))
                    yield cls(properties=mdict)
                mdict = MultiDict(uri=[next_uri])
                curr_uri = next_uri
            # add each var to mdict
            for key, val in zip(preds, row[1:]):
                mdict.add(key, val)
        if mdict is not None:
            logging.debug("making a {0} with props {1}".format(
                cls.__name__, str(mdict)))
            yield cls(properties=mdict)  # the last one
        else:
            logging.debug("no nodes of type {0} found".format(cls.__name__))
Beispiel #16
0
class SolrQuery(object):
    separators = {}

    def __init__(self, connections=None, index=None):
        self.params = MultiDict()
        self.connections = connections
        self.conn = self.connections.get('main', None)
        self.index = index
        self.params_joined = False

    def use_index(self, index):
        self.index = index

    def handle_row(self, row):
        return self.index(row)

    def join_params(self):
        return MultiDict((k,self.separators[k].join([unicode(v) for v in val]) if k in self.separators else unicode(val)) for k,val in self.params.iteritems())

    def execute(self, handler=handle_row, conn='main'):
        if not self.params_joined:
            self.params=self.join_params()
            self.params_joined = True
        return SolrResult(self._execute_search(), handler=partial(handler, self))


    def query(self, query):
        self.params.add('q', query)
        return self

    def filter(self, *filters):
        [self.params.add('fq', filter) for filter in filters]
        return self

    def facet(self, field, method='enum', missing_facet=NotGiven, sort=NotGiven, min_count=1):
        self.params['facet'] = 'true'
        field_name = SolrQuery._name_or_val(field)
        self.params.add('facet.field', field_name)
        self.params['f.%s.facet.method' % field_name] = method
        if sort != NotGiven:
            self.params['f.%s.facet.sort' % field_name] = sort
        if missing_facet != NotGiven:
            self.params['f.%s.facet.missing' % field_name] = missing_facet
        self.params['f.%s.facet.mincount' % field_name] = min_count
        return self

    def facet_date(self, date_field, start_date=NotGiven, end_date=NotGiven,
                   gap=NotGiven, hard_end=NotGiven, other=NotGiven):
        self.params['facet'] = 'true'
        field_name = SolrQuery._name_or_val(date_field)
        self.params.add('facet.date', field_name)

        if start_date != NotGiven:
            self.params['f.%s.facet.date.start' % field_name] = start_date

        if end_date != NotGiven:
            self.params['f.%s.facet.date.end' % field_name] = end_date

        if gap != NotGiven:
            self.params['f.%s.facet.date.gap' % field_name] = gap

        if hard_end != NotGiven:
            self.params['f.%s.facet.date.hardend' % field_name] = hard_end

        if other != NotGiven:
            self.params['f.%s.facet.date.other' % field_name] = other

        return self

    def facet_query(self, *queries):
        self.params['facet'] = 'true'
        [self.params.add('facet.query', query) for query in queries]
        return self

    def facet_prefix(self, field, prefix):
        self.facet(field)
        self.params['facet.prefix'] = prefix
        return self

    def default_operator(self, op):
        self.params['q.op'] = op
        return self

    def default_field(self, field):
        self.params['df'] = field
        return self

    def def_type(self, type):
        self.params['defType'] = type
        return self

    def tie(self, tie):
        self.params['tie'] = tie
        return self

    def default_query(self, query):
        self.params['q.alt'] = query
        return self

    @join_with_separator(separators, 'mm', ' ')
    def min_should_match(self, *args):
        self._add_items_to_key('mm', *args)
        return self

    def query_type(self, query_type):
        self.params['qt'] = query_type
        return self

    def paginate(self, limit, offset):
        self.params['start'] = limit
        self.params['rows'] = offset
        return self

    def limit(self, limit):
        self.params['rows'] = limit
        return self

    def offset(self, offset):
        self.params['start'] = offset
        return self

    @join_with_separator(separators, 'bf', ' ')
    def boost(self, *fields):
        self._add_items_to_key('bf', *fields)
        return self

    @join_with_separator(separators, 'pf', ' ')
    def phrase_boost(self, *fields):
        self._add_items_to_key('pf', *fields)
        return self

    def query_slop(self, slop):
        self.params['qs'] = slop
        return self

    def phrase_slop(self, slop):
        self.params['ps'] = slop
        return self

    def boost_query(self, query):
        self.params.add('bq', query)

    @staticmethod
    def _name_or_val(arg):
        return arg.solr_field_name if hasattr(arg, 'solr_field_name') else arg

    # Emulating defaultdict but on MultiDict
    def _add_items_to_key(self, param, *args):
        list_args = self.params.get(param, [])
        list_args.extend([SolrQuery._name_or_val(arg) for arg in args])
        self.params[param] = list_args

    @join_with_separator(separators, 'qf', ' ')
    def queried_fields(self, *fields):
        self._add_items_to_key('qf', *fields)
        return self

    def dismax_of(self, *fields):
        # Wrapper for queries using the dismax query parser
        self.def_type('dismax')
        self.queried_fields(*fields)
        self.tie(0.1)
        return self

    @join_with_separator(separators, 'fl', ',')
    def fields(self, *args):
        self._add_items_to_key('fl', *args)
        return self

    @join_with_separator(separators, 'sort', ',')
    def sort_by(self, *args):
        self._add_items_to_key('sort', *args)
        return self

    def indent(self, indent):
        self.params['indent']=indent
        return self

    def debug(self, debug):
        self.params['debugQuery'] = debug
        return self

    def echo_handler(self, echo):
        self.params['echoHandler'] = echo
        return self

    def echo_params(self, echo):
        self.params['echoParams'] = echo
        return self

    def _execute_search(self):
        return self.conn.search(self)
Beispiel #17
0
 def __init__(self, connections=None, index=None):
     self.params = MultiDict()
     self.connections = connections
     self.conn = self.connections.get('main', None)
     self.index = index
     self.params_joined = False
Beispiel #18
0
 def __init__(self, properties: MultiDict = MultiDict()) -> None:
     super().__init__(properties)
     self._fmtInfo = None  # MultiDict()
     self._filePatterns = None
     self._logFormatType = None
Beispiel #19
0
class LogSeries(Node):
    rdf_class = "logset:LogSeries"
    getters = {
        'rdfs:label': 'ask',
        'logset:logFormatType': 'select',
        'logset:infoType': 'select',
        'logset:subjectType': 'select',
        'logset:logFormatInfo': 'skip'  # need to work this out
    }
    required_properties = set([
        'rdfs:label', 'logset:logFormatType', 'logset:infoType',
        'logset:subjectType'
    ])
    prompts = {
        'rdfs:label': 'Give {0} a short identifying lael: ',
        'logset:logFormatType':
        'what logformattype is {0}? (TODO get a better question!) ',
        'logset:infoType': 'what type of information does {0} hold? ',
        'logset:subjectType': 'what type of system/component is {0} about? '
    }
    targets = {
        'logset:infoType': InfoType,
        'logset:subjectType': SubjectType,
        'logset:logFormatType': LogFormatType
    }

    #    finder_query = ''' SELECT ?uri (SAMPLE(?label) as ?label)
    #                                   (SAMPLE(?fmttype) as ?fmttype)
    #                                   (SAMPLE(?infotype) as ?infotype)
    #                                   (SAMPLE(?subjtype) as ?subjtype)  WHERE {
    #                          ?uri a logset:LogSeries .
    #                          ?uri rdfs:label ?label .
    #                          ?uri logset:logFormatType ?fmttype .
    #                          ?uri logset:infoType ?infotype .
    #                          ?uri logset:subjectType ?subjtype .
    #                        } GROUP BY ?uri
    #                   '''
    #    finder_fields = [ 'uri', 'rdfs:label', 'logset:logFormatType',
    #                      'logset:infoType', 'logset:subjectType' ]
    label_property: str = 'rdfs:label'
    label_alternate: str = 'log series'

    def __init__(self, properties: MultiDict = MultiDict()) -> None:
        super().__init__(properties)
        self._fmtInfo = None  # MultiDict()
        self._filePatterns = None
        self._logFormatType = None

    @property
    def fmtInfo(self) -> MultiDict:
        if self._fmtInfo is None:
            self._fmtInfo = MultiDict()
            for prop in self.get_values('logset:logFormatInfo'):
                key, sep, val = prop.partition('=')
                self._fmtInfo.add(key, val)
        return self._fmtInfo

#                logging.info("prop is {0}".format(prop))
#                logging.info("values is {0}".format())
#                for value in prop:
#                    key,sep,val = prop.partition('=')
#                    logging.info("found key {0} and val {1}".format(key,val))
#                    self._fmtInfo.add(key, val)

    @property
    def filePatterns(self) -> List:
        """ return a list, sorted by pattern length (as a proxy for "most 
            specific to least specific") of filename patterns that generally 
            match this logseries
        """
        if self._filePatterns is None:
            pats = set()
            tags = re.compile('(<[\w:]+>)')
            for pattern in self.fmtInfo['filepattern']:
                # convert from a tag-based-pattern to a regex:
                parts = tags.split(pattern)
                # every 2nd part will be a tag, replace it with corresponding pattern
                tagpatterns = [tagPattern(t) for t in parts[1::2]]
                regex_p = ''.join(sum(zip(parts[:-1:2], tagpatterns),
                                      ())) + parts[-1]
                pats.add(regex_p)
                self._filePatterns = [
                    re.compile(p) for p in sorted(pats, key=len, reverse=True)
                ]
        return self._filePatterns

    @property
    def logFormatType(self):
        if self._logFormatType is None:
            handlerName = self.get_one_value('logset:logFormatType')
            #self._logFormatType = LogFormatType.handlers[handlerName]
        return self._logFormatType

#    logset:logFormatInfo "filepattern=console-<date:YYYYMMDD>$" ;
#    logset:logFormatInfo "filepattern=console$" ;
#    logset:logFormatInfo "ts_words=0" ;
#    logset:logFormatInfo "part_word=1" ;

    @property
    def uri(self):
        # lazily find uri so there is a chance of deriving a readable one
        if self._uri is None:
            ns = self._namespace or localdict_ns()
            self._uri = self.make_uri(self.label_property, ns)
        return self._uri

    def identify_subjects(self, subject_list: List[URIRef]) -> URIRef:
        """ given a list of high-level subjects, look for specific subject that are
            a partOf one of these and an isSpecific of the subjecttypes relevant to
            this logseries. Returns a list of uris
        """
        as_str = lambda x: "<{0}>".format(str(x)) if isinstance(x, Identifier
                                                                ) else x
        subjtypes = ', '.join(
            [as_str(uri) for uri in self.get_values('logset:subjectType')])
        #logging.debug("my subtypes are: {0}".format(subjtypes))
        subjects = []
        for subj in subject_list:
            # find subjects whose subjecttpye corresponds with subjtype
            # and that are partOf something in the subjectlist
            q = ''' SELECT ?uri 
                    WHERE {{
                        ?uri a logset:Subject .
                        ?uri logset:isSpecific ?type .
                        ?uri logset:partOf* {0} .
                        FILTER ( ?type in ({1}) ) .
                    }}
                '''.format(as_str(subj), subjtypes)
            subjects += [row[0] for row in self.graph.query(q)]
        #logging.debug("query is: {0}".format(q))
        return subjects