Exemple #1
0
def test_suite():
    # pass in a db_name to test a specific database
    client = RexsterClient(db_name=db_name)
    BulbsTestCase.client = client
    BulbsTestCase.vertex_index_proxy = VertexIndexProxy
    BulbsTestCase.edge_index_proxy = EdgeIndexProxy
    BulbsTestCase.index_class = ManualIndex
    BulbsTestCase.graph = Graph(client.config)

    suite = bulbs_test_suite()
    #suite.addTest(unittest.makeSuite(RestTestCase))
    suite.addTest(unittest.makeSuite(GremlinTestCase))
    return suite
def main():
    config = Config('http://localhost:8182/graphs/conlexgraph')
    g = Graph(config)
    #clear(g)
    g.add_proxy("words",Word)
    g.add_proxy("bigram",Bigram)
    g.add_proxy("trigram",Trigram)
    #create_bigrams(g)
    #create_trigrams(g)
    in_bigram_context_entropy('boy','girl',g)
    in_trigram_context_entropy('boy','girl',g)
Exemple #3
0
 def __init__(self, config):
     self._graph = Graph(config)
     self._set_entity_proxy()
     self._set_activity_proxy()
     self._set_generation_proxy()
     self._set_usage_proxy()
     self._set_communication_proxy()
     self._set_start_proxy()
     self._set_end_proxy()
     self._set_invalidation_proxy()
     self._set_derivation_proxy()
     self._set_agent_proxy()
     self._set_attribution_proxy()
     self._set_association_proxy()
     self._set_delegation_proxy()
     self._set_influence_proxy()
     self._set_specialization_proxy()
     self._set_alternate_proxy()
     self._set_mention_proxy()
     self._set_membership_proxy()
     self._set_bundle_proxy()
Exemple #4
0
# -*- coding: utf-8 -*-
'''
Created on 12 авг. 2014 г.

@author: feelosoff
'''

from bulbs.config import Config
from bulbs.rexster import Graph, Edge, Vertex
from bulbs.rexster.client import REXSTER_URI


Edge()
Vertex()
c = Config('http://localhost:8182/graphs/myorientgraph')
g = Graph(c)

james = g.vertices.create(name="James")
julie = g.vertices.create(name="Julie")
g.edges.create(james, "knows", julie)

if __name__ == '__main__':
    pass
Exemple #5
0
def Connect(uri=DEFAULT_URI):
    """
    Establishes a connection to the graph database backend.  It also does 
    a few standard tasks to set up the models and server side scripts we
    depend on, so every utility that calls Connect() has a consistent 
    environment.

    Returns a Graph() object.

    Example:

        g = Connect()  # Connect using the standard default database info
        g = Connect("http://localhost:8182/graphs/myDB") # Use a custom DB
    """
    config = Config(uri)
    g = Graph(config)
    
    # Set up the node and relationship proxies
    g.add_proxy("host", Host)
    g.add_proxy("flow", Flow)
    g.add_proxy("source", Source)
    g.add_proxy("contains", Contains)
    g.add_proxy("dest", Dest)
    g.add_proxy("connectedTo", ConnectedTo)
    g.add_proxy("fqdn", FQDN)
    g.add_proxy("dnsTransaction", DNSTransaction)
    g.add_proxy("resolved", Resolved)
    g.add_proxy("answer", Answer)
    g.add_proxy("queried", Queried)
    g.add_proxy("queriedServer", QueriedServer)
    g.add_proxy("lookedUp", LookedUp)
    g.add_proxy("resolvedTo", ResolvedTo)
    g.add_proxy("file", File)
    g.add_proxy("transferred", Transferred)
    g.add_proxy("sentTo", SentTo)
    g.add_proxy("sentBy", SentBy)
    g.add_proxy("httpTransaction", HTTPTransaction)
    g.add_proxy("uri", URI)
    g.add_proxy("userAgent", UserAgent)
    g.add_proxy("requestedBy", RequestedBy)
    g.add_proxy("requestedOf", RequestedOf)
    g.add_proxy("hostedBy", HostedBy)
    g.add_proxy("identifiedBy", IdentifiedBy)
    g.add_proxy("agent", Agent)
    g.add_proxy("sent", Sent)
    g.add_proxy("received", Received)
    g.add_proxy("account", Account)
    g.add_proxy("requested", Requested)
    g.add_proxy("uses", Uses)
    g.add_proxy("pe", Pe)
    # Load in our groovy scripts
    g.scripts.update("groovy/gremlin.groovy")
    return g
Exemple #6
0
class Interface(object):
    """
    Proxy interface to to the provbulbs model

    Example:

    >>> from xcede.interface import Interface, Config
    >>> config = Config("http://<your-rexster-graph-url>")
    >>> interface = Interface(config)
    >>> interface.parse_prov("provenance.json")
    >>> interface.process_bundle()
    """
    def __init__(self, config):
        self._graph = Graph(config)
        self._set_entity_proxy()
        self._set_activity_proxy()
        self._set_generation_proxy()
        self._set_usage_proxy()
        self._set_communication_proxy()
        self._set_start_proxy()
        self._set_end_proxy()
        self._set_invalidation_proxy()
        self._set_derivation_proxy()
        self._set_agent_proxy()
        self._set_attribution_proxy()
        self._set_association_proxy()
        self._set_delegation_proxy()
        self._set_influence_proxy()
        self._set_specialization_proxy()
        self._set_alternate_proxy()
        self._set_mention_proxy()
        self._set_membership_proxy()
        self._set_bundle_proxy()

    def close_connection(self):
        del self._graph

    def parse_prov (self, prov_json):
        """
        parse a prov.json file using provpy

        cache as an attribute of interface

        return parsed bundle
        """
        self._bundle = prov.json.load(open(prov_json), cls = prov.ProvBundle.JSONDecoder)
        return self._bundle

    def _upload_lookup(self, proxy_type):
        """
        method to lookup the upload function for a given structure
        """
        return getattr(self, '_upload_' + proxy_type, None)

    def process_bundle(self, bundle=None):
        bundle = bundle
        structures = {'element': [],
                     'relation': []}
        responses = {}
        try:
            if not bundle:
                bundle = self._bundle
            for record in bundle.get_records():
                if record.is_element():
                    # TODO logic for processing nested bundles
                    if isinstance(record,prov.ProvBundle):
                        structures['element'].append(record)
                        self.process_bundle(bundle=record)
                    else:
                        structures['element'].append(record)
                elif record.is_relation():
                    structures['relation'].append(record)
                else:
                    pass
        except AttributeError:
            print "self._bundle is None. Did you run interface.parse_prov(<prov.json>)?"

        # first process all the elements
        for element in structures['element']:
            element_type = prov.PROV_N_MAP[element.get_type()]
            element_uri = element.get_identifier().get_uri()
            responses[element_uri] = self._upload_lookup(element_type)(element)

        # relations require that elements are created first
        for relation in structures['relation']:
            relation_type = prov.PROV_N_MAP[relation.get_type()]
            relation_id = relation.get_identifier()
            if relation_id:
                relation_uri = relation_id.get_uri()
                responses[relation_uri] = self._upload_lookup(relation_type)(relation)
            else:
                relation_uri = 'rel_' + str(hash(relation))
                responses[relation_uri] = self._upload_lookup(relation_type)(relation)

        return responses

    def _process_attributes(self, record):
        all_attributes = record.get_attributes()
        attributes = all_attributes[0]
        extra_attributes = all_attributes[1]
        results = {'attrs':{},
                   'extra':{}}

        if attributes:
            attrs = {}
            for k in attributes.keys():
                print 'attr key:', k
                if isinstance(k, prov.QName):
                    key = k.get_namespace().get_prefix() + '_' + k.get_localpart()
                    if isinstance(attributes[k], prov.QName):
                        attrs[key] = attributes[k].get_uri()
                    else:
                        attrs[key] = attributes[k]
                else:
                    if isinstance(k, prov.QName):
                        attrs[k] = attributes[k].get_uri()
                    else:
                        attrs[k] = attributes[k]
                results['attrs'] = attrs

        if extra_attributes:
            for k,v in extra_attributes:
                extra = {}
                if isinstance(k, prov.QName):
                    if isinstance(v, prov.QName):
                        key = k.get_namespace().get_prefix() + '_' + k.get_localpart()
                        extra[key] = v.get_uri()
                    else:
                        key = k.get_namespace().get_prefix() + '_' + k.get_localpart()
                        extra[key] = v
                else:
                    if isinstance(v, prov.QName):
                        extra[k] = v.get_localpart()
                    else:
                        extra[k] = v
                results['extra'] = extra
        return results


### Component 1: Entities and Activities


    def _set_entity_proxy(self):
        """
        initialize entity proxy
        """
        self._graph.add_proxy("entities", ProvEntity)
        self.entities = self._graph.entities

    def _upload_entity(self, record):
        """
        parse entity
        """
        all_attributes = self._process_attributes(record)
        attributes = all_attributes['attrs']
        extra_attributes = all_attributes['extra']

        data = dict(
            identifier = record.get_identifier().get_uri(),
            provn = record.get_provn(),
            asserted_types = [type.get_uri() for type in record.get_asserted_types()],
            attributes = attributes.keys(),
            extra_attributes = extra_attributes.keys()
            )
        response = self.entities.create(data)

        data_update = response.data()

        for attrs in attributes.iterkeys():
            if attrs in prov.PROV_ID_ATTRIBUTES_MAP.keys():
                prov_type = prov.PROV_ID_ATTRIBUTES_MAP[attrs].split(':')[-1]
            else:
                prov_type = attrs
            data_update[prov_type] = attributes[attrs]

        for extra in extra_attributes.iterkeys():
            # bulbs reserves the 'label' keyword
            prov_type = extra
            data_update[prov_type] = extra_attributes[extra]

        if record.is_element:
            self._graph.vertices.update(response.eid,data_update)

        elif record.is_relation:
            self._graph.edges.update(response.eid,data_update)

        print 'entity:',response.eid
        return response

    def _set_activity_proxy(self):
        self._graph.add_proxy("activities", ProvActivity)
        self.activities = self._graph.activities

    def _upload_activity(self, record):
        """
        parse activity
        """
        all_attributes = self._process_attributes(record)
        attributes = all_attributes['attrs']
        extra_attributes = all_attributes['extra']

        data = dict(
            identifier = record.get_identifier().get_uri(),
            provn = record.get_provn(),
            asserted_types = [type.get_uri() for type in record.get_asserted_types()],
            attributes = attributes.keys(),
            extra_attributes = extra_attributes.keys()
        )
        response = self.activities.create(data)

        data_update = response.data()

        for attrs in attributes.iterkeys():
            if attrs in prov.PROV_ID_ATTRIBUTES_MAP.keys():
                prov_type = prov.PROV_ID_ATTRIBUTES_MAP[attrs].split(':')[-1]
            else:
                prov_type = attrs
            data_update[prov_type] = attributes[attrs]

        for extra in extra_attributes.iterkeys():
            # bulbs reserves the 'label' keyword
            prov_type = extra
            data_update[prov_type] = extra_attributes[extra]

        if record.is_element:
            self._graph.vertices.update(response.eid,data_update)

        elif record.is_relation:
            self._graph.edges.update(response.eid,data_update)

        print 'activity:',response.eid
        return response

    def _set_generation_proxy(self):
        self._graph.add_proxy("wasGeneratedBy", ProvGeneration)
        self.wasGeneratedBy = self._graph.wasGeneratedBy

    def _upload_wasGeneratedBy(self,record):
        all_attributes = self._process_attributes(record)
        attributes = all_attributes['attrs']
        extra_attributes = all_attributes['extra']

        if record.get_identifier():
            identifier = record.get_identifier().get_uri()
        else:
            identifier = record.get_identifier()

        # search for entity ids
        if 1 in attributes.iterkeys():
            entity_id =  attributes[1].get_identifier().get_uri()
        else:
            entity_id =  attributes[1].get_identifier()

        # search for activity ids
        if 2 in attributes.iterkeys():
            activity_id = attributes[2].get_identifier().get_uri()
        else:
            activity_id = attributes[2].get_identifier()

        data = dict(
            identifier = identifier,
            entity = entity_id,
            activity = activity_id,
            #startTime = None,
            #endTime = None,
            provn = record.get_provn(),
            asserted_types = [type.get_uri() for type in record.get_asserted_types()],
            attributes = attributes.keys(),
            extra_attributes = extra_attributes.keys()
        )


        outV = self._graph.activities.index.lookup(identifier=activity_id).next()
        inV = self._graph.entities.index.lookup(identifier=entity_id).next()
        response = self.wasGeneratedBy.create(outV,inV,data)

        data_update = response.data()

        for attrs in attributes.iterkeys():
            if attrs in prov.PROV_ID_ATTRIBUTES_MAP.keys():
                prov_type = prov.PROV_ID_ATTRIBUTES_MAP[attrs].split(':')[-1]
            else:
                prov_type = attrs
            data_update[prov_type] = attributes[attrs]

        for extra in extra_attributes.iterkeys():
            # bulbs reserves the 'label' keyword
            prov_type = extra
            data_update[prov_type] = extra_attributes[extra]

        if record.is_relation:
            self._graph.edges.update(response.eid,data_update)

        print 'generation:', response.eid
        return response

    def _set_usage_proxy(self):
        self._graph.add_proxy("used", ProvUsage)
        self.used = self._graph.used

    def _upload_used(self, record):
        provn = record.get_provn()
        activity_id = record.get_attributes()[0][2].get_identifier().get_uri()
        activity = list(self.activities.index.lookup(identifier=activity_id))[0]
        entity_id = record.get_attributes()[0][1].get_identifier().get_uri()
        entity = list(self.entities.index.lookup(identifier=entity_id))[0]
        response = self.used.create(activity, entity,
            activity=activity,
            entity=entity,
            provn=provn
        )
        print 'used:', response.eid
        return response

    def _set_communication_proxy(self):
        self._graph.add_proxy("wasInformedBy", ProvCommunication)
        self.wasInformedBy = self._graph.wasInformedBy

    def _set_start_proxy(self):
        self._graph.add_proxy("wasStartedBy", ProvStart)
        self.wasStartedBy = self._graph.wasStartedBy

    def _upload_wasStartedBy(self,record):
        provn = record.get_provn()

        attributes = record.get_attributes()

        activity_id = attributes[0][2].get_identifier().get_uri()
        activity = list(self.activities.index.lookup(identifier=activity_id))[0]
        starter_id = attributes[0][6].get_identifier().get_uri()
        starter = list(self.activities.index.lookup(identifier=starter_id))[0]
        response = self.wasStartedBy.create(activity, starter,
            activity=activity,
            starter_activity=starter,
            provn=provn
        )
        print 'wasStartedBy:', response.eid
        return response

    def _set_end_proxy(self):
        self._graph.add_proxy("wasEndedBy", ProvEnd)
        self.wasEndedBy = self._graph.wasEndedBy

    def _set_invalidation_proxy(self):
        self._graph.add_proxy("wasInvalidatedBy", ProvInvalidation)
        self.wasInvalidatedBy = self._graph.wasInvalidatedBy

### Component 2: Derivations

    def _set_derivation_proxy(self):
        self._graph.add_proxy("wasDerivedFrom", ProvDerivation)
        self.wasDerivedFrom = self._graph.wasDerivedFrom


### Component 3: Agents, Responsibility, and Influence

    def _set_agent_proxy(self):
        self._graph.add_proxy("agents", ProvAgent)
        self.agents = self._graph.agents

    def _upload_agent(self, record):
        """
        parse activity
        """

        identifier = record.get_identifier().get_uri()
        asserted_types = [type.get_uri() for type in record.get_asserted_types()]
        attributes = record.get_attributes()
        provn = record.get_provn()
        response = self.agents.create(identifier=identifier,
            asserted_types=asserted_types,
            attributes=attributes[1],
            provn=provn,
        )
        print 'agent:', response.eid
        return response

    def _set_attribution_proxy(self):
        self._graph.add_proxy("wasAttributedTo", ProvAttribution)
        self.wasAttributedTo = self._graph.wasAttributedTo

    def _set_association_proxy(self):
        self._graph.add_proxy("wasAssociatedWith", ProvAssociation)
        self.wasAssociatedWith = self._graph.wasAssociatedWith

    def _upload_wasAssociatedWith(self,record):
        provn = record.get_provn()

        attributes = record.get_attributes()

        activity_id = attributes[0][2].get_identifier().get_uri()
        activity = list(self.activities.index.lookup(identifier=activity_id))[0]
        agent_id = attributes[0][8].get_identifier().get_uri()
        agent = list(self.agents.index.lookup(identifier=agent_id))[0]
        response = self.wasAssociatedWith.create(activity, agent,
            activity=activity,
            agent=agent,
            provn=provn
        )
        print 'wasAssociatedWith:', response.eid
        return response

    def _set_delegation_proxy(self):
        self._graph.add_proxy("actedOnBehalfOf", ProvDelegation)
        self.actedOnBehalfOf = self._graph.actedOnBehalfOf

    def _set_influence_proxy(self):
        self._graph.add_proxy("wasInfluencedBy", ProvInfluence)
        self.wasInfluencedBy = self._graph.wasInfluencedBy

### Component 4: Bundles

# See below

### Component 5: Alternate Entities

    def _set_specialization_proxy(self):
        self._graph.add_proxy("specializationOf", ProvSpecialization)
        self.specializationOf = self._graph.specializationOf

    def _set_alternate_proxy(self):
        self._graph.add_proxy("alternateOf", ProvAlternate)
        self.alternateOf = self._graph.alternateOf

    def _set_mention_proxy(self):
        self._graph.add_proxy("mentionOf", ProvMention)
        self.mentionOf = self._graph.mentionOf

### Component 6: Collections

    def _set_membership_proxy(self):
        self._graph.add_proxy("hadMember", ProvMembership)
        self.hadMember = self._graph.hadMember

    def _set_bundle_proxy(self):
        self._graph.add_proxy("bundles", ProvBundle)
        self.bundles = self._graph.bundles

    class _ElementProxy(object):
        def __init__(self):
            pass
Exemple #7
0
from flask import json
from flaskext.login import UserMixin

from dagcuss import app


class RexsterConnectionError(Exception):
    pass


config = Config(app.config["REXSTER_DB_URI"])
if app.config["BULBS_DEBUG"]:
    config.set_logger(logging.DEBUG)
try:
    graph = Graph(config)
except socket.error:
    raise RexsterConnectionError("Are you running Rexster on host/port pair in REXSTER_DB_URI?")
graph.scripts.update(os.path.join(os.path.dirname(__file__), "gremlin.groovy"))


def element_to_model(e, model_cls):
    if e == None:
        return None
    m = model_cls(e._client)
    m._initialize(e._result)
    return m


class User(Node, UserMixin):
    __mode__ = STRICT
Exemple #8
0
def Connect(uri=DEFAULT_URI):
    """
    Establishes a connection to the graph database backend.  It also does 
    a few standard tasks to set up the models and server side scripts we
    depend on, so every utility that calls Connect() has a consistent 
    environment.

    Returns a Graph() object.

    Example:

        g = Connect()  # Connect using the standard default database info
        g = Connect("http://localhost:8182/graphs/myDB") # Use a custom DB
    """
    config = Config(uri)
    g = Graph(config)

    # Set up the node and relationship proxies
    g.add_proxy("host", Host)
    g.add_proxy("flow", Flow)
    g.add_proxy("source", Source)
    g.add_proxy("contains", Contains)
    g.add_proxy("dest", Dest)
    g.add_proxy("connectedTo", ConnectedTo)
    g.add_proxy("fqdn", FQDN)
    g.add_proxy("dnsTransaction", DNSTransaction)
    g.add_proxy("resolved", Resolved)
    g.add_proxy("answer", Answer)
    g.add_proxy("queried", Queried)
    g.add_proxy("queriedServer", QueriedServer)
    g.add_proxy("lookedUp", LookedUp)
    g.add_proxy("resolvedTo", ResolvedTo)
    g.add_proxy("file", File)
    g.add_proxy("transferred", Transferred)
    g.add_proxy("sentTo", SentTo)
    g.add_proxy("sentBy", SentBy)
    g.add_proxy("httpTransaction", HTTPTransaction)
    g.add_proxy("uri", URI)
    g.add_proxy("userAgent", UserAgent)
    g.add_proxy("requestedBy", RequestedBy)
    g.add_proxy("requestedOf", RequestedOf)
    g.add_proxy("hostedBy", HostedBy)
    g.add_proxy("identifiedBy", IdentifiedBy)
    g.add_proxy("agent", Agent)
    g.add_proxy("sent", Sent)
    g.add_proxy("received", Received)
    g.add_proxy("account", Account)
    g.add_proxy("requested", Requested)
    g.add_proxy("uses", Uses)
    # Load in our groovy scripts
    g.scripts.update("groovy/gremlin.groovy")
    return g
Exemple #9
0
from flask import json
from flaskext.login import UserMixin

from dagcuss import app


class RexsterConnectionError(Exception):
    pass


config = Config(app.config['REXSTER_DB_URI'])
if app.config['BULBS_DEBUG']:
    config.set_logger(logging.DEBUG)
try:
    graph = Graph(config)
except socket.error:
    raise RexsterConnectionError(
        "Are you running Rexster on host/port pair in REXSTER_DB_URI?")
graph.scripts.update(os.path.join(os.path.dirname(__file__), 'gremlin.groovy'))


def element_to_model(e, model_cls):
    if e == None:
        return None
    m = model_cls(e._client)
    m._initialize(e._result)
    return m


class User(Node, UserMixin):
Exemple #10
0
###
# Bulbflow code
###
from bulbs.rexster import Graph, Config, REXSTER_URI
from bulbs.model import Node, Relationship
from bulbs.property import String, Integer, Long, Float, List, Dictionary, DateTime
from bulbs.utils import current_datetime
from bulbs.config import DEBUG

import hashlib

config = Config("http://localhost:8182/graphs/bbs", "root", "root")
g = Graph(config)
g.scripts.update('gremlin.groovy')
g.config.set_logger(DEBUG)

class Account(Node):
    element_type = "account"

    screen_name = String(nullable=False)
    password = String(nullable=False)
    email = String(nullable=False)
    first_name = String(nullable=False)
    last_name = String(nullable=False)

class RootFolderContainer(Node):
    element_type = "root_folder_container"

class ContentRoot(Relationship):
    label = "content_root"