def create_models(names=vertex_names): g = Graph(config) g.add_proxy("people", Person) g.add_proxy("knows", Knows) for name in names: model = g.people.create(name=name) return g
def main(): g = Graph() g.edges.create(7, "related", 8) try: g = Graph() keys = ["fqdn", "asn", "ipaddress"] for prodList in getData(): if len(prodList[0]) != 0: values = [prodList[0], prodList[1], prodList[2]] dict_object = dict(zip(keys, values)) g = Graph() g.vertices.create(dict_object) except IndexError: raise IndexError
def __init__(self): db_url = 'http://{}:{}{}'.format( settings.NEO4J_DATABASES['default']['HOST'], settings.NEO4J_DATABASES['default']['PORT'], settings.NEO4J_DATABASES['default']['ENDPOINT']) config = Config(db_url, settings.NEO4J_DATABASES['default']['USER'], settings.NEO4J_DATABASES['default']['PASSWORD']) self.g = Graph(config)
def initialize(): #Make sure you have done a neo4j start before this g = Graph() # uses default server address to connect g.add_proxy("article", Article) g.add_proxy("referral", Referral) return g
import os from flask import Flask, jsonify, g from bulbs.neo4jserver import Graph graph = Graph() def create_app(config_name): """Create an application instance.""" app = Flask(__name__) # apply configuration cfg = os.path.join(os.getcwd(), 'config', config_name + '.py') app.config.from_pyfile(cfg) # register blueprints from .api_v1 import api as api_blueprint app.register_blueprint(api_blueprint, url_prefix='/api/v1') # authentication token route from .auth import auth @app.route('/get-auth-token') @auth.login_required def get_auth_token(): return jsonify({'token': g.user.generate_auth_token()}) return app
import Test from bulbs.config import Config from bulbs.tests import BulbsTestCase, bulbs_test_suite from bulbs.neo4jserver import Graph, Neo4jClient, NEO4J_URI, \ VertexIndexProxy, EdgeIndexProxy, ExactIndex from bulbs.tests import GremlinTestCase config = Config(NEO4J_URI) BulbsTestCase.client = Neo4jClient(config) BulbsTestCase.vertex_index_proxy = VertexIndexProxy BulbsTestCase.edge_index_proxy = EdgeIndexProxy BulbsTestCase.index_class = ExactIndex BulbsTestCase.graph = Graph(config) def test_suite(): suite = bulbs_test_suite() #suite.addTest(unittest.makeSuite(RestTestCase)) suite.addTest(Test.makeSuite(GremlinTestCase)) return suite if __name__ == '__main__': Test.main(defaultTest='test_suite')
label = "knows" created = DateTime(default=current_datetime, nullable=False) #from people import Person, Knows from bulbs.neo4jserver import Graph def build_parser(): parser = argparse.ArgumentParser(conflict_handler='resolve') parser.add_argument('-w', '--worker', action='store_true') return parser if __name__ == '__main__': parser = build_parser() args = parser.parse_args() g = Graph() g.config.set_logger(DEBUG) g.add_proxy("people", Person) g.add_proxy("knows", Knows) james = g.people.create(name="James") julie = g.people.create(name="Julie") relationship = g.knows.create(james, julie) friends = james.outV('knows') friends = julie.inV('knows') print relationship.data() from IPython import Shell Shell.IPShellEmbed(argv=['-noconfirm_exit'])()
from celery import task from bulbs.neo4jserver import Graph from .models import * import pyping g = Graph() g.add_proxy('m', Measure) g.add_proxy('t', Trigger) g.add_proxy('tv', TriggerValue) @task def run(): print "run" for measure in g.m.get_all(): print measure.operation evaluate.delay(measure.operation, measure.uuid) for trigger in g.t.get_all(): control.delay(trigger) @task def evaluate(op, base): print "evaluate" print op print base ops = op.split(')')[0] ops = ops.split('(')
class Knows(Relationship): autoindex=True label = "knows" created = DateTime(default=current_datetime, nullable=False) #from people import Person, Knows from bulbs.neo4jserver import Graph def build_parser(): parser = argparse.ArgumentParser(conflict_handler='resolve') parser.add_argument('-w', '--worker', action='store_true') return parser if __name__ == '__main__': parser = build_parser() args = parser.parse_args() g = Graph() g.config.set_logger(DEBUG) g.add_proxy("people", Person) g.add_proxy("knows", Knows) james = g.people.create(name="James") julie = g.people.create(name="Julie") relationship = g.knows.create(james, julie) friends = james.outV('knows') friends = julie.inV('knows') print relationship.data() from IPython import Shell; Shell.IPShellEmbed(argv=['-noconfirm_exit'])()
print "Creating variable..." varNode = g.variables.create(name=varName) print "Creating describes relationship..." g.describes.create(itf, varNode) print "Adding relationship..." selector = g.selects.create(impl, varNode, value_type=0, value=config[varName]) else: print " Updating current impl" count = count + 1 ## if __name__ == "__main__": g = Graph() print "Initialized graph." g.add_proxy("interfaces", Interface) g.add_proxy("templates", Template) g.add_proxy("implems", TemplateImplem) g.add_proxy("variables", ConfigVariable) g.add_proxy("describes", Describes) g.add_proxy("depends", Depends) g.add_proxy("implements", Implements) g.add_proxy("selects", Selects) print "Added structure to graph" """
from django.http import HttpResponseRedirect from django.shortcuts import render_to_response from django import forms from .models import * from inventory.models import * from monitor.tasks import * from bulbs.neo4jserver import Graph from django.template import RequestContext g = Graph() g.add_proxy('nd', NetworkDevice) g.add_proxy('i', Interface) g.add_proxy('l', Link) g.add_proxy('m', Measure) g.add_proxy('t', Trigger) g.add_proxy('tv', TriggerValue) g.add_proxy('connects', Connects) g.add_proxy('measures', Measures) g.add_proxy('has_trigger', HasTrigger) g.add_proxy('has_value', HasValue) g.add_proxy('has_state', HasState) class MeasureForm(forms.Form): component = forms.ChoiceField(choices = [ ("Nodes",[(x.eid,x.name) for x in g.nd.get_all()]), ("Links", [(z.eid,"%s-%s <-> %s-%s" % ( [b for b in [a for a in z.inV()][0].inV()][0].name,
import re, datetime, neo4j, logging from bizold.input.file import StringReader, StringWriter log = logging.getLogger(__name__) #from neo4j import GraphDatabase, INCOMING, Evaluation from bulbs.neo4jserver import Graph g = Graph() #db = GraphDatabase(r"Q:\neo4j\test") # Only embedded; doesn't scale well!! def test_init(): with db.transaction: # A node to connect customers to customers = db.node() # A node to connect invoices to invoices = db.node() # Connected to the reference node, so # that we can always find them. db.reference_node.CUSTOMERS(customers) # An index, helps us rapidly look up customers if not db.node.indexes.exists('customers'): customer_idx = db.node.indexes.create('customers') print customers def test_read( file_in=r"P:\data\source\kpn\swol_marketing\odin\2012-11-14\extract\odin_dump-small.csv" ): prev = datetime.datetime.now() cnt, total = 1000, 10000
from settlers import Tile, Adjacency, TileEdge, TileVertex from constants import * from bulbs.neo4jserver import Graph, Config import os, random, itertools from collections import defaultdict, Iterable config = Config('http://192.168.1.14:7474/db/data/') g = Graph(config) g.add_proxy('tiles', Tile) g.add_proxy('touches', Adjacency) g.add_proxy('sides', TileEdge) g.add_proxy('corners', TileVertex) def configure_model(): vertex_dict = defaultdict(lambda: defaultdict(dict)) edge_dict = defaultdict(dict) def mk_ti(**kwargs): return g.tiles.create(**kwargs) def mk_to(src, dst): return g.touches.create(src, 'touches', dst) def mk_s(src, dst, **data): h, t = sorted([src, dst], key=lambda x: x._id if hasattr(x, '_id') else 0) if not t in edge_dict[h]: edge_dict[h][t] = g.sides.create(src, dst, **data) return edge_dict[h][t]
""" def _tripletGenerator(): #starts at the first word and goes forward if len(s) < 3: return for i in range(len(s) - 2): #generator, so we're independent of input file size words = [s[i], s[i+1], s[i+2]] yield words sen = "END this is a sentence END" s = sen.split() if __name__ == '__main__': g = Graph() g.clear() g.add_proxy("key", Key) g.add_proxy("link", Link) print(s) prev = '' prevg = g.vertices.create(data=prev) for words in _tripletGenerator(): curr = ' '.join(words) print(curr) currg = g.vertices.create(data=curr) g.edges.create(prevg, "link", currg) prevg = currg
def main(argv=None): # Config processing conf_file = join(expanduser("~"),"scrape.conf") config = ConfigParser.ConfigParser() config.read(conf_file) salt = config.get('info', 'salt') tab_page = config.get('info','tab_page') domain = config.get('info','domain') delay = int(config.get('info', 'delay')) # Start graph config = Config(NEO4J_URI) g = Graph(config) g.clear() # Change if youre working with persistent data store!!!! # Set up local indices users = {} instruments = {} # Make page crawlers tab_loader = TabDownloader(domain, tab_page, salt, delay) user_loader = UserDownloader(domain, delay) # Unofficial iterator resource = TabSpider(domain) # Start crawling! while resource.has_more(): # Get tab info tab_info = tab_loader.tab_download(resource.next_url()) # has_more doesnt actually work because of the retarted logic needed to keep track of tabs if not tab_info: break # If tab is valid if tab_info: # Store base tab tab_node = g.vertices.create(name=tab_info.tab_file) tab_node.tab_file = tab_info.tab_file tab_node.title = tab_info.title tab_node.version = tab_info.version tab_node.rating = tab_info.rating tab_node.num_ratings = tab_info.num_ratings tab_node.num_comments = tab_info.num_comments tab_node.label = "tab" tab_node.save() # Add instruments for instrument in tab_info.instruments: if instrument not in instruments.keys(): i_node = g.vertices.create(name=instrument) i_node.label = "instrument" i_node.save() instruments[instrument] = i_node i_node = instruments[instrument] g.edges.create(tab_node,"has_instrument",i_node) # Add comments (recursive) if tab_info.comments: for comment in tab_info.comments: g.edges.create(tab_node,"has_comment",save_comment(g, comment)) # Get info on the tabber if we don't have it if tab_info.tabber: if tab_info.tabber not in users.keys(): tabber = user_loader.load_user(tab_info.tabber) if not tabber: continue # create user node for tabber tempname = tabber.name if not tempname: tempname = "" u_node = g.vertices.create(name=tabber.tempname) u_node.registration_date = tabber.registration_date u_node.num_contributions = tabber.num_contributions u_node.rank = tabber.rank u_node.save() users[tab_info.tabber] = u_node # Add tab to tabber's transcriptions tabber = users[tab_info.tabber] g.edges.create(tabber,"tabbed",tab_node) print "Finished crawl! Woah!"
import datetime import codecs import os # Import the bulbs graph ORM from bulbs.neo4jserver import Graph, ExactIndex from relationships import * from bulbs.element import Vertex # Import XML parser import xml.etree.ElementTree as ET print "Starting processing at ", datetime.datetime.now() # Neo4J Test g = Graph() g.add_proxy("agents", Agent) g.add_proxy("places", Place) g.add_proxy("documents", Document) g.add_proxy("associated", AssociatedWith) g.add_proxy("corresponded", CorrespondedWith) g.add_proxy("referencedIn", ReferencedIn) g.add_proxy("location", Location) g.vertices.occindex = g.factory.get_index(Vertex, ExactIndex, "occupationIndex") g.vertices.subindex = g.factory.get_index(Vertex, ExactIndex, "subjectIndex") #print g #print "Vertices:", g.V #print "Edges:" , g.E # XML Test namespaces = { "snac" : "urn:isbn:1-931666-33-4" ,
#!/usr/bin/env python # -*- coding: utf-8 -*- # from py2neo import neo4j, node, rel from bulbs.config import DEBUG from bulbs.neo4jserver import Graph, Config, NEO4J_URI from message import Message, IsRetweet # models from datetime import datetime # setup config = Config(NEO4J_URI, "james", "secret") g = Graph(config) # g.config.set_logger(DEBUG) # g.add_proxy("message", Message) g.add_proxy("tweet", Message) g.add_proxy("isRT", IsRetweet) # create node m1 = g.tweet.create(text="salut", created_at=datetime.now()) m2 = g.tweet.create(text="re-salut", created_at=datetime.now()) # nodes = g.tweet.index.lookup(text="salut") # create edge rt = g.isRT.create(m2, m1) # Connect to neo4j # graph_db = neo4j.GraphDatabaseService("http://localhost:7474/db/data/"
from bulbs.model import Node, Relationship from bulbs.property import String, Integer, DateTime from bulbs.utils import current_datetime from bulbs.neo4jserver import Graph nltk.download('wordnet') # wn.synsets('dog') # len(wn.all_lemma_names()) # 140k words class Lemma(Node): element_type = "lemma" name = String(nullable=False) g = Graph() g.add_proxy("lemma", Lemma) # >>> g.add_proxy("knows", Knows) # >>> james = g.people.create(name="James") # >>> julie = g.people.create(name="Julie") # >>> g.knows.create(james, julie) import progressbar as pb N = 150000 for N, ln in enumerate(wn.all_lemma_names()): pass widgets = [pb.Counter(), '%d rows: ' % N, pb.Percentage(), ' ', pb.RotatingMarker(), ' ', pb.Bar(),' ', pb.ETA()]
label = "has_treenode" class PresynapticTo(Relationship): label = "presynaptic_to" class PostsynapticTo(Relationship): label = "postsynaptic_to" #### # Test #### g = Graph() g.add_proxy("neuron", Neuron) g.add_proxy("treenode", Treenode) g.add_proxy("connector", Connector) g.add_proxy("has_child", HasChild) g.add_proxy("has_treenode", HasTreenode) g.add_proxy("presynaptic_to", PresynapticTo) g.add_proxy("postsynaptic_to", PostsynapticTo) # create a few objects neuron1 = g.neuron.create(name="MyNeuron1") neuron2 = g.neuron.create(name="MyNeuron2") neuron3 = g.neuron.create(name="MyNeuron3") treenode1 = g.treenode.create(x=3.3, y=4.3, z=3.2) treenode11 = g.treenode.create(x=3.3, y=4.3, z=3.2)
id = Integer() name = String(nullable=False) region = String(nullable=False) security = Float() class IsConnectedTo(Relationship): label = "is_connected_to" conn = sqlite3.connect(os.path.expanduser("~/eve.db")) conn.row_factory = sqlite3.Row # populate graph from bulbs.neo4jserver import Graph, Config, NEO4J_URI g = Graph(Config(NEO4J_URI, "neo4j", "key")) g.add_proxy("system", System) g.add_proxy("is_connected_to", IsConnectedTo) systems = {} for item in g.V[1::]: systems[item.get("id")] = item def id_to_name(i): c = conn.cursor() c.execute("select solarSystemName from mapSolarSystems where solarSystemID=?", (i, )) return c.fetchone()[0] def distance_between(f, t): print f, t
name = String(nullable=False) region = String(nullable=False) security = Float() class IsConnectedTo(Relationship): label = "is_connected_to" conn = sqlite3.connect(os.path.expanduser("~/eve.db")) conn.row_factory = sqlite3.Row # populate graph from bulbs.neo4jserver import Graph, Config, NEO4J_URI g = Graph(Config(NEO4J_URI, "neo4j", "key")) g.add_proxy("system", System) g.add_proxy("is_connected_to", IsConnectedTo) systems = {} for item in g.V[1::]: systems[item.get("id")] = item def id_to_name(i): c = conn.cursor() c.execute("select solarSystemName from mapSolarSystems where solarSystemID=?", (i,)) return c.fetchone()[0]
from bulbs.property import String, Integer, DateTime from bulbs.utils import current_datetime from bulbs.neo4jserver import Graph nltk.download('wordnet') # wn.synsets('dog') # len(wn.all_lemma_names()) # 140k words class Lemma(Node): element_type = "lemma" name = String(nullable=False) g = Graph() g.add_proxy("lemma", Lemma) # >>> g.add_proxy("knows", Knows) # >>> james = g.people.create(name="James") # >>> julie = g.people.create(name="Julie") # >>> g.knows.create(james, julie) import progressbar as pb N = 150000 for N, ln in enumerate(wn.all_lemma_names()): pass widgets = [ pb.Counter(),
from django.http import HttpResponseRedirect, HttpResponse from django.shortcuts import render_to_response from django import forms import json from .models import * from .tasks import * from bulbs.neo4jserver import Graph from django.template import RequestContext g = Graph() g.add_proxy('nd', NetworkDevice) g.add_proxy('i', Interface) g.add_proxy('st', Storage) g.add_proxy('se', Service) g.add_proxy('a', SSHAgent) g.add_proxy('l', Link) g.add_proxy('contains', Contains) g.add_proxy('connects', Connects) class NetworkDeviceForm(forms.Form): name = forms.CharField() ip=forms.CharField() virtual = forms.BooleanField(required=False) class InterfaceForm(forms.Form): name = forms.CharField()
from bulbs.neo4jserver import Graph, Config, NEO4J_URI from texts.config import NEO4J_URI, NEO4J_USER, NEO4J_PASS from texts.py import * g = Graph(config) def create_link(v1, v2, e1, verb): g.vertices.create(name='v1') g.vertices.create(name='v2') g.edges.create(v1, verb, v2)
label = "has_child" class HasTreenode(Relationship): label = "has_treenode" class PresynapticTo(Relationship): label = "presynaptic_to" class PostsynapticTo(Relationship): label = "postsynaptic_to" #### # Test #### g = Graph() g.add_proxy("neuron", Neuron) g.add_proxy("treenode", Treenode) g.add_proxy("connector", Connector) g.add_proxy("has_child", HasChild) g.add_proxy("has_treenode", HasTreenode) g.add_proxy("presynaptic_to", PresynapticTo) g.add_proxy("postsynaptic_to", PostsynapticTo) # create a few objects neuron1 = g.neuron.create(name="MyNeuron1") neuron2 = g.neuron.create(name="MyNeuron2") neuron3 = g.neuron.create(name="MyNeuron3") treenode1 = g.treenode.create(x=3.3,y=4.3,z=3.2) treenode11 = g.treenode.create(x=3.3,y=4.3,z=3.2)
import sys import urllib2 from urlparse import urljoin from bs4 import BeautifulSoup import Queue from bulbs.neo4jserver import Graph import time from indexer import PageSoup # import robotparser # used to check robot files DEFAULT_URLS = ['http://www.google.com','http://www.amazon.com','http://www.nytimes.com','http://www.racialicious.com','http://www.groupon.com','http://www.yelp.com'] DEFAULT_DEPTH = 2 DEFAULT_GRAPH = Graph() class BFS_Crawler: """Create an instance of Crawler with a root and its tree""" def __init__(self,graph = DEFAULT_GRAPH,start = 'http://www.google.com',depth = DEFAULT_DEPTH): """Initialize the crawler with the starting urls""" self.g = graph self.root = start self.depth = depth self.start = [] self.indexer = Indexer() self.connection = MongoClient() self.page_db = connection.page_db def process_page(self,url): """Retrieve all html data from a webpage,index it and return a list of links""" links = [] try: p = Page(data)
def add_song(graph, song): artist, name, url, ts = song record = graph.song.get_or_create('name', name, name=name, artist=artist, url=url) return record def zip_db(db, prev=()): _tmp_db = copy(db) _tmp_db.insert(0, prev) _tmp_db.pop() return zip(db, _tmp_db) if __name__ == '__main__': g = Graph() g.add_proxy('song', Song) g.add_proxy('followed_by', FollowedBy) # db = pickle.load(open('db/tracks-00001.db')) # add_to_graph(g, db[1], db[0]) prev=() db_files = glob('db/tracks*') for db_file in db_files: print 'processing %s file right now...' % db_file db = pickle.load(open(db_file)) pbar = ProgressBar(widgets=WIDGETS, maxval=200).start() for i, (antes, depois) in enumerate(zip_db(db, prev)): if depois:
import sys, re, datetime, neo4j from bulbs.model import Node, Relationship from bulbs.property import String, Integer, DateTime from bulbs.utils import get_logger from bulbs.utils import current_datetime from bulbs.neo4jserver import Graph from bulbs.neo4jserver.batch import Neo4jBatchClient from bizold.input.file import StringReader, StringWriter log = get_logger(__name__) graph = Graph() class BizBase(object): @classmethod def new(cls, graph=graph, **kwargs): proxy = getattr(graph, cls.element_type) return proxy.create(**kwargs) def __repr__(self): result = [] for prop in self.__class__._get_initial_properties().values(): result.append([prop.name, getattr(self, prop.name)]) return '<{}-{}: {}>'.format( self.__class__.__name__, self.eid, ', '.join(['{}: {}'.format(*t) for t in result])) __str__ = __repr__ class Singleton(BizBase): """Class for umltiple inheritance. Together with a neo4j-Node ensures unique document:
from settlers import Tile, Adjacency, TileEdge, TileVertex from constants import * from bulbs.neo4jserver import Graph, Config import os, random, itertools from collections import defaultdict, Iterable config = Config('http://192.168.1.14:7474/db/data/') g = Graph(config) g.add_proxy('tiles', Tile) g.add_proxy('touches', Adjacency) g.add_proxy('sides', TileEdge) g.add_proxy('corners', TileVertex) def configure_model(): vertex_dict = defaultdict(lambda: defaultdict(dict)) edge_dict = defaultdict(dict) def mk_ti(**kwargs): return g.tiles.create(**kwargs) def mk_to(src, dst): return g.touches.create(src, 'touches', dst) def mk_s(src, dst, **data): h, t = sorted([src, dst], key=lambda x: x._id if hasattr(x, '_id') else 0) if not t in edge_dict[h]: edge_dict[h][t] = g.sides.create(src, dst, **data) return edge_dict[h][t] def mk_c(x,y,v): coords = v_coords(x,y,v) vert = None