def create_models(names=vertex_names):
g = Graph(config)
g.add_proxy("people", Person)
g.add_proxy("knows", Knows)
for name in names:
model = g.people.create(name=name)
return g
def main():
g = Graph()
g.edges.create(7, "related", 8)
try:
g = Graph()
keys = ["fqdn", "asn", "ipaddress"]
for prodList in getData():
if len(prodList[0]) != 0:
values = [prodList[0], prodList[1], prodList[2]]
dict_object = dict(zip(keys, values))
g = Graph()
g.vertices.create(dict_object)
except IndexError:
raise IndexError
def __init__(self):
db_url = 'http://{}:{}{}'.format(
settings.NEO4J_DATABASES['default']['HOST'],
settings.NEO4J_DATABASES['default']['PORT'],
settings.NEO4J_DATABASES['default']['ENDPOINT'])
config = Config(db_url, settings.NEO4J_DATABASES['default']['USER'],
settings.NEO4J_DATABASES['default']['PASSWORD'])
self.g = Graph(config)
def initialize():
#Make sure you have done a neo4j start before this
g = Graph() # uses default server address to connect
g.add_proxy("article", Article)
g.add_proxy("referral", Referral)
return g
import os
from flask import Flask, jsonify, g
from bulbs.neo4jserver import Graph
graph = Graph()
def create_app(config_name):
"""Create an application instance."""
app = Flask(__name__)
# apply configuration
cfg = os.path.join(os.getcwd(), 'config', config_name + '.py')
app.config.from_pyfile(cfg)
# register blueprints
from .api_v1 import api as api_blueprint
app.register_blueprint(api_blueprint, url_prefix='/api/v1')
# authentication token route
from .auth import auth
@app.route('/get-auth-token')
@auth.login_required
def get_auth_token():
return jsonify({'token': g.user.generate_auth_token()})
return app
import Test
from bulbs.config import Config
from bulbs.tests import BulbsTestCase, bulbs_test_suite
from bulbs.neo4jserver import Graph, Neo4jClient, NEO4J_URI, \
VertexIndexProxy, EdgeIndexProxy, ExactIndex
from bulbs.tests import GremlinTestCase
config = Config(NEO4J_URI)
BulbsTestCase.client = Neo4jClient(config)
BulbsTestCase.vertex_index_proxy = VertexIndexProxy
BulbsTestCase.edge_index_proxy = EdgeIndexProxy
BulbsTestCase.index_class = ExactIndex
BulbsTestCase.graph = Graph(config)
def test_suite():
suite = bulbs_test_suite()
#suite.addTest(unittest.makeSuite(RestTestCase))
suite.addTest(Test.makeSuite(GremlinTestCase))
return suite
if __name__ == '__main__':
Test.main(defaultTest='test_suite')
label = "knows"
created = DateTime(default=current_datetime, nullable=False)
#from people import Person, Knows
from bulbs.neo4jserver import Graph
def build_parser():
parser = argparse.ArgumentParser(conflict_handler='resolve')
parser.add_argument('-w', '--worker', action='store_true')
return parser
if __name__ == '__main__':
parser = build_parser()
args = parser.parse_args()
g = Graph()
g.config.set_logger(DEBUG)
g.add_proxy("people", Person)
g.add_proxy("knows", Knows)
james = g.people.create(name="James")
julie = g.people.create(name="Julie")
relationship = g.knows.create(james, julie)
friends = james.outV('knows')
friends = julie.inV('knows')
print relationship.data()
from IPython import Shell
Shell.IPShellEmbed(argv=['-noconfirm_exit'])()
from celery import task
from bulbs.neo4jserver import Graph
from .models import *
import pyping
g = Graph()
g.add_proxy('m', Measure)
g.add_proxy('t', Trigger)
g.add_proxy('tv', TriggerValue)
@task
def run():
print "run"
for measure in g.m.get_all():
print measure.operation
evaluate.delay(measure.operation, measure.uuid)
for trigger in g.t.get_all():
control.delay(trigger)
@task
def evaluate(op, base):
print "evaluate"
print op
print base
ops = op.split(')')[0]
ops = ops.split('(')
class Knows(Relationship):
autoindex=True
label = "knows"
created = DateTime(default=current_datetime, nullable=False)
#from people import Person, Knows
from bulbs.neo4jserver import Graph
def build_parser():
parser = argparse.ArgumentParser(conflict_handler='resolve')
parser.add_argument('-w', '--worker', action='store_true')
return parser
if __name__ == '__main__':
parser = build_parser()
args = parser.parse_args()
g = Graph()
g.config.set_logger(DEBUG)
g.add_proxy("people", Person)
g.add_proxy("knows", Knows)
james = g.people.create(name="James")
julie = g.people.create(name="Julie")
relationship = g.knows.create(james, julie)
friends = james.outV('knows')
friends = julie.inV('knows')
print relationship.data()
from IPython import Shell; Shell.IPShellEmbed(argv=['-noconfirm_exit'])()
print "Creating variable..."
varNode = g.variables.create(name=varName)
print "Creating describes relationship..."
g.describes.create(itf, varNode)
print "Adding relationship..."
selector = g.selects.create(impl, varNode, value_type=0, value=config[varName])
else:
print " Updating current impl"
count = count + 1
##
if __name__ == "__main__":
g = Graph()
print "Initialized graph."
g.add_proxy("interfaces", Interface)
g.add_proxy("templates", Template)
g.add_proxy("implems", TemplateImplem)
g.add_proxy("variables", ConfigVariable)
g.add_proxy("describes", Describes)
g.add_proxy("depends", Depends)
g.add_proxy("implements", Implements)
g.add_proxy("selects", Selects)
print "Added structure to graph"
"""
from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django import forms
from .models import *
from inventory.models import *
from monitor.tasks import *
from bulbs.neo4jserver import Graph
from django.template import RequestContext
g = Graph()
g.add_proxy('nd', NetworkDevice)
g.add_proxy('i', Interface)
g.add_proxy('l', Link)
g.add_proxy('m', Measure)
g.add_proxy('t', Trigger)
g.add_proxy('tv', TriggerValue)
g.add_proxy('connects', Connects)
g.add_proxy('measures', Measures)
g.add_proxy('has_trigger', HasTrigger)
g.add_proxy('has_value', HasValue)
g.add_proxy('has_state', HasState)
class MeasureForm(forms.Form):
component = forms.ChoiceField(choices = [
("Nodes",[(x.eid,x.name) for x in g.nd.get_all()]),
("Links", [(z.eid,"%s-%s <-> %s-%s" % (
[b for b in [a for a in z.inV()][0].inV()][0].name,
import re, datetime, neo4j, logging
from bizold.input.file import StringReader, StringWriter
log = logging.getLogger(__name__)
#from neo4j import GraphDatabase, INCOMING, Evaluation
from bulbs.neo4jserver import Graph
g = Graph()
#db = GraphDatabase(r"Q:\neo4j\test") # Only embedded; doesn't scale well!!
def test_init():
with db.transaction:
# A node to connect customers to
customers = db.node()
# A node to connect invoices to
invoices = db.node()
# Connected to the reference node, so
# that we can always find them.
db.reference_node.CUSTOMERS(customers)
# An index, helps us rapidly look up customers
if not db.node.indexes.exists('customers'):
customer_idx = db.node.indexes.create('customers')
print customers
def test_read(
file_in=r"P:\data\source\kpn\swol_marketing\odin\2012-11-14\extract\odin_dump-small.csv"
):
prev = datetime.datetime.now()
cnt, total = 1000, 10000
from settlers import Tile, Adjacency, TileEdge, TileVertex
from constants import *
from bulbs.neo4jserver import Graph, Config
import os, random, itertools
from collections import defaultdict, Iterable
config = Config('http://192.168.1.14:7474/db/data/')
g = Graph(config)
g.add_proxy('tiles', Tile)
g.add_proxy('touches', Adjacency)
g.add_proxy('sides', TileEdge)
g.add_proxy('corners', TileVertex)
def configure_model():
vertex_dict = defaultdict(lambda: defaultdict(dict))
edge_dict = defaultdict(dict)
def mk_ti(**kwargs):
return g.tiles.create(**kwargs)
def mk_to(src, dst):
return g.touches.create(src, 'touches', dst)
def mk_s(src, dst, **data):
h, t = sorted([src, dst],
key=lambda x: x._id if hasattr(x, '_id') else 0)
if not t in edge_dict[h]:
edge_dict[h][t] = g.sides.create(src, dst, **data)
return edge_dict[h][t]
"""
def _tripletGenerator():
#starts at the first word and goes forward
if len(s) < 3:
return
for i in range(len(s) - 2):
#generator, so we're independent of input file size
words = [s[i], s[i+1], s[i+2]]
yield words
sen = "END this is a sentence END"
s = sen.split()
if __name__ == '__main__':
g = Graph()
g.clear()
g.add_proxy("key", Key)
g.add_proxy("link", Link)
print(s)
prev = ''
prevg = g.vertices.create(data=prev)
for words in _tripletGenerator():
curr = ' '.join(words)
print(curr)
currg = g.vertices.create(data=curr)
g.edges.create(prevg, "link", currg)
prevg = currg
def main(argv=None):
# Config processing
conf_file = join(expanduser("~"),"scrape.conf")
config = ConfigParser.ConfigParser()
config.read(conf_file)
salt = config.get('info', 'salt')
tab_page = config.get('info','tab_page')
domain = config.get('info','domain')
delay = int(config.get('info', 'delay'))
# Start graph
config = Config(NEO4J_URI)
g = Graph(config)
g.clear() # Change if youre working with persistent data store!!!!
# Set up local indices
users = {}
instruments = {}
# Make page crawlers
tab_loader = TabDownloader(domain, tab_page, salt, delay)
user_loader = UserDownloader(domain, delay)
# Unofficial iterator
resource = TabSpider(domain)
# Start crawling!
while resource.has_more():
# Get tab info
tab_info = tab_loader.tab_download(resource.next_url())
# has_more doesnt actually work because of the retarted logic needed to keep track of tabs
if not tab_info:
break
# If tab is valid
if tab_info:
# Store base tab
tab_node = g.vertices.create(name=tab_info.tab_file)
tab_node.tab_file = tab_info.tab_file
tab_node.title = tab_info.title
tab_node.version = tab_info.version
tab_node.rating = tab_info.rating
tab_node.num_ratings = tab_info.num_ratings
tab_node.num_comments = tab_info.num_comments
tab_node.label = "tab"
tab_node.save()
# Add instruments
for instrument in tab_info.instruments:
if instrument not in instruments.keys():
i_node = g.vertices.create(name=instrument)
i_node.label = "instrument"
i_node.save()
instruments[instrument] = i_node
i_node = instruments[instrument]
g.edges.create(tab_node,"has_instrument",i_node)
# Add comments (recursive)
if tab_info.comments:
for comment in tab_info.comments:
g.edges.create(tab_node,"has_comment",save_comment(g, comment))
# Get info on the tabber if we don't have it
if tab_info.tabber:
if tab_info.tabber not in users.keys():
tabber = user_loader.load_user(tab_info.tabber)
if not tabber:
continue
# create user node for tabber
tempname = tabber.name
if not tempname:
tempname = ""
u_node = g.vertices.create(name=tabber.tempname)
u_node.registration_date = tabber.registration_date
u_node.num_contributions = tabber.num_contributions
u_node.rank = tabber.rank
u_node.save()
users[tab_info.tabber] = u_node
# Add tab to tabber's transcriptions
tabber = users[tab_info.tabber]
g.edges.create(tabber,"tabbed",tab_node)
print "Finished crawl! Woah!"
import datetime
import codecs
import os
# Import the bulbs graph ORM
from bulbs.neo4jserver import Graph, ExactIndex
from relationships import *
from bulbs.element import Vertex
# Import XML parser
import xml.etree.ElementTree as ET
print "Starting processing at ", datetime.datetime.now()
# Neo4J Test
g = Graph()
g.add_proxy("agents", Agent)
g.add_proxy("places", Place)
g.add_proxy("documents", Document)
g.add_proxy("associated", AssociatedWith)
g.add_proxy("corresponded", CorrespondedWith)
g.add_proxy("referencedIn", ReferencedIn)
g.add_proxy("location", Location)
g.vertices.occindex = g.factory.get_index(Vertex, ExactIndex, "occupationIndex")
g.vertices.subindex = g.factory.get_index(Vertex, ExactIndex, "subjectIndex")
#print g
#print "Vertices:", g.V
#print "Edges:" , g.E
# XML Test
namespaces = { "snac" : "urn:isbn:1-931666-33-4" ,
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# from py2neo import neo4j, node, rel
from bulbs.config import DEBUG
from bulbs.neo4jserver import Graph, Config, NEO4J_URI
from message import Message, IsRetweet # models
from datetime import datetime
# setup
config = Config(NEO4J_URI, "james", "secret")
g = Graph(config)
# g.config.set_logger(DEBUG)
# g.add_proxy("message", Message)
g.add_proxy("tweet", Message)
g.add_proxy("isRT", IsRetweet)
# create node
m1 = g.tweet.create(text="salut", created_at=datetime.now())
m2 = g.tweet.create(text="re-salut", created_at=datetime.now())
# nodes = g.tweet.index.lookup(text="salut")
# create edge
rt = g.isRT.create(m2, m1)
# Connect to neo4j
# graph_db = neo4j.GraphDatabaseService("http://localhost:7474/db/data/"
from bulbs.model import Node, Relationship
from bulbs.property import String, Integer, DateTime
from bulbs.utils import current_datetime
from bulbs.neo4jserver import Graph
nltk.download('wordnet')
# wn.synsets('dog')
# len(wn.all_lemma_names()) # 140k words
class Lemma(Node):
element_type = "lemma"
name = String(nullable=False)
g = Graph()
g.add_proxy("lemma", Lemma)
# >>> g.add_proxy("knows", Knows)
# >>> james = g.people.create(name="James")
# >>> julie = g.people.create(name="Julie")
# >>> g.knows.create(james, julie)
import progressbar as pb
N = 150000
for N, ln in enumerate(wn.all_lemma_names()):
pass
widgets = [pb.Counter(), '%d rows: ' % N, pb.Percentage(), ' ', pb.RotatingMarker(), ' ', pb.Bar(),' ', pb.ETA()]
label = "has_treenode"
class PresynapticTo(Relationship):
label = "presynaptic_to"
class PostsynapticTo(Relationship):
label = "postsynaptic_to"
####
# Test
####
g = Graph()
g.add_proxy("neuron", Neuron)
g.add_proxy("treenode", Treenode)
g.add_proxy("connector", Connector)
g.add_proxy("has_child", HasChild)
g.add_proxy("has_treenode", HasTreenode)
g.add_proxy("presynaptic_to", PresynapticTo)
g.add_proxy("postsynaptic_to", PostsynapticTo)
# create a few objects
neuron1 = g.neuron.create(name="MyNeuron1")
neuron2 = g.neuron.create(name="MyNeuron2")
neuron3 = g.neuron.create(name="MyNeuron3")
treenode1 = g.treenode.create(x=3.3, y=4.3, z=3.2)
treenode11 = g.treenode.create(x=3.3, y=4.3, z=3.2)
id = Integer()
name = String(nullable=False)
region = String(nullable=False)
security = Float()
class IsConnectedTo(Relationship):
label = "is_connected_to"
conn = sqlite3.connect(os.path.expanduser("~/eve.db"))
conn.row_factory = sqlite3.Row
# populate graph
from bulbs.neo4jserver import Graph, Config, NEO4J_URI
g = Graph(Config(NEO4J_URI, "neo4j", "key"))
g.add_proxy("system", System)
g.add_proxy("is_connected_to", IsConnectedTo)
systems = {}
for item in g.V[1::]:
systems[item.get("id")] = item
def id_to_name(i):
c = conn.cursor()
c.execute("select solarSystemName from mapSolarSystems where solarSystemID=?", (i, ))
return c.fetchone()[0]
def distance_between(f, t):
print f, t
name = String(nullable=False)
region = String(nullable=False)
security = Float()
class IsConnectedTo(Relationship):
label = "is_connected_to"
conn = sqlite3.connect(os.path.expanduser("~/eve.db"))
conn.row_factory = sqlite3.Row
# populate graph
from bulbs.neo4jserver import Graph, Config, NEO4J_URI
g = Graph(Config(NEO4J_URI, "neo4j", "key"))
g.add_proxy("system", System)
g.add_proxy("is_connected_to", IsConnectedTo)
systems = {}
for item in g.V[1::]:
systems[item.get("id")] = item
def id_to_name(i):
c = conn.cursor()
c.execute("select solarSystemName from mapSolarSystems where solarSystemID=?", (i,))
return c.fetchone()[0]
from bulbs.property import String, Integer, DateTime
from bulbs.utils import current_datetime
from bulbs.neo4jserver import Graph
nltk.download('wordnet')
# wn.synsets('dog')
# len(wn.all_lemma_names()) # 140k words
class Lemma(Node):
element_type = "lemma"
name = String(nullable=False)
g = Graph()
g.add_proxy("lemma", Lemma)
# >>> g.add_proxy("knows", Knows)
# >>> james = g.people.create(name="James")
# >>> julie = g.people.create(name="Julie")
# >>> g.knows.create(james, julie)
import progressbar as pb
N = 150000
for N, ln in enumerate(wn.all_lemma_names()):
pass
widgets = [
pb.Counter(),
from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import render_to_response
from django import forms
import json
from .models import *
from .tasks import *
from bulbs.neo4jserver import Graph
from django.template import RequestContext
g = Graph()
g.add_proxy('nd', NetworkDevice)
g.add_proxy('i', Interface)
g.add_proxy('st', Storage)
g.add_proxy('se', Service)
g.add_proxy('a', SSHAgent)
g.add_proxy('l', Link)
g.add_proxy('contains', Contains)
g.add_proxy('connects', Connects)
class NetworkDeviceForm(forms.Form):
name = forms.CharField()
ip=forms.CharField()
virtual = forms.BooleanField(required=False)
class InterfaceForm(forms.Form):
name = forms.CharField()
from bulbs.neo4jserver import Graph, Config, NEO4J_URI
from texts.config import NEO4J_URI, NEO4J_USER, NEO4J_PASS
from texts.py import *
g = Graph(config)
def create_link(v1, v2, e1, verb):
g.vertices.create(name='v1')
g.vertices.create(name='v2')
g.edges.create(v1, verb, v2)
label = "has_child"
class HasTreenode(Relationship):
label = "has_treenode"
class PresynapticTo(Relationship):
label = "presynaptic_to"
class PostsynapticTo(Relationship):
label = "postsynaptic_to"
####
# Test
####
g = Graph()
g.add_proxy("neuron", Neuron)
g.add_proxy("treenode", Treenode)
g.add_proxy("connector", Connector)
g.add_proxy("has_child", HasChild)
g.add_proxy("has_treenode", HasTreenode)
g.add_proxy("presynaptic_to", PresynapticTo)
g.add_proxy("postsynaptic_to", PostsynapticTo)
# create a few objects
neuron1 = g.neuron.create(name="MyNeuron1")
neuron2 = g.neuron.create(name="MyNeuron2")
neuron3 = g.neuron.create(name="MyNeuron3")
treenode1 = g.treenode.create(x=3.3,y=4.3,z=3.2)
treenode11 = g.treenode.create(x=3.3,y=4.3,z=3.2)
import sys import urllib2 from urlparse import urljoin from bs4 import BeautifulSoup import Queue from bulbs.neo4jserver import Graph import time from indexer import PageSoup # import robotparser # used to check robot files DEFAULT_URLS = ['http://www.google.com','http://www.amazon.com','http://www.nytimes.com','http://www.racialicious.com','http://www.groupon.com','http://www.yelp.com'] DEFAULT_DEPTH = 2 DEFAULT_GRAPH = Graph() class BFS_Crawler: """Create an instance of Crawler with a root and its tree""" def __init__(self,graph = DEFAULT_GRAPH,start = 'http://www.google.com',depth = DEFAULT_DEPTH): """Initialize the crawler with the starting urls""" self.g = graph self.root = start self.depth = depth self.start = [] self.indexer = Indexer() self.connection = MongoClient() self.page_db = connection.page_db def process_page(self,url): """Retrieve all html data from a webpage,index it and return a list of links""" links = [] try: p = Page(data)
def add_song(graph, song):
artist, name, url, ts = song
record = graph.song.get_or_create('name', name, name=name, artist=artist, url=url)
return record
def zip_db(db, prev=()):
_tmp_db = copy(db)
_tmp_db.insert(0, prev)
_tmp_db.pop()
return zip(db, _tmp_db)
if __name__ == '__main__':
g = Graph()
g.add_proxy('song', Song)
g.add_proxy('followed_by', FollowedBy)
# db = pickle.load(open('db/tracks-00001.db'))
# add_to_graph(g, db[1], db[0])
prev=()
db_files = glob('db/tracks*')
for db_file in db_files:
print 'processing %s file right now...' % db_file
db = pickle.load(open(db_file))
pbar = ProgressBar(widgets=WIDGETS, maxval=200).start()
for i, (antes, depois) in enumerate(zip_db(db, prev)):
if depois:
import sys, re, datetime, neo4j
from bulbs.model import Node, Relationship
from bulbs.property import String, Integer, DateTime
from bulbs.utils import get_logger
from bulbs.utils import current_datetime
from bulbs.neo4jserver import Graph
from bulbs.neo4jserver.batch import Neo4jBatchClient
from bizold.input.file import StringReader, StringWriter
log = get_logger(__name__)
graph = Graph()
class BizBase(object):
@classmethod
def new(cls, graph=graph, **kwargs):
proxy = getattr(graph, cls.element_type)
return proxy.create(**kwargs)
def __repr__(self):
result = []
for prop in self.__class__._get_initial_properties().values():
result.append([prop.name, getattr(self, prop.name)])
return '<{}-{}: {}>'.format(
self.__class__.__name__, self.eid,
', '.join(['{}: {}'.format(*t) for t in result]))
__str__ = __repr__
class Singleton(BizBase):
"""Class for umltiple inheritance. Together with a neo4j-Node ensures unique document:
from settlers import Tile, Adjacency, TileEdge, TileVertex
from constants import *
from bulbs.neo4jserver import Graph, Config
import os, random, itertools
from collections import defaultdict, Iterable
config = Config('http://192.168.1.14:7474/db/data/')
g = Graph(config)
g.add_proxy('tiles', Tile)
g.add_proxy('touches', Adjacency)
g.add_proxy('sides', TileEdge)
g.add_proxy('corners', TileVertex)
def configure_model():
vertex_dict = defaultdict(lambda: defaultdict(dict))
edge_dict = defaultdict(dict)
def mk_ti(**kwargs):
return g.tiles.create(**kwargs)
def mk_to(src, dst):
return g.touches.create(src, 'touches', dst)
def mk_s(src, dst, **data):
h, t = sorted([src, dst], key=lambda x: x._id if hasattr(x, '_id') else 0)
if not t in edge_dict[h]:
edge_dict[h][t] = g.sides.create(src, dst, **data)
return edge_dict[h][t]
def mk_c(x,y,v):
coords = v_coords(x,y,v)
vert = None
