def createDB():
db = GraphDatabase('dbpedia-graph.db')
if db.node.indexes.exists('dbpedia') == 0:
index = db.node.indexes.create('dbpedia')
else:
index = db.node.indexes.get('dbpedia')
return db, index
def __init__(self,
config,
graphtype,
minoccs=1,
maxcoocs=1,
maxcables=None,
year=None):
self.mongodb = CablegateDatabase(
config['general']['mongodb'])["cablegate"]
self.graphdb = GraphDatabase(config['general']['neo4j'])
self.config = config
if graphtype is None or graphtype == "occurrences":
self.update_occurrences_network(minoccs,
maxcoocs,
maxcables,
year,
documents=False)
elif graphtype == "cooccurrences":
(nodecache,
ngramcache) = self.update_occurrences_network(minoccs,
maxcoocs,
maxcables,
year,
documents=False)
self.update_cooccurrences_network(nodecache, ngramcache, minoccs,
maxcoocs)
def main(argv):
#Parse commandline arguments
#http://www.tutorialspoint.com/python/python_command_line_arguments.htm
start_hash = None
try:
opts, args = getopt.getopt(argv, "h:s:")
except getopt.GetoptError:
print "-s <hash>"
sys.exit(2)
for opt, arg in opts:
if opt == "-h":
print "-s <hash>"
elif opt == "-s":
start_hash = arg
projects = [
d for d in os.listdir(devknowledge.settings.VERSION_CONTROL_REPOS)
if os.path.isdir(
os.path.join(devknowledge.settings.VERSION_CONTROL_REPOS, d))
]
try:
#print "Connecting to database: ",devknowledge.settings.NEO4J_DATABASE
gdb = GraphDatabase(devknowledge.settings.NEO4J_DATABASE)
except Exception, e:
print "Error: ", e.message()
def get_graphdb(location):
import atexit
from neo4j import GraphDatabase
location = os.path.abspath(location)
_db = GraphDatabase(location)
atexit.register(lambda d: d.shutdown(), _db)
return _db
def create_database(database_name): ''' Create neo4j database ''' # TODO: Add db.shutdown() on error (try, except, else)? # Create or open neo4j database db_obj = GraphDatabase(database_name) return db_obj
def get_rel_info(dbname,relid):
from neo4j import GraphDatabase
from neo4j import OUTGOING, INCOMING, ANY
db = GraphDatabase(dbname)
relid = int(relid) #make sure it is an int
rel = db.relationship[relid]
print rel
for key,value in rel.items():
print "\t"+key,value
db.shutdown()
def make_custs_embedded(amount=10000, offset=0, every=10000):
db = GraphDatabase(db_path)
customers = db.node[1]
with db.transaction:
for i in range(offset, offset + amount):
name = 'Customer {:0=5}'.format(i)
customer = db.node(name=name)
customer.INSTANCE_OF(customers)
if i % every == 0:
print datetime.datetime.now(), i
db.shutdown()
def __init__(self,mind,dbpath="data"):
self.mind = mind
self.dbpath = dbpath
self.mind.log("Starting Graph Database at "+dbpath)
self.db = GraphDatabase(dbpath)
self.root = None
memorySize = len(self.db.nodes)
self.log("starting with "+str(memorySize)+" nodes in memory")
self.vars = {}
self.initialGraph(memorySize<=1)
def get_custs_embed(amount=-1, every=10000):
#print len(customers.INSTANCE_OF)
#l = customers.INSTANCE_OF
db = GraphDatabase(db_path)
l = db.nodes
for j in range(2):
for i, node in enumerate(l):
if amount >= 0 and i > amount:
break
if i % every == 0:
print datetime.datetime.now(), node
db.shutdown()
def get_node_info(dbname,nodeid):
from neo4j import GraphDatabase
from neo4j import OUTGOING, INCOMING, ANY
db = GraphDatabase(dbname)
nodeid = int(nodeid) #make sure it is an int
nd = db.node[nodeid]
print nd
for key,value in nd.items():
print "\t"+key,value
if "uniqname" in str(key):
break
db.shutdown()
def get_tree_from_rels(dbname,ottolid,reltype,depth):
from neo4j import GraphDatabase
from neo4j import OUTGOING, INCOMING, ANY
db = GraphDatabase(dbname)
with db.transaction:
#get the node based on the ottolid
node_idx = db.node.indexes.get('graphNamedNodes')
hits = node_idx['name'][name]
stnd = i
for i in hits:
print i
stnd = i
break
hits.close()
def test_create_configured_db(self):
folder_to_put_db_in = tempfile.mkdtemp()
try:
# START SNIPPET: creatingConfiguredDatabase
from neo4j import GraphDatabase
# Example configuration parameters
db = GraphDatabase(folder_to_put_db_in, string_block_size=200, array_block_size=240)
db.shutdown()
# END SNIPPET: creatingConfiguredDatabase
finally:
if os.path.exists(folder_to_put_db_in):
import shutil
shutil.rmtree(folder_to_put_db_in)
def run(self):
#IMPORTANT:
#The python embedded bindings for Neo4j use JPype. Based on the Neo4j mailing list thread
#entitled [Neo4j] Java segfaults using paste and Python bindings, the Neo4j Python binding
#needs to be single threaded. Thus, waiting to import until the thread is ready and running.
from neo4j import GraphDatabase
gdb = None
index_nodes = None
try:
#print "Connecting to database: ",devknowledge.settings.NEO4J_DATABASE
gdb = GraphDatabase(devknowledge.settings.NEO4J_DATABASE)
except Exception, e:
print "Error: ", e.message()
def init_globals():
'''Restore global varibales on a running db'''
global db
global ROOT, PAPER, AUTHOR
global author_idx, source_idx, label_idx, search_idx
db = GraphDatabase(neo4j_db_folder)
label_idx = db.node.indexes.get('label_idx')
source_idx = db.node.indexes.get('source_idx')
author_idx = db.node.indexes.get('author_idx')
search_idx = db.node.indexes.get('search_idx')
AUTHOR = label_idx['label']['AUTHOR'].single
PAPER = label_idx['label']['PAPER'].single
ROOT = db.reference_node
def test_create_db(self):
folder_to_put_db_in = tempfile.mkdtemp()
try:
# START SNIPPET: creatingDatabase
from neo4j import GraphDatabase
# Create db
db = GraphDatabase(folder_to_put_db_in)
# Always shut down your database
db.shutdown()
# END SNIPPET: creatingDatabase
finally:
if os.path.exists(folder_to_put_db_in):
import shutil
shutil.rmtree(folder_to_put_db_in)
def get_nodeinfo_for_name(dbname,name):
from neo4j import GraphDatabase
from neo4j import OUTGOING, INCOMING, ANY
db = GraphDatabase(dbname)
with db.transaction:
node_idx = db.node.indexes.get('graphNamedNodes')
hits = node_idx['name'][name]
for i in hits:
print i
for key,value in i.items():
print "\t"+key,value
#printing after, prints mrcas which are usually too long
#comment to let it go, but don't commit
if "uniqname" in str(key):
break
hits.close()
db.shutdown()
def test_hello_world(self):
folder_to_put_db_in = tempfile.mkdtemp()
try:
# START SNIPPET: helloworld
from neo4j import GraphDatabase
# Create a database
db = GraphDatabase(folder_to_put_db_in)
# All write operations happen in a transaction
with db.transaction:
firstNode = db.node(name='Hello')
secondNode = db.node(name='world!')
# Create a relationship with type 'knows'
relationship = firstNode.knows(secondNode, name='graphy')
# Read operations can happen anywhere
message = ' '.join(
[firstNode['name'], relationship['name'], secondNode['name']])
print message
# Delete the data
with db.transaction:
firstNode.knows.single.delete()
firstNode.delete()
secondNode.delete()
# Always shut down your database when your application exits
db.shutdown()
# END SNIPPET: helloworld
finally:
if os.path.exists(folder_to_put_db_in):
import shutil
shutil.rmtree(folder_to_put_db_in)
self.assertEqual(message, 'Hello graphy world!')
def graph_factory():
"""Utility to store the db graph conneciton object
"""
settings = get_current_registry().settings
connection_string = settings and settings['graphdb_connection_string'] \
or "testdb"
import os
os.environ['NEO4J_PYTHON_JVMARGS'] = '-Xms128M -Xmx512M'
from neo4j import GraphDatabase
db = GraphDatabase(connection_string)
def close_db():
"""Make sure to always close the database
"""
try:
db.shutdown()
print("db shut down")
except NameError:
print 'Could not shutdown Neo4j database.'
atexit.register(close_db)
return EmbeddedGraph(db)
from gexf import *
import os
import time
import pprint
import random
os.environ['JAVA_HOME'] ='/usr/lib/jvm/java-7-oracle'
from neo4j import GraphDatabase
import requests
gexf = Gexf("Sean Voss","Crunchbase")
graph=gexf.addGraph("directed","static","a graph")
created = []
i = 0
db = GraphDatabase('/var/www/spinner/db1')
print len(db.nodes)
for a in db.nodes:
if a.has_key('name'):
node = graph.addNode(a.id,a['name'].encode('ascii', 'ignore'))
node.setColor(unicode(random.randrange(1,255)),unicode(random.randrange(1,255)),unicode(random.randrange(1,255)))
#node.addAttribute('size',len(a.relationships))
for a in db.nodes:
if a.has_key('name'):
#graph.addNode(a.id,a['name'])
#gexf_sm = Gexf("Sean Voss","Crunchbase")
#graph_sm = gexf_sm.addGraph("directed","static","a graph")
def connect_to_neo4j(self, db_path):
try:
db = GraphDatabase(db_path)
except Exception, e:
print "%s: Could not connect to database." % e
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from neo4j import GraphDatabase
#open neo4j database
db = GraphDatabase('test.db')
#read
a_node = db.node[97]
print(a_node)
# Loop property values
for value in a_node.values():
print(value)
#shutdown database
db.shutdown()
def __init__(self, neo4j_url="graph.db"):
self.db = GraphDatabase(neo4j_url)
self.nodes = {}
self.rels = []
os.environ['JAVA_HOME'] = '/usr/lib/jvm/java-6-openjdk/jre/'
from neo4j import GraphDatabase, INCOMING, Evaluation
import sys
sys.path.append('../MetaImport')
from MetaRead import get_json_from_dir
sys.path.append('../tools')
from shared import group_generator
from time import time
DEBUG = 1
db = GraphDatabase('../DATA/NEO4J/')
ROOT = None
match_file = '../DATA/ALL_MATCH.txt'
meta_json_dir = '../DATA/META/JSON/'
def main():
if not db.node.indexes.exists('author_idx'):
print "Setup new db"
setup_db()
else:
print "Reading db"
init_globals()
# Add paper nodes with information from meta_pkl_feed
def test_invoice_app(self):
folder_to_put_db_in = tempfile.mkdtemp()
try:
# START SNIPPET: invoiceapp-setup
from neo4j import GraphDatabase, INCOMING, Evaluation
# Create a database
db = GraphDatabase(folder_to_put_db_in)
# All write operations happen in a transaction
with db.transaction:
# A node to connect customers to
customers = db.node()
# A node to connect invoices to
invoices = db.node()
# Connected to the reference node, so
# that we can always find them.
db.reference_node.CUSTOMERS(customers)
db.reference_node.INVOICES(invoices)
# An index, helps us rapidly look up customers
customer_idx = db.node.indexes.create('customers')
# END SNIPPET: invoiceapp-setup
# START SNIPPET: invoiceapp-domainlogic-create
def create_customer(name):
with db.transaction:
customer = db.node(name=name)
customer.INSTANCE_OF(customers)
# Index the customer by name
customer_idx['name'][name] = customer
return customer
def create_invoice(customer, amount):
with db.transaction:
invoice = db.node(amount=amount)
invoice.INSTANCE_OF(invoices)
invoice.SENT_TO(customer)
return customer
# END SNIPPET: invoiceapp-domainlogic-create
# START SNIPPET: invoiceapp-domainlogic-get-by-idx
def get_customer(name):
return customer_idx['name'][name].single
# END SNIPPET: invoiceapp-domainlogic-get-by-idx
# START SNIPPET: invoiceapp-domainlogic-get-by-cypher
def get_invoices_with_amount_over(customer, min_sum):
# Find all invoices over a given sum for a given customer.
# Note that we return an iterator over the "invoice" column
# in the result (['invoice']).
return db.query('''START customer=node({customer_id})
MATCH invoice-[:SENT_TO]->customer
WHERE has(invoice.amount) and invoice.amount >= {min_sum}
RETURN invoice''',
customer_id = customer.id, min_sum = min_sum)['invoice']
# END SNIPPET: invoiceapp-domainlogic-get-by-cypher
# START SNIPPET: invoiceapp-create-and-search
for name in ['Acme Inc.', 'Example Ltd.']:
create_customer(name)
# Loop through customers
for relationship in customers.INSTANCE_OF:
customer = relationship.start
for i in range(1,12):
create_invoice(customer, 100 * i)
# Finding large invoices
large_invoices = get_invoices_with_amount_over(get_customer('Acme Inc.'), 500)
# Getting all invoices per customer:
for relationship in get_customer('Acme Inc.').SENT_TO.incoming:
invoice = relationship.start
# END SNIPPET: invoiceapp-create-and-search
self.assertEqual(len(list(large_invoices)), 7)
db.shutdown()
finally:
if os.path.exists(folder_to_put_db_in):
import shutil
shutil.rmtree(folder_to_put_db_in)
def __init__(self, path):
self.db = GraphDatabase(path)
import time
import natLanProc
tags = ['#ows','#anonymous']
per_page = 1000
la_ID=None
jsonInfo=None
l_id = None
#contained within this comment are the functions for neo4j
server_folder = "/var/lib/neo4j/data/graph.db"
test_folder = "/var/www/twitter/DB/anon-ows-clean"
folder_to_put_db_in = test_folder
# Create a database
db = GraphDatabase(folder_to_put_db_in)
#small class to hold our tags
class tag:
def __init__(self,tag):
self.tag=tag
def setLA(self,la):
self.la = la
#threaded url-parser using natLanProc class to be able to timeout requests
class threadParser(threading.Thread):
def setLink(self,link,natLanProc):
self.link = link
self.natLanProc = natLanProc
self.summary = None
def __init__(self, databasePath, datasetDict):
#increment to existing db
self.db = GraphDatabase(databasePath)
self.datasetDict = datasetDict
import os,jpype,time,random, cPickle
os.environ['NEO4J_PYTHON_JVMARGS'] = '-Xms4G -Xmx8G -XX:MaxPermSize=1G'
os.environ['JAVA_HOME']='/usr/lib/jvm/jdk1.7.0/'
from neo4j import GraphDatabase
db = GraphDatabase('data')
print 'deleting'
with db.transaction:
db.query('start r=relationship(*) delete r');
db.query('start n=node(*) delete n;')
print 'reading'
friends = cPickle.load(open('friends.pickle'))
print 'nodes'
nodes = {}
with db.transaction:
for a,data in friends.items():
n = db.node(noscenda_name='person%s'%a)
nodes[a]=n
if a%1000==0:
print 'n ',a
i= 0
print 'relations'
with db.transaction:
for a,targets in friends.items():
for b in targets:
i+=1
getattr(nodes[a],'friend')(nodes[b])
if i%1000==0:
File: database.py
Author: huxuan - i(at)huxuan.org
Created: 2012-11-28
Last modified: 2012-11-30
Description:
Actions related to database only
Copyrgiht (c) 2012 by huxuan. All rights reserved.
License GPLv3
"""
from neo4j import GraphDatabase
from config import DBNAME
GRAPHDB = GraphDatabase(DBNAME)
if GRAPHDB.node.indexes.exists('user'):
USER_IDX = GRAPHDB.node.indexes.get('user')
else:
USER_IDX = GRAPHDB.node.indexes.create('user')
if GRAPHDB.node.indexes.exists('tweet'):
TWEET_IDX = GRAPHDB.node.indexes.get('tweet')
else:
TWEET_IDX = GRAPHDB.node.indexes.create('tweet')
TWEET_REF_ID = 1000000
if len(TWEET_IDX['tid'][TWEET_REF_ID]) == 0:
with GRAPHDB.transaction:
TWEET_REF = GRAPHDB.node()
from time import clock
os.environ['JAVA_HOME'] = '/usr/lib/jvm/java-7-oracle/'
#os.environ['JAVA_HOME'] = '/usr/lib/jvm/java-7-openjdk-amd64/'
from neo4j import GraphDatabase, INCOMING, Evaluation, OUTGOING, ANY
def TipoVar(v):
tipo = type(v)
if tipo == int:
return 'int'
elif tipo == float:
return 'float'
elif tipo == str:
return 'string'
elif tipo == list:
return 'list'
else:
return None
db = GraphDatabase('teste')
for node in db.nodes:
for item in node.items():
for a in item:
print type(a).decode('ascii')
db.shutdown()
