def makeNeoDriver(self, neo_uri, neo_user, neo_pass):
if neo_uri is not None:
self.driverUuid = self.get_uuid()
self.driverStartedAt = datetime.now(timezone.utc)
if neoVersion[0] == '4':
self.neoDriver = GraphDatabase.driver(
uri=neo_uri,
auth=basic_auth(
user=neo_user,
password=neo_pass,
),
max_transaction_retry_time=2
# max_connection_lifetime=200,
# encrypted=True,
)
if neoVersion[0] == '1':
self.neoDriver = GraphDatabase.driver(
uri=neo_uri,
auth=basic_auth(
user=neo_user,
password=neo_pass,
),
# max_connection_lifetime=200,
encrypted=True,
max_retry_time=2)
def connect(self, url=None, user=None, password=None, **kw):
"""
Parse a Neo4J URL and attempt to connect using Bolt
Note: If the user and password arguments are provided, they
will only be used in case no auth information is provided as
part of the connection URL.
"""
if url is None:
url = 'bolt://localhost'
if user is None:
user = '******'
if password is None:
password = '******'
try:
protocol, url = url.split('://')
if protocol.lower() != 'bolt':
warnings.warn('Switching protocols. Only Bolt is supported.')
except ValueError:
pass
try:
credentials, url = url.split('@')
except ValueError:
kw['auth'] = basic_auth(user, password)
else:
kw['auth'] = basic_auth(*credentials.split(':', 1))
self.driver = GraphDatabase.driver('bolt://%s' % url, **kw)
def makeNeoDriver(self, neo_uri, neo_user, neo_pass):
if neo_uri is not None:
self.driverUuid = self.get_uuid()
self.driverStartedAt = datetime.now(timezone.utc)
if self.verbose:
print(
f'serpentmonkee.makeNeoDriver: neoVersion = {neoVersion}')
if neoVersion[0] == '4':
if self.verbose:
print(
f'serpentmonkee.makeNeoDriver: max_transaction_retry_time:{self.maxTransactionRetryTime}, max_connection_lifetime:{self.maxConnectionLifetime}, connection_timeout:{self.connectionTimeout}, keep_alive:{self.keepAlive}'
)
self.neoDriver = GraphDatabase.driver(
uri=neo_uri,
auth=basic_auth(
user=neo_user,
password=neo_pass,
),
max_transaction_retry_time=self.maxTransactionRetryTime,
max_connection_lifetime=self.maxConnectionLifetime,
connection_timeout=self.connectionTimeout,
keep_alive=self.keepAlive)
if neoVersion[0] == '1':
self.neoDriver = GraphDatabase.driver(
uri=neo_uri,
auth=basic_auth(
user=neo_user,
password=neo_pass,
),
max_connection_lifetime=self.maxConnectionLifetime,
encrypted=True,
max_retry_time=self.maxTransactionRetryTime)
def connect(self, authentication_options):
"""
The connect method to create a new instance of the db_driver
:param authentication_options: Username, password, host and encrypted option
:return: None
:rtype: None
:raises: InvalidOptionsException
"""
if authentication_options is None:
raise InvalidOptionsException()
user_name = authentication_options.get("user_name")
user_pass = authentication_options.get("user_password")
encrypted = authentication_options.get("encrypted")
host = authentication_options.get("host")
if encrypted is None:
encrypted = False
if user_name is None or user_pass is None or host is None:
raise InvalidOptionsException()
try:
self.driver = GraphDatabase.driver("bolt://{}".format(host),
encrypted=encrypted,
auth=basic_auth(
user_name, user_pass))
except SessionError as e:
raise InvalidOptionsException(e)
self._create_session()
def build_dataset():
neo_client = GraphDatabase.driver("bolt://139.88.179.199:7687", auth=basic_auth("neo4j", "testing"), encrypted=False)
image_dir = '../../../data/images/'
files = []
ids = dict()
for filename in os.listdir(image_dir):
uuid = filename.split('_')[0]
with neo_client.session() as session:
records = list(session.run('match (s) where s.uuid = \'{uuid}\' return s.name'.format(uuid=uuid)).records())
name = records[0]['s.name']
if 'Encephalartos' in name:
ids[uuid] = 0
else:
ids[uuid] = 1
filepath = image_dir + filename
try:
Image.open(filepath)
files.append(filepath)
except OSError as e:
print(e)
pass
shuffle(files)
cutoff = int(0.8 * len(files))
print(len(files), cutoff)
return Dataset(files[:cutoff], ids), Dataset(files[cutoff:], ids)
def set_connection(self, url):
"""
Sets the connection URL to the address a Neo4j server is set up at
"""
u = urlparse(url)
if u.netloc.find('@') > -1 and (u.scheme == 'bolt'
or u.scheme == 'bolt+routing'
or u.scheme == 'neo4j'):
credentials, hostname = u.netloc.rsplit('@', 1)
username, password, = credentials.split(':')
else:
raise ValueError(
"Expecting url format: bolt://user:password@localhost:7687"
" got {0}".format(url))
self.database = config.DATABASE_NAME
self.driver = GraphDatabase.driver(
u.scheme + '://' + hostname,
auth=basic_auth(username, password),
encrypted=config.ENCRYPTED_CONNECTION,
max_connection_pool_size=config.MAX_POOL_SIZE)
self.url = url
self._pid = os.getpid()
self._active_transaction = None
def __create_course(self, semester_record, course_id, semester,
course_name, dept_id, credits, desc, geneds,
relationships):
try:
with self._driver.session() as session:
query_create_rel = """
MATCH (s:Semester{id: $semester_id}), (d:Department{id: $department_id})
CREATE (c:Course{id: $course_id, name: $course_name, credits: $credits, description: $desc}),
(c) -[:DURING]-> (s), (c) -[:WITHIN]-> (d)
RETURN c, d, s
"""
result = session.run(query_create_rel,
parameters={
"semester_id": semester,
"department_id": dept_id,
"course_id": course_id,
"course_name": course_name,
"credits": credits,
"desc": desc
})
return result
except:
self._driver.close()
self._driver = GraphDatabase.driver(os.getenv("BOLT_URL"),
auth=basic_auth(
os.getenv("USER"),
os.getenv("PASSWORD")))
self.__create_course(record[0], course["course_id"], semester,
course["name"], course["dept_id"],
course["credits"], course["description"],
course["gen_ed"], course["relationships"])
def scrape_news(user_id, lang):
driver = GraphDatabase.driver(NEO4J_CONN,
auth=basic_auth(NEO4J_USER, NEO4J_PASS))
session = driver.session()
keywords = get_keywords(session, user_id, lower=True)
if lang == 'es':
links = [
scrap_bbc_spanish(keywords),
scrape_investing_spanish(keywords),
scrape_expansion_spanish(keywords),
scrape_gizmodo_spanish(keywords),
scrape_elpais_spanish(keywords),
scrape_cnet_spanish(keywords),
scrape_eleconomista_spanish(keywords),
scrape_computerhoy_spanish(keywords),
scrape_hoy_spanish(keywords),
scrape_genbeta_spanish(keywords)
]
else:
links = [
scrap_techcrunch(keywords),
scrap_business_insider(keywords),
scrap_reuters(keywords)
]
links = list(itertools.chain.from_iterable(links))
session.close()
return links
def start(bot, update):
driver = GraphDatabase.driver(NEO4J_CONN,
auth=basic_auth(NEO4J_USER, NEO4J_PASS))
session = driver.session()
user_id = update.message.chat_id
user_first_name = update.effective_user.first_name
# delete user
session.run(
"MATCH (n:User {{id: '{}'}})-[r]->(k) DELETE r".format(user_id))
session.run("MATCH (n:User {{id: '{}'}}) DELETE n".format(user_id))
if user_exists(session, user_id):
lang = get_lang(session, user_id)
bot.sendMessage(chat_id=user_id,
text=lang.welcome_back + user_first_name +
"! \U0001F496")
else:
keyboard = [[
InlineKeyboardButton("English", callback_data='en'),
InlineKeyboardButton("Spanish", callback_data='es')
]]
reply_markup = InlineKeyboardMarkup(keyboard)
bot.sendMessage(chat_id=user_id,
text='First choose a language:',
reply_markup=reply_markup)
session.close()
return LANGUAGE
session.close()
return -1
def get_db():
if 'db' not in g:
g.db = GraphDatabase.driver(NEO4J_URI,
auth=basic_auth(user=NEO4J_CREDS[0],
password=NEO4J_CREDS[1]),
encrypted=False)
return g.db
def set_neo4j_workflow_roles(workflow_uuid):
wf_obj = WorkflowNeo4j.objects.get(uuid=workflow_uuid)
bolt_driver = GraphDatabase.driver(bolt_url,
auth=basic_auth(neo_user, neo_pass))
db = bolt_driver.session()
statement = "MATCH path = (n)-[r]->(m) WHERE n.GraphID = '" + str(workflow_uuid) + \
"' AND n.SAFEType in ['common-set', 'template-user-set', 'None'] RETURN path"
results = db.run(statement).graph()
db.close()
for node in results.nodes:
for key in node.keys():
if key == 'Role':
roles = str(node[key])
for role in roles.split(','):
if role == 'None':
pass
elif role == 'PI':
# if not wf_obj.roles.filter(id=int(getattr(Role, 'PI_ADMIN'))).exists():
# wf_obj.roles.add(int(getattr(Role, 'PI_ADMIN')))
if not wf_obj.roles.filter(id=int(
getattr(WorkflowRole, 'PI_MEMBER'))).exists():
wf_obj.roles.add(
int(getattr(WorkflowRole, 'PI_MEMBER')))
# print("Adding role " + str(WorkflowRole.objects.get(id=int(getattr(WorkflowRole, 'PI_MEMBER')))))
else:
if not wf_obj.roles.filter(
id=int(getattr(WorkflowRole, role))).exists():
wf_obj.roles.add(int(getattr(WorkflowRole, role)))
# print("Adding role " + str(WorkflowRole.objects.get(id=int(getattr(WorkflowRole, role)))))
wf_obj.save()
def get_db():
if not hasattr(g, "neo4j_db"):
driver = GraphDatabase.driver(DATABASE_URL,
auth=basic_auth(DATABASE_USERNAME,
DATABASE_PASSWORD))
g.neo4j_db = driver.session()
return g.neo4j_db
def cypher(plpy, query, params, url, login, password):
"""
Make cypher query and return JSON result
"""
driver = GraphDatabase.driver(url, auth=basic_auth(login, password))
session = driver.session()
log_to_postgres(
"Cypher function with query " + query + " and params " +
unicode(params), DEBUG)
# Execute & retrieve neo4j data
try:
for record in session.run(query, ast.literal_eval(params)):
jsonResult = "{"
for key in record.keys():
if len(jsonResult) > 1:
jsonResult += ","
jsonResult += '"' + key + '":'
object = record[key]
if object.__class__.__name__ == "Node":
jsonResult += node2json(object)
elif object.__class__.__name__ == "Relationship":
jsonResult += relation2json(object)
elif object.__class__.__name__ == "Path":
jsonResult += path2json(object)
else:
jsonResult += json.dumps(object)
jsonResult += "}"
yield jsonResult
except CypherError:
raise RuntimeError("Bad cypher query : " + statement)
finally:
session.close()
def main(argv):
driver = GraphDatabase.driver("neo4j://localhost:7687",
auth=basic_auth("neo4j", "password"),
encrypted=False)
main_dir = os.path.dirname(argv[0]) if os.path.dirname(argv[0]) else "."
data_dir = main_dir + "/../../../Lectures/SEMESTER_2/neo4j/data_integration/test_data"
print("Importing data from %s directory" % data_dir)
with driver.session() as session:
importer = Neo4jPatientsMutationsImporter(session)
print("Importing Patients' data...")
importer.run_queries(
importer.input_data(data_dir + "/Sample.csv", "Patient"))
print("Importing HugoSymbol...")
importer.run_queries(
importer.input_data(data_dir + "/HugoSymbol.csv", "HugoSymbol"))
print("Importing Patients' mutations...")
importer.run_queries(
importer.relate_patient_and_hugo_symbol(data_dir +
"/mutation.csv"))
print("Importing done.")
print(
"Visualise imported data with the following CQL: MATCH (a:Patient)--(b:HugoSymbol) RETURN a, b LIMIT 20"
)
def __init__(self, options, columns):
# Calling super constructor
super(Neo4jForeignDataWrapper, self).__init__(options, columns)
# Managed server option
if 'url' not in options:
log_to_postgres('The Bolt url parameter is required and the default is "bolt://localhost:7687"', WARNING)
self.url = options.get("url", "bolt://localhost:7687")
if 'user' not in options:
log_to_postgres('The user parameter is required and the default is "neo4j"', ERROR)
self.user = options.get("user", "neo4j")
if 'password' not in options:
log_to_postgres('The password parameter is required for Neo4j', ERROR)
self.password = options.get("password", "")
if 'cypher' not in options:
log_to_postgres('The cypher parameter is required', ERROR)
self.cypher = options.get("cypher", "MATCH (n) RETURN n LIMIT 100")
# Setting table columns
self.columns = columns
# Create a neo4j driver instance
self.driver = GraphDatabase.driver( self.url, auth=basic_auth(self.user, self.password))
self.columns_stat = self.compute_columns_stat()
self.table_stat = int(options.get("estimated_rows", -1))
if(self.table_stat < 0):
self.table_stat = self.compute_table_stat()
log_to_postgres('Table estimated rows : ' + unicode(self.table_stat), DEBUG)
def test_should_generate_base_auth_with_realm_correctly(self):
auth = basic_auth("molly", "meoooow", "cat_cafe")
assert auth.scheme == "basic"
assert auth.principal == "molly"
assert auth.credentials == "meoooow"
assert auth.realm == "cat_cafe"
assert not hasattr(auth, "parameters")
def __init__(self, **kwargs):
"""Initialize."""
super().__init__(**kwargs)
self.url = f'bolt://{self.hostname}:{self.port}'
self.driver = GraphDatabase.driver(self.url,
auth=self.auth
and basic_auth(*self.auth))
def registration():
form = Registration()
if form.validate_on_submit():
flash('Account successfully created')
gdb1 = GraphDatabase.driver(bolt_url,
auth=basic_auth(usr_nm_db, pswrd_db))
gdb = gdb1.session()
query = '''CREATE(:users{username:'******', email:'%s', password:'******'})''' % (
form.username.data, form.email.data, form.password.data)
gdb.run(query)
if (form.Animation.data == True):
query = '''MATCH (a:users),(g:Genre) where a.username= '******' AND g.name='Animation' CREATE (a)-[:likes]->(g)
return g.name''' % (form.username.data)
result = gdb.run(query)
print(result.values())
if (form.Adventure.data == True):
query = '''MATCH (a:users),(g:Genre) where a.username= '******' AND g.name='Adventure' CREATE (a)-[:likes]->(g)
return g.name''' % (form.username.data)
result = gdb.run(query)
print(result.values())
if (form.Comedy.data == True):
query = '''MATCH (a:users),(g:Genre) where a.username= '******' AND g.name='Comedy' CREATE (a)-[:likes]->(g)
return g.name''' % (form.username.data)
result = gdb.run(query)
print(result.values())
if (form.Children.data == True):
query = '''MATCH (a:users),(g:Genre) where a.username= '******' AND g.name='Children' CREATE (a)-[:likes]->(g)
return g.name''' % (form.username.data)
result = gdb.run(query)
print(result.values())
if (form.Action.data == True):
query = '''MATCH (a:users),(g:Genre) where a.username= '******' AND g.name='Action' CREATE (a)-[:likes]->(g)
return g.name''' % (form.username.data)
result = gdb.run(query)
print(result.values())
if (form.Crime.data == True):
query = '''MATCH (a:users),(g:Genre) where a.username= '******' AND g.name='Crime' CREATE (a)-[:likes]->(g)
return g.name''' % (form.username.data)
result = gdb.run(query)
print(result.values())
if (form.Romance.data == True):
query = '''MATCH (a:users),(g:Genre) where a.username= '******' AND g.name='Romance' CREATE (a)-[:likes]->(g)
return g.name''' % (form.username.data)
result = gdb.run(query)
print(result.values())
if (form.Drama.data == True):
query = '''MATCH (a:users),(g:Genre) where a.username= '******' AND g.name='Drama' CREATE (a)-[:likes]->(g)
return g.name''' % (form.username.data)
result = gdb.run(query)
print(result.values())
print(form.username.data)
return redirect(url_for('login'))
return render_template('registration.html', form=form)
def login():
form = Login()
if form.validate_on_submit():
user = form.username.data
passw = form.password.data
gdb1 = GraphDatabase.driver(bolt_url,
auth=basic_auth(usr_nm_db, pswrd_db))
gdb = gdb1.session()
query = '''MATCH(m:users)WHERE m.username='******' RETURN m.password AS pass''' % (
user)
result = gdb.run(query, parameters={})
result = result.value()
print(result)
if result[0] == passw:
print('Successfully logged in')
session['username'] = user
query = '''MATCH((u:users)-[:VIEWED]->(m:Movie)) where u.username ="******" RETURN m''' % (
user)
results = gdb.run(query)
results = results.value()
print("haba", results)
if len(results) == 0:
print("right")
return redirect(url_for('choose'))
else:
print("wrong")
return redirect(url_for('index'))
else:
print('password does not match')
return render_template('login.html', form=form)
def __init__(self):
self.driver = GraphDatabase.driver('bolt://' + self.HOST,
auth=basic_auth(
self.NEO4J_USERNAME,
self.NEO4J_PASSWORD),
max_connecion_pool_size=9999999999,
connection_acquisition_timeout=30)
self.db = self.get_db()
def connect_to_graphenedb():
global gp
graphenedb_url = Config.GRAPHENEDB_BOLT_URL
graphenedb_user = Config.GRAPHENEDB_BOLT_USER
graphenedb_pass = Config.GRAPHENEDB_BOLT_PASSWORD
gp = GraphDatabase.driver(graphenedb_url,
auth=basic_auth(graphenedb_user,
graphenedb_pass))
def run():
ping('metadata_database', verbose=True)
ping('temporal_index_database', verbose=True)
ping('spatial_index_database', verbose=True)
ping('thematic_index_solr', verbose=True)
engine1 = create_engine(config.model_config["db_connection"])
with engine1.connect() as conn:
test = conn.execute("""SELECT table_name
FROM information_schema.tables
WHERE table_schema='public'
AND table_type='BASE TABLE';""").fetchall()
if len(test) < 2:
raise Exception("As tabelas não foram criadas ainda.")
test = conn.execute("SELECT trigger_name FROM information_schema.triggers;").fetchall()
if len(test) < 2:
raise Exception("Os gatilhos não foram criadas ainda.")
del engine1
engine2 = create_engine(config.temporal_config["db_connection"])
with engine2.connect() as conn:
test = conn.execute("""SELECT table_name
FROM information_schema.tables
WHERE table_schema='public'
AND table_type='BASE TABLE';""").fetchall()
if len(test) < 1:
raise Exception("As tabelas não foram criadas ainda.")
test = conn.execute("""SELECT routine_name
FROM information_schema.routines
WHERE routine_type='FUNCTION'
AND specific_schema='public';""").fetchall()
if len(test) < 4:
raise Exception("Os procedimentos armazenados não foram criados ainda.")
del engine2
driver = GraphDatabase.driver(config.spatial_config["db_connection"]["bolt_uri"],
auth=basic_auth(config.spatial_config["db_connection"]["user"],
config.spatial_config["db_connection"]["password"]),
encrypted=False)
with driver.session() as session:
test = session.run("match(p:Place) return count(p);").value()
if test[0] < 5596:
raise Exception("O grafo de lugares não foi totalmente criado.")
del driver
solr_resource = pysolr.Solr(config.thematic_config["resource_solr_core_uri"], always_commit=True)
solr_resource.ping()
solr_dataset = pysolr.Solr(config.thematic_config["dataset_solr_core_uri"], always_commit=True)
solr_dataset.ping()
del solr_dataset
del solr_resource
def neo4jConn(neo4jUri,neo4jUser,neo4jPass):
try:
driver = GraphDatabase.driver(neo4jUri, auth=basic_auth(user=neo4jUser, password=neo4jPass))
#print("[+] Successful connection with database")
return(driver)
except Exception as e:
print("[-] %s" % e)
sys.exit(1)
def search(var):
gdb1 = GraphDatabase.driver(bolt_url, auth=basic_auth(usr_nm_db, pswrd_db))
gdb = gdb1.session()
query = '''MATCH (m:Movie)
WHERE toLower(m.title) contains '%s'
RETURN m.title,m.imdbId''' % var
results = gdb.run(query)
results = results.values()
return render_template('search.html', results=results)
def __init__(self, uri, user, password, log_level=logging.INFO):
# logging.basicConfig(filename="neo4j.log",
# format='%(asctime)s %(message)s',
# filemode='w')
self._logger = logging.getLogger()
self._logger.setLevel(log_level)
self._logger.info(f'Connecting to {uri}')
self._driver = GraphDatabase.driver(uri,
auth=basic_auth(user, password))
def import_data(person1, person2):
with GraphDatabase.driver("bolt://localhost:7687",
auth=basic_auth("neo4j", password)) as driver:
with driver.session() as session:
result = session.run(query, {
"person1": person1,
"person2": person2
})
print(result.consume().counters)
def size_of_given_type_in_KP(self,
node_type,
use_cypher_command=True,
kg='KG1'):
"""
find all nodes of a certain type in KP
:param node_type: the query node type
:param use_cypher_command: Boolean (True or False). If True, it used cypher command to query all nodes otherwise used kgNodeIndex
:param kg: only allowed for choosing 'KG1' or 'KG2' now. Will extend to BTE later
"""
# TODO: extend this to KG2, BTE, and other KP's we know of
size_of_total = None
if kg == 'KG1' or kg == 'KG2':
pass
else:
self.response.error(
f"Only KG1 or KG2 is allowable to calculate the Fisher's exact test temporally"
)
return size_of_total
if kg == 'KG1':
if use_cypher_command:
rtxConfig = RTXConfiguration()
# Connection information for the neo4j server, populated with orangeboard
driver = GraphDatabase.driver(rtxConfig.neo4j_bolt,
auth=basic_auth(
rtxConfig.neo4j_username,
rtxConfig.neo4j_password))
session = driver.session()
query = "MATCH (n:%s) return count(distinct n)" % (node_type)
res = session.run(query)
size_of_total = res.single()["count(distinct n)"]
return size_of_total
else:
kgNodeIndex = KGNodeIndex()
size_of_total = kgNodeIndex.get_total_entity_count(node_type,
kg_name=kg)
return size_of_total
else:
if use_cypher_command:
self.response.warning(
f"KG2 is only allowable to use kgNodeIndex to query the total number of node with query type. It was set to use kgNodeIndex"
)
kgNodeIndex = KGNodeIndex()
size_of_total = kgNodeIndex.get_total_entity_count(node_type,
kg_name=kg)
return size_of_total
else:
kgNodeIndex = KGNodeIndex()
size_of_total = kgNodeIndex.get_total_entity_count(node_type,
kg_name=kg)
return size_of_total
def __init__(self, config=None):
if not config:
config = DEFAULT_CONF
config = config.copy()
username = config.pop('username')
password = config.pop('password')
config['auth'] = basic_auth(username, password)
self.driver = GraphDatabase.driver(**config)
self._cli = self.driver.session()
self._cql_stack = None
def _run(self, cypher_query):
if self._driver is None:
config = self.config
self._driver = GraphDatabase.driver(
config['URI'],
auth=basic_auth(config["USER"], config["PASSWORD"])
)
with self._driver.session() as session:
return session.run(cypher_query)
def __init__(self, hostname='localhost', user='******', password='******'):
## create a driver to the db
self._driver = GraphDB.driver(
'bolt://{}'.format(hostname),
auth=basic_auth(user, password)
)
## blank out the password
password=None
self.is_closed = False
