def get_similarity_network(self):
plot = None
try:
cwd = os.path.abspath(os.path.dirname(__file__))
query_path = os.path.join(cwd, self.queries_file)
project_cypher = query_utils.read_queries(query_path)
query = query_utils.get_query(project_cypher,
query_id="projects_subgraph")
list_projects = []
driver = connector.getGraphDatabaseConnectionConfiguration()
if "other_id" in self.similar_projects:
list_projects = self.similar_projects[
"other_id"].values.tolist()
list_projects.append(self.identifier)
list_projects = ",".join(['"{}"'.format(i) for i in list_projects])
query = query.replace("LIST_PROJECTS", list_projects)
path = connector.sendQuery(driver, query, parameters={}).data()
G = acore_utils.neo4j_path_to_networkx(path, key='path')
args = {}
style, layout = self.get_similarity_network_style()
args['stylesheet'] = style
args['layout'] = layout
args['title'] = "Projects subgraph"
net, mouseover = acore_utils.networkx_to_cytoscape(G)
plot = viz.get_cytoscape_network(net, "projects_subgraph", args)
except Exception as err:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logger.error(
"Error: {}. Reading queries from file {}: {}, file: {},line: {}"
.format(err, query_path, sys.exc_info(), fname,
exc_tb.tb_lineno))
return plot
def query_data(self):
data = {}
try:
cwd = os.path.abspath(os.path.dirname(__file__))
queries_path = os.path.join(cwd, self.queries_file)
project_cypher = query_utils.read_queries(queries_path)
driver = connector.getGraphDatabaseConnectionConfiguration()
replace = [("PROJECTID", self.identifier)]
for query_name in project_cypher:
title = query_name.lower().replace('_', ' ')
query = project_cypher[query_name]['query']
query_type = project_cypher[query_name]['query_type']
for r, by in replace:
query = query.replace(r, by)
if query_type == "pre":
data[title] = connector.getCursorData(driver, query)
except Exception as err:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logger.error(
"Reading queries from file {}: {}, file: {},line: {}, error: {}"
.format(queries_path, sys.exc_info(), fname, exc_tb.tb_lineno,
err))
return data
def parseUsersFile(importDirectory, expiration=365):
"""
Creates new user in the graph database and corresponding node, through the following steps:
1. Generates new user identifier
2. Checks if a user with given properties already exists in the database. If not:
3. Creates new local user (access to graph database)
4. Saves data to tab-delimited file.
:param str importDirectory: path to the directory where all the import files are generated.
:param int expiration: number of days a user is given access.
:return: Writes relevant .tsv file for the users in the provided file.
"""
usersDir = os.path.join(cwd, config['usersDirectory'])
usersFile = os.path.join(usersDir, config['usersFile'])
usersImportFile = os.path.join(importDirectory, config['import_file'])
driver = connector.getGraphDatabaseConnectionConfiguration(database=None)
data = pd.read_excel(usersFile).applymap(str)
date = datetime.today() + timedelta(days=expiration)
df = []
try:
user_identifier = get_new_user_identifier(driver)
if user_identifier is None:
user_identifier = 'U1'
new_id = int(re.search('\d+', user_identifier).group())
for index, row in data.iterrows():
username = check_if_node_exists(driver, 'username',
row['username'])
name = check_if_node_exists(driver, 'name', row['name'])
email = check_if_node_exists(driver, 'email', row['email'])
if username.empty and name.empty and email.empty:
row['ID'] = 'U{}'.format(new_id)
row['acronym'] = ''.join(
[c for c in row['name'] if c.isupper()])
row['rolename'] = 'reader'
row['expiration_date'] = date.strftime('%Y-%m-%d')
row['image'] = ''
#create_db_user(driver, row)
row['password'] = bcrypt.encrypt(row['password'])
df.append(row)
new_id += 1
except Exception as err:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logger.error("Extracting users info: {}, file: {},line: {}".format(
sys.exc_info(), fname, exc_tb.tb_lineno))
if len(df) > 0:
data = pd.DataFrame(df)
data['phone_number'] = data['phone_number'].str.split('.').str[0]
data = data[[
'ID', 'acronym', 'name', 'username', 'password', 'email',
'secondary_email', 'phone_number', 'affiliation',
'expiration_date', 'rolename', 'image'
]]
GenerateGraphFiles(data, usersImportFile)
def create_new_project(identifier, data, separator='|'):
driver = connector.getGraphDatabaseConnectionConfiguration()
project_result, projectId = projectCreation.create_new_project(driver, identifier, pd.read_json(data), separator=separator)
if projectId is not None:
result = {str(projectId): str(project_result)}
else:
result = {}
return result
def getMappingFromDatabase(id_list, node, attribute_from='id', attribute_to='name'):
id_list = ["'{}'".format(i) for i in id_list]
driver = connector.getGraphDatabaseConnectionConfiguration()
mapping_query = "MATCH (n:{}) WHERE n.{} IN [{}] RETURN n.{} AS from, n.{} AS to"
mapping = connector.getCursorData(driver, mapping_query.format(node, attribute_from, ','.join(id_list), attribute_from, attribute_to))
if not mapping.empty:
mapping = dict(zip(mapping['from'], mapping['to']))
return mapping
def create_user(data, output_file, expiration=365):
"""
Creates new user in the graph database and corresponding node, through the following steps:
1. Checks if a user with given properties already exists in the database. If not:
2. Generates new user identifier
3. Creates new local user (access to graph database)
4. Creates new user node
5. Saves data to users.tsv
:param data: pandas dataframe with users as rows and arguments and columns.
:param str output_file: path to output csv file.
:param int expiration: number of days users is given access.
:return: Writes relevant .tsv file for the users in data.
"""
driver = connector.getGraphDatabaseConnectionConfiguration(database=None)
date = datetime.today() + timedelta(days=expiration)
df = []
try:
for index, row in data.iterrows():
username = uh.check_if_node_exists(driver, 'username',
row['username'])
name = uh.check_if_node_exists(driver, 'name', row['name'])
email = uh.check_if_node_exists(driver, 'email', row['email'])
if username.empty and name.empty and email.empty:
user_identifier = uh.get_new_user_identifier(driver)
if user_identifier is None:
user_identifier = 'U1'
row['ID'] = user_identifier
row['acronym'] = ''.join(
[c for c in row['name'] if c.isupper()])
row['rolename'] = 'reader'
row['expiration_date'] = date.strftime('%Y-%m-%d')
row['image'] = ''
uh.create_db_user(driver, row)
row['password'] = bcrypt.encrypt(row['password'])
create_user_node(driver, row)
df.append(row)
except Exception as err:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logger.error("Creating users: file: {},line: {}, error: {}".format(
fname, exc_tb.tb_lineno, err))
if len(df) > 0:
data = pd.DataFrame(df)
data['phone_number'] = data['phone_number'].str.split('.').str[0]
data = data[[
'ID', 'acronym', 'name', 'username', 'password', 'email',
'secondary_email', 'phone_number', 'affiliation',
'expiration_date', 'rolename', 'image'
]]
uh.GenerateGraphFiles(data, output_file)
def get_mapping_analytical_samples(project_id):
from graphdb_connector import connector
driver = connector.getGraphDatabaseConnectionConfiguration()
mapping = {}
query = "MATCH (p:Project)-[:HAS_ENROLLED]-(:Subject)-[:BELONGS_TO_SUBJECT]-()-[:SPLITTED_INTO]-(a:Analytical_sample) WHERE p.id='{}' RETURN a.external_id, a.id".format(project_id)
mapping = connector.getCursorData(driver, query)
if not mapping.empty:
mapping = mapping.set_index("a.external_id").to_dict(orient='dict')["a.id"]
return mapping
def create_new_identifiers(project_id, data, directory, filename):
driver = connector.getGraphDatabaseConnectionConfiguration()
upload_result = dataUpload.create_experiment_internal_identifiers(
driver, project_id,
pd.read_json(data,
dtype={
'subject external_id': object,
'biological_sample external_id': object,
'analytical_sample external_id': object
}), directory, filename)
res_n = dataUpload.check_samples_in_project(driver, project_id)
return {str(project_id): str(upload_result), 'res_n': res_n.to_dict()}
def fullUpdate():
"""
Main method that controls the population of the graph database. Firstly, it gets a connection \
to the database (driver) and then initiates the update of the entire database getting \
all the graph entities to update from configuration. Once the graph database has been \
populated, the imports folder in data/ is compressed and archived in the archive/ folder \
so that a backup of the imports files is kept (full).
"""
imports = config["graph"]
driver = connector.getGraphDatabaseConnectionConfiguration()
logger.info("Full update of the database - Updating: {}".format(
",".join(imports)))
updateDB(driver, imports)
logger.info(
"Full update of the database - Update took: {}".format(datetime.now() -
START_TIME))
logger.info("Full update of the database - Archiving imports folder")
archiveImportDirectory(archive_type="full")
logger.info("Full update of the database - Archiving took: {}".format(
datetime.now() - START_TIME))
def partialUpdate(imports, specific=[]):
"""
Method that controls the update of the graph database with the specified entities and \
relationships. Firstly, it gets a connection to the database (driver) and then initiates \
the update of the specified graph entities. \
Once the graph database has been populated, the data files uploaded to the graph are compressed \
and archived in the archive/ folder (partial).
:param list imports: list of entities to update
"""
driver = connector.getGraphDatabaseConnectionConfiguration()
logger.info("Partial update of the database - Updating: {}".format(
",".join(imports)))
updateDB(driver, imports, specific)
logger.info("Partial update of the database - Update took: {}".format(
datetime.now() - START_TIME))
logger.info("Partial update of the database - Archiving imports folder")
#archiveImportDirectory(archive_type="partial")
logger.info(
"Partial update of the database - Archiving {} took: {}".format(
",".join(imports),
datetime.now() - START_TIME))
from passlib.hash import bcrypt
from datetime import datetime
import uuid
from graphdb_connector import connector
driver = connector.getGraphDatabaseConnectionConfiguration()
class User:
def __init__(self, username):
self.username = username
def find(self):
user = connector.find_node(driver,
node_type="User",
username=self.username)
return user
def register(self, password):
if not self.find():
return connector.create_node(driver,
"User",
username=self.username,
password=bcrypt.encrypt(password))
else:
return False
def verify_password(self, password):
user = self.find()
if user:
return bcrypt.verify(password, user['password'])
def send_query(self, query):
driver = connector.getGraphDatabaseConnectionConfiguration()
data = connector.getCursorData(driver, query)
return data
