def test_get_provenance_history(self):
# in general, the display id (ex: TEST0010-LK-1-1-1) will dictate the corresponding
# number of entity entries returned. Basically, the entity entry count should equal the number of hyphens
# in the display id plus one. For example, TEST0010 will have 1 entity entry. TEST0010-LK will have two: one for the donor TEST0010
# and one for the organ TEST0010-LK.
confdata = Provenance.load_config_file()
conn = Neo4jConnection(confdata['neo4juri'], confdata['neo4jusername'],
confdata['neo4jpassword'])
driver = conn.get_driver()
donor_list = Entity.get_entities_by_type(driver, 'Donor')
prov = Provenance(confdata['appclientid'], confdata['appclientsecret'],
confdata['UUID_WEBSERVICE_URL'])
# walk through the first 5 donors and test them
for x in range(6):
history_data_str = prov.get_provenance_history(
driver, donor_list[x])
history_data = eval(history_data_str)
self.assertEqual(len(history_data['entity']), 1)
sample_list = Entity.get_entities_by_type(driver, 'Sample')
# walk through the first 20 samples and test them
for x in range(20):
sample_item = Entity.get_entity(driver, sample_list[x])
history_data_str = prov.get_provenance_history(
driver, sample_list[x])
history_data = eval(history_data_str)
display_id = sample_item['hubmap_identifier']
hypen_count = str(display_id).count('-')
self.assertEqual(len(history_data['entity']), hypen_count + 1)
def write_data_to_neo4j(conf_data, token, data_dict):
conn = Neo4jConnection(conf_data['NEO4J_SERVER'],
conf_data['NEO4J_USERNAME'],
conf_data['NEO4J_PASSWORD'])
driver = conn.get_driver()
with driver.session() as session:
tx = None
try:
tx = session.begin_transaction()
for dict_item in data_dict:
current_uuid = []
try:
current_uuid = Entity.get_uuid_list(
conf_data['UUID_WEBSERVICE_URL'], token, [dict_item])
except ValueError as ve:
print("Unable to resolve UUID for: " + str(dict_item))
continue
# remove single quotes and replace with double quotes
metadata_entry = str(data_dict[dict_item]).replace('\'', '"')
stmt = "MATCH (e)-[:{has_metadata_rel}]-(m) WHERE e.{uuid_attr} = '{uuid}' SET m += {{ {metadata_attr} : '{metadata_entry}' }}".format(
has_metadata_rel=HubmapConst.HAS_METADATA_REL,
uuid_attr=HubmapConst.UUID_ATTRIBUTE,
uuid=current_uuid[0],
metadata_attr=HubmapConst.METADATA_ATTRIBUTE,
metadata_entry=metadata_entry)
print('Executing stmt: ' + stmt)
tx.run(stmt)
except Exception as e:
print("Exception encountered: " + str(e))
tx.rollback()
tx.commit()
def initialize_all_dataset_access_levels(confdata, nexus_token):
conn = Neo4jConnection(confdata['NEO4J_SERVER'], confdata['NEO4J_USERNAME'], confdata['NEO4J_PASSWORD'])
driver = conn.get_driver()
header={'Content-Type':'application/json', 'Authorization': 'Bearer {token}'.format(token=nexus_token )}
with driver.session() as session:
tx = None
try:
tx = session.begin_transaction()
stmt = """MATCH (e)-[:{has_metadata_rel}]-(m) WHERE e.{entity_type_attr} = 'Dataset'
RETURN e.{uuid_attr} AS uuid, m.{source_uuid_attr} AS source_uuid,
m.{group_uuid_attr} AS group_uuid, m.{has_phi_attr} AS {has_phi_attr}""".format(
has_metadata_rel=HubmapConst.HAS_METADATA_REL,entity_type_attr=HubmapConst.ENTITY_TYPE_ATTRIBUTE,
uuid_attr=HubmapConst.UUID_ATTRIBUTE, source_uuid_attr=HubmapConst.SOURCE_UUID_ATTRIBUTE,
group_uuid_attr=HubmapConst.PROVENANCE_GROUP_UUID_ATTRIBUTE, has_phi_attr=HubmapConst.HAS_PHI_ATTRIBUTE)
for record in session.run(stmt):
uuid = record['uuid']
source_uuid = record['source_uuid']
if str(source_uuid).startswith('[') == False:
source_uuid = [source_uuid]
group_uuid = record['group_uuid']
has_phi = record['phi']
dataset_record = {'uuid': uuid, 'source_uuid':source_uuid, 'group_uuid':group_uuid, 'phi': has_phi}
dataset = Dataset(conf_data)
dataset.modify_dataset(driver, header, uuid, dataset_record, dataset_record['group_uuid'])
except Exception as e:
print ('An exception occurred: while initializing dataset access levels: ' + str(e))
tx.rollback()
def get_create_activity_metadata_statement(activity_metadata_record, activity_metadata_uuid_record, activity_uuid_record, metadata_userinfo, provenance_group):
activity_metadata_record[HubmapConst.UUID_ATTRIBUTE] = activity_metadata_uuid_record[HubmapConst.UUID_ATTRIBUTE]
activity_metadata_record[HubmapConst.ENTITY_TYPE_ATTRIBUTE] = HubmapConst.METADATA_TYPE_CODE
activity_metadata_record[HubmapConst.REFERENCE_UUID_ATTRIBUTE] = activity_uuid_record[HubmapConst.UUID_ATTRIBUTE]
if HubmapConst.PROVENANCE_SUB_ATTRIBUTE in metadata_userinfo:
activity_metadata_record[HubmapConst.PROVENANCE_SUB_ATTRIBUTE] = metadata_userinfo[HubmapConst.PROVENANCE_SUB_ATTRIBUTE]
activity_metadata_record[HubmapConst.PROVENANCE_USER_EMAIL_ATTRIBUTE] = metadata_userinfo[HubmapConst.PROVENANCE_USER_EMAIL_ATTRIBUTE]
activity_metadata_record[HubmapConst.PROVENANCE_USER_DISPLAYNAME_ATTRIBUTE] = metadata_userinfo[
HubmapConst.PROVENANCE_USER_DISPLAYNAME_ATTRIBUTE]
# do some processing to handle older provenance_group objects:
prov_group_name = None
prov_group_uuid = None
if HubmapConst.PROVENANCE_GROUP_NAME_ATTRIBUTE in provenance_group:
prov_group_name = provenance_group[HubmapConst.PROVENANCE_GROUP_NAME_ATTRIBUTE]
else:
prov_group_name = provenance_group['displayname']
if HubmapConst.PROVENANCE_GROUP_UUID_ATTRIBUTE in provenance_group:
prov_group_name = provenance_group[HubmapConst.PROVENANCE_GROUP_UUID_ATTRIBUTE]
else:
prov_group_name = provenance_group['uuid']
activity_metadata_record[HubmapConst.PROVENANCE_GROUP_NAME_ATTRIBUTE] = prov_group_name
activity_metadata_record[HubmapConst.PROVENANCE_GROUP_UUID_ATTRIBUTE] = prov_group_uuid
stmt = Neo4jConnection.get_create_statement(
activity_metadata_record, HubmapConst.METADATA_NODE_NAME, HubmapConst.METADATA_TYPE_CODE, True)
return stmt
def initialize_all_entity_access_levels(confdata):
conn = Neo4jConnection(confdata['NEO4J_SERVER'], confdata['NEO4J_USERNAME'], confdata['NEO4J_PASSWORD'])
driver = conn.get_driver()
with driver.session() as session:
tx = None
try:
tx = session.begin_transaction()
stmt = """MATCH (e)-[:{has_metadata_rel}]-(m) WHERE e.{entity_type_attr} IN ['Donor','Sample']
SET m += {{ {data_access_attr}: '{access_level}' }} RETURN e.{uuid_attr}""".format(
has_metadata_rel=HubmapConst.HAS_METADATA_REL,entity_type_attr=HubmapConst.ENTITY_TYPE_ATTRIBUTE,
data_access_attr=HubmapConst.DATA_ACCESS_LEVEL, access_level=HubmapConst.ACCESS_LEVEL_CONSORTIUM,
uuid_attr=HubmapConst.UUID_ATTRIBUTE)
print ("EXECUTING UPDATE: " + stmt)
tx.run(stmt)
stmt = """MATCH (c:Collection)
SET c += {{ {data_access_attr}: '{access_level}' }} RETURN c.{uuid_attr}""".format(
data_access_attr=HubmapConst.DATA_ACCESS_LEVEL, access_level=HubmapConst.ACCESS_LEVEL_CONSORTIUM,
uuid_attr=HubmapConst.UUID_ATTRIBUTE)
print ("EXECUTING UPDATE: " + stmt)
tx.run(stmt)
tx.commit()
except Exception as e:
print ('An exception occurred while initializing entity access levels: ' + str(e))
traceback.print_exc(file=sys.stdout)
tx.rollback()
def get_create_activity_statements(self, current_token, activity_type, inputUUID_list, outputUUID, metadata_userinfo, provenance_group):
ret_object = {}
stmt_list = []
# create the Activity Entity node
activity_uuid_record_list = None
activity_uuid_record = None
ug = UUID_Generator(self.UUID_WEBSERVICE_URL)
try:
activity_uuid_record_list = ug.getNewUUID(current_token, activity_type)
if (activity_uuid_record_list == None) or (len(activity_uuid_record_list) != 1):
raise ValueError("UUID service did not return a value")
activity_uuid_record = activity_uuid_record_list[0]
except requests.exceptions.ConnectionError as ce:
raise ConnectionError("Unable to connect to the UUID service: " + str(ce.args[0]))
activity_record = {HubmapConst.UUID_ATTRIBUTE: activity_uuid_record[HubmapConst.UUID_ATTRIBUTE],
HubmapConst.DOI_ATTRIBUTE: activity_uuid_record[HubmapConst.DOI_ATTRIBUTE],
HubmapConst.DISPLAY_DOI_ATTRIBUTE: activity_uuid_record['displayDoi'],
HubmapConst.ACTIVITY_TYPE_ATTRIBUTE: activity_type}
stmt = Neo4jConnection.get_create_statement(activity_record, HubmapConst.ACTIVITY_NODE_NAME, activity_type, False)
stmt_list.append(stmt)
ret_object['activity_uuid'] = activity_uuid_record
# create the Activity Metadata node
activity_metadata_record = {}
activity_metadata_uuid_record_list = None
activity_metadata_uuid_record = None
try:
activity_metadata_uuid_record_list = ug.getNewUUID(current_token, HubmapConst.METADATA_TYPE_CODE)
if (activity_metadata_uuid_record_list == None) or (len(activity_metadata_uuid_record_list) != 1):
raise ValueError("UUID service did not return a value")
activity_metadata_uuid_record = activity_metadata_uuid_record_list[0]
except requests.exceptions.ConnectionError as ce:
raise ConnectionError("Unable to connect to the UUID service: " + str(ce.args[0]))
ret_object['activity_metadata_uuid'] = activity_metadata_uuid_record
stmt = Activity.get_create_activity_metadata_statement(activity_metadata_record, activity_metadata_uuid_record, activity_uuid_record, metadata_userinfo, provenance_group)
stmt_list.append(stmt)
stmt = Neo4jConnection.create_relationship_statement(ret_object['activity_uuid']['hm_uuid'], HubmapConst.HAS_METADATA_REL, ret_object['activity_metadata_uuid']['hm_uuid'])
stmt_list.append(stmt)
for inputUUID in inputUUID_list:
stmt = Neo4jConnection.create_relationship_statement(inputUUID, HubmapConst.ACTIVITY_INPUT_REL, ret_object['activity_uuid'][HubmapConst.UUID_ATTRIBUTE])
stmt_list.append(stmt)
stmt = Neo4jConnection.create_relationship_statement(ret_object['activity_uuid'][HubmapConst.UUID_ATTRIBUTE], HubmapConst.ACTIVITY_OUTPUT_REL, outputUUID)
stmt_list.append(stmt)
ret_object['statements'] = stmt_list
return ret_object
def status():
response_data = {
# Use strip() to remove leading and trailing spaces, newlines, and tabs
'version': (Path(__file__).parent / 'VERSION').read_text().strip(),
'build': (Path(__file__).parent / 'BUILD').read_text().strip(),
'neo4j_connection': False
}
conn = Neo4jConnection(app.config['NEO4J_SERVER'], app.config['NEO4J_USERNAME'], app.config['NEO4J_PASSWORD'])
driver = conn.get_driver()
is_connected = conn.check_connection(driver)
if is_connected:
response_data['neo4j_connection'] = True
return jsonify(response_data)
def get_create_metadata_statement(self, current_token, metadata_record):
metadata_uuid_record_list = None
metadata_uuid_record = None
ug = UUID_Generator(self.md_config['UUID_WEBSERVICE_URL'])
try:
metadata_uuid_record_list = ug.getNewUUID(current_token, HubmapConst.METADATA_TYPE_CODE)
if (metadata_uuid_record_list == None) or (len(metadata_uuid_record_list) != 1):
raise ValueError("UUID service did not return a value")
metadata_uuid_record = metadata_uuid_record_list[0]
except requests.exceptions.ConnectionError as ce:
raise ConnectionError("Unable to connect to the UUID service: " + str(ce.args[0]))
metadata_record[HubmapConst.UUID_ATTRIBUTE] = metadata_uuid_record[HubmapConst.UUID_ATTRIBUTE]
stmt = Neo4jConnection.get_create_statement(
metadata_record, HubmapConst.METADATA_NODE_NAME, HubmapConst.METADATA_TYPE_CODE, True)
# NOTE: I need to return a list of the newly created uud plus the stmt: {'uuid':'', 'doi':'', 'display_doi':'', 'stmt':stmt}
# otherwise, the new uuid might get lost
return {'uuid_data': metadata_uuid_record_list, 'stmt':stmt}
def setUp(self):
self.config = load_config_file()
self.conn = Neo4jConnection(self.config['neo4juri'], self.config['neo4jusername'], self.config['neo4jpassword'])
self.driver = self.conn.get_driver()
def create_collection(self, driver, current_token, collection_record):
ug = UUID_Generator(self.confdata['UUID_WEBSERVICE_URL'])
with driver.session() as session:
tx = None
collection_uuid_record = None
try:
tx = session.begin_transaction()
try:
collection_uuid_record_list = ug.getNewUUID(
current_token, HubmapConst.COLLECTION_TYPE_CODE)
if (collection_uuid_record_list
== None) or (len(collection_uuid_record_list)
== 0):
raise ValueError("UUID service did not return a value")
if len(collection_uuid_record_list) > 1:
raise ValueError(
"UUID service returned more than one UUID value")
collection_uuid_record = collection_uuid_record_list[0]
except requests.exceptions.ConnectionError as ce:
raise ConnectionError(
"Unable to connect to the UUID service: " +
str(ce.args[0]))
collection_record[
HubmapConst.UUID_ATTRIBUTE] = collection_uuid_record[
HubmapConst.UUID_ATTRIBUTE]
collection_record[
HubmapConst.DOI_ATTRIBUTE] = collection_uuid_record[
HubmapConst.DOI_ATTRIBUTE]
collection_record[
HubmapConst.
DISPLAY_DOI_ATTRIBUTE] = collection_uuid_record[
'displayDoi']
collection_record[
HubmapConst.
ENTITY_TYPE_ATTRIBUTE] = HubmapConst.COLLECTION_TYPE_CODE
if 'name' in collection_record:
collection_record[
HubmapConst.NAME_ATTRIBUTE] = collection_record['name']
stmt = Neo4jConnection.get_create_statement(
collection_record, HubmapConst.COLLECTION_NODE_NAME,
HubmapConst.COLLECTION_TYPE_CODE, False)
print('Collection Create statement: ' + stmt)
tx.run(stmt)
tx.commit()
return collection_record[HubmapConst.UUID_ATTRIBUTE]
except ConnectionError as ce:
print('A connection error occurred: ', str(ce.args[0]))
if tx.closed() == False:
tx.rollback()
raise ce
except ValueError as ve:
print('A value error occurred: ', ve.value)
if tx.closed() == False:
tx.rollback()
raise ve
except TransactionError as te:
print('A transaction error occurred: ', te.value)
if tx.closed() == False:
tx.rollback()
except:
print('A general error occurred: ')
traceback.print_exc()
if tx.closed() == False:
tx.rollback()
msg = msg + " Cannot read line: {0}".format(syntaxError.text)
print(msg + " Program stopped.")
exit(0)
except AttributeError as attrError:
msg = "Error reading the config.ini file. Check config.ini file to make sure it matches the structure in config.ini.example: {0}".format(
attrError)
msg = msg + " Cannot read line: {0}".format(attrError.text)
print(msg + " Program stopped.")
exit(0)
except:
traceback.print_exc()
exit(0)
if __name__ == "__main__":
confdata = Metadata.load_config_file()
conn = Neo4jConnection(confdata['neo4juri'], confdata['neo4jusername'], confdata['neo4jpassword'])
driver = conn.get_driver()
uuid = "cafd03e784d2fd091dd2bafc71db911d"
record = Metadata.get_metadata_by_source(driver, uuid)
pprint(record)
record_list = Metadata.get_metadata_by_source_type(driver, 'entitytype', 'Donor')
pprint(record_list)
record_list = Metadata.get_metadata_by_source_type(driver, 'entitytype', 'Tissue Sample')
pprint(record_list)
record_list = Metadata.get_metadata_by_source_type(driver, 'entitytype', 'bad entity')
pprint(record_list)
conf_data = {
'NEO4J_SERVER': NEO4J_SERVER,
'NEO4J_USERNAME': NEO4J_USERNAME,
'NEO4J_PASSWORD': NEO4J_PASSWORD,
'APP_CLIENT_ID': APP_CLIENT_ID,
'APP_CLIENT_SECRET': APP_CLIENT_SECRET,
'UUID_WEBSERVICE_URL': UUID_WEBSERVICE_URL,
'HUBMAP_WEBSERVICE_FILEPATH': HUBMAP_WEBSERVICE_FILEPATH
}
prov = Provenance(conf_data['APP_CLIENT_ID'],
conf_data['APP_CLIENT_SECRET'],
conf_data['UUID_WEBSERVICE_URL'])
# this is a Vanderbilt uuid: uuid = '4614ea24338ec820569f988196a5c503'
uuid = ''
conn = Neo4jConnection(NEO4J_SERVER, NEO4J_USERNAME, NEO4J_PASSWORD)
nexus_token = ''
driver = conn.get_driver()
provenance_data = prov.get_provenance_history(driver, uuid)
print("Provenance data: " + str(provenance_data))
#print('max depth')
#history_data = prov.get_provenance_history(driver, uuid, None)
#g.serialize(format='rdf', rdf_format='trig')
#print(history_data.serialize(indent=2))
#print(history_data.serialize(format='rdf', rdf_format='trig'))
#print(history_data.serialize(format='provn'))
donor_uuid,
headers={'Authorization': "Bearer " + token})
except:
print("Error happened when calling reindex web service")
# reindex the donor
pass
if __name__ == "__main__":
file_path = '/home/chb69/git/ingest-ui/src/ingest-api/instance'
filename = 'app.cfg'
confdata = load_config(file_path, filename)
conn = Neo4jConnection(confdata['NEO4J_SERVER'],
confdata['NEO4J_USERNAME'],
confdata['NEO4J_PASSWORD'])
driver = conn.get_driver()
dataset_uuid_list = [
'b17694503bcbdd2458d3e96373ce9fbc', 'c34acecd4dbd9fc6a41a6ba4563bffc5',
'ed8a4dbbb1554a5e3227d6dfb2368828', 'b40eb3abccf2341f274cfd4ba809c03e'
]
try:
delete_dataset(confdata, driver, dataset_uuid_list)
except:
print('A general error occurred: ')
traceback.print_exc()
raise
finally:
if conn != None: