def cluster_2_linkset_metadata(specs):
# METADATA
# A TARGET COMBINES A DATATYPE AND A LIST OF PROPERTIES
alignment_targets = target_datatype_properties(specs[St.targets],
"alignmentTarget",
specs[St.linkset_name])
query = """
# CREATION OF A LINKSET OF MIXED-RESOURCES
PREFIX ll: <{0}>
PREFIX void: <{1}>
PREFIX rdfs: <{2}>
PREFIX bdb: <{3}>
PREFIX prov: <{4}>
PREFIX singleton: <{5}>
prefix linkset: <{6}>
PREFIX llTarget: <{7}>
prefix stardog: <tag:stardog:api:context:>
INSERT
{{
# GENERIC METADATA
linkset:{8}
rdfs:label "{8}" ;
a void:Linkset ;
ll:alignsMechanism <{9}exact> .
{10}
}}
WHERE
{{
{11}
}}
""".format(
Ns.alivocab,
Ns.void,
Ns.rdfs,
Ns.bdb,
Ns.prov,
Ns.singletons,
Ns.linkset,
Ns.alignmentTarget,
# 8 9 10 11
specs[St.linkset_name],
Ns.mechanism,
alignment_targets["list"],
alignment_targets["binds"])
specs["metadata"] = query
Qry.boolean_endpoint_response(query)
def register_evolution(research_question_uri, alignment_uri, evolution_str):
if alignment_uri.__contains__("<<"):
alignment_uri = str(alignment_uri).replace("<<",
"<").replace(">>", ">")
bind = "BIND(iri(\"{}\") AS ?LINK)".format(alignment_uri)
query = """
PREFIX alivocab: <http://risis.eu/alignment/predicate/>
INSERT DATA
{{
{0}
GRAPH <{1}>
{{
?LINK alivocab:evolution ""\"{2}\""" .
}}
}}
""".format(bind, research_question_uri, evolution_str)
else:
query = """
PREFIX alivocab: <http://risis.eu/alignment/predicate/>
INSERT DATA
{{
GRAPH <{0}>
{{
<{1}> alivocab:evolution ""\"{2}\""" .
}}
}}
""".format(research_question_uri, alignment_uri, evolution_str)
# print query
registered = Qry.boolean_endpoint_response(query)
print "\t>>> IS EVOLUTION REGISTERED FOR {}?: {}".format(
alignment_uri, registered)
def refine_metadata(specs):
# GENERATE GENERIC METADATA
metadata = Gn.linkset_refined_metadata(specs)
if int(specs[St.triples]) > 0:
# print metadata
is_inserted = Qry.boolean_endpoint_response(metadata["query"])
print ">>> THE METADATA IS SUCCESSFULLY INSERTED:", is_inserted
# GENERATE LINKSET CONSTRUCT QUERY
construct_query = "\n{}\n{}\n{}\n{}\n{}\n".format(
"PREFIX predicate: <{}>".format(Ns.alivocab),
"PREFIX src{}: <{}>".format(specs[St.source][St.graph_name],
specs[St.source][St.graph_ns]),
"PREFIX trg{}: <{}>".format(specs[St.target][St.graph_name],
specs[St.target][St.graph_ns]),
"construct { ?x ?y ?z }",
"where {{ graph <{}> {{ ?x ?y ?z }} }}".format(
specs[St.refined]),
)
# GENERATE LINKSET SINGLETON METADATA QUERY
singleton_metadata_query = "\n{}\n{}\n{}\n{}\n{}\n{}\n\n".format(
"PREFIX singMetadata: <{}>".format(Ns.singletons),
"PREFIX predicate: <{}>".format(Ns.alivocab),
"PREFIX prov: <{}>".format(Ns.prov),
"PREFIX rdf: <{}>".format(Ns.rdf),
"construct { ?x ?y ?z }",
"where {{ graph <{}> {{ ?x ?y ?z }} }}".format(
specs[St.singleton]),
)
# GET THE CORRESPONDENCES INSERTED USING A THE CONSTRUCT QUERY
singleton_construct = Qry.endpointconstruct(singleton_metadata_query)
if singleton_construct is not None:
singleton_construct = \
singleton_construct.replace('{', "singMetadata:{}\n{{".format(specs[St.refined_name]), 1)
# GET THE SINGLETON METADATA USING THE CONSTRUCT QUERY
construct_response = Qry.endpointconstruct(construct_query)
if construct_response is not None:
construct_response = construct_response.replace(
'{', "<{}>\n{{".format(specs[St.refined]), 1)
# WRITE TO FILE
print "\t>>> WRITING THE METADATA YO FILE TO FILE"
write_to_file(graph_name=specs[St.refined_name],
metadata=metadata["query"].replace("INSERT DATA", ""),
correspondences=construct_response,
singletons=singleton_construct,
directory=DIRECTORY)
return metadata["message"]
def register_dataset_mapping(question_uri, mapping, activated=True):
if activated:
print "\nREGISTERING A [DATASET-MAPPING]" \
"\n======================================================" \
"========================================================"
ds_mapping_query = ds_mapping(question_uri, mapping)
inserted = Qry.boolean_endpoint_response(ds_mapping_query)
message = "THE DATASET MAPPING WAS SUCCESSFULLY INSERTED." if inserted \
else "DUE TO A SYSTEM FAILURE, THE MAPPING COULD NOT BE INSERTED."
print message
return {St.message: message, St.result: message}
def register_lens(specs, is_created=True):
# inverse = ""
if is_created is True:
created = "alivocab:created"
inverse = "prov:used"
print "REGISTERING [{}] AS CREATED".format(specs[St.lens])
else:
created = "prov:used\t\t"
inverse = "alivocab:created"
print "REGISTERING [{}] AS IMPORTED".format(specs[St.lens])
query = PREFIX + """
INSERT
{{
GRAPH <{0}>
{{
<{0}> {1} <{2}> .
<{2}> a bdb:Lens .
}}
}}
WHERE
{{
GRAPH <{0}>
{{
FILTER NOT EXISTS
{{
<{0}> {3} <{2}> .
}}
}}
}}""".format(specs[St.researchQ_URI], created, specs[St.lens], inverse)
# print query
registered = Qry.boolean_endpoint_response(query)
print "\t>>> IS THE LENS REGISTERED?:", registered
def geo_match(specs):
# geo_query(ls_specs_1, True)
# geo_query(ls_specs_1, False)
# geo_match_query(ls_specs_1)
drop_1 = """
PREFIX tmp: <{0}>
DROP SILENT GRAPH tmp:load_{1}_1 ;
drop silent graph tmp:load_{1}_2
""".format(Ns.tmpgraph, specs[St.lens_name], Ns.lens, Ns.singletons)
drop_2 = """
PREFIX lens: <{0}>
PREFIX singletons: <{1}>
drop silent graph lens:{2} ;
drop silent graph singletons:{2}
""".format(Ns.lens, Ns.singletons, specs[St.lens_name])
print "\n\t4.1 >>> DROPPING GRAPH LOAD_1 & LOAD_2 IF THEY EXIST"
# print drop_1
print "\t", Qry.boolean_endpoint_response(drop_1)
# print drop_2
print "\t", Qry.boolean_endpoint_response(drop_2)
print "\n\t4.2 >>> LOADING SOURCE INTO GRAPH LOAD-1"
# print geo_load_query(specs, True)
print "\t", Qry.boolean_endpoint_response(geo_load_query(specs, True))
print "\n\t4.3 >>> LOADING SOURCE INTO GRAPH LOAD-2"
# print geo_load_query(specs, False)
print "\t", Qry.boolean_endpoint_response(geo_load_query(specs, False))
print "\n\t4.4 >>> LOOKING FOR GEO-SIM BETWEEN SOURCE AND TARGET"
print geo_match_query(specs)
print "\t", Qry.boolean_endpoint_response(geo_match_query(specs))
print "\n\t4.5 >>> DROPPING GRAPH LOAD_1 & LOAD_2"
print "\t", Qry.boolean_endpoint_response(drop_1)
def run_checks(specs, query):
print "\n3. RUNNING GOOD TO GO CHECK"
# print "QUERY FOR CHECK:", query
# CHECK-1: CHECK WHETHER THE LENS EXIST BY ASKING ITS METADATA WHETHER IT IS COMPOSED OF THE SAME GRAPHS
print "QUERY:", query
ask = Qry.sparql_xml_to_matrix(query)
print "\t3.1 ANSWER 1:", ask['message']
# ASK IS NOT SUPPOSED TO BE NONE
# CHECK-1-RESULT: PROBLEM CONNECTING WITH THE SERVER
if ask is None:
# print "IN 1"
print Ec.ERROR_CODE_1
return {St.message: Ec.ERROR_CODE_1, St.error_code: 1, St.result: None}
# CHECK-1-RESULT: ASK HAS A RESULT, MEANING THE LENS EXIT UNDER THE SAME COMPOSITION OF GRAPHS
elif ask[St.message] != "NO RESPONSE":
print "\tFOUND"
if ask[St.result]:
for i in range(1, len(ask[St.result])):
print "\t\t- {}".format(ask[St.result][i][0])
# IF THERE IS RESULT WITH THE SAME NUMBER OF TARGETS THEN THE LENS ALREADY EXISTS
if ask[St.result] and len(ask[St.result]) - 1 == len(
specs[St.datasets]):
message = Ec.ERROR_CODE_7.replace("#", specs[St.lens]).replace(
"@", ask[St.result][1][0])
print message
return {
St.message: message.replace("\n", "<br/>"),
St.error_code: 1,
St.result: specs[St.lens]
}
print "\tCHECK 1: THERE IS NO METADATA FOR TIS LENS"
# ELSE
# WITH THE UNSTATED ELSE, WE GET OUT AND PROCEED TO THE CREATION OF A NEW LENS
else:
print "IN 3"
print Ec.ERROR_CODE_1
return {St.message: Ec.ERROR_CODE_1, St.error_code: 1, St.result: None}
# print "GOT OUT!!!"
update_specification(specs)
# print "CHECK 2: CHECK WHETHER THE ACTUAL LENS EXISTS UNDER THIS NAME"
check_02 = "\nASK {{ graph <{}> {{ ?S ?p ?o . }} }}".format(specs[St.lens])
ask = Qry.boolean_endpoint_response(check_02)
# print specs
# print check_02
# print ask
if ask is None:
# PROBLEM CONNECTING WITH THE SERVER
print " CHECK 2: PROBLEM CONNECTING WITH THE SERVER"
print Ec.ERROR_CODE_1
return {
St.message: Ec.ERROR_CODE_1,
St.error_code: 1,
St.result: specs[St.lens]
}
if ask == "true":
print " CHECK 2: THE LINKSET ALREADY EXISTS"
message = Ec.ERROR_CODE_6.replace("#", specs[St.lens])
print message
return {
St.message: message.replace("\n", "<br/>"),
St.error_code: 1,
St.result: specs[St.lens]
}
print "\n\tDIAGNOSTICS: GOOD TO GO\n"
return {
St.message: "GOOD TO GO",
St.error_code: 0,
St.result: "GOOD TO GO"
}
def register_research_question(question):
print "REGISTERING A RESEARCH QUESTION." \
"\n======================================================" \
"========================================================"
if True:
# CHECK WHETHER THE RESEARCH QUESTION ALREADY EXISTS
question = to_bytes(to_unicode(question, "utf-8"))
existence_query = check_rq_existence(question)
check = Qry.boolean_endpoint_response(existence_query)
# LOOK FOR A RESEARCH QUESTION OF THE SAME NAMES GRAPH
find_query = find_rq(question)
# AN INTERNAL PROBLEM OCCURRED
if check is None:
return check
# THE RESEARCH QUESTION WAS ALREADY REGISTERED
elif check == "true":
find = Qry.sparql_xml_to_matrix(find_query)
# print find
if find:
if find[St.result]:
message = MESSAGE_1.replace("@", find[St.result][1][0])
print message
return {
St.message: message.replace("@", "<br/>"),
St.result: find[St.result][1][0]
}
return find
else:
return find
# REGISTERING YOUR RESEARCH QUESTION
else:
print "REGISTERING THE RESEARCH QUESTION"
ins_rq = research_question(question)
# print ins_rq
inserted = Qry.boolean_endpoint_response(ins_rq)
print "INSERTED RESULT:", inserted
# THE REGISTRATION WAS NOT SUCCESSFUL
if inserted is None:
print "THE RESEARCH QUESTION WAS REGISTERED"
print MESSAGE_3
# THE REGISTRATION WAS SUCCESSFUL. RETRIEVE THE URI
if inserted == "true" or inserted == STARDOG_BOOLEAN_BUG_MESSAGE:
print "THE RESEARCH QUESTION IS REGISTERED"
find = Qry.sparql_xml_to_matrix(find_query)
if find:
if find[St.result]:
message = MESSAGE_2.replace("@", find[St.result][1][0])
print message
return {
St.message: message.replace("@", "<br/>"),
St.result: find[St.result][1][0]
}
return {
St.message: MESSAGE_4.replace("@", "<br/>"),
St.result: None
}
else:
return find
print {
St.message: MESSAGE_3.replace("@", "<br/>"),
St.result: None
}
def register_alignment_mapping(alignment_mapping, created):
print "\nREGISTERING AN [ALIGNMENT-MAPPING]"
question_uri = alignment_mapping[St.researchQ_URI]
# MAKE SURE THE WRITE URI IS USED WHEN REGISTERING A REFINED LINKSET
linkset_uri = alignment_mapping[
St.refined] if St.refined in alignment_mapping else alignment_mapping[
St.linkset]
print "\tLINKSET TO REGISTER:", linkset_uri
# LINKSET EXISTS
if linkset_uri:
# 1 CHECK WHETHER THE ALIGNMENT WAS REGISTERED
ask_query = linkset_composition(alignment_mapping,
request_ask_select_or_insert="ask")
# print ask_query
if ask_query is None:
return
ask = Qry.boolean_endpoint_response(ask_query)
# print ask_query
print "\t>>> ASK WHETHER THE [ALIGNMENT] WAS REGISTERED:", ask
# 2 THE ALIGNMENT WAS NOT REGISTERED
if ask == "false":
# REGISTER THE ALIGNMENT-MAPPING
insert_alignment_query = linkset_composition(
alignment_mapping, request_ask_select_or_insert="insert")
insert_alignment = Qry.boolean_endpoint_response(
insert_alignment_query)
# print insert_alignment_query
print "\t>>> IS THE [ALIGNMENT] NOW INSERTED?:", insert_alignment
# 2.1 RETRIEVE THE ALIGNMENT-MAPPING URI
alignment_uri = None
alignment_uri_query = ask_query.replace(
"ASK", "SELECT ?alignmentMapping")
alignment_uri_resp = Qry.sparql_xml_to_matrix(alignment_uri_query)
if alignment_uri_resp:
if alignment_uri_resp[St.result]:
alignment_uri = alignment_uri_resp[St.result][1][0]
print "\t>>> ALIGNMENT REGISTERED AS:", alignment_uri
if alignment_uri:
# IF WE ARE DEALING WITH A REFINED LINKSET, REGISTER ITS EVOLUTION
if St.refined in alignment_mapping:
print "REGISTERING THE EVOLUTION OF THIS REFINED LINKSET TO\n\t{}".format(
alignment_uri)
evolution_str = linkset_evolution(question_uri,
linkset_uri)
register_evolution(question_uri, alignment_uri,
evolution_str)
# 2.2 ADD THE LINKSET TO THE ALIGNMENT
assign_ls_query = linkset_createdorused(question_uri,
alignment_uri,
alignment_mapping,
is_created=created)
is_linkset_registered = Qry.boolean_endpoint_response(
assign_ls_query)
print ">>> IS THE [LINKSET] REGISTERED?:", is_linkset_registered
# 3 THE ALIGNMENT WAS REGISTERED
else:
# CHECK IF THE LINKSET WAS REGISTERED
# is_linkset_registered_query = ask_query.replace("> .", "> ;\n\t\t?pred\t<{}> .".format(linkset_uri))
# is_linkset_registered_query = is_linkset_registered_query.replace(">\" .", ">\" ;\n\t\t?pred\t<{}> .".format(linkset_uri))
is_linkset_registered_query = ask_query.replace(
"###@SLOT",
"\n\t\t\t?alignmentMapping ?pred\t<{}> .".format(linkset_uri))
# print "CHECKING WHETHER THE LINKSET WAS TRULY REGISTERED QUERY:", is_linkset_registered_query
is_linkset_registered = Qry.boolean_endpoint_response(
is_linkset_registered_query)
# print is_linkset_registered_query
print "\t>>> ASK WHETHER [LINKSET] WAS REGISTERED?:", is_linkset_registered
if is_linkset_registered == "false":
# RETRIEVE THE ALIGNMENT-MAPPING URI
alignment_uri = None
alignment_uri_query = ask_query.replace(
"ASK", "SELECT ?alignmentMapping")
# print "alignment_uri_query:", alignment_uri_query
alignment_uri_resp = Qry.sparql_xml_to_matrix(
alignment_uri_query)
if alignment_uri_resp:
if alignment_uri_resp[St.result]:
alignment_uri = alignment_uri_resp[St.result][1][0]
if alignment_uri:
# IF WE ARE DEALING WITH A REFINED LINKSET,
# REGISTER ITS EVOLUTION IF NOT REGISTERED YET
if St.refined in alignment_mapping:
print "REGISTERING THE EVOLUTION OF THIS REFINED LINKSET"
evolution_str = linkset_evolution(
question_uri, linkset_uri)
register_evolution(question_uri, alignment_uri,
evolution_str)
# 2.3 ADD THE LINKSET TO THE ALIGNMENT
assign_ls_query = linkset_createdorused(question_uri,
alignment_uri,
alignment_mapping,
is_created=created)
is_linkset_registered = Qry.boolean_endpoint_response(
assign_ls_query)
print "\t>>> IS LINKSET NOW REGISTERED?:", is_linkset_registered
def run_checks_cluster(specs, check_type):
heading = "RUNNING LINKSET SPECS CHECK" \
"\n======================================================" \
"========================================================"
print heading
ask = "ASK {{ <#> ?p ?o . }}"
linkset = specs[St.refined] if St.refined in specs else specs[St.linkset]
# print linkset
"""
# CHECK WHETHER THE GRAPHS EXIST
"""
g_exist_q = "ASK { GRAPH <@> {?s ?p ?o} }"
response = "false"
for target in specs[St.targets]:
# query = g_exist_q.replace("@", target[St.graph])
# print query
response = Qry.boolean_endpoint_response(
g_exist_q.replace("@", target[St.graph]))
if response == "false":
break
# print response
if response == "false":
print Ec.ERROR_CODE_10
return {
St.message: Ec.ERROR_CODE_10,
St.error_code: 10,
St.result: None
}
"""
# CHECK THE TASK SPECIFIC PREDICATE COUNT
"""
if specs[St.sameAsCount] is None:
print Ec.ERROR_CODE_1
return {St.message: Ec.ERROR_CODE_1, St.error_code: 1, St.result: None}
# print "sameAsCount", specs[St.sameAsCount]
"""
# CHECK WHETHER THE LINKSET WAS ALREADY CREATED AND ITS ALTERNATIVE NAME REPRESENTATION
"""
# CHECK WHETHER THE CURRENT LINKSET NAME EXIST
print "CHECK WHETHER THE CURRENT LINKSET NAME EXIST"
print ask.replace("#", linkset)
ask_1 = Qry.boolean_endpoint_response(ask.replace("#", linkset))
# print ask_1
# THE CURRENT AME EXIST
if ask_1 == "true":
# print specs[St.linkset_name]
print "ASK_1: YES, THE CURRENT NAME EXIST"
message = Ec.ERROR_CODE_2.replace('#', linkset)
if St.refined in specs:
# GET LINKSET DERIVED FROM
subjects_target = linkset_wasderivedfrom(linkset)
# CHECK THE RESULT OF THE DIFFERENCE AND OUT PUT BOTH THE REFINED AND THE DIFFERENCE
if subjects_target is not None:
print "\t>>> NOT GOOD TO GO, IT ALREADY EXISTS"
diff_lens_specs = {
St.researchQ_URI:
specs[St.researchQ_URI],
# THE OBJECT IS THE LINKSET THE REFINED LINKSET WAS DERIVED FROM
St.subjectsTarget:
subjects_target,
# THE TARGET IS THE REFINED LINKSET
St.objectsTarget:
linkset
}
Lu.diff_lens_name(diff_lens_specs)
message2 = Ec.ERROR_CODE_7.replace('#',
diff_lens_specs[St.lens])
refined = {
St.message: message,
St.error_code: 0,
St.result: linkset
}
difference = {
St.message: message2,
St.error_code: 7,
St.result: diff_lens_specs[St.lens]
}
# print "difference", difference
# REGISTER THE ALIGNMENT
if refined[St.message].__contains__("ALREADY EXISTS"):
Ura.register_alignment_mapping(specs, created=False)
else:
Ura.register_alignment_mapping(specs, created=True)
# REGISTER THE LENS
Ura.register_lens(diff_lens_specs, is_created=False)
return {
St.message: "NOT GOOD TO GO",
'refined': refined,
'difference': difference
}
print message
return {
St.message: message.replace("\n", "<br/>"),
St.error_code: 2,
St.result: linkset
}
# CHECK WHETHER THE ALTERNATIVE LINKSET NAME EXIST
# elif ask_1 == "false" and str(check_type).lower() != "subset":
#
# print "ASK 2: CHECK WHETHER THE ALTERNATIVE LINKSET NAME EXIST"
# if St.refined not in specs:
# print "\t- NOT REFINED"
# GENERATE ALTERNATIVE NAME. THIS DOS NOT APPLY TO SUBSET BECAUSE WE ASSUME
# A LINKSET BY SUBSET DOES NOT NEED THE TARGET ALIGNS TO BE SET AS IT IS OFTEN UNKNOWN
# counter_check = set_linkset_name(specs, inverse=True)
#
# CHECK WHETHER THE CURRENT LINKSET EXIST UNDER A DIFFERENT NAME
# ask_2 = Qry.boolean_endpoint_response(ask.replace("#", counter_check))
#
# if ask_2 == "true":
# message = Ec.ERROR_CODE_3.replace('#', linkset).replace("@", counter_check)
# print "\n>>> NOT GOOD TO GO, IT ALREADY EXISTS UNDER THE NAME {}".format(counter_check)
# print message
# return {St.message: message.replace("\n", "<br/>"), St.error_code: 3, St.result: counter_check}
print "\t>>> NO IT DOES NOT EXIST YET..."
if str(check_type).lower() == "subset":
print "\nSETTING THE LINKSET SUBSET NAME"
set_subset_name(specs, inverse=False)
elif str(check_type).lower() == "linkset":
print "\nSETTING THE LINKSET NAME"
set_cluster_linkset_name(specs)
elif str(check_type).lower() == "refine":
print "\nSETTING THE LINKSET REFINE NAME"
set_refined_name(specs)
# print specs[St.linkset_name]
print "\n>>> GOOD TO GO !!!"
return {
St.message: "GOOD TO GO",
St.error_code: 0,
St.result: "GOOD TO GO"
}
def run_checks_id(specs):
heading = "\n======================================================" \
"========================================================"\
"\nRUNNING LINKSET SPECS CHECK" \
"\n======================================================" \
"========================================================"
print heading
ask = "ASK {{ <#> ?p ?o . }}"
linkset = specs[St.refined] if St.refined in specs else specs[St.linkset]
# print linkset
"""
# CHECK WHETHER THE SOURCE & TARGET GRAPHS EXIST
"""
g_exist_q = "ASK { GRAPH <@> {?s ?p ?o} }"
src_g_exist = Qry.boolean_endpoint_response(
g_exist_q.replace("@", specs[St.source][St.graph]))
trg_g_exist = Qry.boolean_endpoint_response(
g_exist_q.replace("@", specs[St.target][St.graph]))
if (src_g_exist == "false") or (trg_g_exist == "false"):
print Ec.ERROR_CODE_10
return {
St.message: Ec.ERROR_CODE_10,
St.error_code: 10,
St.result: None
}
"""
# CHECK THE TASK SPECIFIC PREDICATE COUNT
"""
if specs[St.sameAsCount] is None:
print Ec.ERROR_CODE_1
return {St.message: Ec.ERROR_CODE_1, St.error_code: 1, St.result: None}
"""
# CHECK WHETHER THE LINKSET WAS ALREADY CREATED AND ITS ALTERNATIVE NAME REPRESENTATION
"""
# CHECK WHETHER THE CURRENT LINKSET NAME EXIST
ask_1 = Qry.boolean_endpoint_response(ask.replace("#", linkset))
if ask_1 == "true":
# print "ASK_1"
message = Ec.ERROR_CODE_2.replace('#', linkset)
if St.refined in specs:
# GET LINKSET DERIVED FROM
subjects_target = linkset_wasderivedfrom(linkset)
# CHECK THE RESULT OF THE DIFFERENCE AND OUT PUT BOTH THE REFINED AND THE DIFFERENCE
if subjects_target is not None:
diff_lens_specs = {
St.researchQ_URI:
specs[St.researchQ_URI],
# THE OBJECT IS THE LINKSET THE REFINED LINKSET WAS DERIVED FROM
St.subjectsTarget:
subjects_target,
# THE TARGET IS THE REFINED LINKSET
St.objectsTarget:
linkset
}
Lu.diff_lens_name(diff_lens_specs)
message2 = Ec.ERROR_CODE_7.replace('#',
diff_lens_specs[St.lens])
refined = {
St.message: message,
St.error_code: 0,
St.result: linkset
}
difference = {
St.message: message2,
St.error_code: 7,
St.result: diff_lens_specs[St.lens]
}
# REGISTER THE ALIGNMENT
if refined[St.message].__contains__("ALREADY EXISTS"):
Ura.register_alignment_mapping(specs, created=False)
else:
Ura.register_alignment_mapping(specs, created=True)
# REGISTER THE LENS
Ura.register_lens(diff_lens_specs, is_created=False)
print "\n>>> NOT GOOD TO GO, IT ALREADY EXISTS"
return {
St.message: "NOT GOOD TO GO",
'refined': refined,
'difference': difference
}
print message
return {
St.message: message.replace("\n", "<br/>"),
St.error_code: 2,
St.result: linkset
}
# CHECK WHETHER THE alternative LINKSET NAME EXIST
elif ask_1 == "false" and str(linkset).__contains__("subset") is False:
# print "ASK 2"
if St.refined not in specs:
# GENERATE ALTERNATIVE NAME. THIS DOS NOT APPLY JTO SUBSET BECAUSE WE ASSUME
# A LINKSET BY SUBSET DOES NEEDS NOT THE TARGET ALIGNS TO BE SET AS IT IS OFTEN UNKNOWN
counter_check = set_linkset_identity_name(specs, inverse=True)
# CHECK WHETHER THE CURRENT LINKSET EXIST UNDER A DIFFERENT NAME
ask_2 = Qry.boolean_endpoint_response(
ask.replace("#", counter_check))
if ask_2 == "true":
message = Ec.ERROR_CODE_3.replace('#', linkset).replace(
"@", counter_check)
print "\n>>> NOT GOOD TO GO, IT ALREADY EXISTS UNDER THE NAME {}".format(
counter_check)
print message
return {
St.message: message.replace("\n", "<br/>"),
St.error_code: 3,
St.result: counter_check
}
print "NO PROBLEM"
set_linkset_identity_name(specs, inverse=False)
print "NAME: " + specs[St.linkset]
print "\n>>> GOOD TO GO !!!"
return {
St.message: "GOOD TO GO",
St.error_code: 0,
St.result: "GOOD TO GO"
}
def spa_linkset_subset(specs, activated=False):
if activated is True:
check = Ls.run_checks(specs, check_type="subset")
if check[St.result] != "GOOD TO GO":
return check
# THE LINKSET DOES NOT EXIT, LETS CREATE IT NOW
print Ls.linkset_info(specs, specs[St.sameAsCount])
##########################################################
""" 1. GENERATE SUBSET LINKSET INSERT QUERY """
##########################################################
insert_query = spa_subset_insert(specs)
# print insert_query
#############################################################
""" 2. EXECUTING INSERT SUBSET LINKSET QUERY AT ENDPOINT """
#############################################################
Qry.endpoint(insert_query)
#############################################################
""" 3. LINKSET SIZE (NUMBER OF TRIPLES) """
#############################################################
# LINKSET SIZE (NUMBER OF TRIPLES)
specs[St.triples] = Qry.get_namedgraph_size(specs[St.linkset])
print "\t>>> {} TRIPLES INSERTED".format(specs[St.triples])
# NO MATCH FOUND
if specs[St.triples] == "0":
# logger.warning("WE DID NOT INSERT A METADATA AS NO TRIPLE WAS INSERTED.")
print "WE DID NOT INSERT A METADATA AS NO TRIPLE WAS INSERTED."
specs[St.insert_query] = insert_query
# metadata = spa_subset_metadata(source, target, data, size)
explain_q = "ask {{ GRAPH <{}> {{ ?s <{}> ?o }} }}".format(
specs[St.linkset], specs[St.source][St.link_old])
response = Qry.boolean_endpoint_response(explain_q)
explain = True if response == "true" else False
# print explain
if explain is False:
# logger.warning("{} DOES NOT EXIST IS {}.".format(data[St.link_old], source[St.graph]))
print "{} DOES NOT EXIST IS {}.".format(
specs[St.source][St.link_old], specs[St.source][St.graph])
message = "{} DOES NOT EXIST IS {}.".format(
specs[St.source][St.link_old], specs[St.source][St.graph])
return {St.message: message, St.error_code: 1, St.result: None}
# SOME MATCHES WHERE FOUND
construct_query = "\n{}\n{}\n{}\n".format(
"PREFIX predicate: <{}>".format(Ns.alivocab),
"construct { ?x ?y ?z }",
"where {{ graph <{}> {{ ?x ?y ?z }} }}".format(
specs[St.linkset]),
)
# print construct_query
construct_response = Qry.endpointconstruct(construct_query)
if construct_response is not None:
construct_response = construct_response.replace(
'{', "<{}>\n{{".format(specs[St.linkset]), 1)
# GENERATE LINKSET SINGLETON METADATA QUERY
singleton_metadata_query = "\n{}\n{}\n{}\n{}\n{}\n{}\n\n".format(
"PREFIX singMetadata: <{}>".format(Ns.singletons),
"PREFIX predicate: <{}>".format(Ns.alivocab),
"PREFIX prov: <{}>".format(Ns.prov),
"PREFIX rdf: <{}>".format(Ns.rdf),
"construct { ?x ?y ?z }",
"where {{ graph <{}{}> {{ ?x ?y ?z }} }}".format(
Ns.singletons, specs[St.linkset_name]),
)
# GET THE SINGLETON METADATA USING THE CONSTRUCT QUERY
singleton_construct = Qry.endpointconstruct(singleton_metadata_query)
if singleton_construct is not None:
singleton_construct = singleton_construct.replace(
'{', "singMetadata:{}\n{{".format(specs[St.linkset_name]), 1)
#############################################################
""" 4. LINKSET METADATA """
#############################################################
# METADATA
specs[St.insert_query] = insert_query
metadata = Gn.spa_subset_metadata(specs)
###############################################################
""" 5. EXECUTING INSERT LINKSET METADATA QUERY AT ENDPOINT """
###############################################################
# EXECUTING METADATA QUERY AT ENDPOINT
Qry.endpoint(metadata)
print "\t>>> WRITING TO FILE"
write_to_file(graph_name=specs[St.linkset_name],
metadata=metadata.replace("INSERT DATA", ""),
correspondences=construct_response,
singletons=singleton_construct,
directory=DIRECTORY)
print "\tLinkset created as [SUBSET]: ", specs[St.linkset]
print "\t*** JOB DONE! ***"
message = "The linkset was created as [{}] with {} triples found!".format(
specs[St.linkset], specs[St.triples])
return {
St.message: message,
St.error_code: 0,
St.result: specs[St.linkset]
}
def union(specs, activated=False):
if activated is False:
# logger.warning("THE FUNCTION IS NOT ACTIVATED")
print("THE FUNCTION IS NOT ACTIVATED")
return {
St.message: "THE FUNCTION IS NOT ACTIVATED.",
St.error_code: 1,
St.result: None
}
print "\nEXECUTING UNION SPECS" \
"\n======================================================" \
"========================================================"
"""
THE generate_lens_name FUNCTION RETURNS THE NAME OF THE UNION AND A
QUERY THAT ALLOWS TO ASk WHETHER THE LENS TO BE CREATED EXIST BY CHECKING
WHETHER THERE EXISTS A LENS WITH THE SAME COMPOSITION IN TERMS GRAPHS USED FOR THE UNION
"""
# SET THE NAME OF THE UNION-LENS
print "1. DATASETS:", len(specs[St.datasets])
for ds in specs[St.datasets]:
print "\t- {}".format(ds)
info = Lu.generate_lens_name(specs[St.datasets])
specs[St.lens] = "{}{}".format(Ns.lens, info["name"])
print "\n2. LENS: ", info["name"]
# CHECK WHETHER THE LENS EXISTS
check = run_checks(specs, info["query"])
if check[St.result] != "GOOD TO GO":
if check[St.message].__contains__("ALREADY EXISTS"):
Urq.register_lens(specs, is_created=False)
return check
# print "AFTER CHECK"
# PREPARATION FOR THE CREATION OF THE LENS
specs[St.lens_target_triples] = ""
specs[St.expectedTriples] = 0
specs[St.insert_query] = ""
lens = specs[St.lens]
source = "{}{}".format(Ns.tmpgraph, "load00")
message_2 = Ec.ERROR_CODE_8.replace("#", specs[St.lens])
count = -1
insert_ans = False
try:
# GO THROUGH THE LINKSETS/LENSES IN THE LENS
# 1-SUM UP THE EXPECTED NUMBER OF TRIPLES
# 2-GENERATE THE TRIPLES REPRESENTATION OF GHE GRAPHS COMPOSING THIS LENS
# 3-GENERATE THE INSERT QUERY FOR MOVING BOTH LINKSET AND SINGLETON GRAPHS TO THE UNION GRAPH
total_size = 0
# LOAD ALL GRAPHS IN LOAD00
specs[St.insert_query] += "DROP SILENT GRAPH <{}{}> ;\n".format(
Ns.tmpgraph, "load00")
# ITERATE THROUGH THE PROVIDED GRAPHS
for linkset in specs[St.datasets]:
# print "TARGET: ", linkset
count += 1
# GET THE TOTAL NUMBER OF CORRESPONDENCE TRIPLES INSERTED
curr_triples = Qry.get_triples(linkset)
# PROBABLY THE LINKSET HAS NO SUCH PROPERTY " void:triples ?triples ."
if curr_triples is None:
curr_triples = Qry.get_triples_count(linkset)
total_size += int(curr_triples)
print "{} Contains {} triples".format(linkset, curr_triples)
if curr_triples is not None:
specs[St.expectedTriples] += int(curr_triples)
else:
# THE IS A PROBLEM WITH THE GRAPH FOR SEVERAL POSSIBLE REASONS
return {
St.message: message_2.replace("\n", "<br/>"),
St.error_code: 1,
St.result: None
}
# GENERATE TRIPLES OUT OF THE TARGETS
specs[
St.
lens_target_triples] += "\n\t void:target <{}> ;".format(
linkset)
# GET THE INSERT QUERY
# BOTH THE LINKSET AND THE SINGLETONS ARE MOVED TO A SINGLE GRAPH
partial_query = Qry.q_copy_graph(source, source, linkset)
if count == 0:
specs[St.insert_query] += partial_query
else:
specs[St.insert_query] += " ;\n{}".format(partial_query)
# INTERSECTION MANIPULATION OVER THE UNION (SOURCE)
insert_query = union_insert_q(lens, source, specs[St.lens_name])
# print "manipulation:", manipulation
specs[St.insert_query] += " ;\n{}".format(insert_query)
# GENERATE THE LENS UNION
if activated is True:
# print specs[St.insert_query]
insert_ans = Qry.boolean_endpoint_response(specs[St.insert_query])
specs[St.triples] = Qry.get_namedgraph_size(lens, isdistinct=False)
if specs[St.triples] == "0":
message = Ec.ERROR_CODE_9
print message
# return None
return {
St.message: message.replace("\n", "<br/>"),
St.error_code: 1,
St.result: None
}
# CHECK WHETHER THE RESULT CONTAINS DUPLICATES
contains_duplicated = Qry.contains_duplicates(lens)
print "Contains Opposite Direction Duplicated:", contains_duplicated
# IF IT DOES, REMOVE THE DUPLICATES
if contains_duplicated is True:
# logger.warning("THE LENS CONTAINS DUPLICATES.")
print "THE LENS CONTAINS DUPLICATES."
Qry.remove_duplicates(lens)
# logger.warning("THE DUPLICATES ARE NOW REMOVED.")
print "THE DUPLICATES ARE NOW REMOVED."
print "Number of triples loaded : {}".format(
total_size)
specs[St.triples] = Qry.get_namedgraph_size(lens, isdistinct=False)
print "\t>>> INSERTED: {}\n\t>>> INSERTED TRIPLES: {}".format(
insert_ans, specs[St.triples])
print "Inserted : {}".format(specs[St.triples])
print "Removed : {}".format(total_size - int(specs[St.triples]))
# LOAD THE METADATA
# NOT GOOD AS THE LENS ALSO HAS A SINGLETON GRAPH
# inserted_correspondences = int(Qry.get_union_triples(lens))
inserted_correspondences = int(specs[St.triples])
# print "inserted_correspondences:", inserted_correspondences
specs[St.removedDuplicates] = specs[
St.expectedTriples] - inserted_correspondences
metadata = Gn.union_meta(specs)
# print "METADATA:", metadata
meta_ans = Qry.boolean_endpoint_response(metadata)
print "\t>>> IS THE METADATA GENERATED AND INSERTED? {}".format(
meta_ans)
construct_response = Qry.get_constructed_graph(specs[St.lens])
if construct_response is not None:
print "\t>>> WRITING TO FILE"
construct_response = construct_response.replace(
'{', "<{}>\n{{".format(specs[St.lens]), 1)
write_to_file(graph_name=specs[St.lens_name],
metadata=None,
correspondences=construct_response,
directory=DIRECTORY)
print "\tLens created as : ", specs[St.lens]
# REGISTER THE LINKSET
Urq.register_lens(specs, is_created=True)
# return specs[St.lens]
message = "THE LENS WAS CREATED as {}. " \
"With initially {} triples loaded, {} duplicated triples were found and removed.".\
format(specs[St.lens], total_size, total_size - int(specs[St.triples]))
print "\t*** JOB DONE! ***"
return {
St.message: message,
St.error_code: 0,
St.result: specs[St.lens]
}
except Exception as err:
# logger.warning(err)
if insert_ans == "true":
"DROP THE INSERTED UNION"
drop_linkset(lens, activated=True)
print "ERROR IN UNION LENS CREATION:", err
return {St.message: ERROR_CODE_11, St.error_code: 11, St.result: None}
def lens_transitive(specs, activated=False):
# CHECK BOTH DATASETS FOR SAME MECHANISM
print "GENERATE THE LENS NAME"
Lu.composition_lens_name(specs)
print "GET THE SAME AS COUNT"
specs[St.sameAsCount] = Qry.get_same_as_count(specs[St.lens_operation])
# print same_as_count
# GENERATE THE INSERT QUERY FOR TRANSITIVITY
# transitive_analyses = lens_transitive_query(specs)
# if transitive_analyses is None:
# return
# specs[St.insert_query] = transitive_analyses[1]
# print insert_query
# exit(0)
# specs['is_transitive_by'] = transitive_analyses[0]
ln = get_uri_local_name(specs[St.lens])
sg = specs[St.subjectsTarget]
tg = specs[St.objectsTarget]
ssg = "{}{}".format(Ns.singletons, get_uri_local_name(sg))
tsg = "{}{}".format(Ns.singletons, get_uri_local_name(tg))
print "SOURCE: {}".format(sg)
print "TARGET: {}".format(tg)
print "1. GENERATING THE INSERT QUERY"
specs[St.insert_query] = transitive_insert_query(ln, sg, tg, ssg, tsg)
if activated is True:
# RUN THE QUERY AT THE END POINT
print "2. RUNNING THE INSERT QUERY"
Qry.boolean_endpoint_response(specs[St.insert_query])
# GET THE SIZE OF THE LENS JUST CREATED ABOVE
print "3. ETTING THE SIZE OF THE LENS JUST INSERTED"
size = Qry.get_namedgraph_size(specs[St.lens], isdistinct=False)
# IF ACTIVATED, INSERT THE METADATA
if size > 0:
# GENERATE THE METADATA ABOUT THE LENS JUST CREATED
print "4. SOME {} TRANSITIVE TRIPLES WERE FOUND".format(size)
metadata = transitive_metadata(specs, size)
# print metadata
print "5. INSERTING THE METADATA"
Qry.boolean_endpoint_response(metadata)
print "6. REGISTER THE LENS"
Urq.register_lens(specs, is_created=True)
# RUN A CORRESPONDENCE CONSTRUCT QUERY FOR BACKING UP THE DATA TO DISC
print "7. GENERATE THE CONSTRUCT FOR FILE DUMP"
construct_correspondence = Qry.endpointconstruct(
Qry.construct_namedgraph(specs[St.lens]))
if construct_correspondence is not None:
construct_correspondence = construct_correspondence.replace(
'{', "<{}>\n{{".format(specs[St.lens]), 1)
# RUN A SINGLETON METADATA CONSTRUCT QUERY FOR BACKING UP THE DATA TO DISC
construct_singletons = Qry.endpointconstruct(
Qry.construct_namedgraph("{}{}".format(Ns.singletons,
specs[St.lens_name])))
if construct_singletons is not None:
construct_singletons = construct_singletons. \
replace('{', "<{}{}>\n{{".format(Ns.singletons, specs[St.lens_name]), 1)
# WRITE TO FILE
print "WRITING TO FILE"
write_to_file(graph_name=ln,
metadata=metadata,
directory=DIRECTORY,
correspondences=construct_correspondence,
singletons=construct_singletons)
# return specs[St.lens]
message = "THE LENS WAS CREATED!<br/>URI = {}".format(
specs[St.lens])
print message
print "\t*** JOB DONE! ***"
return {
St.message: message,
St.error_code: 0,
St.result: specs[St.lens]
}
if activated is False:
logger.warning(
"THE FUNCTION IS NOT ACTIVATED BUT THE METADATA THAT IS "
"SUPPOSED TO BE ENTERED IS WRITEN TO THE CONSOLE.")
print "6. RUNNING THE BATCH FILE FOR LOADING THE CORRESPONDENCES INTO THE TRIPLE STORE\n\t\t{}", writers[
St.batch_output_path]
if Svr.settings[St.split_sys] is True:
print "THE DATA IS BEING LOADED OVER HTTP POST."
else:
print "THE DATA IS BEING LOADED AT THE STARDOG LOCAL HOST FROM BATCH."
# os.system(writers[St.batch_output_path])
Ut.batch_load(writers[St.batch_output_path])
# inserted = Qry.insert_size(specs[St.linkset], isdistinct=False)
metadata = Gn.linkset_metadata(specs, display=False).replace("INSERT DATA", "")
writers[St.meta_writer].write(to_unicode(metadata))
if int(specs[St.triples]) > 0:
Qry.boolean_endpoint_response(metadata)
writers[St.meta_writer].close()
# REGISTER THE ALIGNMENT
# if check[St.result].__contains__("ALREADY EXISTS"):
# Urq.register_alignment_mapping(specs, created=False)
# else:
# Urq.register_alignment_mapping(specs, created=True)
Urq.register_alignment_mapping(specs, created=False)
# WRITE TO FILE
# check_rdf_file(writers[St.crpdce_writer_path])
# check_rdf_file(writers[St.meta_writer_path])
# check_rdf_file(writers[St.singletons_writer_path])
else:
writers[St.meta_writer].close()
def refining(specs, insert_query, activated=False):
refined = {St.message: Ec.ERROR_CODE_1, St.error_code: 5, St.result: None}
diff = {St.message: Ec.ERROR_CODE_4, St.error_code: 1, St.result: None}
# UPDATE THE SPECS VARIABLE
# print "UPDATE THE SPECS VARIABLE"
update_specification(specs)
update_specification(specs[St.source])
update_specification(specs[St.target])
# ACCESS THE TASK SPECIFIC PREDICATE COUNT
specs[St.sameAsCount] = Qry.get_same_as_count(specs[St.mechanism])
# print "sameAsCount:", specs[St.sameAsCount]
if specs[St.sameAsCount] is None:
return {'refined': refined, 'difference': diff}
# GENERATE THE NAME OF THE LINKSET
Ls.set_refined_name(specs)
# print "\nREFINED NAME:", specs[St.refined]
# print "LINKSET TO REFINE BEFORE CHECK:", specs[St.linkset]
# CHECK WHETHER OR NOT THE LINKSET WAS ALREADY CREATED
check = Ls.run_checks(specs, check_type="refine")
# print "\nREFINED NAME:", specs[St.refined]
# print "LINKSET TO REFINE:", specs[St.linkset]
if check[St.message] == "NOT GOOD TO GO":
# refined = check[St.refined]
# difference = check["difference"]
return check
# print "\nREFINED:", specs[St.refined]
# print "LINKSET TO REFINE:", specs[St.linkset]
# print "CHECK:", check
# THE LINKSET DOES NOT EXIT, LETS CREATE IT NOW
print Ls.refined_info(specs, specs[St.sameAsCount])
# POINT TO THE LINKSET THE CURRENT LINKSET WAS DERIVED FROM
print "1. wasDerivedFrom {}".format(specs[St.linkset])
specs[St.derivedfrom] = "\t\tprov:wasDerivedFrom\t\t\t<{}> ;".format(
specs[St.linkset])
# print "REFINED NAME:", specs[St.refined_name]
# print "REFINED:", specs[St.refined]
# print "LINKSET TO BE REFINED:", specs[St.linkset]
print "\n2. RETRIEVING THE METADATA ABOUT THE GRAPH TO REFINE"
# metadata_q = Qry.q_linkset_metadata(specs[St.linkset])
metadata_q = """
prefix ll: <{}>
SELECT DISTINCT ?type ?singletonGraph
{{
# LINKSET METADATA
<{}>
a ?type ;
ll:singletonGraph ?singletonGraph .
}}
""".format(Ns.alivocab, specs[St.linkset])
print "QUERY:", metadata_q
matrix = Qry.sparql_xml_to_matrix(metadata_q)
# print "\nMETA DATA: ", matrix
if matrix:
if matrix[St.message] == "NO RESPONSE":
print Ec.ERROR_CODE_1
print matrix[St.message]
return {'refined': refined, 'difference': diff}
elif matrix[St.result] is None:
print matrix[St.message]
returned = {
St.message: matrix[St.message],
St.error_code: 666,
St.result: None
}
return {'refined': returned, 'difference': diff}
else:
print Ec.ERROR_CODE_1
return {'refined': refined, 'difference': diff}
# GET THE SINGLETON GRAPH OF THE LINKSET TO BE REFINED
print "\n3. GETTING THE SINGLETON GRAPH OF THE GRAPH TO REFINE"
specs[St.singletonGraph] = matrix[St.result][1][1]
# print matrix[St.result][1][0]
specs[St.insert_query] = insert_query(specs)
print specs[St.insert_query]
if type(specs[St.insert_query]) == str:
is_run = Qry.boolean_endpoint_response(specs[St.insert_query])
else:
print "\n4. RUNNING THE EXTRACTION QUERY"
print specs[St.insert_query][0]
# is_run = Qry.boolean_endpoint_response(specs[St.insert_query][0])
Qry.boolean_endpoint_response(specs[St.insert_query][0])
print "\n5. RUNNING THE FINDING QUERY"
print specs[St.insert_query][1]
is_run = Qry.boolean_endpoint_response(specs[St.insert_query][1])
print "\n>>> RUN SUCCESSFULLY:", is_run.upper()
# NO INSERTION HAPPENED
if is_run == "true" or is_run == Ec.ERROR_STARDOG_1:
# GENERATE THE
# (1) LINKSET METADATA
# (2) LINKSET OF CORRESPONDENCES
# (3) SINGLETON METADATA
# AND WRITE THEM ALL TO FILE
print "GENERATING THE METADATA"
pro_message = refine_metadata(specs)
# SET THE RESULT ASSUMING IT WENT WRONG
refined = {
St.message: Ec.ERROR_CODE_4,
St.error_code: 4,
St.result: None
}
diff = {St.message: Ec.ERROR_CODE_4, St.error_code: 4, St.result: None}
server_message = "Linksets created as: [{}]".format(specs[St.refined])
message = "The linkset was created as [{}]. <br/>{}".format(
specs[St.refined], pro_message)
# MESSAGE ABOUT THE INSERTION STATISTICS
print "\t", server_message
if int(specs[St.triples]) > 0:
# UPDATE THE REFINED VARIABLE AS THE INSERTION WAS SUCCESSFUL
refined = {
St.message: message,
St.error_code: 0,
St.result: specs[St.linkset]
}
print "REGISTERING THE ALIGNMENT"
if refined[St.message].__contains__("ALREADY EXISTS"):
register_alignment_mapping(specs, created=False)
else:
register_alignment_mapping(specs, created=True)
try:
print "\nCOMPUTE THE DIFFERENCE AND DOCUMENT IT"
diff_lens_specs = {
St.researchQ_URI: specs[St.researchQ_URI],
St.subjectsTarget: specs[St.linkset],
St.objectsTarget: specs[St.refined]
}
diff = Df.difference(diff_lens_specs, activated=activated)
message_2 = "\t>>> {} CORRESPONDENCES INSERTED AS THE DIFFERENCE".format(
diff_lens_specs[St.triples])
print message_2
except Exception as err:
print "THE DIFFERENCE FAILED: ", str(err.message)
print "\tLinkset created as: ", specs[St.refined]
print "\t*** JOB DONE! ***"
return {'refined': refined, 'difference': diff}
else:
print ">>> NO TRIPLE WAS INSERTED BECAUSE NO MATCH COULD BE FOUND"
return {'refined': refined, 'difference': diff}
else:
print "NO MATCH COULD BE FOUND."
def intersecting(specs, activated=False):
if activated is False:
print("THE FUNCTION [intersecting] IS NOT ACTIVATED")
return {
St.message: "THE FUNCTION [intersecting] IS NOT ACTIVATED.",
St.error_code: 1,
St.result: None
}
print Ut.headings("EXECUTING INTERSECTION SPECS...")
# 1. GENERATE THE LENS NAME
lens_name = generate_lens_name(specs['datasets'], operator="intersection")
specs[St.lens] = "{}{}".format(Ns.lens, lens_name['name'])
Ut.update_specification(specs)
# **********************************
# 3. GOOD TO GO CHECK
# **********************************
query = """
SELECT *
{{
<{}> ?predicate ?object .
}}
""".format(specs[St.lens])
check = Lens_Union.run_checks(specs, query, operator="intersection")
# NOT GOOD TO GO, IT ALREADY EXISTS
if check[St.message].__contains__("ALREADY EXISTS"):
return {
St.message: check[St.message],
St.error_code: 71,
St.result: specs[St.lens]
}
# **********************************
# GOOD TO GO
# **********************************
else:
try:
specs[St.lens_target_triples] = ""
# DOCUMENTING START TIME
lens_start = time.time()
print "\n4. GENERATE THE INSERT QUERY"
specs[St.insert_query] = intersection_extended(
specs, lens_name=specs[St.lens_name])
print specs[St.insert_query]
print "\n5. >>> LOOKING FOR INTERSECTING LINKS"
print "\t", Qry.boolean_endpoint_response(specs[St.insert_query])
print "\n6. EXTRACTING THE NUMBER OF TRIPLES"
specs[St.triples] = Qry.get_namedgraph_size("{0}{1}".format(
Ns.lens, specs[St.lens_name]))
lens_end = time.time()
diff = lens_end - lens_start
print " \n>>> Executed so far in : {:<14}".format(
str(datetime.timedelta(seconds=diff)))
if int(specs[St.triples]) > 0:
for linkset in specs[St.datasets]:
specs[St.lens_target_triples] += \
"\n\t void:target <{}> ;".format(linkset)
print "\n7. INSERTING THE GENERIC METADATA"
metadata = Gn.intersection_meta(specs)
# print metadata
Qry.boolean_endpoint_response(metadata)
# print "\n8. WRITING TO FILE"
server_message = "Linksets created as: {}".format(
specs[St.lens])
message = "The linkset was created as [{}] with {} triples found!".format(
specs[St.lens], specs[St.triples])
print "\n\t", server_message
Urq.register_lens(specs, is_created=True)
ls_end_2 = time.time()
diff = ls_end_2 - lens_end
print ">>> Executed in : {:<14}".format(
str(datetime.timedelta(seconds=diff)))
print "\t*** JOB DONE! ***"
return {
St.message: message,
St.error_code: 0,
St.result: specs[St.lens]
}
else:
print "The linkset was not generated as no match could be found"
print "\t*** JOB DONE! ***"
return {
St.message:
"The linkset was not generated as no match could be found",
St.error_code: 4,
St.result: None
}
except Exception as err:
traceback.print_exc()
return {
St.message: Ec.ERROR_CODE_1,
St.error_code: 5,
St.result: None
}
# specs = {
# 'lens_operation': u'intersection',
# 'datasets': [u'http://risis.eu/linkset/grid_20170712_eter_2014_approxStrSim_Organization_altLabel_P1079405301',
# u'http://risis.eu/linkset/grid_20170712_eter_2014_approxStrSim_Organization_altLabel_P1661430032',
# u'http://risis.eu/linkset/grid_20170712_eter_2014_approxStrSim_Organization_label_N1860664105'],
# 'researchQ_URI': u'http://risis.eu/activity/idea_67a6ce'}
# specs_2 = {'lens_operation': u'intersection',
# 'datasets': [u'http://risis.eu/lens/union_Grid_20170712_Eter_2014_N291690309',
# u'http://risis.eu/lens/union_Orgreg_20170718_Eter_2014_P1061032980',
# u'http://risis.eu/lens/union_Orgreg_20170718_Grid_20170712_N1966224323'],
# 'researchQ_URI': u'http://risis.eu/activity/idea_67a6ce'}
#
# specs_3 = {'lens_operation': u'intersection',
# 'datasets': [
# u'http://risis.eu/linkset/orgreg_20170718_grid_20170712_approxStrSim_University_Entity_current_name_English_N682223883',
# u'http://risis.eu/linkset/orgreg_20170718_grid_20170712_approxStrSim_University_Entity_current_name_English_P2117262605',
# u'http://risis.eu/lens/union_Grid_20170712_Eter_2014_N291690309',
# u'http://risis.eu/lens/union_Orgreg_20170718_Eter_2014_P1061032980',
# u'http://risis.eu/lens/union_Orgreg_20170718_Grid_20170712_N1966224323'],
# 'researchQ_URI': u'http://risis.eu/activity/idea_67a6ce'}
#
#
#
# print intersection_extended(specs_3, "lens_name", display=False)
# import Alignments.Manage.AdminGraphs as adm
# adm.drop_a_lens("http://risis.eu/lens/intersection_Grid_20170712_Eter_2014_P1326988364", display=True, activated=True)
# print intersecting(specs, activated=True)
def refine_lens(specs, activated=False, check_file=False):
try:
message = Ec.ERROR_CODE_0.replace('\n', "<br/>")
if activated is False:
print Ut.headings("THE FUNCTION [refine_lens] IS NOT ACTIVATED")
return {St.message: message, St.error_code: 4, St.result: None}
# 1. UPDATING THE SPECS BY CHANGING LINKSET TO TENS
specs[St.refined] = specs['linkset']
specs.pop('linkset')
Ut.update_specification(specs)
# CHECKING WHETHER THE LENS IS REFINENABLE
# Refine.is_refinable(specs[St.refined])
# PRINTING THE SPECIFICATIONS
# lensUt.print_specs(specs)
# ASSIGN THE SAME AS COUNT
specs[St.sameAsCount] = Qry.get_same_as_count(specs[St.mechanism])
message = Ec.ERROR_CODE_4.replace('\n', "<br/>")
if specs[St.sameAsCount]:
source = specs[St.source]
target = specs[St.target]
# 2. SET THE LENS NAME
# *******************************
print "\n2. SET THE LENS NAME"
# *******************************
lensUt.lens_refine_name(specs, 'refine')
#*******************************
# GOOD TO GO CHECK
# *******************************
query = """
SELECT *
{{
<{}> ?predicate ?object .
}}
""".format(specs[St.lens])
check = Lens_Union.run_checks(specs, query, operator="refine")
# NOT GOOD TO GO, IT ALREADY EXISTS
if check[St.message].__contains__("ALREADY EXISTS"):
return {
St.message: check[St.message],
St.error_code: 71,
St.result: specs[St.lens]
}
# *******************************
# GOOD TO GO
# *******************************
else:
lens_start = time.time()
# UPDATE THE SPECIFICATION
Ut.update_specification(specs[St.source])
Ut.update_specification(specs[St.target])
# PRINTING THE SPECIFICATIONS
lensUt.print_specs(specs)
########################################################################
print """\n4. EXECUTING THE GEO-MATCH """
########################################################################
geo_match(specs)
########################################################################
print """\n5. EXTRACT THE NUMBER OF TRIPLES """
########################################################################
specs[St.triples] = Qry.get_namedgraph_size("{0}{1}".format(
Ns.lens, specs[St.lens_name]))
########################################################################
print """\n6. ASSIGN THE SPARQL INSERT QUERY """
########################################################################
specs[St.insert_query] = "{} ;\n{};\n{}".format(
geo_load_query(specs, True), geo_load_query(specs, False),
geo_match_query(specs))
lens_end = time.time()
diff = lens_end - lens_start
print "\n\t>>> Executed so far in : {:<14}".format(
str(datetime.timedelta(seconds=diff)))
if int(specs[St.triples]) > 0:
########################################################################
print """\n4. INSERTING THE GENERIC METADATA """
########################################################################
metadata = Gn.lens_refine_geo_metadata(specs)
Qry.boolean_endpoint_response(metadata)
########################################################################
print """\n5. WRITING TO FILE """
########################################################################
src = [source[St.graph_name], "", source[St.entity_ns]]
trg = [target[St.graph_name], "", target[St.entity_ns]]
# linkset_path = "D:\datasets\Linksets\ExactName"
linkset_path = DIRECTORY
writelinkset(src,
trg,
specs[St.lens_name],
linkset_path,
metadata,
check_file=check_file)
server_message = "Linksets created as: {}".format(
specs[St.lens])
message = "The linkset was created as [{}] with {} triples found!".format(
specs[St.lens], specs[St.triples])
print "\n\t", server_message
Urq.register_lens(specs, is_created=True)
ls_end_2 = time.time()
diff = ls_end_2 - lens_end
print ">>> Executed in : {:<14}".format(
str(datetime.timedelta(seconds=diff)))
print "\t*** JOB DONE! ***"
return {
St.message: message,
St.error_code: 0,
St.result: specs[St.lens]
}
else:
print "\tThe linkset was not generated as no match could be found"
print "\t*** JOB DONE! ***"
return {
St.message: message,
St.error_code: 4,
St.result: None
}
except Exception as err:
traceback.print_exc()
return {St.message: Ec.ERROR_CODE_1, St.error_code: 5, St.result: None}
# print geo_load_query(specs, is_source=True)
# print geo_load_query(specs, is_source=False)
# geo_match_query(specs)
# traceback.print_exception()
# import Alignments.Manage.AdminGraphs as adm
# adm.drop_a_lens("http://risis.eu/lens/refine_union_Grid_20170712_Eter_2014_N291690309", display=True, activated=True)
# refine_lens(specs_example, activated=True, check_file=False)
#
# adm.drop_a_lens("http://risis.eu/lens/refine_union_Orgreg_20170718_Eter_2014_P1061032980", display=True, activated=True)
# refine_lens(specs_example_2, activated=True, check_file=False)
#
# adm.drop_a_lens("http://risis.eu/lens/refine_union_Orgreg_20170718_Grid_20170712_N1966224323", display=True, activated=True)
# refine_lens(specs_example_3, activated=True, check_file=False)
def run_checks_expands(specs, check_type):
heading = "\nRUNNING LINKSET SPECS CHECK FOR EXPANSION" \
"\n=========================================="
print heading
ask = "ASK {{ <#> ?p ?o . }}"
linkset = specs[St.linkset]
"""
# CHECK WHETHER THE SOURCE & TARGET GRAPHS EXIST
"""
g_exist_q = "ASK { GRAPH <@> {?s ?p ?o} }"
src_g_exist = Qry.boolean_endpoint_response(
g_exist_q.replace("@", specs[St.source][St.graph]))
trg_g_exist = Qry.boolean_endpoint_response(
g_exist_q.replace("@", specs[St.target][St.graph]))
if (src_g_exist == "false") or (trg_g_exist == "false"):
print Ec.ERROR_CODE_10
return {
St.message: Ec.ERROR_CODE_10,
St.error_code: 10,
St.result: None,
'inserted': 0
}
"""
# CHECK THE TASK SPECIFIC PREDICATE COUNT
"""
if specs[St.sameAsCount] is None:
print Ec.ERROR_CODE_1
return {
St.message: Ec.ERROR_CODE_1,
St.error_code: 1,
St.result: None,
'inserted': 0
}
"""
# CHECK WHETHER THE LINKSET WAS ALREADY CREATED AND ITS ALTERNATIVE NAME REPRESENTATION
"""
# CHECK WHETHER THE CURRENT LINKSET NAME EXIST IN THE GENERIC METADATA
# print ask.replace("#", linkset)
ask_1 = Qry.boolean_endpoint_response(ask.replace("#", linkset))
# print ask_1
# THE CURRENT AME EXIST
if ask_1 == "true":
print "ASK_1:\n\t- YES, THE CURRENT NAME EXIST"
message = Ec.ERROR_CODE_2.replace('#', linkset)
print "\t", message
return {
St.message: message.replace("\n", "<br/>"),
St.error_code: 2,
St.result: linkset,
'inserted': 0
}
elif ask_1 == "false":
print "\nASK_1:\n\t- NO, THE CURRENT NAME DOES NOT EXIST"
print "\t- LINKSET {} \n\t- DOES NOT HAVE ANY GENERIC METADATA.".format(
linkset)
return {
St.message: "GOOD TO GO",
St.result: "GOOD TO GO",
St.error_code: 0,
'inserted': 0
}
def difference(specs, save=False, activated=False):
# print "LINKSET FUNCTION ACTIVATED: {}".format(activated)
if activated is False:
print "THE FUNCTION IS NOT ACTIVATED" \
"\n======================================================" \
"========================================================"
return {St.message: "THE FUNCTION IS NOT ACTIVATED.", St.error_code: 1, St.result: None}
else:
print "COMPUTING THE DIFFERENCE BETWEEN:\n\t{}\n\t{}".format(specs[St.subjectsTarget], specs[St.objectsTarget])
Lu.diff_lens_name(specs)
# GENERATE THE LINKSET OF THE DIFFERENCE
diff_insert_query = """
PREFIX prov: <{0}>
PREFIX specific:<{8}>
### Difference between
### <{1}> and
### <{2}>
INSERT
{{
GRAPH <{3}>
{{
?subject ?newSingletons ?object .
}}
GRAPH specific:{9}
{{
?newSingletons prov:wasDerivedFrom ?predicate1 .
?predicate1 ?pre ?obj .
}}
}}
WHERE
{{
{{
GRAPH <{4}>
{{
?subject ?predicate1 ?object .
bind( iri(replace("{5}{6}_#", "#", strafter(str(uuid()), "uuid:") )) as ?newSingletons )
}}
GRAPH ?g
{{
?predicate1 ?pre ?obj .
}}
}}
MINUS
{{
GRAPH <{7}>
{{
?subject ?predicate2 ?object .
}}
}}
}}
""".format(Ns.prov, specs[St.subjectsTarget], specs[St.objectsTarget], specs[St.lens],
specs[St.subjectsTarget], Ns.alivocab, "diff", specs[St.objectsTarget], Ns.singletons,
specs[St.lens_name])
# print diff_insert_query
# print specs[St.lens_name]
specs[St.insert_query] = diff_insert_query
# RUN THE INSERT QUERY
insertion_result = Qry.boolean_endpoint_response(diff_insert_query)
print "\tDIFFERENCE INSERTED: {}".format(insertion_result)
# RUN THE METADATA
metadata = Gn.diff_meta(specs)
# print metadata
if int(specs[St.triples]) > 0:
insertion_metadata = Qry.boolean_endpoint_response(metadata)
print "\tDIFFERENCE INSERTED METADATA: {}".format(insertion_metadata)
if save is True:
# GENERATE LINKSET CONSTRUCT QUERY
construct_query = "\n{}\n{}\n{}\n".format(
"PREFIX alivocab:<{}>".format(Ns.alivocab),
"construct { ?x ?y ?z }",
"where {{ graph <{}> {{ ?x ?y ?z }} }}".format(specs[St.lens]),
)
# print construct_query
# GET THE SINGLETON METADATA USING THE CONSTRUCT QUERY
construct_response = Qry.endpointconstruct(construct_query)
if construct_response is not None:
construct_response = construct_response.replace('{', "<{}>\n{{".format(specs[St.lens]), 1)
# print construct_response
# WRITE TO FILE
print "\t>>> WRITING THE DIFFERENCE TO FILE"
write_to_file(graph_name=specs[St.lens_name], metadata=metadata.replace("INSERT DATA", ""),
correspondences=construct_response, singletons=None, directory=DIRECTORY)
server_message = "LENS created as: {}".format(specs[St.lens])
message = "The LENS DIFFERENCE was created as<br/> {}" \
"<br/>with {} CORRESPONDENCES INSERTED AS THE DIFFERENCE".format(specs[St.lens], specs[St.triples])
print "\t", server_message
print "\t*** JOB DONE! ***"
# REGISTER THE LENS
register_lens(specs, is_created=True)
return {St.message: message, St.error_code: 0, St.result: specs[St.lens]}
else:
return {St.message: Ec.ERROR_CODE_4, St.error_code: 4, St.result: None}