def __init__(self, conf, stats):
# save the parameters
self.stats = stats
self.conf = conf
# create a KP
self.kp = SEPAClient()
def __init__(self, conf):
# store the conf
self.conf = conf
self.expirationTime = self.conf.caching['expiration-time']
# create a KP
self.kp = SEPAClient()
def __init__(self, ysap, thingName):
"""This method is used to initialize the WebThing.
Mandatory parameters are the name of the YSAP file
with the configuration and the name of the WebThing
to create"""
# debug message
logging.debug("Device::__init__() invoked")
# read the ysap provided
logging.debug("Device::__init__() -- Reading configuration file")
self.ysap = YSAPObject(ysap, 40)
# store things data
self.thingName = thingName
self.thingURI = self.getRandomURI()
self.thingID = self.thingURI.split("#")[1]
self.thingDescURI = self.thingURI + "_TD"
# initialize an empty dictionary for events, props and actions
# keys are represented by names, values are the related URIs
self.events = {}
self.actions = {}
self.properties = {}
# initialize a list of the custom statements
self.statements = []
# create a KP
logging.debug("Device::__init__() -- Creating a new KP")
self.kp = SEPAClient(None, 40)
# save the important URIs
self.updateURI = self.ysap.updateURI
self.subscribeURI = self.ysap.subscribeURI
# call TD_INIT
u = self.ysap.getUpdate("TD_INIT", {
"thingName": " '%s' " % self.thingName,
"thingURI": " <%s> " % self.thingURI,
"thingDescURI": " <%s> " % self.thingDescURI
})
print(u)
self.kp.update(self.updateURI, u)
def __init__(self, jsap, thingName):
"""This method is used to initialize the WebThing.
Mandatory parameters are the name of the JSAP file
with the configuration and the name of the WebThing
to create"""
# debug message
logging.debug("Device::__init__() invoked")
# read the jsap provided
logging.debug("Device::__init__() -- Reading configuration file")
self.jsap = JSAPObject(jsap, 40)
# setting a namespace
self.defaultNS = "http://eecs.qmul.ac.uk/wot#"
# store things data
self.thingName = thingName
self.thingURI = self.getRandomURI()
self.thingID = self.thingURI.split("#")[1]
# initialize an empty dictionary for events, props and actions
# keys are represented by names, values are the related URIs
self.events = {}
self.actions = {}
self.properties = {}
# initialize a list of the custom statements
self.statements = []
# create a KP
logging.debug("Device::__init__() -- Creating a new KP")
self.kp = SEPAClient(None, 40)
# save the important URIs
self.updateURI = self.jsap.updateUri
self.subscribeURI = self.jsap.subscribeUri
# call TD_INIT
u = self.jsap.getUpdate("ADD_NEW_THING", {
"name": self.thingName,
"thing": self.thingURI
})
print(u)
self.kp.update(self.updateURI, u)
def __init__(self, conf, stats):
# save the parameters
self.stats = stats
self.conf = conf # configuration
self.gs = None # Graphstore
# create a KP
self.kp = SEPAClient()
if 'graphstore' in self.conf.tools:
self.gs = GraphStoreClient(self.conf.tools['graphstore'])
colaboFlowAuditConfig = self.conf.getExtensionConfig(
"space.colabo.flow.audit")
if self.conf.isExtensionActive("space.colabo.flow.audit"):
import uuid
# from colabo.flow.audit import ColaboFlowAudit, audit_pb2
from colabo.flow.audit import audit_pb2
from colabo.flow.audit import ColaboFlowAudit
self.audit_pb2 = audit_pb2
self.colaboFlowAudit = ColaboFlowAudit()
colaboFlowGoConfig = self.conf.getExtensionConfig(
"space.colabo.flow.go")
if self.conf.isExtensionActive("space.colabo.flow.go"):
import uuid
from colabo.flow.go import go_pb2
from colabo.flow.go import ColaboFlowGo
gRpcUrl = colaboFlowGoConfig['host'] + ':' + str(
colaboFlowGoConfig['port'])
# https://docs.python.org/2/library/uuid.html
# flowInstanceId = str(uuid.uuid1())
self.go_pb2 = go_pb2
self.colaboFlowGo = ColaboFlowGo(socketUrl=gRpcUrl)
##############################################################
# initialize the logging system
logger = logging.getLogger('europeanaWT')
logging.basicConfig(format='[%(levelname)s] %(message)s',
level=logging.DEBUG)
logging.debug("Logging subsystem initialized")
# read europeana key
config = configparser.ConfigParser()
config.read("europeanaWT.conf")
clientID = config["Europeana"]["clientId"]
searchLimit = config["Europeana"]["limit"]
# create a new KP
kp = SEPAClient(None, 40)
# create an YSAPObject
ysap = YSAPObject("europeanaTD.yaml", 40)
# create URIs for
# - thing
# - thingDescription
# - search action
# - search action input and output dataschema
thingURI = ysap.getNamespace("qmul") + "EuropeanaWT"
thingDescURI = ysap.getNamespace("qmul") + "EuropeanaWT_TD"
actionURI = ysap.getNamespace("qmul") + "searchAction"
indataSchemaURI = ysap.getNamespace("qmul") + "searchAction_IDS"
outdataSchemaURI = ysap.getNamespace("qmul") + "searchAction_ODS"
# main
if __name__ == "__main__":
##############################################################
#
# Basic initialization
#
##############################################################
# initialize the logging system
logger = logging.getLogger('annotatorWT')
logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.DEBUG)
logging.debug("Logging subsystem initialized")
# 1 - create an instance of the YSAP and the KP
kp = SEPAClient(None, 40)
ysap = YSAPObject(CONFIG_FILE, 40)
##############################################################
#
# Put TD into SEPA
#
##############################################################
# generate URIs and literals for:
logging.debug("Pushing Thing Description to SEPA")
# 1 - thing and thing description
thingName = "Sonic Annotator WT"
thingURI = getRandomURI(ysap.namespaces["qmul"])
thingDescURI = getRandomURI(ysap.namespaces["qmul"])
class AudioClipProcessor:
def __init__(self, conf, stats):
# save the parameters
self.stats = stats
self.conf = conf # configuration
self.gs = None # Graphstore
# create a KP
self.kp = SEPAClient()
if 'graphstore' in self.conf.tools:
self.gs = GraphStoreClient(self.conf.tools['graphstore'])
colaboFlowAuditConfig = self.conf.getExtensionConfig(
"space.colabo.flow.audit")
if self.conf.isExtensionActive("space.colabo.flow.audit"):
import uuid
# from colabo.flow.audit import ColaboFlowAudit, audit_pb2
from colabo.flow.audit import audit_pb2
from colabo.flow.audit import ColaboFlowAudit
self.audit_pb2 = audit_pb2
self.colaboFlowAudit = ColaboFlowAudit()
colaboFlowGoConfig = self.conf.getExtensionConfig(
"space.colabo.flow.go")
if self.conf.isExtensionActive("space.colabo.flow.go"):
import uuid
from colabo.flow.go import go_pb2
from colabo.flow.go import ColaboFlowGo
gRpcUrl = colaboFlowGoConfig['host'] + ':' + str(
colaboFlowGoConfig['port'])
# https://docs.python.org/2/library/uuid.html
# flowInstanceId = str(uuid.uuid1())
self.go_pb2 = go_pb2
self.colaboFlowGo = ColaboFlowGo(socketUrl=gRpcUrl)
def error(params, msg):
return {
"@type": "schema:SearchAction",
"query": params["pattern"],
# "schema:startTime": startTime ,
# "schema:endTime": endTime ,
"actionStatus": "schema:FailedActionStatus",
"error": msg,
"object": {
"@type": "doap:Version",
"revision": VERSION,
"releaseOf": "https://m2.audiocommons.org/"
}
}
def search(self, path, params, cacheEntryUuid, sources):
"""
Performs search for audioclips
:param path:
:param params: dictionary of query parameteres (pattern: what we are searching against)
:param cacheEntryUuid: a string reference to a cache entry stored in a graphstore (it forms: `graphURI = "http://ns#%s" % cacheEntryUuid`)
:param sources: sources we should search against, can be None (=all)
:return results: a JSON response of metadata and results
:return req_id: UUID of the request
"""
# debug print
logging.debug("New audioclip search request")
# update stats
if not path in self.stats.requests["paths"]:
self.stats.requests["paths"][path] = {
"total": 0,
"failed": 0,
"successful": 0
}
self.stats.requests["total"] += 1
self.stats.requests["paths"][path]["total"] += 1
# if the cache missed
if not cacheEntryUuid:
if self.conf.isExtensionActive("space.colabo.flow.audit"):
cfAReqSWoCache = self.audit_pb2.SubmitAuditRequest(
name='searchSoundsNoCache')
cfAResSWoCache = self.colaboFlowAudit.audit_create(
cfAReqSWoCache)
print("cfAResSWoCache = %s" % (cfAResSWoCache))
# generate an UUID for the request
newCacheEntryUuid = str(uuid4())
graphURI = "http://m2.audiocommons.org/graphs/%s" % newCacheEntryUuid
mainActionURI = "http://m2.audiocommons.org/actions/%s" % newCacheEntryUuid
# init a thread list
threads = []
# define the thread-worker function
def worker(conf, sg_query, cp):
"""
1. calls sparql-generate to access the search provider and generate RDF as a result
2. stores results in either
a) graphstore (self.gs) if exists or
b) SEPA otherwise
Arguments:
conf - configuration
sg_query - sparql-generate query
cp - sound provider name (`jamendo`, `freesound`, `europeana`, ...)
"""
logging.debug("Sending query to SPARQL Generate")
# logging.debug(sg_query)
# do the request to SPARQL-Generate
try:
data = {"query": sg_query}
logging.debug("Request sent to the SPARQL Generate:")
logging.debug(data)
st = time.time()
if self.conf.isExtensionActive("space.colabo.flow.go"):
actionName = 'sparql-gen'
flowId = 'search-sounds'
logging.debug(
"Calling Sparql-gen through ColaboFlow.Go action: '%s' in flow: '%s' "
% (actionName, flowId))
sg_requestDataStr = json.dumps(data)
sg_request = self.go_pb2.ActionExecuteRequest(
flowId=flowId,
name=actionName,
flowInstanceId='fa23',
dataIn=sg_requestDataStr)
sg_response = self.colaboFlowGo.executeActionSync(
sg_request)
sg_respones_text = sg_response.dataOut
else:
logging.debug("Calling Sparql-gen directly")
# NOTE: self.conf.tools["sparqlgen"] == self.config["sparql-generate"]["URI"]
sg_req = requests.post(self.conf.tools["sparqlgen"],
data=data)
sg_respones_text = sg_req.text
et = time.time()
print(et - st)
except Exception as e:
logging.error(
"Exception during request to SPARQL-Generate server: %s"
% (e))
self.stats.requests["failed"] += 1
self.stats.requests["paths"][path]["failed"] += 1
print(traceback.print_exc())
return
logging.debug("Result from the SPARQL Generate:")
logging.debug(sg_respones_text)
if self.gs is not None: # if graphstore exists use it
try:
logging.debug(
"Posting data to graphstore for newCacheEntryUuid: %s"
% (newCacheEntryUuid))
self.gs.insertRDF(sg_respones_text, graphURI)
except Exception as e:
msg = "Error while posting RDF data on graphstore"
logging.error(msg)
print(traceback.print_exc())
logging.debug("Process %s completed!" % cp)
else: # otherwise use SEPA
# from the turtle output create a SPARQL INSERT DATA
logging.debug("Creating INSERT DATA query")
triples = QueryUtils.getTriplesFromTurtle(sg_respones_text)
update = QueryUtils.getInsertDataFromTriples(
triples, graphURI)
# put data in SEPA
try:
logging.debug("Sending INSERT DATA query to SEPA")
self.kp.update(self.conf.tools["sepa"]["update"],
update)
except:
logging.error("Error while connecting to SEPA")
logging.debug("Process %s completed!" % cp)
# read the mappings
results = {}
# cp are search engines (`jamendo`, `freesound`, `europeana`, ...)
for cp in self.conf.mappings["audioclips"]["search"]:
if (sources and cp in sources) or (not sources):
logging.debug("Searching for %s on %s" %
(params["pattern"], cp))
datetimeNow = "\"" + datetime.datetime.now().isoformat(
) + "\"" + "^^<http://www.w3.org/2001/XMLSchema#dateTime>"
# build the SPARQL-generate query
# baseQuery is a content of the file referred to in the config file. So for the freesound service it is the content of the file: `src/services/mediator/lib/mappings/freesound-to-audiocommons/data-adapters/audio-search-by-text.rq`
baseQuery = self.conf.mappings["audioclips"]["search"][cp]
sg_query = QueryUtils.bindInGenerateQuery(
baseQuery, {
"pattern": "\"" + params["pattern"] + "\"",
"startTime": datetimeNow,
"limit": str(params["limit"]
or DEFAULT_RESULTS_LIMIT),
"page": str(params["page"] or 1)
})
# sg_query = baseQuery.replace("$pattern", "\"" + pattern + "\"").replace("$startTime", datetimeNow)
logging.debug('Modified query')
logging.debug(sg_query)
# for every mapping spawn a thread
t = threading.Thread(target=worker,
args=(self.conf, sg_query, cp))
threads.append(t)
t.start()
# wait for results
for t in threads:
t.join()
logging.debug("Ready to query SEPA")
if self.conf.isExtensionActive("space.colabo.flow.audit"):
cfAResSWoCache = self.colaboFlowAudit.audit_finish(
cfAReqSWoCache)
else: # is cached
# graphURI = "http://ns#%s" % cacheEntryUuid
graphURI = "http://m2.audiocommons.org/graphs/%s" % cacheEntryUuid
mainActionURI = "http://m2.audiocommons.org/actions/%s" % cacheEntryUuid
# get results from graphstore or ...
if self.gs is not None: # if self.gs exists, use it
try:
msg = "Collating results ..."
logging.debug(msg)
self.gs.sparqlUpdate("""
PREFIX schema: <http://schema.org/>
PREFIX doap: <http://usefulinc.com/ns/doap#>
INSERT {
GRAPH <%s> {
<%s>
a schema:SearchAction ;
schema:query "%s" ;
schema:actionStatus schema:CompletedActionStatus ;
schema:object <https://m2.audiocommons.org/api/v%s> ;
schema:result ?result ;
schema:error ?error .
<https://m2.audiocommons.org/api/v%s>
a doap:Version;
doap:revision "%s".
<https://m2.audiocommons.org/api/> doap:release <https://m2.audiocommons.org/api/v%s>.
}
}
WHERE {
BIND(BNODE() AS ?searchAction)
GRAPH <%s> {
{?action schema:result ?result}
UNION
{?action schema:error ?error}
}
}""" % (graphURI, mainActionURI, params["pattern"],
VERSION, VERSION, VERSION, VERSION, graphURI))
except Exception as e:
msg = "Error while collating results: " + e.text
logging.error(msg)
self.stats.requests["failed"] += 1
self.stats.requests["paths"][path]["failed"] += 1
print(traceback.print_exc())
return json.dumps(error(params, msg)), -1
try:
msg = "Getting RDF data as JSON-LD ..."
logging.debug(msg)
resultsTurtle = self.gs.getGraph(graphURI)
g = rdflib.Graph()
g.parse(data=resultsTurtle, format="n3")
resultsJsonLd = g.serialize(format="json-ld")
logging.debug(json.loads(resultsJsonLd))
frameTemplate = self.conf.resources["jsonld-frames"][
"audioclips"]["search"]
frame = json.loads(
frameTemplate.replace("$mainAction", mainActionURI))
context = json.loads(self.conf.resources["jsonld-context"])
jres = QueryUtils.frameAndCompact(json.loads(resultsJsonLd),
frame, context)
logging.debug(jres)
except Exception as e:
msg = "Error while getting RDF data as JSON-LD: " + e.text
logging.error(msg)
self.stats.requests["failed"] += 1
self.stats.requests["paths"][path]["failed"] += 1
print(traceback.print_exc())
return json.dumps(error(params, msg)), -1
# ... or from SEPA
else: # no self.gs, use SEPA
# assembly results
query = None
# if not sources:
query = """PREFIX prov: <http://www.w3.org/ns/prov#>
CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <%s> { ?s ?p ?o } }""" % graphURI
# else:
# filters = []
# for s in sources:
# filters.append(" ?pp = <%s> " % self.conf.cps[s])
# query = """PREFIX prov: <http://www.w3.org/ns/prov#>
# CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <%s> { ?s ?p ?o . ?s prov:wasAttributedTo ?pp . FILTER( %s ) } }""" % (graphURI, " || ".join(filters))
# print(query)
try:
msg = "Querying SEPA ..."
logging.debug(msg)
status, results = self.kp.query(
self.conf.tools["sepa"]["query"], query)
frame = json.loads(self.conf.resources["jsonld-frames"]
["audioclips"]["search"])
context = json.loads(self.conf.resources["jsonld-context"])
jres = QueryUtils.getJsonLD(results, frame, context)
except:
msg = "Error while connecting to SEPA"
logging.error(msg)
self.stats.requests["failed"] += 1
self.stats.requests["paths"][path]["failed"] += 1
return json.dumps(error(params, msg)), -1
# return results and cache id
self.stats.requests["successful"] += 1
self.stats.requests["paths"][path]["successful"] += 1
if cacheEntryUuid: # previousely cached
return json.dumps(jres), cacheEntryUuid
else: # not cached already
return json.dumps(jres), newCacheEntryUuid
def show(self, path, audioclipId, source, cacheEntryUuid):
# debug print
logging.debug("New audioclip show request")
# update stats
if not path in self.stats.requests["paths"]:
self.stats.requests["paths"][path] = {
"total": 0,
"failed": 0,
"successful": 0
}
self.stats.requests["total"] += 1
self.stats.requests["paths"][path]["total"] += 1
# verify if cache exists
if not cacheEntryUuid:
# generate an UUID for the request
req_id = str(uuid4())
graphURI = "http://ns#%s" % req_id
# verify if source is one of the supported CPs
if source in self.conf.mappings["audioclips"]["show"]:
logging.debug("Showing audioclip %s from %s" %
(audioclipId, source))
# get the query
sg_query = self.conf.mappings["audioclips"]["show"][
source].replace("$trackId", audioclipId)
# do the request to SPARQL-Generate
try:
data = {"query": sg_query}
sg_req = requests.post(self.conf.tools["sparqlgen"],
data=data)
except Exception as e:
logging.error(
"Exception during request to SPARQL-Generate server")
self.stats.requests["failed"] += 1
self.stats.requests["paths"][path]["failed"] += 1
print(traceback.print_exc())
return
# from the turtle output create a SPARQL INSERT DATA
triples = QueryUtils.getTriplesFromTurtle(sg_req.text)
update = QueryUtils.getInsertDataFromTriples(triples, graphURI)
# put data in SEPA
try:
self.kp.update(self.conf.tools["sepa"]["update"], update)
except:
logging.error("Error while connecting to SEPA")
logging.debug("Process %s completed!" % source)
else:
graphURI = "http://ns#%s" % cacheEntryUuid
logging.debug("Ready to query SEPA")
# assembly results
query = """PREFIX rdf:<http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX dc: <http://purl.org/dc/elements/1.1/>
PREFIX ac: <http://audiocommons.org/ns/audiocommons#>
SELECT ?audioClip ?title
WHERE { GRAPH <%s> { ?audioClip rdf:type ac:AudioClip .
?audioClip dc:title ?title }}""" % graphURI
try:
status, results = self.kp.query(self.conf.tools["sepa"]["query"],
query)
except:
msg = "Error while getting RDF data as JSON-LD: " + exception.text
logging.error(msg)
self.stats.requests["failed"] += 1
self.stats.requests["paths"][path]["failed"] += 1
self.write(json.dumps({"status": "failure", "cause": msg}))
return
# return
self.stats.requests["successful"] += 1
self.stats.requests["paths"][path]["successful"] += 1
if cacheEntryUuid:
return json.dumps({
"status": "ok",
"results": results
}), cacheEntryUuid
else:
return json.dumps({"status": "ok", "results": results}), req_id
def analyse(self, path, audioclipId, cp, descriptor, cacheEntryUuid):
# update stats
if not path in self.stats.requests["paths"]:
self.stats.requests["paths"][path] = {
"total": 0,
"failed": 0,
"successful": 0
}
self.stats.requests["total"] += 1
self.stats.requests["paths"][path]["total"] += 1
# verify if cache exists
# if not cacheEntryUuid:
# generate an UUID for the request
req_id = str(uuid4())
headers = {"Content-Type": "application/json"}
graphURI = "http://ns#%s" % req_id
# invoke the tool
fullURI = self.conf.tools["ac-analysis"][
"baseURI"] + "?provider=%s&id=%s&descriptor=%s" % (cp, audioclipId,
descriptor)
print(fullURI)
req = requests.get(fullURI, headers=headers)
# else:
# graphURI = "http://ns#%s" % cacheEntryUuid
# TODO -- parse and "semanticize" results
results = req.text
logging.info(req.text)
# return
self.stats.requests["successful"] += 1
self.stats.requests["paths"][path]["successful"] += 1
if cacheEntryUuid:
return json.dumps({
"status": "ok",
"results": results
}), cacheEntryUuid
else:
return json.dumps({"status": "ok", "results": results}), req_id
# Initialization
#
##############################################################
# initialize the logging system
logger = logging.getLogger('freesoundWT')
logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.DEBUG)
logging.debug("Logging subsystem initialized")
# read freesound key
config = configparser.ConfigParser()
config.read("freesoundWT.conf")
clientID = config["Freesound"]["clientId"]
# create a new KP
kp = SEPAClient(None, 40)
# create an YSAPObject
if len(sys.argv) < 2:
sys.exit("You need to specify a yaml configuration file!")
yamlFile = sys.argv[1]
ysap = YSAPObject(yamlFile, 40)
# read the qmul namespace
qmul = ysap.getNamespace("qmul")
# create URIs and Literals for the thing and its TD
thingName = "FreesoundWT"
thingURI = getRandomURI(qmul)
thingDescURI = getRandomURI(qmul)
class CollectionProcessor:
def __init__(self, conf, stats):
# save the parameters
self.stats = stats
self.conf = conf
# create a KP
self.kp = SEPAClient()
def search(self, path, pattern, cacheEntry, sources):
# debug print
logging.debug("New collection search request")
# initialize
if cacheEntry:
graphURI = "http://ns#%s" % cacheEntry
else:
graphURI = None
if not cacheEntry:
# generate an UUID for the request
req_id = str(uuid4())
graphURI = "http://ns#%s" % req_id
# update stats
if not path in self.stats.requests["paths"]:
self.stats.requests["paths"][path] = {
"total": 0,
"failed": 0,
"successful": 0
}
self.stats.requests["total"] += 1
self.stats.requests["paths"][path]["total"] += 1
# init a thread list
threads = []
# define the worker function
def worker(conf, sg_query, cp):
# do the request to SPARQL-Generate
try:
data = {"query": sg_query}
sg_req = requests.post(self.conf.tools["sparqlgen"],
data=data)
except Exception as e:
logging.error(
"Exception during request to SPARQL-Generate server")
self.stats.requests["failed"] += 1
self.stats.requests["paths"][path]["failed"] += 1
print(traceback.print_exc())
return
# from the turtle output create a SPARQL INSERT DATA
triples = QueryUtils.getTriplesFromTurtle(sg_req.text)
update = QueryUtils.getInsertDataFromTriples(triples, graphURI)
# put data in SEPA
try:
self.kp.update(self.conf.tools["sepa"]["update"], update)
except:
logging.error("Error while connecting to SEPA")
logging.debug("Process %s completed!" % cp)
# read the mappings
results = {}
for cp in self.conf.mappings["collections"]["search"]:
if (sources and cp in sources) or (not sources):
logging.debug("Searching for %s on %s" % (pattern, cp))
# build the SPARQL-generate query
sg_query = self.conf.mappings["collections"]["search"][
cp].replace("$pattern", pattern)
# for every mapping spawn a thread
t = threading.Thread(target=worker,
args=(self.conf, sg_query, cp))
threads.append(t)
t.start()
# wait for results
for t in threads:
t.join()
logging.debug("Ready to query SEPA")
# assembly results
query = None
if not sources:
query = """PREFIX prov: <http://www.w3.org/ns/prov#>
CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <%s> { ?s ?p ?o } }"""
else:
filters = []
for s in sources:
filters.append(" ?pp = <%s> " % self.conf.cps[s])
query = """PREFIX prov: <http://www.w3.org/ns/prov#>
CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <%s> { ?s ?p ?o . ?s prov:wasAttributedTo ?pp . FILTER( %s ) } }""" % (
graphURI, " || ".join(filters))
print(query)
try:
status, results = self.kp.query(self.conf.tools["sepa"]["query"],
query % graphURI)
frame = json.loads(
self.conf.resources["jsonld-frames"]["collections"]["search"])
context = json.loads(self.conf.resources["jsonld-context"])
jres = QueryUtils.getJsonLD(results, frame, context)
except:
msg = "Error while connecting to SEPA"
logging.error(msg)
self.stats.requests["failed"] += 1
self.stats.requests["paths"][path]["failed"] += 1
self.write(json.dumps({"status": "failure", "cause": msg}))
return
# return
self.stats.requests["successful"] += 1
self.stats.requests["paths"][path]["successful"] += 1
if cacheEntry:
return json.dumps({"status": "ok", "results": jres}), cacheEntry
else:
return json.dumps({"status": "ok", "results": jres}), req_id
class CacheManager:
entries = {}
def __init__(self, conf):
# store the conf
self.conf = conf
self.expirationTime = self.conf.caching['expiration-time']
# create a KP
self.kp = SEPAClient()
def setEntry(self, path, params, sources, uuid):
"""
Sets UUID and timestap for the cache entry.
Cache itself is stored in the triplestore.
Cache itself is stored in the triplestore and addressed through the UUID which represents namespaced graph.
Arguments:
path - request.path
params - search params (pattern, limit, page, ...)
source - what sources to search from
uuid - UUID of the cache (graph) entry
"""
paramStr = json.dumps(params, sort_keys=True)
# add the key if missing
if not path in self.entries:
self.entries[path] = {}
pathEntries = self.entries[path]
if not paramStr in pathEntries:
pathEntries[paramStr] = {}
paramEntries = pathEntries[paramStr]
# store the uuid
paramEntries[str(sources)] = {"id": uuid, "timestamp": datetime.datetime.now()}
def getEntryUuid(self, path, params, sources):
"""
Returns the UUID for the specific search parameters (if matched)
Cache itself is stored in the triplestore and addressed through the UUID which represents namespaced graph.
Arguments:
path - request.path
params - search params (pattern, limit, page, ...)
source - what sources to search from
"""
paramStr = json.dumps(params, sort_keys=True)
# if a request to that path, with that paramStr is found
# in the cache, then return the results
if path in self.entries:
if paramStr in self.entries[path]:
sourcesStr = str(sources)
if sourcesStr in self.entries[path][paramStr]:
uuidStr = self.entries[path][paramStr][sourcesStr]["id"]
if (datetime.datetime.now() - self.entries[path][paramStr][sourcesStr]["timestamp"]).total_seconds() < self.expirationTime: # if not expired
return uuidStr
else:
# SPARQL update to delete the subgraph
graphURI = "http://ns#%s" % uuidStr
update = """DELETE { ?s ?p ?o } WHERE { GRAPH <%s> { ?s ?p ?o } }""" % graphURI
self.kp.update(self.conf.tools["sepa"]["update"], update)
# delete cache entry
del self.entries[path][paramStr][sourcesStr]
if len(self.entries[path][paramStr]) == 0:
del self.entries[path][paramStr]
if len(self.entries[path]) == 0:
del self.entries[path]
# return
return None
class Device:
"""This class represents a generic WebThing"""
# constructor
def __init__(self, ysap, thingName):
"""This method is used to initialize the WebThing.
Mandatory parameters are the name of the YSAP file
with the configuration and the name of the WebThing
to create"""
# debug message
logging.debug("Device::__init__() invoked")
# read the ysap provided
logging.debug("Device::__init__() -- Reading configuration file")
self.ysap = YSAPObject(ysap, 40)
# store things data
self.thingName = thingName
self.thingURI = self.getRandomURI()
self.thingID = self.thingURI.split("#")[1]
self.thingDescURI = self.thingURI + "_TD"
# initialize an empty dictionary for events, props and actions
# keys are represented by names, values are the related URIs
self.events = {}
self.actions = {}
self.properties = {}
# initialize a list of the custom statements
self.statements = []
# create a KP
logging.debug("Device::__init__() -- Creating a new KP")
self.kp = SEPAClient(None, 40)
# save the important URIs
self.updateURI = self.ysap.updateURI
self.subscribeURI = self.ysap.subscribeURI
# call TD_INIT
u = self.ysap.getUpdate("TD_INIT", {
"thingName": " '%s' " % self.thingName,
"thingURI": " <%s> " % self.thingURI,
"thingDescURI": " <%s> " % self.thingDescURI
})
print(u)
self.kp.update(self.updateURI, u)
def addProperty(self, isUri, propertyName, propertyValue, propertyURI=None, dataschema="-", writable=True, stability=0):
"""This method adds a new property to the thing"""
# debug message
logging.debug("Debug::addProperty() invoked")
# generate an URI and store data
if not propertyURI:
self.properties[propertyName] = self.getRandomURI()
else:
self.properties[propertyName] = propertyURI
# add the property
u = None
if isUri:
u = self.ysap.getUpdate("ADD_URI_PROPERTY", {
"thing": self.thingURI,
"property": propertyURI,
"newName": propertyName,
"newStability": stability,
"newWritable": writable,
"newDataSchema": dataschema,
"newValue": propertyValue
})
else:
u = self.ysap.getUpdate("ADD_PROPERTY", {
"thing": self.thingURI,
"property": propertyURI,
"newName": propertyName,
"newStability": stability,
"newWritable": writable,
"newDataSchema": dataschema,
"newValue": propertyValue
})
self.kp.update(self.updateURI, u)
# add new action
def addAction(self, actionName, actionURI=None, fields=[]):
"""This method is used to put the description of
an action into the SEPA instance. Mandatory is the
name of the action to publish. The eventURI is instead
optional. If not provided, will be automatically generated"""
# debug message
logging.debug("Debug::addAction() invoked")
u = self.ysap.getUpdate("TD_ADD_ACTION_GRAPH_INPUT_GRAPH_OUTPUT", {
"thingDescURI": self.thingDescURI,
"actionURI": self.actionURI,
"inField": fieldURI,
"fieldType": f["fieldType"],
"fieldName": f["fieldName"]
})
self.kp.update(self.updateURI, u)
print(u)
class Device:
"""This class represents a generic WebThing"""
# constructor
def __init__(self, jsap, thingName):
"""This method is used to initialize the WebThing.
Mandatory parameters are the name of the JSAP file
with the configuration and the name of the WebThing
to create"""
# debug message
logging.debug("Device::__init__() invoked")
# read the jsap provided
logging.debug("Device::__init__() -- Reading configuration file")
self.jsap = JSAPObject(jsap, 40)
# setting a namespace
self.defaultNS = "http://eecs.qmul.ac.uk/wot#"
# store things data
self.thingName = thingName
self.thingURI = self.getRandomURI()
self.thingID = self.thingURI.split("#")[1]
# initialize an empty dictionary for events, props and actions
# keys are represented by names, values are the related URIs
self.events = {}
self.actions = {}
self.properties = {}
# initialize a list of the custom statements
self.statements = []
# create a KP
logging.debug("Device::__init__() -- Creating a new KP")
self.kp = SEPAClient(None, 40)
# save the important URIs
self.updateURI = self.jsap.updateUri
self.subscribeURI = self.jsap.subscribeUri
# call TD_INIT
u = self.jsap.getUpdate("ADD_NEW_THING", {
"name": self.thingName,
"thing": self.thingURI
})
print(u)
self.kp.update(self.updateURI, u)
def addProperty(self,
isUri,
propertyName,
propertyValue,
propertyURI=None,
dataschema="-",
writable=True,
stability=0):
"""This method adds a new property to the thing"""
# debug message
logging.debug("Debug::addProperty() invoked")
# generate an URI and store data
if not propertyURI:
self.properties[propertyName] = self.getRandomURI()
else:
self.properties[propertyName] = propertyURI
# add the property
u = None
if isUri:
u = self.jsap.getUpdate(
"ADD_URI_PROPERTY", {
"thing": self.thingURI,
"property": propertyURI,
"newName": propertyName,
"newStability": stability,
"newWritable": writable,
"newDataSchema": dataschema,
"newValue": propertyValue
})
else:
u = self.jsap.getUpdate(
"ADD_PROPERTY", {
"thing": self.thingURI,
"property": propertyURI,
"newName": propertyName,
"newStability": stability,
"newWritable": writable,
"newDataSchema": dataschema,
"newValue": propertyValue
})
self.kp.update(self.updateURI, u)
# add new action
def addAction(self, actionName, actionURI=None, fields=[]):
"""This method is used to put the description of
an action into the SEPA instance. Mandatory is the
name of the action to publish. The eventURI is instead
optional. If not provided, will be automatically generated"""
# debug message
logging.debug("Debug::addAction() invoked")
# generate an URI and store data
self.actions[actionName] = {}
if not actionURI:
self.actions[actionName]["uri"] = self.getRandomURI()
else:
self.actions[actionName]["uri"] = actionURI
# generate an URI for the data schema
dataSchema = self.getRandomURI()
# generate and perform the update
u = self.jsap.getUpdate(
"ADD_NEW_ACTION", {
"thing": self.thingURI,
"action": self.actions[actionName]["uri"],
"newName": actionName,
"newInDataSchema": dataSchema,
"newOutDataSchema": "-"
})
self.kp.update(self.updateURI, u)
print(u)
# add fields to the action
self.actions[actionName]["fields"] = fields
for f in fields:
# generate an URI for the field
fieldURI = self.getRandomURI()
# read field name and type
u = self.jsap.getUpdate(
"ADD_INPUT_FIELD_TO_ACTION", {
"thing": self.thingURI,
"action": self.actions[actionName]["uri"],
"inField": fieldURI,
"fieldType": f["fieldType"],
"fieldName": f["fieldName"]
})
self.kp.update(self.updateURI, u)
print(u)
# add event
def addEvent(self, eventName, eventURI=None):
"""This method is used to put the description of
an event into the SEPA instance. Mandatory is the
name of the event to publish. The eventURI is instead
optional. If not provided, will be automatically generated"""
# debug message
logging.debug("Device::addEvent() invoked")
# generate an URI and store data
if not eventURI:
self.events[eventName] = self.getRandomURI()
else:
self.events[eventName] = eventURI
# generate and perform the update
u = self.jsap.getUpdate(
"ADD_EVENT", {
"event": self.events[eventName],
"thing": self.thingURI,
"eName": eventName,
"outDataSchema": "-"
})
print(u)
self.kp.update(self.updateURI, u)
def addCustomStatement(self, statement):
"""This method is used to add custom statements not present
in the jsap file used by this class"""
# debug message
logging.debug("Device::addCustomStatement() invoked")
# store and do the update
self.statements.append(statement)
self.kp.update(self.updateURI, "INSERT DATA { %s }" % statement)
def subscribeToAction(self):
logging.error("Device::subscribeToAction yet to implement!")
# delete Web Thing
def deleteWT(self):
"""This method is used to delete a Thing from the KB. It should be
called before shutting down the Thing"""
# debug message
logging.debug("Device::deleteWT() invoked")
# delete properties
logging.debug("Device::deleteWT() -- removing all the properties")
for p in self.properties:
u = self.jsap.getUpdate("DELETE_PROPERTY", {
"thing": self.thingURI,
"property": self.properties[p]
})
self.kp.update(self.updateURI, u)
# delete actions
logging.debug("Device::deleteWT() -- removing all the actions")
for a in self.actions:
# delete fields, then the action
for f in self.actions[a]["fields"]:
print(f)
u = self.jsap.getUpdate(
"DELETE_INPUT_FIELD_FROM_ACTION", {
"thing": self.thingURI,
"action": self.actions[a]["uri"],
"fieldType": f["fieldType"],
"fieldName": f["fieldName"]
})
print(u)
self.kp.update(self.updateURI, u)
u = self.jsap.getUpdate("DELETE_ACTION", {
"thing": self.thingURI,
"action": self.actions[a]["uri"]
})
self.kp.update(self.updateURI, u)
# delete events
logging.debug("Device::deleteWT() -- removing all the events")
for e in self.events:
u = self.jsap.getUpdate("DELETE_EVENT", {
"thing": self.thingURI,
"event": self.events[e]
})
self.kp.update(self.updateURI, u)
# delete thing
logging.debug("Device::deleteWT() -- removing thing")
u = self.jsap.getUpdate("DELETE_THING", {
"thing": self.thingURI,
"name": self.thingName
})
self.kp.update(self.updateURI, u)
# delete custom statements
logging.debug("Device::deleteWT() -- removing custom statements")
for statement in self.statements:
self.kp.update(self.updateURI, "DELETE DATA { %s }" % statement)
def waitForActions(self, handlerClass):
"""This method is used to subscribe to all the actions request
for this Web Thing. The handler is the only parameter required"""
# get subscription
s = self.jsap.getQuery("GET_ACTION_REQUEST", {"thing": self.thingURI})
# subscribe
self.kp.subscribe(self.subscribeURI, s, "actions",
handlerClass(self.kp, self.jsap))
def getRandomURI(self):
"""This method is used to create a random URI"""
# debug message
logging.debug("Device::getRandomURI() invoked")
# return
return self.defaultNS + str(uuid4())