def parse_txt_queries(p_pol_size, u_pol_size):
"""Parses textual policies files to a set of queries"""
root_path = "./conf/workloads/policies/"
queries_str = []
for i in range(1, p_pol_size + 1):
with open(root_path + 'p' + str(i) + '.rq', 'r') as f:
queries_str.append(f.read())
for i in range(1, u_pol_size + 1):
with open(root_path + 'u' + str(i) + '.rq', 'r') as f:
queries_str.append(f.read())
queries = []
for q_str in queries_str:
q = Query([], [])
fyzz_q = fyzz.parse(q_str)
for sel in fyzz_q.selected:
q.select.append('?' + sel.name)
for wh in fyzz_q.where:
wh_str = ""
for wh_part in range(0, 3):
if type(wh[wh_part]) is fyzz.ast.SparqlVar:
wh_str += '?' + wh[wh_part].name + ' '
elif type(wh[wh_part]) is fyzz.ast.SparqlLiteral:
wh_str += wh[wh_part].value + ' '
elif type(wh[wh_part]) is tuple: #for IRIs
wh_str += wh[wh_part][0] + wh[wh_part][1] + ' '
elif type(wh[wh_part]) is str: #for expanded URIs
wh_str += wh[wh_part][1:-1] + ' '
wh_str += "."
q.where.append(wh_str)
queries.append(q)
return queries
def getVars (query):
vrs=[]
ast = fyzz.parse(query)
if ast.selected == ['*']:
varNDXs = [i for i in range(len(query)) if query.startswith('?', i)]
for v in varNDXs:
var = query[v+1:v+2]
if var not in vrs:
vrs.append(var)
else:
for v in ast.selected:
vrs.append(v.name)
return vrs
def handle(self,added, removed):
"""M3 handler
"""
# May arrive more requests at the same time!
for index in added:
logger.info("New request:")
logger.info(index)
# Query to obtain the sparql query to subscribe to
#query = theNode.CreateQueryTransaction(theSmartSpace)
q = "select ?sparql where {<%s> <%s> ?sparql .}" %(index[2], HAS_SPARQL)
result = self.m3.load_query_sparql(q)
#result = query.sparql_query(q)
if len(result):
sparqlToSubscribeTo = str(result[0][0][2])
# If SPARQL is not good remove the request and continue the loop
parsed = fyzz.parse(sparqlToSubscribeTo)
# Subscribe to SPARQL query issued by the history request
#subscription = theNode.CreateSubscribeTransaction(theSmartSpace)
subscription = self.M3.load_subscribe_sparql(sparqlToSubscribeTo,
HistorySparqlHandler(parsed.where))
#results = subscription.subscribe_sparql(sparqlToSubscribeTo,
# HistorySparqlHandler(parsed.where))
logger.info("Subscribed to: %s" % sparqlToSubscribeTo)
# Serve the history request with existing data, otherwise when
# the read request gets an error (table not found)
HistorySparqlHandler(parsed.where).handle(self.m3.result_sparql_first_sub,
[])
# Track all the subscriptions in an array
global sparqlSubscriptions
sparqlSubscriptions.append(subscription)
else:
logger.warning('Request has no sparql query to subscribe to')
for index in removed:
logger.info("Removed record:")
logger.info(index)
def subscribe_sparql(self, query_text, handlerClass):
"""Performs a SPARQL query"""
# check if it is a select - currently the only supported for subscriptions
# pay attention: fyzz is not reliable! -- see SP2B
try:
fyzz_parsed = parse(query_text)
if fyzz_parsed.type.lower() != "select":
return False, None, None
except:
pass
# build a message
msg_dict = {
"tt" : "SUBSCRIBE",
"mt" : "REQUEST",
"tid" : self.transaction_id,
"sid" : self.space_id,
"nid" : self.node_id,
"enc" : "sparql",
"qt" : query_text
}
# increment the transaction_id
self.transaction_id += 1
# send the request
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((self.sib_host, self.sib_port))
s.send(json.dumps(msg_dict))
s.send("\n")
# wait for the reply
complete_reply = ""
complete = False
while not complete:
reply = s.recv(4096)
complete_reply = complete_reply + reply
try:
# parse the reply
reply_dict = json.loads(complete_reply)
complete = True
except Exception as e:
# uncomplete message
pass
res_list = [reply_dict["results"]]
# return
if reply_dict["code"].lower() == "m3:success":
# spawn a thread
t = threading.Thread(target = self.indication_handler, args = (s, handlerClass,))
t.start()
# add the thread to a proper structure
self.subscriptions[reply_dict["subid"]] = {}
self.subscriptions[reply_dict["subid"]]["thread"] = t
self.subscriptions[reply_dict["subid"]]["socket"] = s
# return
return True, reply_dict["subid"], res_list
else:
return False, None, None
def query_sparql(self, query_text):
"""Performs a SPARQL query"""
# check if it is a select / ask - currently the only supported
# pay attention: fyzz is not reliable! -- see SP2B
try:
fyzz_parsed = parse(query_text)
if not(fyzz_parsed.type.lower() in ["select", "ask"]):
return False, None
except:
pass
# build a message
msg_dict = {
"tt" : "QUERY",
"mt" : "REQUEST",
"tid" : self.transaction_id,
"sid" : self.space_id,
"nid" : self.node_id,
"enc" : "sparql",
"qt" : query_text
}
# increment the transaction_id
self.transaction_id += 1
# send the request
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((self.sib_host, self.sib_port))
s.send(json.dumps(msg_dict))
s.send("\n")
# wait for the reply
reply = ""
msg_check = False
while not msg_check:
# receive a chunk
reply = reply + s.recv(4096)
# parse the reply
try:
reply_dict = json.loads(reply)
msg_check = True
except:
# message incomplete
pass
# close the connection
s.close()
# if it is a select
if reply_dict["qt"].lower() == "select":
res_list = reply_dict["results"]
# if it is an ask
elif reply_dict["qt"].lower() == "ask":
res_list = [reply_dict["results"]]
# return
if reply_dict["code"].lower() == "m3:success":
return True, res_list
else:
return False, None
def readHistoryData(self, sparql, read_request_uri):
"""Make a SPARQL query in a SQL DB
"""
# Parse SPARQL query
# parsed.selected => selected vars
# parsed.variables => all vars
parsed = fyzz.parse(sparql)
# Incoming and outgoing connection to and from a node
# {'car': [(user1, 'hasCar', None), # => Incoming
# (None, 'hasTire', tire10)] } # => Outgoing
adjacency_list = {}
# Select clause is given by selected vars (parsed.selected)
# NULL AS car, NULL AS tire
select_clause_sql = []
# Contains the selected vars prepared for being injected into the SQL
# select clause
selected_vars_sql = []
for var in parsed.selected:
selected_vars_sql.append({'name': var.name, 'uri': None})
# used for joining to property table, use array to save some IF
join_on = ('SubjectID', None, 'Object')
# Creation of the adjacencies list
# ================================
for triple in parsed.where:
_s ,_p, _o = triple
if _s.__class__.__name__ == 'SparqlVar':
_s = _s.name
if _o.__class__.__name__ == 'SparqlVar':
_o = _o.name
# Init incoming and outgoing adjacencies for a node, if not exist
if not _s in adjacency_list:
adjacency_list[_s] = []
if not _o in adjacency_list:
adjacency_list[_o] = []
# Insert both connections
adjacency_list[_s].append( (None,_p,_o) )
adjacency_list[_o].append( (_s,_p,None) )
# Main cycle. Yield 1 union each cycle
# ====================================
# The number of UNIONs is the number of triples that contain selected
# variables
unions = []
for triple in parsed.where:
_s ,_p, _o = triple
if ( _s not in parsed.selected ) and\
( _o not in parsed.selected ): continue
if _s.__class__.__name__ == 'SparqlVar': _s = _s.name
if _o.__class__.__name__ == 'SparqlVar': _o = _o.name
# Table name of the property of the current triple
table_name = self.db.getPropertyTableName(_p)
if table_name:
table_name = table_name[0]
else:
logger.critical('Table "%s" not found, but must be there!' % _p)
return None
# The first element after the timestamps is the removed flag
select = []
select.append('`%s`.%s AS removed' % (table_name, 'Removed'))
for var in selected_vars_sql:
var_name = var['name']
if var_name == _s:
# Get table name. If this handler has been called, the table
# must exist! If it doesn't it's a big problem!!!
table_name = self.db.getPropertyTableName(_p)
if table_name:
table_name = table_name[0]
else:
logger.critical('Table "%s" not found, but must be there!' % _p)
return None
select.append('`%s`.%s AS %s' % (table_name, 'SubjectID', var_name))
# This is a subj so for sure is an URI
var['uri'] = True
elif var_name == _o:
table_name = self.db.getPropertyTableName(_p)
if table_name:
table_name, _t = table_name
else:
logger.critical('Table "%s" not found, but must be there!' % _p)
return None
select.append('`%s`.%s AS %s' % (table_name, 'Object', var_name))
# Property type from table
var['uri'] = _t
else:
select.append('NULL AS %s' % var_name)
# 'NULL AS .., NULL AS ..., smth AS asdf'
select_clause_sql.append(','.join(select))
# Need table name, not the property name
table_name = self.db.getPropertyTableName(_p)
if table_name:
table_name = table_name[0]
else:
logger.warning('Table "%s" not found, but handler has been called!' % _p)
return None
join = " JOIN `%(p)s` ON r.ID = `%(p)s`.RecordID" % ({'p':table_name})
if _s.__class__.__name__ == 'SparqlVar': _s = _s.name
if _o.__class__.__name__ == 'SparqlVar': _o = _o.name
# Contains the nodes where to start the adjacencies walk and also
# the path you come from, to reach that node: you don't want to walk
# back on the same path!
nodes = {_s: (None,_p,_o), _o: (_s,_p,None)}
# Iterate till the stack is not empty
while len(nodes):
# Randomly pop a key-value pair. Order doesn't matter!
node, relFrom = nodes.popitem()
for rel in adjacency_list[node]:
if not rel == relFrom:
_s,_p,_o = rel
# Found a new node on the adjacencies walk. The
# returning path has the verse which is opposite to the
# adjacency
p = self.db.getPropertyTableName(_p)
p_from = self.db.getPropertyTableName(relFrom[1])
if p and p_from:
p = p[0]
p_from = p_from[0]
else:
logger.warning('Tables "%s" or "%s" not found, but handler has been called!'
% (_p,relFrom[1]))
return None
# The JOIN on subj/obj depends on the positions
# of the None in 'rel' and 'relFrom'
# j_on/_from are set to 'SubjectID' or 'Object'
# This trick handle properties chaining in both direction
# and parallel properties
j_on = join_on[ rel.index(None) ]
j_on_from = join_on[ relFrom.index(None) ]
join_properties = {'p': p, 'p_from': p_from,
'j_on': j_on, 'j_on_from': j_on_from}
join_properties_debug = {'p': _p, 'p_from': relFrom[1],
'j_on': j_on, 'j_on_from': j_on_from}
# Add the path to the query with a JOIN clause
join += """
JOIN `%(p)s` ON `%(p_from)s`.%(j_on_from)s = `%(p)s`.%(j_on)s""" % (
join_properties)
_join = """
JOIN `%(p)s` ON `%(p_from)s`.%(j_on)s = `%(p)s`.%(j_on_from)s""" % (
join_properties_debug)
# Incoming adjacency
if _o == None:
nodes[_s] = (None, _p, node)
# Outgoing adjacency
elif _s == None:
nodes[_o] = (node, _p, None)
unions.append('SELECT r.ID AS rec, unix_timestamp(r.Timestamp), ' +
select_clause_sql.pop(0) +
' FROM `Records` AS r' +
join )
sql_query = ' UNION '.join(unions)
sql_query+= ' ORDER BY rec ASC'
cur = self.db.connection.cursor()
cur.execute(sql_query)
# SPARQL response: root
sparql_response = ET.Element('sparql',
{'xmlns':"http://www.w3.org/2005/sparql-results#"})
# SPARQL response: head
sparql_response_head = ET.SubElement(sparql_response, 'head')
ET.SubElement(sparql_response_head,
'variable', {'name': 'HistoryServiceTimestamp'})
for var in parsed.variables:
ET.SubElement(sparql_response_head, 'variable', {'name': var})
# SPARQL response: results
sparql_response_results = ET.SubElement(sparql_response, 'results')
# row is something like:
#
# (2L, datetime.datetime(2013, 4, 17, 15, 7, 13), removed
# None, None, 2L, 1L, None, None, None, None)
#
# (RecordID, date, your_data, lot of NULL/None)
#
row = cur.fetchone()
while row:
sparql_response_result = ET.SubElement(sparql_response_results, 'result')
for i,col in enumerate(row):
# RecordID
if i == 0: continue
# Record Timestamp
elif i == 1:
sparql_response_result_binding = ET.SubElement(
sparql_response_result, 'binding',
{'name': 'HistoryServiceTimestamp'})
sparql_response_result_binding_type = ET.SubElement(
sparql_response_result_binding, 'literal')
sparql_response_result_binding_type.text = str(col)
# Removed
elif i == 2:
sparql_response_result_binding = ET.SubElement(
sparql_response_result, 'binding',
{'name': 'HistoryServiceRemoved'})
sparql_response_result_binding_type = ET.SubElement(
sparql_response_result_binding, 'literal')
sparql_response_result_binding_type.text = str(col)
# Variables
else:
if col == None: continue
sparql_response_result_binding = ET.SubElement(
sparql_response_result,
'binding',
{ 'name': selected_vars_sql[i-3]['name'] } )
if selected_vars_sql[i-3]['uri']:
type = 'uri'
var_value = str( self.db.getInstanceURI(int(col))[0] )
else:
type = 'literal'
var_value = str(col)
sparql_response_binding_type = ET.SubElement(
sparql_response_result_binding, type)
sparql_response_binding_type.text = var_value
row = cur.fetchone()
# Alert, ET.dump() is a print, use ET.tostring() to save in variable
result = ET.tostring(sparql_response)
insert = theNode.CreateInsertTransaction(theSmartSpace)
insert.insert([
Triple( URI(read_request_uri),
URI(HAS_HISTORY_READ_RESPONSE),
Literal(result) ) ])
theNode.CloseQueryTransaction(insert)
a=[]
for line in inp:
m=re.search("PREFIX(.)*",line)
if m!=None:
a.append(str(m.group(0)))
out.write(m.group(0)+"\n")
print m.group(0)
"""
g=[16, 30, 37, 65, 78, 84, 98, 99, 114, 121, 131, 135, 136, 139, 140, 143, 149, 150, 152, 162, 168, 169, 170, 173, 175, 182, 186]
i=1
with open ("./requetes.txt", "r") as inp:
with open("./res.txt",'w') as out:
for elmt in inp:
#if re.search("^#",elmt)==None:
if i not in g :
#print "ligne "+ str(i)
out.write(str(parse(str(elmt)).where)+'\n')
else :
g.append(i)
i+=1
with open ("./res.txt", "r+") as file:
old = file.read()
file.seek(0)
file.write(str(g)+'\n'+old)
with open ("./res.txt", "r+") as file:
print str(file.seek(2))
def subscribe_sparql(self, query_text, handlerClass):
"""Performs a SPARQL query"""
# check if it is a select - currently the only supported for subscriptions
# pay attention: fyzz is not reliable! -- see SP2B
try:
fyzz_parsed = parse(query_text)
if fyzz_parsed.type.lower() != "select":
return False, None, None
except:
pass
# build a message
msg_dict = {
"tt": "SUBSCRIBE",
"mt": "REQUEST",
"tid": self.transaction_id,
"sid": self.space_id,
"nid": self.node_id,
"enc": "sparql",
"qt": query_text,
}
# increment the transaction_id
self.transaction_id += 1
# send the request
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((self.sib_host, self.sib_port))
s.send(json.dumps(msg_dict))
s.send("\n")
# wait for the reply
complete_reply = ""
complete = False
while not complete:
reply = s.recv(4096)
complete_reply = complete_reply + reply
try:
# parse the reply
reply_dict = json.loads(complete_reply)
complete = True
except Exception as e:
# uncomplete message
pass
res_list = [reply_dict["results"]]
# return
if reply_dict["code"].lower() == "m3:success":
# spawn a thread
t = threading.Thread(target=self.indication_handler, args=(s, handlerClass))
t.start()
# add the thread to a proper structure
self.subscriptions[reply_dict["subid"]] = {}
self.subscriptions[reply_dict["subid"]]["thread"] = t
self.subscriptions[reply_dict["subid"]]["socket"] = s
# return
return True, reply_dict["subid"], res_list
else:
return False, None, None
def query_sparql(self, query_text):
"""Performs a SPARQL query"""
# check if it is a select / ask - currently the only supported
# pay attention: fyzz is not reliable! -- see SP2B
try:
fyzz_parsed = parse(query_text)
if not (fyzz_parsed.type.lower() in ["select", "ask"]):
return False, None
except:
pass
# build a message
msg_dict = {
"tt": "QUERY",
"mt": "REQUEST",
"tid": self.transaction_id,
"sid": self.space_id,
"nid": self.node_id,
"enc": "sparql",
"qt": query_text,
}
# increment the transaction_id
self.transaction_id += 1
# send the request
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((self.sib_host, self.sib_port))
s.send(json.dumps(msg_dict))
s.send("\n")
# wait for the reply
reply = ""
msg_check = False
while not msg_check:
# receive a chunk
reply = reply + s.recv(4096)
# parse the reply
try:
reply_dict = json.loads(reply)
msg_check = True
except:
# message incomplete
pass
# close the connection
s.close()
# if it is a select
if reply_dict["qt"].lower() == "select":
res_list = reply_dict["results"]
# if it is an ask
elif reply_dict["qt"].lower() == "ask":
res_list = [reply_dict["results"]]
# return
if reply_dict["code"].lower() == "m3:success":
return True, res_list
else:
return False, None
