def test_RegexFilter(self):
testType = 'id'
testVal = '1_114'
want = '''FILTER regex(?id, "1_114", "i")\n'''
self.assertTrue(qbuilder.regex_filter(testType, custom=testVal)==want)
#test to string
testVal = '1_114'
want = '''FILTER regex(str(?id), "1_114", "i")\n'''
self.assertTrue(qbuilder.regex_filter(testType,toString=True, custom=testVal)==want)
#test prepend
testVal = '1_114'
want = '''FILTER regex(?id, "addhere1_114", "i")\n'''
self.assertTrue(qbuilder.regex_filter(testType,prepend='addhere', custom=testVal)==want)
#test prepend and tostring
testVal = '1_114'
want = '''FILTER regex(str(?id), "addhere1_114", "i")\n'''
self.assertTrue(qbuilder.regex_filter(testType,toString=True,prepend='addhere', custom=testVal)==want)
def item_results():
'''Like results(), but for items not speakers. Functionally similar.'''
session = bottle.request.environ.get('beaker.session') #@UndefinedVariable
client = session.get('client',None)
if client is None:
session['message'] = "You must log in to view this page!"
bottle.redirect('/')
if len(session.get('partlist',[]))==0 or session.get('partcount',0)==0:
session['message'] = "Please first select some speakers."
bottle.redirect('/psearch')
quer = alquery.AlQuery(client)
query = PREFIXES + """
SELECT distinct ?id ?item ?prompt ?componentName ?media
WHERE {
?item a ausnc:AusNCObject .
?item olac:speaker ?id .
?item austalk:prototype ?prototype .
?prototype austalk:prompt ?prompt .
?item austalk:componentName ?componentName .
?item ausnc:document ?media .
?media austalk:version 1 .
?media austalk:channel "ch6-speaker16" .
?id a foaf:Person .
?id austalk:recording_site ?recording_site .
?recording_site austalk:city ?city .
?id foaf:age ?age .
?id foaf:gender ?gender .
"""
if bottle.request.forms.get('anno') == "required":
query = query + """?ann dada:annotates ?item ."""
partList = session['partlist']
resultsCount = 0
#if there is a complete sentance (prompt) selected from the dropdown, use that and not the 'prompt' regex.
if bottle.request.forms.get('fullprompt'):
query = query + qbuilder.regex_filter('prompt',custom='^'+bottle.request.forms.get('fullprompt')+'$')
else:
query = query + qbuilder.regex_filter('prompt')
query = query + qbuilder.simple_filter('componentName')
query = query + qbuilder.to_str_filter('prototype',prepend="https://app.alveo.edu.au/protocol/item/")
if bottle.request.forms.get('comptype') != "":
query=query+"""FILTER regex(?componentName, "%s", "i")""" % (bottle.request.forms.get('comptype'))
hVdWords = {
'monopthongs': ['head', 'had', 'hud', 'heed', 'hid', 'hood', 'hod', 'whod', 'herd', 'haired', 'hard', 'horde'],
'dipthongs': ['howd', 'hoyd', 'hide', 'hode', 'hade', 'heared']
}
if bottle.request.forms.get('wlist')=='hvdwords':
query=query+'FILTER regex(?prompt, "^%s$", "i")' % "$|^".join(hVdWords['monopthongs'] + hVdWords['dipthongs'])
if bottle.request.forms.get('wlist')=='hvdmono':
query=query+'FILTER regex(?prompt, "^%s$", "i")' % "$|^".join(hVdWords['monopthongs'])
if bottle.request.forms.get('wlist')=='hvddip':
query=query+'FILTER regex(?prompt, "^%s$", "i")' % "$|^".join(hVdWords['dipthongs'])
#This partcount greater than 65 isn't arbitrary,
#The amount of or's in the filter to select 70 or more
#speakers results in a HTTP 414
#Basically the url is too long for the server to handle
#ie: the query is too long.
#This 65 should be a decent compromise between keeping
#a short and potentially more efficient search query
#and over querying for what the user wants.
#When querying for more than 65 speaker we will
#search using the search filters, any further modifications
#to the speaker list after are ignored and items from
#those who are unselected will also be returned.
#later on those extra results will be dumped.
if session['partcount']>65 or session['partcount']==session['searchedcount']:
query = query + "\n"+session.get('partfilters','')+"\n}"
else:
query = query + "\nFILTER ("
for p in partList:
query = query + 'str(?id) = "%s" || ' % p['id']
query = query[:-3] + ')\n}'
results = quer.results_dict_list(session.get('corpus','austalk'), query)
resultsCount = 0
#Converting speaker list into a dict so it's O(1) to add an item to this participant
#Without this, worst case complexity will be O(num_parts*num_items)
partDict = {}
for part in partList:
partDict[part['id']] = part
partDict[part['id']]['item_results'] = []
for row in results:
try:
partDict[row['id']]['item_results'].append(row)
resultsCount = resultsCount + 1
except KeyError:
#Incase we get a result from a speaker we haven't selected
#This will only happen if the speaker filters are applied
#instead of selecting each of the speakers.
pass
partList = partDict.values()
session['itemcount'] = resultsCount
undoExists = 'backupItemList' in session and len(session['backupItemList'])>0
return bottle.template('itemresults',
partList=partList,
resultsCount=resultsCount,
message=session.pop('message',''),
undo=undoExists,
name=session.get('name',None))
def results():
'''Perfoms a search of speakers and display the results as a table. Saves the results in the session so they can be retrieved later'''
session = bottle.request.environ.get('beaker.session') #@UndefinedVariable
client = session.get('client',None)
if client is None:
session['message'] = "You must log in to view this page!"
bottle.redirect('/')
quer = alquery.AlQuery(client)
query = PREFIXES+ """
SELECT distinct ?id ?gender ?age ?institution ?first_language_name ?pob_country ?pob_town"""
query = query + """
WHERE {
?id a foaf:Person .
?id austalk:recording_site ?recording_site .
?recording_site austalk:institution ?institution .
?id foaf:age ?age .
?id foaf:gender ?gender .
OPTIONAL { ?id austalk:residential_history ?rh .
OPTIONAL { ?rh austalk:country ?hist_country . }
OPTIONAL { ?rh austalk:state ?hist_state . }
OPTIONAL { ?rh austalk:town ?hist_town . }
OPTIONAL { ?rh austalk:age_from ?age_from . }
OPTIONAL { ?rh austalk:age_to ?age_to . } }
OPTIONAL { ?id austalk:first_language ?first_language .
?first_language iso639schema:name ?first_language_name . }
OPTIONAL { ?id austalk:pob_country ?pob_country . }
OPTIONAL { ?id austalk:pob_town ?pob_town . }
"""
#special args is anything all the form arguments that need something more than a simple filter.
filterTable = {
'simple':['gender','institution','cultural_heritage','ses','professional_category',
'education_level','mother_cultural_heritage','father_cultural_heritage','pob_town',
'mother_professional_category','father_professional_category','mother_education_level',
'father_education_level','mother_pob_town','father_pob_town'],
'regex':['id','other_languages','hobbies_details'],
'boolean':['has_vocal_training','is_smoker','has_speech_problems','has_piercings','has_health_problems',
'has_hearing_problems','has_dentures','is_student','is_left_handed','has_reading_problems',],
'multiselect':['pob_country','pob_state','father_pob_country','mother_pob_country','hist_town','hist_state',
'hist_country','mother_pob_state','father_pob_state',],
'to_str':['first_language','mother_first_language','father_first_language'],
'num_range':['age','age_from','age_to'],
'recording_site':['institution'],
'original_where':['id','institution','age','gender','first_language','pob_country','pob_town','age_from','age_to',
'hist_town','hist_state','hist_country']
}
searchArgs = [arg for arg in bottle.request.forms.allitems() if len(arg[1])>0]
qfilter=""
#build up the where clause
for item in searchArgs:
#to avoid having two lines of the same thing rfom multiselect and the predefined elements.
if item[0] in filterTable['multiselect'] or item[0] in filterTable['original_where']:
if query.find(item[0])==-1:
qfilter = qfilter + """?id austalk:%s ?%s .\n""" % (item[0],item[0])
else:
qfilter = qfilter + """?id austalk:%s ?%s .\n""" % (item[0],item[0])
#now build the filters
regexList = [arg for arg in searchArgs if arg[0] in filterTable['regex']]
for item in regexList:
if item[0]=='id':
id = item[1].strip()
if not ((id[0]=='"' and id[-1]=='"') or (id[0]=="'" and id[-1]=="'")):
id = '"'+id+'"'
qfilter = qfilter + qbuilder.regex_filter('id',custom=id,toString=True,prepend="https://app.alveo.edu.au/speakers/%s/"%session.get('corpus','austalk'))
else:
qfilter = qfilter + qbuilder.regex_filter(item[0])
#we want only unique listings in the multiselect list
unique = []
multiselectList = [arg for arg in searchArgs if arg[0] in filterTable['multiselect']]
for item in multiselectList:
if item[0] not in unique:
unique.append(item[0])
#since birth country is a multipple select, it can be gotten as a list. We can now put it together so it's as
#if it's a normal user entered list of items.
customStr = "".join('''"%s",''' % s for s in bottle.request.forms.getall(item[0]))[0:-1]
qfilter = qfilter + qbuilder.regex_filter(item[0],custom=customStr)
numRangeList = [arg for arg in searchArgs if arg[0] in filterTable['num_range']]
for item in numRangeList:
qfilter = qfilter + qbuilder.num_range_filter(item[0])
toStrList = [arg for arg in searchArgs if arg[0] in filterTable['to_str']]
for item in toStrList:
qfilter = qfilter + qbuilder.to_str_filter(item[0])
booleanList = [arg for arg in searchArgs if arg[0] in filterTable['boolean']]
for item in booleanList:
qfilter = qfilter + qbuilder.boolean_filter(item[0])
simpleList = [arg for arg in searchArgs if arg[0] in filterTable['simple']]
for item in simpleList:
qfilter = qfilter + qbuilder.simple_filter(item[0])
query = query + qfilter + "} \nORDER BY ?id"
resultsList = quer.results_dict_list(session.get('corpus','austalk'), query)
session['partfilters'] = qfilter #so we can use the filters later again
session['partlist'] = resultsList
session['partcount'] = session['resultscount']
session['searchedcount'] = session['resultscount']
undoExists = 'backupPartList' in session.itervalues()
return bottle.template('presults',
resultsList=resultsList,
resultCount=session['partcount'],
message=session.pop('message',''),
undo=undoExists,
name=session.get('name',None))
def item_results():
'''Like results(), but for items not participants. Functionally similar.'''
session = bottle.request.environ.get('beaker.session') #@UndefinedVariable
try:
apiKey = session['apikey']
client = pyalveo.Client(apiKey, BASE_URL)
quer = alquery.AlQuery(client)
except KeyError:
global USER_MESSAGE
USER_MESSAGE = "You must log in to view this page!"
bottle.redirect('/')
query = PREFIXES + """
SELECT distinct ?item ?prompt ?componentName ?media
WHERE {
?item a ausnc:AusNCObject .
?item olac:speaker <%s> .
?item austalk:prompt ?prompt .
?item austalk:prototype ?prototype .
?item austalk:componentName ?componentName .
?item ausnc:document ?media .
?media austalk:version 1 .
?media austalk:channel "ch6-speaker16"
"""
if bottle.request.forms.get('anno') == "required":
query = query + """?ann dada:annotates ?item ."""
partList = session['partlist']
resultsCount = 0
query = query + qbuilder.regex_filter('prompt')
query = query + qbuilder.simple_filter('componentName')
query = query + qbuilder.to_str_filter('prototype',prepend="http://id.austalk.edu.au/protocol/item/")
if bottle.request.forms.get('comptype') != "":
query=query+"""FILTER regex(?componentName, "%s", "i")""" % (bottle.request.forms.get('comptype'))
if bottle.request.forms.get('wlist')=='hvdwords':
query=query+"""FILTER regex(?prompt, "^head$|^had$|^hud$|^hard$|^heared$|^heed$|^hid$|^herd$|^howd$|^hoyd$|^haired$|^hood$|^hod|^whod$", "i")"""
if bottle.request.forms.get('wlist')=='hvdmono':
query=query+"""FILTER regex(?prompt, "^head$|^had$|^hud$|^heed$|^hid$|^herd$|^hood$|^hod|^whod$", "i")"""
if bottle.request.forms.get('wlist')=='hvddip':
query=query+"""FILTER regex(?prompt, "^hard$|^heared$|^herd$|^howd$|^hoyd$|^haired$", "i")"""
query = query + "}"
for row in partList:
row['item_results'] = quer.results_dict_list("austalk", query % (row['id']))
resultsCount = resultsCount + session['resultscount']
session['itemcount'] = resultsCount
session.save()
try:
message = session['message']
session['message'] = ""
session.save()
except KeyError:
session['message'] = ""
message = session['message']
undoExists = 'backupItemList' in session and len(session['backupPartList'])>0
return bottle.template('itemresults', partList=partList, resultsCount=resultsCount, message=message,undo=undoExists, apiKey=apiKey)
def results():
'''Perfoms a search of participants and display the results as a table. Saves the results in the session so they can be retrieved later'''
session = bottle.request.environ.get('beaker.session') #@UndefinedVariable
try:
apiKey = session['apikey']
client = pyalveo.Client(apiKey, BASE_URL)
quer = alquery.AlQuery(client)
except KeyError:
global USER_MESSAGE
USER_MESSAGE = "You must log in to view this page!"
bottle.redirect('/')
try:
message = session['message']
session['message'] = ""
session.save()
except KeyError:
session['message'] = ""
message = session['message']
query = PREFIXES+ """
SELECT ?id ?gender ?age ?city ?first_language ?pob_country ?pob_town"""
query = query + """
WHERE {
?id a foaf:Person .
?id austalk:recording_site ?recording_site .
?recording_site austalk:city ?city .
?id foaf:age ?age .
?id foaf:gender ?gender .
OPTIONAL { ?id austalk:first_language ?fl . }
OPTIONAL { ?fl iso639schema:name ?first_language . }
OPTIONAL { ?id austalk:pob_country ?pob_country . }
OPTIONAL { ?id austalk:pob_town ?pob_town . }
"""
#special args is anything all the form arguments that need something more than a simple filter.
filterTable = {
'simple':['gender','city','pob_state','cultural_heritage','ses','professional_category',
'education_level','mother_cultural_heritage','father_cultural_heritage','pob_town',
'mother_professional_category','father_professional_category','mother_education_level',
'father_education_level','mother_pob_state','mother_pob_town','father_pob_state',
'father_pob_town'],
'regex':['id','other_languages','hobbies_details'],
'boolean':['has_vocal_training','is_smoker','has_speech_problems','has_piercings','has_health_problems',
'has_hearing_problems','has_dentures','is_student','is_left_handed','has_reading_problems',],
'multiselect':['pob_country','father_pob_country','mother_pob_country'],
'to_str':['first_language','mother_first_language','father_first_language'],
'num_range':['age'],
'original_where':['id','city','age','gender','first_language','pob_country','pob_town']
}
searchArgs = [arg for arg in bottle.request.forms.allitems() if len(arg[1])>0]
qfilter=""
#build up the where clause
for item in searchArgs:
#to avoid having two lines of the same thing rfom multiselect and the predefined elements.
if item[0] in filterTable['multiselect'] or item[0] in filterTable['original_where']:
if query.find(item[0])==-1:
qfilter = qfilter + """?id austalk:%s ?%s .\n""" % (item[0],item[0])
else:
qfilter = qfilter + """?id austalk:%s ?%s .\n""" % (item[0],item[0])
#now build the filters
regexList = [arg for arg in searchArgs if arg[0] in filterTable['regex']]
for item in regexList:
if item[0]=='id':
qfilter = qfilter + qbuilder.regex_filter('id',toString=True,prepend="http://id.austalk.edu.au/participant/")
else:
qfilter = qfilter + qbuilder.regex_filter(item[0])
multiselectList = [arg for arg in searchArgs if arg[0] in filterTable['multiselect']]
for item in multiselectList:
#since birth country is a multipple select, it can be gotten as a list. We can now put it together so it's as
#if it's a normal user entered list of items.
customStr = "".join('''"%s",''' % s for s in bottle.request.forms.getall(item[0]))[0:-1]
qfilter = qfilter + qbuilder.regex_filter(item[0],custom=customStr)
numRangeList = [arg for arg in searchArgs if arg[0] in filterTable['num_range']]
for item in numRangeList:
qfilter = qfilter + qbuilder.num_range_filter(item[0])
toStrList = [arg for arg in searchArgs if arg[0] in filterTable['to_str']]
for item in toStrList:
qfilter = qfilter + qbuilder.to_str_filter(item[0])
booleanList = [arg for arg in searchArgs if arg[0] in filterTable['boolean']]
for item in booleanList:
qfilter = qfilter + qbuilder.boolean_filter(item[0])
simpleList = [arg for arg in searchArgs if arg[0] in filterTable['simple']]
for item in simpleList:
qfilter = qfilter + qbuilder.simple_filter(item[0])
query = query + qfilter + "} \nORDER BY ?id"
resultsList = quer.results_dict_list("austalk", query)
session['partfilters'] = qfilter #so we can use the filters later again
session['partlist'] = resultsList
session['partcount'] = session['resultscount']
session.save()
undoExists = 'backupPartList' in session.itervalues()
return bottle.template('presults', resultsList=resultsList, resultCount=session['partcount'],message=message,undo=undoExists, apiKey=apiKey)
