def __init__(self, qq, address, sock=None):
Thread.__init__(self)
if sock is None:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
else:
self.sock = sock
self.addr = address
self.qb = QueryBuilder(
'https://www.tutti.ch/api/v10/list.json?aggregated=1',
'https://www.tutti.ch/de/li/', qq)
def handle_search(request, resource_type):
'''
handle FHIR search operation
'''
query_builder = QueryBuilder(request.authorizer)
search_query = query_builder.build_query(resource_type, request.args)
ttam_resource = None
if (resource_type in ('Patient', 'Sequence') and
g.ttam_client is not None):
ttam_resource = resource_type
resp_bundle = FHIRBundle(search_query, request, ttam_resource=ttam_resource)
return resp_bundle.as_response()
def push_data_keywords(pub_ids=None, index=None):
""" Go through all the publications and their datatables and move data
keywords from tables to their parent publications. """
if not pub_ids:
body = {'query': {'match_all': {}}}
results = es.search(index=index,
doc_type=CFG_PUB_TYPE,
body=body,
_source=False)
pub_ids = [i['_id'] for i in results['hits']['hits']]
for pub_id in pub_ids:
query_builder = QueryBuilder()
query_builder.add_child_parent_relation(
'publication',
relation='parent',
must=True,
related_query={'match': {
'recid': pub_id
}})
tables = es.search(index=index,
doc_type=CFG_DATA_TYPE,
body=query_builder.query,
_source_include='keywords')
keywords = [
d['_source'].get('keywords', None) for d in tables['hits']['hits']
]
# Flatten the list
keywords = [i for inner in keywords for i in inner]
# Aggregate
agg_keywords = defaultdict(list)
for kw in keywords:
agg_keywords[kw['name']].append(kw['value'])
# Remove duplicates
for k, v in agg_keywords.items():
agg_keywords[k] = list(set(v))
body = {"doc": {'data_keywords': dict(agg_keywords)}}
try:
es.update(index=index, doc_type=CFG_PUB_TYPE, id=pub_id, body=body)
except Exception as e:
log.error(e.message)
def suggest(request):
query = request.GET.get('query')
if (query is None) or (len(query.strip()) == 0):
return HttpResponseBadRequest(json.dumps({'response': 'bad request'}))
else:
size = request.GET.get('size')
query_obj = QueryBuilder(query, size)
results = query_obj.fetch_result()['hits']['hits']
data = []
for result in results:
data.append({
'cin': result['_id'],
'name': result['_source']['name']
})
response = HttpResponse(json.dumps(data))
response['Access-Control-Allow-Origin'] = '*'
return response
def get(self):
query = self.get_argument('query')
if (query is None) or (len(query.strip()) == 0):
self.set_status(400)
self.write(json.dumps('bad request'))
else:
size = self.get_argument('size', 5)
query_obj = QueryBuilder(query, size)
results = query_obj.fetch_result()['hits']['hits']
data = []
for result in results:
data.append({
'cin': result['_id'],
'name': result['_source']['name']
})
self.write(json.dumps(data))
def main():
url = "https://nhlstatisticsforohtu.herokuapp.com/players.txt"
reader = PlayerReader(url)
stats = Statistics(reader)
#matcher = And(
# HasAtLeast(5, "goals"),
# HasAtLeast(5, "assists"),
# PlaysIn("PHI")
#)
query = QueryBuilder()
matcher = query.playsIn("NYR").hasAtLeast(5, "goals").build() #
#print(typeof(matcher))
for player in stats.matches(matcher):
print(player)
def answer_question(self, question):
query_builder = QueryBuilder()
pipeline_steps = [
dissect_sentence, capital_filter.process,
independence_date_filter.process, country_code_filter.process,
national_symbol_filter.process, urban_areas_filter.process,
religion_filter.process, geographic_coordinates_filter.process,
national_anthem_filter.process, unemployment_rate_filter.process,
population_growth_rate_filter.process, total_area_filter.process,
population_filter.process, telephone_lines_filter.process,
language_filter.process, public_debt_filter.process,
illicit_drugs_filter.process, industries_filter.process,
climate_filter.process, importation_filter.process,
inflation_rate_filter.process,
electricity_production_filter.process, death_rate_filter.process,
birth_rate_filter.process, internet_users_filter.process,
exportation_filter.process, self.fetch_answer
]
pipeline = combine_pipeline(question, query_builder, pipeline_steps)
consume(pipeline)
def main():
args = get_arguments()
password = args.password
query_source = []
if "file" in args:
with open(args.file, encoding='utf-8') as json_file:
query_source = json.load(json_file)
else:
query_source.append({
"namespace": args.metric_name,
"query": args.query
})
try:
sql = sql_client.SqlClient.SqlClient(args.host, args.username,
password, args.database)
except (pypyodbc.DatabaseError, pypyodbc.DataError) as e:
print("error connecting")
print(type(e))
print(e)
sys.exit(errno.EACCES)
query_builder = QueryBuilder.QueryBuilder()
queries = query_builder.check(query_source)
result = []
for query in queries:
try:
result_query = sql.run_query(query["query"])
except:
print("Error executing namespace {} - query: {} ".format(
query["namespace"], query["query"]))
result_query = []
if result_query is None:
result_query = 0
single_value = isinstance(result_query, int)
if not single_value:
for k, v in result_query:
current_namespace = (".".join(
[query["namespace"],
k.replace(".", "_")])).lower()
result.append({"namespace": current_namespace, "value": v})
print("{}: {}".format(current_namespace, v))
else:
result.append({
"namespace": query["namespace"].lower(),
"value": result_query
})
print("{}: {}".format(query["namespace"].lower(), result_query))
if args.datadog_apikey:
try:
for metric in result:
datadog_metrics.submit_custom_metric(metric["namespace"],
metric["value"],
args.datadog_apikey,
args.datadog_appkey)
except Exception as e:
print("ERROR sending to datadog: %s" % e)
sys.exit(errno.EACCES)
def setUp(self):
self.builder = QueryBuilder()
# Conference Manager App
# Author: Piotr Maślankowski
# Databases, UWr, June 2017
import sys
import json
from database import Database
from query import Query
from query_builder import QueryBuilder
db = Database(False)
openline = sys.stdin.readline()
opendict = json.loads(openline)['open']
db.open(opendict['baza'], opendict['login'], opendict['password'])
if db.opened:
db.create_tables()
builder = QueryBuilder()
for line in sys.stdin:
try:
query = builder.build_query(line)
db.execute(query)
query.parse_result()
print(query.result)
except NotImplementedError:
print(r'{"status": "NOT IMPLEMENTED"}')
db.close()
def search(query,
index=None,
filters=list(),
size=10,
include="*",
exclude="authors",
offset=0,
sort_field=None,
sort_order='',
post_filter=None):
""" Perform a search query.
:param query: [string] query string e.g. 'higgs boson'
:param index: [string] name of the index. If None a default is used
:param filters: [list of tuples] list of filters for the query.
Currently supported: ('author', author_fullname),
('collaboration', collaboration_name), ('date', date)
:param size: [int] max number of hits that should be returned
:param offset: [int] offset for the results (used for pagination)
:param sort_by: [string] sorting field. Currently supported fields:
"title", "collaboration", "date", "relevance"
:param sort_order: [string] order of the sorting either original
(for a particular field) or reversed. Supported:
'' or 'rev'
:return: [dict] dictionary with processed results and facets
"""
# If empty query then sort by date
if query == '' and not sort_field:
sort_field = 'date'
query = HEPDataQueryParser.parse_query(query)
# Build core query
data_query = get_query_by_type(CFG_DATA_TYPE, query)
pub_query = get_query_by_type(CFG_PUB_TYPE, query)
authors_query = get_authors_query(query)
query_builder = QueryBuilder()
query_builder.add_child_parent_relation(
CFG_DATA_TYPE,
relation="child",
related_query=data_query,
other_queries=[pub_query, authors_query])
# Add additional options
query_builder.add_pagination(size=size, offset=offset)
query_builder.add_sorting(sort_field=sort_field, sort_order=sort_order)
query_builder.add_filters(filters)
query_builder.add_post_filter(post_filter)
query_builder.add_aggregations()
query_builder.add_source_filter(include, exclude)
pub_result = es.search(index=index,
body=query_builder.query,
doc_type=CFG_PUB_TYPE)
parent_filter = {
"filtered": {
"filter": {
"terms": {
"_id": [hit["_id"] for hit in pub_result['hits']['hits']]
}
}
}
}
query_builder = QueryBuilder()
query_builder.add_child_parent_relation(CFG_PUB_TYPE,
relation="parent",
related_query=parent_filter,
must=True,
other_queries=[data_query])
query_builder.add_pagination(size=size * 50)
data_result = es.search(index=index,
body=query_builder.query,
doc_type=CFG_DATA_TYPE)
merged_results = merge_results(pub_result, data_result)
return map_result(merged_results)
def main():
args = get_arguments()
password = args.password
query_source = []
if "file" in args:
with open(args.file, encoding='utf-8') as json_file:
query_source = json.load(json_file)
else:
query_source.append({
"namespace": args.metric_name,
"query": args.query
})
try:
sql = sql_client.SqlClient.SqlClient(args.host, args.username,
password, args.database)
except (pypyodbc.DatabaseError, pypyodbc.DataError) as e:
print("error connecting")
print(type(e))
print(e)
sys.exit(errno.EACCES)
query_builder = QueryBuilder.QueryBuilder()
queries = query_builder.check(query_source)
result = []
columns = None
for query in queries:
try:
result_query, columns = sql.run_query(query["query"])
except Exception as e:
print(e)
print("Error executing namespace {} - query: {} ".format(
query["namespace"], query["query"]))
result_query = []
if result_query is None:
result_query = 0
single_value = isinstance(result_query, int)
if single_value:
result.append({
"namespace": query["namespace"].lower(),
"value": result_query
})
print("{}: {}".format(query["namespace"].lower(), result_query))
elif not single_value and ('format' not in query
or query['format'] == 'namespace'):
# original query format does not contain "format" field.
global_namespace = query["namespace"]
for row in result_query:
local_namespace = ""
v = None
cols = []
for col in row:
if isinstance(col, str):
cols.append(col)
else:
v = col
local_namespace = ".".join(cols)
current_namespace = ".".join(
[global_namespace, local_namespace]).lower()
result.append({
"namespace": current_namespace,
"value": v,
"tags": None
})
print("{}: {}".format(current_namespace, v))
elif 'format' in query and query['format'] == 'tags':
# tagged format has specific field "format" in query document to select it
for row in result_query: # row contain the data row. not a dict.
data = dict(zip(columns, row))
tags = [] # string list of tags
v = data['value']
for tag_key, tag_value in data.items():
if tag_key != "value":
a = ""
a.replace(" ", "")
tags.append("{}:{}".format(tag_key, tag_value))
result.append({
"namespace": query["namespace"].lower(),
"value": v,
"tags": tags
})
if args.datadog_apikey:
try:
for metric in result:
datadog_metrics.submit_custom_metric(
metric["namespace"],
metric["value"],
args.datadog_apikey,
args.datadog_appkey,
timestamp=int(time.time()),
tags=metric["tags"] if "tags" in metric else None)
except Exception as e:
print("ERROR sending to datadog: %s" % e)
sys.exit(errno.EACCES)
first_request = False
user_input = input("See more results? (y/n): ").lower()
# If the user input is 'y' or 'yes', then we continue requesting data from the API.
if user_input == "y":
pass
else:
print("Not printing more results. Breaking.")
break
else:
# Handling unsuccessful request.
print("Request unsuccessful with status =", response.text)
break
if __name__ == "__main__":
base_url = "http://data.sfgov.org/resource/bbb8-hzi6.json"
fields = ["applicant", "location"]
n_results = 10
order_by = "applicant"
ascending = True
query_builder = QueryBuilder(base_url, fields, n_results, order_by,
ascending)
handle_requests(base_url, query_builder, fields, n_results)
