def records_v2(domain_id,
dataset_id,
where='',
search='',
refine='',
exclude='',
rows=10,
start=0,
sort='',
select='',
api_key=None):
params = {
'where': where,
'search': search,
'refine': refine,
'exclude': exclude,
'rows': rows,
'start': start,
'sort': sort,
'select': select,
'apikey': api_key
}
request = requests.get(
f'https://{domain_id}.opendatasoft.com/api/v2/catalog/datasets/{dataset_id}/records',
params,
timeout=requester.get_timeout(),
headers=requester.create_ods_headers())
request.raise_for_status()
return request.json()
def lookup_uri(uri, term_type='class', language=None):
if uri:
if term_type == 'class':
url = SEARCH_CLASS_URL
else:
url = SEARCH_PROPERTY_URL
if language:
language_selection_query = "language = '{}' OR language = 'undefined' OR language = 'en'".format(
language)
where = "(uri = '{}') AND ({})".format(uri,
language_selection_query)
else:
where = "uri = '{}'".format(uri)
params = {
'where': where,
'rows': ROWS,
'apikey': settings.DATA_API_KEY
}
request = requests.get(url,
params,
timeout=Requester.get_timeout(),
headers=Requester.create_ods_headers())
request.raise_for_status()
return request.json()
else:
raise QueryParameterMissing
def export_records_v2(domain_id,
dataset_id,
where='',
search='',
rows=-1,
start=0,
sort='',
select='',
format='jsonl',
api_key=None):
# Implemented for JSON line only (strongly advised for streaming operations)
params = {
'where': where,
'search': search,
'rows': rows,
'start': start,
'sort': sort,
'select': select,
'apikey': api_key
}
with requests.get(
f'https://{domain_id}.opendatasoft.com/api/v2/catalog/datasets/{dataset_id}/exports/{format}',
params,
timeout=requester.get_timeout(),
headers=requester.create_ods_headers(),
stream=True) as request:
request.raise_for_status()
for line in request.iter_lines():
if line:
yield json.loads(line)
def term_request(query, term_type='class'):
if query:
params = {'q': query, 'type': term_type}
request = requests.get(SEARCH_TERM_URL, params, timeout=Requester.get_timeout(), headers=Requester.create_ods_headers())
request.raise_for_status()
return request.json()
else:
raise QueryParameterMissing
def lookup_v2(domain_id, dataset_id, api_key=None):
params = {'apikey': api_key}
request = requests.get(f"https://{domain_id}.opendatasoft.com/api/v2/catalog/datasets/{dataset_id}",
params,
timeout=requester.get_timeout(),
headers=requester.create_ods_headers())
request.raise_for_status()
return request.json()
def datasets_meta_request(start=0, rows=100):
params = {'start': start, 'rows': rows, 'apikey': settings.DATA_API_KEY}
request = requests.get(DATA_CATALOG_API_SEARCH_URL,
params,
timeout=Requester.get_timeout(),
headers=Requester.create_ods_headers())
request.raise_for_status()
return request.json()
def vocabulary_request(query, lang=None):
if query:
params = {'q': query}
if lang:
params['lang'] = lang
request = requests.get(SEARCH_VOCAB_URL, params, timeout=Requester.get_timeout(), headers=Requester.create_ods_headers())
request.raise_for_status()
return request.json()
else:
raise QueryParameterMissing
def dataset_meta_request(dataset_id):
if dataset_id:
params = {'apikey': settings.DATA_API_KEY}
request = requests.get(DATA_CATALOG_API_URL.format(dataset_id),
params,
timeout=Requester.get_timeout(),
headers=Requester.create_ods_headers())
request.raise_for_status()
return request.json()
else:
raise DatasetIdMissing
def datasets_search_v2(search='', rows=10, sort='explore.popularity_score'):
params = {
'search': search,
'rows': rows,
'sort': sort,
'apikey': settings.DATA_API_KEY
}
request = requests.get(DATA_CATALOG_API_SEARCH_V2_URL,
params,
timeout=Requester.get_timeout(),
headers=Requester.create_ods_headers())
request.raise_for_status()
return request.json()
def is_valid(lov_result):
"""
Checks if a result from linked open vocabulary (class or property) is good enough to be used by the chatbot
Conditions are:
- URI is available (returns HTTP code 200) activated if CHECK_URI_AVAILABILITY is set to True in configuration
- Chatbot score of the result is equal or higher to the MINIMAL_CHATBOT_SCORE defined in configuration
configuration is in your chatbot_app.local_settings.
chatbot score is computed in utils.lov_ods_api._chatbot_score function.
"""
if settings.CHECK_URI_AVAILABILITY:
try:
if requests.get(lov_result['fields']['uri'], timeout=Requester.get_timeout()).status_code != 200:
return False
except (requests.Timeout, ConnectionError):
return False
if lov_result['chatbot_score'] < settings.MINIMAL_CHATBOT_SCORE:
return False
return True
def term_request(query, term_type='class', language='en'):
language_selection_query = "language = '{}' OR language = 'undefined' OR language = 'en'".format(
language)
if query:
query = _clean(query)
splited_query = query.split()
if term_type == 'class':
field_priority = FIELD_CLASS_PRIORITY
field_weight = FIELD_CLASS_WEIGHT
filter_query = _build_filter_query(FIELD_CLASS_PRIORITY,
splited_query)
url = SEARCH_CLASS_URL
else:
field_priority = FIELD_PROPERTY_PRIORITY
field_weight = FIELD_PROPERTY_WEIGHT
filter_query = _build_filter_query(FIELD_PROPERTY_PRIORITY,
splited_query)
url = SEARCH_PROPERTY_URL
ontology_query = _build_ontology_query()
where = "({}) {} AND ({})".format(filter_query, ontology_query,
language_selection_query)
params = {
'where': where,
'rows': ROWS,
'apikey': settings.DATA_API_KEY
}
request = requests.get(url,
params,
timeout=Requester.get_timeout(),
headers=Requester.create_ods_headers())
request.raise_for_status()
result_set = _chatbot_score(request.json(), field_priority,
field_weight, query)
return result_set
else:
raise QueryParameterMissing
