def _edit_collection_addall(request, session_id, api_code):
"Add all results from a search into the collections table."
# Find the index in the cache table for the min and max opus_ids
(selections, extras) = url_to_search_params(request.GET)
if selections is None:
log.error('_edit_collection_addall: Could not find selections for'
+' request %s', str(request.GET))
return 'bad search'
user_query_table = get_user_query_table(selections, extras,
api_code=api_code)
if not user_query_table:
log.error('_edit_collection_addall: get_user_query_table failed '
+'*** Selections %s *** Extras %s',
str(selections), str(extras))
return 'search failed'
cursor = connection.cursor()
values = [session_id]
q = connection.ops.quote_name
sql = 'REPLACE INTO '+q('collections')+' ('
sql += q('session_id')+','+q('obs_general_id')+','+q('opus_id')+')'
sql += ' SELECT %s,'
sql += q('obs_general')+'.'+q('id')+','+q('obs_general')+'.'+q('opus_id')
sql += ' FROM '+q('obs_general')
# INNER JOIN because we only want rows that exist in the
# user_query_table
sql += ' INNER JOIN '+q(user_query_table)+' ON '
sql += q(user_query_table)+'.'+q('id')+'='+q('obs_general')+'.'+q('id')
log.debug('_edit_collection_addall SQL: %s %s', sql, values)
cursor.execute(sql, values)
return False
def get_result_count_helper(request, api_code):
(selections, extras) = url_to_search_params(request.GET)
if selections is None or throw_random_http404_error():
log.error(
'get_result_count_helper: Could not find selections for ' +
'request %s', str(request.GET))
ret = Http404(HTTP404_SEARCH_PARAMS_INVALID(request))
exit_api_call(api_code, ret)
raise ret
table = get_user_query_table(selections, extras, api_code=api_code)
if not table or throw_random_http500_error(): # pragma: no cover
log.error(
'get_result_count_helper: Could not find/create query table ' +
'for request %s', str(request.GET))
ret = HttpResponseServerError(HTTP500_SEARCH_CACHE_FAILED(request))
return None, None, ret
cache_key = (settings.CACHE_SERVER_PREFIX + settings.CACHE_KEY_PREFIX +
':resultcount:' + table)
count = cache.get(cache_key)
if count is None:
cursor = connection.cursor()
sql = 'SELECT COUNT(*) FROM ' + connection.ops.quote_name(table)
try:
cursor.execute(sql)
count = cursor.fetchone()[0]
if throw_random_http500_error(): # pragma: no cover
raise DatabaseError('random')
except DatabaseError as e: # pragma: no cover
log.error(
'get_result_count_helper: SQL query failed for request ' +
'%s: SQL "%s" ERR "%s"', str(request.GET), sql, str(e))
ret = HttpResponseServerError(HTTP500_DATABASE_ERROR(request))
return None, None, ret
cache.set(cache_key, count)
return count, table, None
def _edit_collection_addall(request, session_id, api_code):
"Add all results from a search into the collections table."
# Find the index in the cache table for the min and max opus_ids
(selections, extras) = url_to_search_params(request.GET)
if selections is None:
log.error(
'_edit_collection_addall: Could not find selections for' +
' request %s', str(request.GET))
return 'bad search'
user_query_table = get_user_query_table(selections,
extras,
api_code=api_code)
if not user_query_table:
log.error(
'_edit_collection_addall: get_user_query_table failed ' +
'*** Selections %s *** Extras %s', str(selections), str(extras))
return 'search failed'
cursor = connection.cursor()
values = [session_id]
q = connection.ops.quote_name
sql = 'REPLACE INTO ' + q('collections') + ' ('
sql += q('session_id') + ',' + q('obs_general_id') + ',' + q(
'opus_id') + ')'
sql += ' SELECT %s,'
sql += q('obs_general') + '.' + q('id') + ',' + q('obs_general') + '.' + q(
'opus_id')
sql += ' FROM ' + q('obs_general')
# INNER JOIN because we only want rows that exist in the
# user_query_table
sql += ' INNER JOIN ' + q(user_query_table) + ' ON '
sql += q(user_query_table) + '.' + q('id') + '=' + q(
'obs_general') + '.' + q('id')
log.debug('_edit_collection_addall SQL: %s %s', sql, values)
cursor.execute(sql, values)
return False
def api_get_range_endpoints(request, slug, fmt, internal=False):
r"""Compute and return range widget endpoints (min, max, nulls)
This is a PUBLIC API.
Compute and return range widget endpoints (min, max, nulls) for the
widget defined by [slug] based on current search defined in request.
Format: api/meta/range/endpoints/(?P<slug>[-\w]+).(?P<fmt>json|html|csv)
__api/meta/range/endpoints/(?P<slug>[-\w]+).json
Arguments: Normal search arguments
units=<unit> (Optional, gives units to return in)
reqno=<N> (Required for internal, ignored for external)
Can return JSON, HTML, or CSV (external) or JSON (internal)
Returned JSON:
{"min": 63.592, "max": 88.637, "nulls": 2365, units: "km"}
or
{"min": 63.592, "max": 88.637, "nulls": 2365, units: "km", "reqno": 123}
Note that min and max can be strings, not just real numbers. This
happens, for example, with spacecraft clock counts, and may also happen
with floating point values when we want to force a particular display
format (such as full-length numbers instead of exponential notation).
{"min": "0.0000", "max": "50000.0000", "nulls": 11}
Returned HTML:
<body>
<dl>
<dt>min</dt><dd>0.0000</dd>
<dt>max</dt><dd>50000.0000</dd>
<dt>nulls</dt><dd>11</dd>
<dt>units</dt><dd>km</dd>
</dl>
</body>
Returned CSV:
min,max,nulls,units
0.0000,50000.0000,11,km
"""
api_code = enter_api_call('api_get_range_endpoints', request)
if not request or request.GET is None:
ret = Http404(
HTTP404_NO_REQUEST(f'/api/meta/range/endpoints/{slug}.{fmt}'))
exit_api_call(api_code, ret)
raise ret
param_info = get_param_info_by_slug(slug, 'widget')
if not param_info or throw_random_http404_error():
log.error(
'get_range_endpoints: Could not find param_info entry for ' +
'slug %s', str(slug))
ret = Http404(HTTP404_UNKNOWN_SLUG(slug, request))
exit_api_call(api_code, ret)
raise ret
(form_type, form_type_format,
form_type_unit_id) = parse_form_type(param_info.form_type)
units = request.GET.get('units', get_default_unit(form_type_unit_id))
if ((form_type_unit_id and not is_valid_unit(form_type_unit_id, units))
or throw_random_http404_error()):
log.error('get_range_endpoints: Bad units "%s" for ' + 'slug %s',
str(units), str(slug))
ret = Http404(HTTP404_UNKNOWN_UNITS(units, slug, request))
exit_api_call(api_code, ret)
raise ret
param_name = param_info.name # Just name
param_qualified_name = param_info.param_qualified_name() # category.name
(form_type, form_type_format,
form_type_unit_id) = parse_form_type(param_info.form_type)
table_name = param_info.category_name
try:
table_model = apps.get_model('search',
table_name.title().replace('_', ''))
if throw_random_http500_error(): # pragma: no cover
raise LookupError
except LookupError: # pragma: no cover
log.error('api_get_range_endpoints: Could not get_model for %s',
table_name.title().replace('_', ''))
ret = HttpResponseServerError(HTTP500_INTERNAL_ERROR(request))
exit_api_call(api_code, ret)
return ret
param_no_num = strip_numeric_suffix(param_name)
param1 = param_no_num + '1'
param2 = param_no_num + '2'
if (form_type in settings.RANGE_FORM_TYPES
and param_info.slug[-1] not in '12'):
param1 = param2 = param_no_num # single column range query
(selections, extras) = url_to_search_params(request.GET)
if selections is None or throw_random_http404_error():
log.error(
'api_get_range_endpoints: Could not find selections for ' +
'request %s', str(request.GET))
ret = Http404(HTTP404_SEARCH_PARAMS_INVALID(request))
exit_api_call(api_code, ret)
raise ret
# Remove this param from the user's query if it is constrained.
# This keeps the green hinting numbers from reacting to changes to its
# own field.
qualified_param_name_no_num = strip_numeric_suffix(param_qualified_name)
for to_remove in [
qualified_param_name_no_num, qualified_param_name_no_num + '1',
qualified_param_name_no_num + '2'
]:
if to_remove in selections:
del selections[to_remove]
if selections:
user_table = get_user_query_table(selections,
extras,
api_code=api_code)
if (user_table is None
or throw_random_http500_error()): # pragma: no cover
log.error(
'api_get_range_endpoints: Count not retrieve query table' +
' for *** Selections %s *** Extras %s', str(selections),
str(extras))
ret = HttpResponseServerError(HTTP500_SEARCH_CACHE_FAILED(request))
exit_api_call(api_code, ret)
return ret
else:
user_table = None
# Is this result already cached?
cache_key = (settings.CACHE_SERVER_PREFIX + settings.CACHE_KEY_PREFIX +
':rangeep:' + qualified_param_name_no_num + ':units:' +
str(units))
if user_table:
cache_num, cache_new_flag = set_user_search_number(selections, extras)
if cache_num is None: # pragma: no cover
log.error(
'api_get_range_endpoints: Failed to create cache table ' +
'for *** Selections %s *** Extras %s', str(selections),
str(extras))
exit_api_call(api_code, Http404)
raise Http404
# We're guaranteed the table actually exists here, since
# get_user_query_table has already returned for the same search.
cache_key += ':' + str(cache_num)
range_endpoints = cache.get(cache_key)
if range_endpoints is None:
# We didn't find a cache entry, so calculate the endpoints
results = table_model.objects
if selections:
# There are selections, so tie the query to user_table
if table_name == 'obs_general':
where = (connection.ops.quote_name(table_name) + '.id=' +
connection.ops.quote_name(user_table) + '.id')
else:
where = (connection.ops.quote_name(table_name) + '.' +
connection.ops.quote_name('obs_general_id') + '=' +
connection.ops.quote_name(user_table) + '.id')
range_endpoints = (results.extra(where=[where],
tables=[user_table]).aggregate(
min=Min(param1),
max=Max(param2)))
where += ' AND ' + param1 + ' IS NULL AND ' + param2 + ' IS NULL'
range_endpoints['nulls'] = (results.extra(where=[where],
tables=[user_table
]).count())
else:
# There are no selections, so hit the whole table
range_endpoints = results.all().aggregate(min=Min(param1),
max=Max(param2))
where = param1 + ' IS NULL AND ' + param2 + ' IS NULL'
range_endpoints['nulls'] = (results.all().extra(
where=[where]).count())
# The returned range endpoints are converted to the destination
# unit
range_endpoints['min'] = format_unit_value(range_endpoints['min'],
form_type_format,
form_type_unit_id, units)
range_endpoints['max'] = format_unit_value(range_endpoints['max'],
form_type_format,
form_type_unit_id, units)
cache.set(cache_key, range_endpoints)
if internal:
reqno = get_reqno(request)
if reqno is None or throw_random_http404_error():
log.error(
'api_get_range_endpoints: Missing or badly formatted reqno')
ret = Http404(HTTP404_BAD_OR_MISSING_REQNO(request))
exit_api_call(api_code, ret)
raise ret
range_endpoints['reqno'] = reqno
range_endpoints['units'] = units
if fmt == 'json':
ret = json_response(range_endpoints)
elif fmt == 'html':
ret = render_to_response('metadata/endpoints.html',
{'data': range_endpoints})
elif fmt == 'csv':
ret = csv_response(slug, [[
range_endpoints['min'], range_endpoints['max'],
range_endpoints['nulls'], range_endpoints['units']
]], ['min', 'max', 'nulls', 'units'])
else:
log.error('api_get_range_endpoints: Unknown format "%s"', fmt)
ret = Http404(HTTP404_UNKNOWN_FORMAT(fmt, request))
exit_api_call(api_code, ret)
raise ret
exit_api_call(api_code, ret)
return ret
def api_get_mult_counts(request, slug, fmt, internal=False):
r"""Return the mults for a given slug along with result counts.
This is a PUBLIC API.
Format: api/meta/mults/(?P<slug>[-\w]+).(?P<fmt>json|html|csv)
__api/meta/mults/(?P<slug>[-\w]+).json
Arguments: Normal search arguments
reqno=<N> (Required for internal, ignored for external)
Can return JSON, HTML, or CSV (external) or JSON (internal)
Returned JSON:
{'field_id': slug, 'mults': mults}
or:
{'field_id': slug, 'mults': mults, 'reqno': reqno}
mult is a list of entries pairing mult name and result count.
Returned HTML:
<body>
<dl>
<dt>Atlas</dt><dd>2</dd>
<dt>Daphnis</dt><dd>4</dd>
</dl>
</body>
Returned CSV:
name1,name2,name3
number1,number2,number3
"""
api_code = enter_api_call('api_get_mult_counts', request)
if not request or request.GET is None:
ret = Http404(HTTP404_NO_REQUEST(f'/api/meta/mults/{slug}.{fmt}'))
exit_api_call(api_code, ret)
raise ret
(selections, extras) = url_to_search_params(request.GET)
if selections is None or throw_random_http404_error():
log.error(
'api_get_mult_counts: Failed to get selections for slug %s, ' +
'URL %s', str(slug), request.GET)
ret = Http404(HTTP404_SEARCH_PARAMS_INVALID(request))
exit_api_call(api_code, ret)
raise ret
param_info = get_param_info_by_slug(slug, 'col')
if not param_info or throw_random_http404_error():
log.error(
'api_get_mult_counts: Could not find param_info entry for ' +
'slug %s *** Selections %s *** Extras %s', str(slug),
str(selections), str(extras))
ret = Http404(HTTP404_UNKNOWN_SLUG(slug, request))
exit_api_call(api_code, ret)
raise ret
table_name = param_info.category_name
param_qualified_name = param_info.param_qualified_name()
# If this param is in selections already we want to remove it
# We want mults for a param as they would be without itself
if param_qualified_name in selections:
del selections[param_qualified_name]
cache_num, cache_new_flag = set_user_search_number(selections, extras)
if cache_num is None or throw_random_http500_error(): # pragma: no cover
log.error(
'api_get_mult_counts: Failed to create user_selections entry' +
' for *** Selections %s *** Extras %s', str(selections),
str(extras))
ret = HttpResponseServerError(HTTP500_DATABASE_ERROR(request))
exit_api_call(api_code, ret)
return ret
# Note we don't actually care here if the cache table even exists, because
# if it's in the cache, it must exist, and if it's not in the cache, it
# will be created if necessary by get_user_query_table below.
cache_key = (settings.CACHE_SERVER_PREFIX + settings.CACHE_KEY_PREFIX +
':mults_' + param_qualified_name + ':' + str(cache_num))
cached_val = cache.get(cache_key)
if cached_val is not None:
mults = cached_val
else:
mult_name = get_mult_name(param_qualified_name)
try:
mult_model = apps.get_model('search',
mult_name.title().replace('_', ''))
if throw_random_http500_error(): # pragma: no cover
raise LookupError
except LookupError: # pragma: no cover
log.error('api_get_mult_counts: Could not get_model for %s',
mult_name.title().replace('_', ''))
ret = HttpResponseServerError(HTTP500_INTERNAL_ERROR(request))
exit_api_call(api_code, ret)
return ret
try:
table_model = apps.get_model('search',
table_name.title().replace('_', ''))
if throw_random_http500_error(): # pragma: no cover
raise LookupError
except LookupError: # pragma: no cover
log.error('api_get_mult_counts: Could not get_model for %s',
table_name.title().replace('_', ''))
ret = HttpResponseServerError(HTTP500_INTERNAL_ERROR(request))
exit_api_call(api_code, ret)
return ret
results = (table_model.objects.values(mult_name).annotate(
Count(mult_name)))
user_table = get_user_query_table(selections,
extras,
api_code=api_code)
if ((selections and not user_table)
or throw_random_http500_error()): # pragma: no cover
log.error(
'api_get_mult_counts: has selections but no user_table ' +
'found *** Selections %s *** Extras %s', str(selections),
str(extras))
ret = HttpResponseServerError(HTTP500_SEARCH_CACHE_FAILED(request))
exit_api_call(api_code, ret)
return ret
if selections:
# selections are constrained so join in the user_table
if table_name == 'obs_general':
where = [
connection.ops.quote_name(table_name) + '.id=' +
connection.ops.quote_name(user_table) + '.id'
]
else:
where = [
connection.ops.quote_name(table_name) +
'.obs_general_id=' +
connection.ops.quote_name(user_table) + '.id'
]
results = results.extra(where=where, tables=[user_table])
mult_result_list = []
for row in results:
mult_id = row[mult_name]
try:
mult = mult_model.objects.get(id=mult_id)
mult_disp_order = mult.disp_order
mult_label = mult.label
if throw_random_http500_error(): # pragma: no cover
raise ObjectDoesNotExist
except ObjectDoesNotExist: # pragma: no cover
log.error(
'api_get_mult_counts: Could not find mult entry for ' +
'mult_model %s id %s', str(mult_model), str(mult_id))
ret = HttpResponseServerError(HTTP500_INTERNAL_ERROR(request))
exit_api_call(api_code, ret)
return ret
mult_result_list.append(
(mult_disp_order, (mult_label, row[mult_name + '__count'])))
mult_result_list.sort()
mults = OrderedDict() # info to return
for _, mult_info in mult_result_list:
mults[mult_info[0]] = mult_info[1]
cache.set(cache_key, mults)
data = {'field_id': slug, 'mults': mults}
if internal:
reqno = get_reqno(request)
if reqno is None or throw_random_http404_error():
log.error('api_get_mult_counts: Missing or badly formatted reqno')
ret = Http404(HTTP404_BAD_OR_MISSING_REQNO(request))
exit_api_call(api_code, ret)
raise ret
data['reqno'] = reqno
if fmt == 'json':
ret = json_response(data)
elif fmt == 'html':
ret = render_to_response('metadata/mults.html', data)
elif fmt == 'csv':
ret = csv_response(slug, [list(mults.values())],
column_names=list(mults.keys()))
else:
log.error('api_get_mult_counts: Unknown format "%s"', fmt)
ret = Http404(HTTP404_UNKNOWN_FORMAT(fmt, request))
exit_api_call(api_code, ret)
raise ret
exit_api_call(api_code, ret)
return ret
def _edit_collection_range(request, session_id, action, api_code):
"Add or remove a range of opus_ids based on the current sort order."
id_range = request.GET.get('range', False)
if not id_range:
return 'no range given'
ids = id_range.split(',')
if len(ids) != 2:
return 'bad range'
if not ids[0] or not ids[1]:
return 'bad range'
# Find the index in the cache table for the min and max opus_ids
(selections, extras) = url_to_search_params(request.GET)
if selections is None:
log.error('_edit_collection_range: Could not find selections for'
+' request %s', str(request.GET))
return 'bad search'
user_query_table = get_user_query_table(selections, extras,
api_code=api_code)
if not user_query_table:
log.error('_edit_collection_range: get_user_query_table failed '
+'*** Selections %s *** Extras %s',
str(selections), str(extras))
return 'search failed'
cursor = connection.cursor()
sort_orders = []
q = connection.ops.quote_name
for opus_id in ids:
sql = 'SELECT '+q('sort_order')+' FROM '+q('obs_general')
# INNER JOIN because we only want rows that exist in the
# user_query_table
sql += ' INNER JOIN '+q(user_query_table)+' ON '
sql += q(user_query_table)+'.'+q('id')+'='
sql += q('obs_general')+'.'+q('id')
sql += ' WHERE '+q('obs_general')+'.'+q('opus_id')+'=%s'
values = [opus_id]
log.debug('_edit_collection_range SQL: %s %s', sql, values)
cursor.execute(sql, values)
results = cursor.fetchall()
if len(results) == 0:
log.error('_edit_collection_range: No OPUS ID "%s" in obs_general',
opus_id)
return 'opusid not found'
sort_orders.append(results[0][0])
if action == 'addrange':
values = [session_id]
sql = 'REPLACE INTO '+q('collections')+' ('
sql += q('session_id')+','+q('obs_general_id')+','+q('opus_id')+')'
sql += ' SELECT %s,'
sql += q('obs_general')+'.'+q('id')+','
sql += q('obs_general')+'.'+q('opus_id')+' FROM '+q('obs_general')
# INNER JOIN because we only want rows that exist in the
# user_query_table
sql += ' INNER JOIN '+q(user_query_table)+' ON '
sql += q(user_query_table)+'.'+q('id')+'='+q('obs_general')+'.'+q('id')
elif action == 'removerange':
values = []
sql = 'DELETE '
sql += q('collections')+' FROM '+q('collections')+' INNER JOIN '
sql += q(user_query_table)+' ON '
sql += q(user_query_table)+'.'+q('id')+'='
sql += q('collections')+'.'+q('obs_general_id')
else:
assert False
sql += ' WHERE '
sql += q(user_query_table)+'.'+q('sort_order')
sql += ' >= '+str(min(sort_orders))+' AND '
sql += q(user_query_table)+'.'+q('sort_order')
sql += ' <= '+str(max(sort_orders))
log.debug('_edit_collection_range SQL: %s %s', sql, values)
cursor.execute(sql, values)
return False
def _edit_collection_range(request, session_id, action, api_code):
"Add or remove a range of opus_ids based on the current sort order."
id_range = request.GET.get('range', False)
if not id_range:
return 'no range given'
ids = id_range.split(',')
if len(ids) != 2:
return 'bad range'
if not ids[0] or not ids[1]:
return 'bad range'
# Find the index in the cache table for the min and max opus_ids
(selections, extras) = url_to_search_params(request.GET)
if selections is None:
log.error(
'_edit_collection_range: Could not find selections for' +
' request %s', str(request.GET))
return 'bad search'
user_query_table = get_user_query_table(selections,
extras,
api_code=api_code)
if not user_query_table:
log.error(
'_edit_collection_range: get_user_query_table failed ' +
'*** Selections %s *** Extras %s', str(selections), str(extras))
return 'search failed'
cursor = connection.cursor()
sort_orders = []
q = connection.ops.quote_name
for opus_id in ids:
sql = 'SELECT ' + q('sort_order') + ' FROM ' + q('obs_general')
# INNER JOIN because we only want rows that exist in the
# user_query_table
sql += ' INNER JOIN ' + q(user_query_table) + ' ON '
sql += q(user_query_table) + '.' + q('id') + '='
sql += q('obs_general') + '.' + q('id')
sql += ' WHERE ' + q('obs_general') + '.' + q('opus_id') + '=%s'
values = [opus_id]
log.debug('_edit_collection_range SQL: %s %s', sql, values)
cursor.execute(sql, values)
results = cursor.fetchall()
if len(results) == 0:
log.error('_edit_collection_range: No OPUS ID "%s" in obs_general',
opus_id)
return 'opusid not found'
sort_orders.append(results[0][0])
if action == 'addrange':
values = [session_id]
sql = 'REPLACE INTO ' + q('collections') + ' ('
sql += q('session_id') + ',' + q('obs_general_id') + ',' + q(
'opus_id') + ')'
sql += ' SELECT %s,'
sql += q('obs_general') + '.' + q('id') + ','
sql += q('obs_general') + '.' + q('opus_id') + ' FROM ' + q(
'obs_general')
# INNER JOIN because we only want rows that exist in the
# user_query_table
sql += ' INNER JOIN ' + q(user_query_table) + ' ON '
sql += q(user_query_table) + '.' + q('id') + '=' + q(
'obs_general') + '.' + q('id')
elif action == 'removerange':
values = []
sql = 'DELETE '
sql += q('collections') + ' FROM ' + q('collections') + ' INNER JOIN '
sql += q(user_query_table) + ' ON '
sql += q(user_query_table) + '.' + q('id') + '='
sql += q('collections') + '.' + q('obs_general_id')
else:
assert False
sql += ' WHERE '
sql += q(user_query_table) + '.' + q('sort_order')
sql += ' >= ' + str(min(sort_orders)) + ' AND '
sql += q(user_query_table) + '.' + q('sort_order')
sql += ' <= ' + str(max(sort_orders))
log.debug('_edit_collection_range SQL: %s %s', sql, values)
cursor.execute(sql, values)
return False
def api_get_result_count(request, fmt):
"""Return the result count for a given search.
You can specify a sort order as well as search arguments because the result
of the search (including the sort order) is cached for future use.
This is a PUBLIC API.
Format: [__]api/meta/result_count.(?P<fmt>json|html|csv)
Arguments: Normal search arguments
reqno=<N> (Optional)
Can return JSON, HTML, or CSV.
Returned JSON:
{"data": [{"result_count": 47}]}
or
{"data": [{"result_count": 47, "reqno": 1}]}
Returned HTML:
<body>
<dl>
<dt>result_count</dt><dd>47</dd>
</dl>
</body>
Returned CSV:
result count,47
"""
api_code = enter_api_call('api_get_result_count', request)
if not request or request.GET is None:
ret = Http404(settings.HTTP404_NO_REQUEST)
exit_api_call(api_code, ret)
raise ret
(selections, extras) = url_to_search_params(request.GET)
if selections is None:
log.error('api_get_result_count: Could not find selections for '
+'request %s', str(request.GET))
ret = Http404(settings.HTTP404_SEARCH_PARAMS_INVALID)
exit_api_call(api_code, ret)
raise ret
table = get_user_query_table(selections, extras, api_code=api_code)
if not table: # pragma: no cover
log.error('api_get_result_count: Could not find/create query table for '
+'request %s', str(request.GET))
ret = HttpResponseServerError(settings.HTTP500_SEARCH_FAILED)
exit_api_call(api_code, ret)
return ret
cache_key = settings.CACHE_KEY_PREFIX + ':resultcount:' + table
count = cache.get(cache_key)
if count is None:
cursor = connection.cursor()
sql = 'SELECT COUNT(*) FROM ' + connection.ops.quote_name(table)
cursor.execute(sql)
try:
count = cursor.fetchone()[0]
except DatabaseError as e: # pragma: no cover
log.error('api_get_result_count: SQL query failed for request %s: '
+' SQL "%s" ERR "%s"',
str(request.GET), sql, str(e))
ret = HttpResponseServerError(settings.HTTP500_SQL_FAILED)
exit_api_call(api_code, ret)
return ret
cache.set(cache_key, count)
data = {'result_count': count}
reqno = get_reqno(request)
if reqno is not None and fmt == 'json':
data['reqno'] = reqno
if fmt == 'json':
ret = json_response({'data': [data]})
elif fmt == 'html':
ret = render_to_response('metadata/result_count.html', {'data': data})
elif fmt == 'csv':
ret = csv_response('result_count', [['result count', count]])
else:
log.error('api_get_result_count: Unknown format "%s"', fmt)
ret = Http404(settings.HTTP404_UNKNOWN_FORMAT)
exit_api_call(api_code, ret)
raise ret
exit_api_call(api_code, ret)
return ret
def api_get_range_endpoints(request, slug, fmt):
"""Compute and return range widget endpoints (min, max, nulls)
This is a PUBLIC API.
Compute and return range widget endpoints (min, max, nulls) for the
widget defined by [slug] based on current search defined in request.
Format: [__]api/meta/range/endpoints/(?P<slug>[-\w]+)
.(?P<fmt>json|html|csv)
Arguments: Normal search arguments
reqno=<N> (Optional)
Can return JSON, HTML, or CSV.
Returned JSON:
{"min": 63.592, "max": 88.637, "nulls": 2365}
or
{"min": 63.592, "max": 88.637, "nulls": 2365, "reqno": 123}
Note that min and max can be strings, not just real numbers. This
happens, for example, with spacecraft clock counts, and may also happen
with floating point values when we want to force a particular display
format (such as full-length numbers instead of exponential notation).
{"min": "0.0000", "max": "50000.0000", "nulls": 11}
Returned HTML:
<body>
<dl>
<dt>min</dt><dd>0.0000</dd>
<dt>max</dt><dd>50000.0000</dd>
<dt>nulls</dt><dd>11</dd>
</dl>
</body>
Returned CSV:
min,max,nulls
0.0000,50000.0000,11
"""
api_code = enter_api_call('api_get_range_endpoints', request)
if not request or request.GET is None:
ret = Http404(settings.HTTP404_NO_REQUEST)
exit_api_call(api_code, ret)
raise ret
param_info = get_param_info_by_slug(slug, from_ui=True)
if not param_info:
log.error('get_range_endpoints: Could not find param_info entry for '+
'slug %s', str(slug))
ret = Http404(settings.HTTP404_UNKNOWN_SLUG)
exit_api_call(api_code, ret)
raise ret
param_name = param_info.name # Just name
param_qualified_name = param_info.param_qualified_name() # category.name
(form_type, form_type_func,
form_type_format) = parse_form_type(param_info.form_type)
table_name = param_info.category_name
try:
table_model = apps.get_model('search',
table_name.title().replace('_',''))
except LookupError:
log.error('api_get_range_endpoints: Could not get_model for %s',
table_name.title().replace('_',''))
ret = HttpResponseServerError(settings.HTTP500_INTERNAL_ERROR)
exit_api_call(api_code, ret)
return ret
param_no_num = strip_numeric_suffix(param_name)
param1 = param_no_num + '1'
param2 = param_no_num + '2'
if (form_type in settings.RANGE_FORM_TYPES and
param_info.slug[-1] not in '12'):
param1 = param2 = param_no_num # single column range query
(selections, extras) = url_to_search_params(request.GET)
if selections is None:
log.error('api_get_range_endpoints: Could not find selections for '
+'request %s', str(request.GET))
ret = Http404(settings.HTTP404_SEARCH_PARAMS_INVALID)
exit_api_call(api_code, ret)
raise ret
# Remove this param from the user's query if it is constrained.
# This keeps the green hinting numbers from reacting to changes to its
# own field.
qualified_param_name_no_num = strip_numeric_suffix(param_qualified_name)
for to_remove in [qualified_param_name_no_num,
qualified_param_name_no_num + '1',
qualified_param_name_no_num + '2']:
if to_remove in selections:
del selections[to_remove]
if selections:
user_table = get_user_query_table(selections, extras, api_code=api_code)
if user_table is None:
log.error('api_get_range_endpoints: Count not retrieve query table'
+' for *** Selections %s *** Extras %s',
str(selections), str(extras))
ret = HttpResponseServerError(settings.HTTP500_SEARCH_FAILED)
exit_api_call(api_code, ret)
return ret
else:
user_table = None
# Is this result already cached?
cache_key = (settings.CACHE_KEY_PREFIX + ':rangeep:'
+ qualified_param_name_no_num)
if user_table:
cache_num, cache_new_flag = set_user_search_number(selections, extras)
if cache_num is None:
log.error('api_get_range_endpoints: Failed to create cache table '
+'for *** Selections %s *** Extras %s',
str(selections), str(extras))
exit_api_call(api_code, Http404)
raise Http404
# We're guaranteed the table actually exists here, since
# get_user_query_table has already returned for the same search.
cache_key += ':' + str(cache_num)
range_endpoints = cache.get(cache_key)
if range_endpoints is None:
# We didn't find a cache entry, so calculate the endpoints
results = table_model.objects
if selections:
# There are selections, so tie the query to user_table
if table_name == 'obs_general':
where = (connection.ops.quote_name(table_name)+'.id='
+connection.ops.quote_name(user_table)+'.id')
else:
where = (connection.ops.quote_name(table_name)
+'.'+connection.ops.quote_name('obs_general_id')+'='
+connection.ops.quote_name(user_table)+'.id')
range_endpoints = (results.extra(where=[where],
tables=[user_table]).
aggregate(min=Min(param1), max=Max(param2)))
where += ' AND ' + param1 + ' IS NULL AND ' + param2 + ' IS NULL'
range_endpoints['nulls'] = (results.extra(where=[where],
tables=[user_table])
.count())
else:
# There are no selections, so hit the whole table
range_endpoints = results.all().aggregate(min=Min(param1),
max=Max(param2))
where = param1 + ' IS NULL AND ' + param2 + ' IS NULL'
range_endpoints['nulls'] = (results.all().extra(where=[where])
.count())
range_endpoints['min'] = format_metadata_number_or_func(
range_endpoints['min'],
form_type_func,
form_type_format)
range_endpoints['max'] = format_metadata_number_or_func(
range_endpoints['max'],
form_type_func,
form_type_format)
cache.set(cache_key, range_endpoints)
reqno = get_reqno(request)
if reqno is not None and fmt == 'json':
range_endpoints['reqno'] = reqno
if fmt == 'json':
ret = json_response(range_endpoints)
elif fmt == 'html':
ret = render_to_response('metadata/endpoints.html',
{'data': range_endpoints})
elif fmt == 'csv':
ret = csv_response(slug, [[range_endpoints['min'],
range_endpoints['max'],
range_endpoints['nulls']]],
['min', 'max', 'nulls'])
else:
log.error('api_get_range_endpoints: Unknown format "%s"', fmt)
ret = Http404(settings.HTTP404_UNKNOWN_FORMAT)
exit_api_call(api_code, ret)
raise ret
exit_api_call(api_code, ret)
return ret
def api_get_mult_counts(request, slug, fmt):
"""Return the mults for a given slug along with result counts.
This is a PUBLIC API.
Format: [__]api/meta/mults/(?P<slug>[-\w]+).(?P<fmt>json|html|csv)
Arguments: Normal search arguments
reqno=<N> (Optional)
Can return JSON, HTML, or CSV.
Returned JSON:
{'field': slug, 'mults': mults}
or:
{'field': slug, 'mults': mults, 'reqno': reqno}
mult is a list of entries pairing mult name and result count.
Returned HTML:
<body>
<dl>
<dt>Atlas</dt><dd>2</dd>
<dt>Daphnis</dt><dd>4</dd>
</dl>
</body>
Returned CSV:
name1,name2,name3
number1,number2,number3
"""
api_code = enter_api_call('api_get_mult_counts', request)
if not request or request.GET is None:
ret = Http404(settings.HTTP404_NO_REQUEST)
exit_api_call(api_code, ret)
raise ret
(selections, extras) = url_to_search_params(request.GET)
if selections is None:
log.error('api_get_mult_counts: Failed to get selections for slug %s, '
+'URL %s', str(slug), request.GET)
ret = Http404(settings.HTTP404_SEARCH_PARAMS_INVALID)
exit_api_call(api_code, ret)
raise ret
param_info = get_param_info_by_slug(slug)
if not param_info:
log.error('api_get_mult_counts: Could not find param_info entry for '
+'slug %s *** Selections %s *** Extras %s', str(slug),
str(selections), str(extras))
ret = Http404(settings.HTTP404_UNKNOWN_SLUG)
exit_api_call(api_code, ret)
raise ret
table_name = param_info.category_name
param_qualified_name = param_info.param_qualified_name()
# If this param is in selections already we want to remove it
# We want mults for a param as they would be without itself
if param_qualified_name in selections:
del selections[param_qualified_name]
cache_num, cache_new_flag = set_user_search_number(selections, extras)
if cache_num is None:
log.error('api_get_mult_counts: Failed to create user_selections entry'
+' for *** Selections %s *** Extras %s',
str(selections), str(extras))
ret = HttpResponseServerError(settings.HTTP500_DATABASE_ERROR)
exit_api_call(api_code, ret)
return ret
# Note we don't actually care here if the cache table even exists, because
# if it's in the cache, it must exist, and if it's not in the cache, it
# will be created if necessary by get_user_query_table below.
cache_key = (settings.CACHE_KEY_PREFIX + ':mults_' + param_qualified_name
+ ':' + str(cache_num))
cached_val = cache.get(cache_key)
if cached_val is not None:
mults = cached_val
else:
mult_name = get_mult_name(param_qualified_name)
try:
mult_model = apps.get_model('search',
mult_name.title().replace('_',''))
except LookupError:
log.error('api_get_mult_counts: Could not get_model for %s',
mult_name.title().replace('_',''))
ret = HttpResponseServerError(settings.HTTP500_INTERNAL_ERROR)
exit_api_call(api_code, ret)
return ret
try:
table_model = apps.get_model('search',
table_name.title().replace('_',''))
except LookupError:
log.error('api_get_mult_counts: Could not get_model for %s',
table_name.title().replace('_',''))
ret = HttpResponseServerError(settings.HTTP500_INTERNAL_ERROR)
exit_api_call(api_code, ret)
return ret
results = (table_model.objects.values(mult_name)
.annotate(Count(mult_name)))
user_table = get_user_query_table(selections, extras, api_code=api_code)
if selections and not user_table:
log.error('api_get_mult_counts: has selections but no user_table '
+'found *** Selections %s *** Extras %s',
str(selections), str(extras))
ret = HttpResponseServerError(settings.HTTP500_SEARCH_FAILED)
exit_api_call(api_code, ret)
return ret
if selections:
# selections are constrained so join in the user_table
if table_name == 'obs_general':
where = [connection.ops.quote_name(table_name) + '.id='
+ connection.ops.quote_name(user_table) + '.id']
else:
where = [connection.ops.quote_name(table_name)
+ '.obs_general_id='
+ connection.ops.quote_name(user_table) + '.id']
results = results.extra(where=where, tables=[user_table])
mult_result_list = []
for row in results:
mult_id = row[mult_name]
try:
mult = mult_model.objects.get(id=mult_id)
mult_disp_order = mult.disp_order
mult_label = mult.label
except ObjectDoesNotExist:
log.error('api_get_mult_counts: Could not find mult entry for '
+'mult_model %s id %s', str(mult_model), str(mult_id))
ret = HttpResponseServerError(settings.HTTP500_INTERNAL_ERROR)
exit_api_call(api_code, ret)
return ret
mult_result_list.append((mult_disp_order,
(mult_label,
row[mult_name + '__count'])))
mult_result_list.sort()
mults = OrderedDict() # info to return
for _, mult_info in mult_result_list:
mults[mult_info[0]] = mult_info[1]
cache.set(cache_key, mults)
data = {'field': slug,
'mults': mults}
reqno = get_reqno(request)
if reqno is not None and fmt == 'json':
data['reqno'] = reqno
if fmt == 'json':
ret = json_response(data)
elif fmt == 'html':
ret = render_to_response('metadata/mults.html', data)
elif fmt == 'csv':
ret = csv_response(slug, [list(mults.values())],
column_names=list(mults.keys()))
else:
log.error('api_get_mult_counts: Unknown format "%s"', fmt)
ret = Http404(settings.HTTP404_UNKNOWN_FORMAT)
exit_api_call(api_code, ret)
raise ret
exit_api_call(api_code, ret)
return ret