def __init__(self, *args, **kwargs):
grouped = 'grouping' in kwargs # for the grouping of mult widgets
grouping = kwargs.pop(
'grouping',
None) # this is how you pass kwargs to the form class, yay!
super(SearchForm, self).__init__(*args, **kwargs)
for slug, values in args[0].items():
if slug.startswith('qtype-'):
continue
param_info = get_param_info_by_slug(slug)
if not param_info:
log.error(
"SearchForm: Could not find param_info entry for slug %s",
str(slug))
continue # todo this should raise end user error
try:
form_type = param_info.form_type
except ParamInfo.DoesNotExist:
continue # this is not a query param, probably a qtype, move along
(form_type, form_type_func,
form_type_format) = parse_form_type(param_info.form_type)
if form_type == 'STRING':
choices = (('contains', 'contains'), ('begins', 'begins'),
('ends', 'ends'), ('matches', 'matches'),
('excludes', 'excludes'))
self.fields[slug] = forms.CharField(
widget=forms.TextInput(attrs={
'class': 'STRING',
'size': '50',
'tabindex': 0
}),
required=False,
label='')
self.fields['qtype-' + slug] = forms.CharField(
required=False,
label='',
widget=forms.Select(choices=choices,
attrs={
'tabindex': 0,
'class': 'STRING'
}),
)
if form_type in settings.RANGE_FORM_TYPES:
choices = (('any', 'any'), ('all', 'all'), ('only', 'only'))
slug_no_num = strip_numeric_suffix(slug)
num = get_numeric_suffix(slug)
if not num:
slug = slug + '1'
label = 'max' if num == '2' else 'min'
pi = get_param_info_by_slug(slug)
self.fields[slug] = MultiFloatField(
required=False,
label=label,
widget=forms.TextInput(attrs={'class': label + ' RANGE'}),
)
if not is_single_column_range(pi.param_qualified_name()):
self.fields['qtype-' + slug_no_num] = forms.CharField(
required=False,
label='',
widget=forms.Select(choices=choices,
attrs={
'tabindex': 0,
'class': "RANGE"
}),
)
self.fields.keyOrder = [
slug_no_num + '1', slug_no_num + '2',
'qtype-' + slug_no_num
] # makes sure min is first! boo ya!
else:
self.fields.keyOrder = [
slug_no_num + '1', slug_no_num + '2'
] # makes sure min is first! boo ya!
elif form_type in settings.MULT_FORM_TYPES:
#self.fields[slug]= MultiStringField(forms.Field)
try:
param_qualified_name = ParamInfo.objects.get(
slug=slug).param_qualified_name()
except ParamInfo.DoesNotExist:
param_qualified_name = ParamInfo.objects.get(
old_slug=slug).param_qualified_name()
except ParamInfo.DoesNotExist:
continue # XXX
mult_param = get_mult_name(param_qualified_name)
model = apps.get_model('search',
mult_param.title().replace('_', ''))
#grouped mult fields:
if grouped:
choices = [(mult.label, mult.label)
for mult in model.objects.filter(
grouping=grouping, display='Y').order_by(
'disp_order')]
else:
choices = [(mult.label, mult.label)
for mult in model.objects.filter(
display='Y').order_by('disp_order')]
if param_qualified_name == 'obs_surface_geometry.target_name':
self.fields[slug] = forms.CharField(
# label = ParamInfo.objects.get(slug=slug).label,
label='',
widget=forms.RadioSelect(
attrs={'class': 'singlechoice'}, choices=choices),
required=False)
else:
self.fields[slug] = forms.CharField(
# label = ParamInfo.objects.get(slug=slug).label,
label='',
widget=forms.CheckboxSelectMultiple(
attrs={'class': 'multichoice'}, choices=choices),
required=False)
# XXX RF - This is awful. It takes the last form_type from the above loop, but
# is it possible the loop went more than once??
# hack to get range fields into the right orde since after Django 1.7 this is deprecated:
# self.fields.keyOrder = [slug_no_num+'1', slug_no_num+'2', 'qtype-'+slug_no_num] # makes sure min is first! boo ya!
if form_type in settings.RANGE_FORM_TYPES:
my_fields = self.fields
self.fields = OrderedDict()
self.fields[slug_no_num + '1'] = my_fields[slug_no_num + '1']
self.fields[slug_no_num + '2'] = my_fields[slug_no_num + '2']
if 'qtype-' + slug_no_num in my_fields:
self.fields['qtype-' + slug_no_num] = my_fields['qtype-' +
slug_no_num]
def __init__(self, *args, **kwargs):
grouped = 'grouping' in kwargs # for the grouping of mult widgets
grouping = kwargs.pop(
'grouping',
None) # this is how you pass kwargs to the form class, yay!
super(SearchForm, self).__init__(*args, **kwargs)
# this makes getMultName() below not choke, circular import issue? not sure..but this fixes
from metadata.views import getMultName
for slug, values in args[0].items():
param_info = get_param_info_by_slug(slug)
try:
form_type = param_info.form_type
except ParamInfo.DoesNotExist:
continue # this is not a query param, probably a qtype, move along
if form_type == 'STRING':
choices = (('contains', 'contains'), ('begins', 'begins'),
('ends', 'ends'), ('matches', 'matches'),
('excludes', 'excludes'))
self.fields[slug] = forms.CharField(
widget=forms.TextInput(attrs={
'class': 'STRING',
'size': '50',
'tabindex': 0
}),
required=False,
label='')
self.fields['qtype-' + slug] = forms.CharField(
required=False,
label='',
widget=forms.Select(choices=choices,
attrs={
'tabindex': 0,
'class': 'STRING'
}),
)
if form_type in settings.RANGE_FIELDS:
choices = (('any', 'any'), ('all', 'all'), ('only', 'only'))
slug_no_num = stripNumericSuffix(slug)
num = getNumericSuffix(slug)
if not num:
slug = slug + '1'
label = 'max' if num == '2' else 'min'
self.fields[slug] = MultiFloatField(
required=False,
label=label,
widget=forms.TextInput(attrs={'class': label + ' RANGE'}),
)
self.fields['qtype-' + slug_no_num] = forms.CharField(
required=False,
label='',
widget=forms.Select(choices=choices,
attrs={
'tabindex': 0,
'class': "RANGE"
}),
)
self.fields.keyOrder = [
slug_no_num + '1', slug_no_num + '2',
'qtype-' + slug_no_num
] # makes sure min is first! boo ya!
elif form_type in settings.MULT_FIELDS:
#self.fields[slug]= MultiStringField(forms.Field)
param_name = ParamInfo.objects.get(slug=slug).param_name()
mult_param = getMultName(param_name)
model = apps.get_model('search',
mult_param.title().replace('_', ''))
#grouped mult fields:
if grouped:
choices = [(mult.label, mult.label)
for mult in model.objects.filter(
grouping=grouping, display='Y')]
else:
choices = [(mult.label, mult.label)
for mult in model.objects.filter(display='Y')]
self.fields[slug] = forms.MultipleChoiceField(
# label = ParamInfo.objects.get(slug=slug).label,
label='',
choices=choices,
widget=forms.CheckboxSelectMultiple(
attrs={'class': 'multichoice'}),
required=False)
# hack to get range fields into the right orde since after Django 1.7 this is deprecated:
# self.fields.keyOrder = [slug_no_num+'1', slug_no_num+'2', 'qtype-'+slug_no_num] # makes sure min is first! boo ya!
if form_type in settings.RANGE_FIELDS:
my_fields = self.fields
self.fields = OrderedDict()
self.fields[slug_no_num + '1'] = my_fields[slug_no_num + '1']
self.fields[slug_no_num + '2'] = my_fields[slug_no_num + '2']
if not param_info.special_query:
self.fields['qtype-' + slug_no_num] = my_fields['qtype-' +
slug_no_num]
def api_get_range_endpoints(request, slug, fmt, internal=False):
r"""Compute and return range widget endpoints (min, max, nulls)
This is a PUBLIC API.
Compute and return range widget endpoints (min, max, nulls) for the
widget defined by [slug] based on current search defined in request.
Format: api/meta/range/endpoints/(?P<slug>[-\w]+).(?P<fmt>json|html|csv)
__api/meta/range/endpoints/(?P<slug>[-\w]+).json
Arguments: Normal search arguments
units=<unit> (Optional, gives units to return in)
reqno=<N> (Required for internal, ignored for external)
Can return JSON, HTML, or CSV (external) or JSON (internal)
Returned JSON:
{"min": 63.592, "max": 88.637, "nulls": 2365, units: "km"}
or
{"min": 63.592, "max": 88.637, "nulls": 2365, units: "km", "reqno": 123}
Note that min and max can be strings, not just real numbers. This
happens, for example, with spacecraft clock counts, and may also happen
with floating point values when we want to force a particular display
format (such as full-length numbers instead of exponential notation).
{"min": "0.0000", "max": "50000.0000", "nulls": 11}
Returned HTML:
<body>
<dl>
<dt>min</dt><dd>0.0000</dd>
<dt>max</dt><dd>50000.0000</dd>
<dt>nulls</dt><dd>11</dd>
<dt>units</dt><dd>km</dd>
</dl>
</body>
Returned CSV:
min,max,nulls,units
0.0000,50000.0000,11,km
"""
api_code = enter_api_call('api_get_range_endpoints', request)
if not request or request.GET is None:
ret = Http404(
HTTP404_NO_REQUEST(f'/api/meta/range/endpoints/{slug}.{fmt}'))
exit_api_call(api_code, ret)
raise ret
param_info = get_param_info_by_slug(slug, 'widget')
if not param_info or throw_random_http404_error():
log.error(
'get_range_endpoints: Could not find param_info entry for ' +
'slug %s', str(slug))
ret = Http404(HTTP404_UNKNOWN_SLUG(slug, request))
exit_api_call(api_code, ret)
raise ret
(form_type, form_type_format,
form_type_unit_id) = parse_form_type(param_info.form_type)
units = request.GET.get('units', get_default_unit(form_type_unit_id))
if ((form_type_unit_id and not is_valid_unit(form_type_unit_id, units))
or throw_random_http404_error()):
log.error('get_range_endpoints: Bad units "%s" for ' + 'slug %s',
str(units), str(slug))
ret = Http404(HTTP404_UNKNOWN_UNITS(units, slug, request))
exit_api_call(api_code, ret)
raise ret
param_name = param_info.name # Just name
param_qualified_name = param_info.param_qualified_name() # category.name
(form_type, form_type_format,
form_type_unit_id) = parse_form_type(param_info.form_type)
table_name = param_info.category_name
try:
table_model = apps.get_model('search',
table_name.title().replace('_', ''))
if throw_random_http500_error(): # pragma: no cover
raise LookupError
except LookupError: # pragma: no cover
log.error('api_get_range_endpoints: Could not get_model for %s',
table_name.title().replace('_', ''))
ret = HttpResponseServerError(HTTP500_INTERNAL_ERROR(request))
exit_api_call(api_code, ret)
return ret
param_no_num = strip_numeric_suffix(param_name)
param1 = param_no_num + '1'
param2 = param_no_num + '2'
if (form_type in settings.RANGE_FORM_TYPES
and param_info.slug[-1] not in '12'):
param1 = param2 = param_no_num # single column range query
(selections, extras) = url_to_search_params(request.GET)
if selections is None or throw_random_http404_error():
log.error(
'api_get_range_endpoints: Could not find selections for ' +
'request %s', str(request.GET))
ret = Http404(HTTP404_SEARCH_PARAMS_INVALID(request))
exit_api_call(api_code, ret)
raise ret
# Remove this param from the user's query if it is constrained.
# This keeps the green hinting numbers from reacting to changes to its
# own field.
qualified_param_name_no_num = strip_numeric_suffix(param_qualified_name)
for to_remove in [
qualified_param_name_no_num, qualified_param_name_no_num + '1',
qualified_param_name_no_num + '2'
]:
if to_remove in selections:
del selections[to_remove]
if selections:
user_table = get_user_query_table(selections,
extras,
api_code=api_code)
if (user_table is None
or throw_random_http500_error()): # pragma: no cover
log.error(
'api_get_range_endpoints: Count not retrieve query table' +
' for *** Selections %s *** Extras %s', str(selections),
str(extras))
ret = HttpResponseServerError(HTTP500_SEARCH_CACHE_FAILED(request))
exit_api_call(api_code, ret)
return ret
else:
user_table = None
# Is this result already cached?
cache_key = (settings.CACHE_SERVER_PREFIX + settings.CACHE_KEY_PREFIX +
':rangeep:' + qualified_param_name_no_num + ':units:' +
str(units))
if user_table:
cache_num, cache_new_flag = set_user_search_number(selections, extras)
if cache_num is None: # pragma: no cover
log.error(
'api_get_range_endpoints: Failed to create cache table ' +
'for *** Selections %s *** Extras %s', str(selections),
str(extras))
exit_api_call(api_code, Http404)
raise Http404
# We're guaranteed the table actually exists here, since
# get_user_query_table has already returned for the same search.
cache_key += ':' + str(cache_num)
range_endpoints = cache.get(cache_key)
if range_endpoints is None:
# We didn't find a cache entry, so calculate the endpoints
results = table_model.objects
if selections:
# There are selections, so tie the query to user_table
if table_name == 'obs_general':
where = (connection.ops.quote_name(table_name) + '.id=' +
connection.ops.quote_name(user_table) + '.id')
else:
where = (connection.ops.quote_name(table_name) + '.' +
connection.ops.quote_name('obs_general_id') + '=' +
connection.ops.quote_name(user_table) + '.id')
range_endpoints = (results.extra(where=[where],
tables=[user_table]).aggregate(
min=Min(param1),
max=Max(param2)))
where += ' AND ' + param1 + ' IS NULL AND ' + param2 + ' IS NULL'
range_endpoints['nulls'] = (results.extra(where=[where],
tables=[user_table
]).count())
else:
# There are no selections, so hit the whole table
range_endpoints = results.all().aggregate(min=Min(param1),
max=Max(param2))
where = param1 + ' IS NULL AND ' + param2 + ' IS NULL'
range_endpoints['nulls'] = (results.all().extra(
where=[where]).count())
# The returned range endpoints are converted to the destination
# unit
range_endpoints['min'] = format_unit_value(range_endpoints['min'],
form_type_format,
form_type_unit_id, units)
range_endpoints['max'] = format_unit_value(range_endpoints['max'],
form_type_format,
form_type_unit_id, units)
cache.set(cache_key, range_endpoints)
if internal:
reqno = get_reqno(request)
if reqno is None or throw_random_http404_error():
log.error(
'api_get_range_endpoints: Missing or badly formatted reqno')
ret = Http404(HTTP404_BAD_OR_MISSING_REQNO(request))
exit_api_call(api_code, ret)
raise ret
range_endpoints['reqno'] = reqno
range_endpoints['units'] = units
if fmt == 'json':
ret = json_response(range_endpoints)
elif fmt == 'html':
ret = render_to_response('metadata/endpoints.html',
{'data': range_endpoints})
elif fmt == 'csv':
ret = csv_response(slug, [[
range_endpoints['min'], range_endpoints['max'],
range_endpoints['nulls'], range_endpoints['units']
]], ['min', 'max', 'nulls', 'units'])
else:
log.error('api_get_range_endpoints: Unknown format "%s"', fmt)
ret = Http404(HTTP404_UNKNOWN_FORMAT(fmt, request))
exit_api_call(api_code, ret)
raise ret
exit_api_call(api_code, ret)
return ret
def get_fields_info(fmt, request, api_code, slug=None, collapse=False):
"Helper routine for api_get_fields."
cache_key = (settings.CACHE_SERVER_PREFIX + settings.CACHE_KEY_PREFIX +
':getFields:field:' + str(slug) + ':' + str(collapse))
return_obj = cache.get(cache_key)
if return_obj is None:
if slug:
fields = ParamInfo.objects.filter(slug=slug)
else:
fields = ParamInfo.objects.all()
fields.order_by('category_name', 'slug')
# We cheat with the HTML return because we want to collapse all the
# surface geometry down to a single target version to save screen
# space. This is a horrible hack, but for right now we just assume
# there will always be surface geometry data for Saturn.
return_obj = {}
for f in fields:
if not f.slug:
# Include referred slug
if f.referred_slug is not None:
referred_slug = f.referred_slug
category = f.category_name
disp_order = f.disp_order
f = get_param_info_by_slug(referred_slug, 'col')
f.label = f.body_qualified_label()
f.label_results = f.body_qualified_label_results(True)
f.referred_slug = referred_slug
f.category_name = category
f.disp_order = disp_order
else:
continue
if (collapse and f.slug.startswith('SURFACEGEO')
and not f.slug.startswith('SURFACEGEOsaturn')):
continue
if f.slug.startswith('**'):
# Internal use only
continue
table_name = TableNames.objects.get(table_name=f.category_name)
cat = table_name.label
if collapse and cat.find('Surface Geometry Constraints') != -1:
cat = cat.replace('Saturn', '<TARGET>')
return_obj[cat] = return_obj.get(cat, OrderedDict())
entry = OrderedDict()
return_obj[cat]['table_order'] = table_name.disp_order
entry['disp_order'] = f.disp_order
collapsed_slug = f.slug
if collapse:
collapsed_slug = entry['field_id'] = f.slug.replace(
'saturn', '<TARGET>')
entry['category'] = table_name.label.replace(
'Saturn', '<TARGET>')
else:
entry['field_id'] = f.slug
entry['category'] = table_name.label
f_type = None
(form_type, form_type_format,
form_type_unit_id) = parse_form_type(f.form_type)
if form_type in settings.RANGE_FORM_TYPES:
if form_type == 'LONG':
f_type = 'range_longitude'
elif form_type_format is not None:
if form_type_format[-1] == 'd':
f_type = 'range_integer'
elif form_type_format[-1] == 'f':
f_type = 'range_float'
else:
log.warning('Unparseable form type ' +
str(f.form_type))
elif form_type_unit_id == 'datetime':
f_type = 'range_time'
elif form_type_unit_id is not None:
f_type = 'range_special'
elif form_type in settings.MULT_FORM_TYPES:
f_type = 'multiple'
elif form_type == 'STRING':
f_type = 'string'
else:
log.warning('Unparseable form type ' + str(f.form_type))
entry['type'] = f_type
entry['label'] = f.label_results
entry['search_label'] = f.label
entry['full_label'] = f.body_qualified_label_results()
entry['full_search_label'] = f.body_qualified_label()
(form_type, form_type_format,
form_type_unit_id) = parse_form_type(f.form_type)
entry['default_units'] = get_default_unit(form_type_unit_id)
entry['available_units'] = get_valid_units(form_type_unit_id)
if f.old_slug and collapse: # Backwards compatibility
entry['old_slug'] = f.old_slug.replace('saturn', '<TARGET>')
else:
entry['old_slug'] = f.old_slug
entry['slug'] = entry['field_id'] # Backwards compatibility
entry['linked'] = True if f.referred_slug else False
return_obj[cat][collapsed_slug] = entry
# Organize return_obj before returning
# Sort categories by table_order
return_obj = OrderedDict(
sorted(return_obj.items(), key=lambda x: x[1]['table_order']))
for cat, cat_data in return_obj.items():
del cat_data['table_order']
# Sort slugs of each category by disp_order
cat_data = OrderedDict(
sorted(cat_data.items(), key=lambda x: x[1]['disp_order']))
return_obj[cat] = cat_data
for key, val in cat_data.items():
del val['disp_order']
cache.set(cache_key, return_obj)
if fmt == 'raw':
ret = return_obj
elif fmt == 'json':
ret = json_response({'data': return_obj})
elif fmt == 'csv':
labels = [
'Field ID', 'Category', 'Type', 'Search Label', 'Results Label',
'Full Search Label', 'Full Results Label', 'Default Units',
'Available Units', 'Old Field ID', 'Linked'
]
rows = []
for cat, cat_data in return_obj.items():
for k, v in cat_data.items():
# In csv, we will store the linked field value as 0 or 1.
linked = 1 if v['linked'] else 0
row_data = [
(v['field_id'], v['category'], v['type'],
v['search_label'], v['label'], v['full_search_label'],
v['full_label'], v['default_units'], v['available_units'],
v['old_slug'], linked)
]
rows += row_data
ret = csv_response('fields', rows, labels)
else:
log.error('get_fields_info: Unknown format "%s"', fmt)
ret = Http404(HTTP404_UNKNOWN_FORMAT(fmt, request))
exit_api_call(api_code, ret)
raise ret
return ret
def api_get_mult_counts(request, slug, fmt, internal=False):
r"""Return the mults for a given slug along with result counts.
This is a PUBLIC API.
Format: api/meta/mults/(?P<slug>[-\w]+).(?P<fmt>json|html|csv)
__api/meta/mults/(?P<slug>[-\w]+).json
Arguments: Normal search arguments
reqno=<N> (Required for internal, ignored for external)
Can return JSON, HTML, or CSV (external) or JSON (internal)
Returned JSON:
{'field_id': slug, 'mults': mults}
or:
{'field_id': slug, 'mults': mults, 'reqno': reqno}
mult is a list of entries pairing mult name and result count.
Returned HTML:
<body>
<dl>
<dt>Atlas</dt><dd>2</dd>
<dt>Daphnis</dt><dd>4</dd>
</dl>
</body>
Returned CSV:
name1,name2,name3
number1,number2,number3
"""
api_code = enter_api_call('api_get_mult_counts', request)
if not request or request.GET is None:
ret = Http404(HTTP404_NO_REQUEST(f'/api/meta/mults/{slug}.{fmt}'))
exit_api_call(api_code, ret)
raise ret
(selections, extras) = url_to_search_params(request.GET)
if selections is None or throw_random_http404_error():
log.error(
'api_get_mult_counts: Failed to get selections for slug %s, ' +
'URL %s', str(slug), request.GET)
ret = Http404(HTTP404_SEARCH_PARAMS_INVALID(request))
exit_api_call(api_code, ret)
raise ret
param_info = get_param_info_by_slug(slug, 'col')
if not param_info or throw_random_http404_error():
log.error(
'api_get_mult_counts: Could not find param_info entry for ' +
'slug %s *** Selections %s *** Extras %s', str(slug),
str(selections), str(extras))
ret = Http404(HTTP404_UNKNOWN_SLUG(slug, request))
exit_api_call(api_code, ret)
raise ret
table_name = param_info.category_name
param_qualified_name = param_info.param_qualified_name()
# If this param is in selections already we want to remove it
# We want mults for a param as they would be without itself
if param_qualified_name in selections:
del selections[param_qualified_name]
cache_num, cache_new_flag = set_user_search_number(selections, extras)
if cache_num is None or throw_random_http500_error(): # pragma: no cover
log.error(
'api_get_mult_counts: Failed to create user_selections entry' +
' for *** Selections %s *** Extras %s', str(selections),
str(extras))
ret = HttpResponseServerError(HTTP500_DATABASE_ERROR(request))
exit_api_call(api_code, ret)
return ret
# Note we don't actually care here if the cache table even exists, because
# if it's in the cache, it must exist, and if it's not in the cache, it
# will be created if necessary by get_user_query_table below.
cache_key = (settings.CACHE_SERVER_PREFIX + settings.CACHE_KEY_PREFIX +
':mults_' + param_qualified_name + ':' + str(cache_num))
cached_val = cache.get(cache_key)
if cached_val is not None:
mults = cached_val
else:
mult_name = get_mult_name(param_qualified_name)
try:
mult_model = apps.get_model('search',
mult_name.title().replace('_', ''))
if throw_random_http500_error(): # pragma: no cover
raise LookupError
except LookupError: # pragma: no cover
log.error('api_get_mult_counts: Could not get_model for %s',
mult_name.title().replace('_', ''))
ret = HttpResponseServerError(HTTP500_INTERNAL_ERROR(request))
exit_api_call(api_code, ret)
return ret
try:
table_model = apps.get_model('search',
table_name.title().replace('_', ''))
if throw_random_http500_error(): # pragma: no cover
raise LookupError
except LookupError: # pragma: no cover
log.error('api_get_mult_counts: Could not get_model for %s',
table_name.title().replace('_', ''))
ret = HttpResponseServerError(HTTP500_INTERNAL_ERROR(request))
exit_api_call(api_code, ret)
return ret
results = (table_model.objects.values(mult_name).annotate(
Count(mult_name)))
user_table = get_user_query_table(selections,
extras,
api_code=api_code)
if ((selections and not user_table)
or throw_random_http500_error()): # pragma: no cover
log.error(
'api_get_mult_counts: has selections but no user_table ' +
'found *** Selections %s *** Extras %s', str(selections),
str(extras))
ret = HttpResponseServerError(HTTP500_SEARCH_CACHE_FAILED(request))
exit_api_call(api_code, ret)
return ret
if selections:
# selections are constrained so join in the user_table
if table_name == 'obs_general':
where = [
connection.ops.quote_name(table_name) + '.id=' +
connection.ops.quote_name(user_table) + '.id'
]
else:
where = [
connection.ops.quote_name(table_name) +
'.obs_general_id=' +
connection.ops.quote_name(user_table) + '.id'
]
results = results.extra(where=where, tables=[user_table])
mult_result_list = []
for row in results:
mult_id = row[mult_name]
try:
mult = mult_model.objects.get(id=mult_id)
mult_disp_order = mult.disp_order
mult_label = mult.label
if throw_random_http500_error(): # pragma: no cover
raise ObjectDoesNotExist
except ObjectDoesNotExist: # pragma: no cover
log.error(
'api_get_mult_counts: Could not find mult entry for ' +
'mult_model %s id %s', str(mult_model), str(mult_id))
ret = HttpResponseServerError(HTTP500_INTERNAL_ERROR(request))
exit_api_call(api_code, ret)
return ret
mult_result_list.append(
(mult_disp_order, (mult_label, row[mult_name + '__count'])))
mult_result_list.sort()
mults = OrderedDict() # info to return
for _, mult_info in mult_result_list:
mults[mult_info[0]] = mult_info[1]
cache.set(cache_key, mults)
data = {'field_id': slug, 'mults': mults}
if internal:
reqno = get_reqno(request)
if reqno is None or throw_random_http404_error():
log.error('api_get_mult_counts: Missing or badly formatted reqno')
ret = Http404(HTTP404_BAD_OR_MISSING_REQNO(request))
exit_api_call(api_code, ret)
raise ret
data['reqno'] = reqno
if fmt == 'json':
ret = json_response(data)
elif fmt == 'html':
ret = render_to_response('metadata/mults.html', data)
elif fmt == 'csv':
ret = csv_response(slug, [list(mults.values())],
column_names=list(mults.keys()))
else:
log.error('api_get_mult_counts: Unknown format "%s"', fmt)
ret = Http404(HTTP404_UNKNOWN_FORMAT(fmt, request))
exit_api_call(api_code, ret)
raise ret
exit_api_call(api_code, ret)
return ret
def __init__(self, *args, **kwargs):
grouped = 'grouping' in kwargs # for the grouping of mult widgets
grouping = kwargs.pop('grouping', None) # this is how you pass kwargs to the form class, yay!
super(SearchForm, self).__init__(*args, **kwargs)
for slug,values in args[0].items():
if slug.startswith('qtype-'):
continue
param_info = get_param_info_by_slug(slug)
if not param_info:
log.error(
"SearchForm: Could not find param_info entry for slug %s",
str(slug))
continue # todo this should raise end user error
try:
form_type = param_info.form_type
except ParamInfo.DoesNotExist:
continue # this is not a query param, probably a qtype, move along
(form_type, form_type_func,
form_type_format) = parse_form_type(param_info.form_type)
if form_type == 'STRING':
choices = (('contains','contains'),('begins','begins'),('ends','ends'),('matches','matches'),('excludes','excludes'))
self.fields[slug] = forms.CharField(
widget = forms.TextInput(
attrs={'class':'STRING', 'size':'50', 'tabindex':0}),
required=False,
label = '')
self.fields['qtype-'+slug] = forms.CharField(
required=False,
label = '',
widget=forms.Select(
choices=choices,
attrs={'tabindex':0, 'class':'STRING'}
),
)
if form_type in settings.RANGE_FORM_TYPES:
choices = (('any','any'),('all','all'),('only','only'))
slug_no_num = strip_numeric_suffix(slug)
num = get_numeric_suffix(slug)
if not num:
slug = slug + '1'
label = 'max' if num == '2' else 'min'
pi = get_param_info_by_slug(slug)
self.fields[slug] = MultiFloatField(
required=False,
label = label,
widget = forms.TextInput(attrs={'class':label + ' RANGE'}),
)
if not is_single_column_range(pi.param_qualified_name()):
self.fields['qtype-'+slug_no_num] = forms.CharField(
required=False,
label = '',
widget=forms.Select(
choices=choices,
attrs={'tabindex':0, 'class':"RANGE"}
),
)
self.fields.keyOrder = [slug_no_num+'1', slug_no_num+'2', 'qtype-'+slug_no_num] # makes sure min is first! boo ya!
else:
self.fields.keyOrder = [slug_no_num+'1', slug_no_num+'2'] # makes sure min is first! boo ya!
elif form_type in settings.MULT_FORM_TYPES:
#self.fields[slug]= MultiStringField(forms.Field)
try:
param_qualified_name = ParamInfo.objects.get(slug=slug).param_qualified_name()
except ParamInfo.DoesNotExist:
param_qualified_name = ParamInfo.objects.get(old_slug=slug).param_qualified_name()
except ParamInfo.DoesNotExist:
continue # XXX
mult_param = get_mult_name(param_qualified_name)
model = apps.get_model('search',mult_param.title().replace('_',''))
#grouped mult fields:
if grouped:
choices = [(mult.label, mult.label) for mult in model.objects.filter(grouping=grouping, display='Y').order_by('disp_order')]
else:
choices = [(mult.label, mult.label) for mult in model.objects.filter(display='Y').order_by('disp_order')]
if param_qualified_name == 'obs_surface_geometry.target_name':
self.fields[slug] = forms.CharField(
# label = ParamInfo.objects.get(slug=slug).label,
label = '',
widget = forms.RadioSelect(attrs={'class':'singlechoice'}, choices = choices),
required=False)
else:
self.fields[slug] = forms.CharField(
# label = ParamInfo.objects.get(slug=slug).label,
label = '',
widget = forms.CheckboxSelectMultiple(attrs={'class':'multichoice'}, choices = choices),
required=False)
# XXX RF - This is awful. It takes the last form_type from the above loop, but
# is it possible the loop went more than once??
# hack to get range fields into the right orde since after Django 1.7 this is deprecated:
# self.fields.keyOrder = [slug_no_num+'1', slug_no_num+'2', 'qtype-'+slug_no_num] # makes sure min is first! boo ya!
if form_type in settings.RANGE_FORM_TYPES:
my_fields = self.fields
self.fields = OrderedDict()
self.fields[slug_no_num+'1'] = my_fields[slug_no_num+'1']
self.fields[slug_no_num+'2'] = my_fields[slug_no_num+'2']
if 'qtype-'+slug_no_num in my_fields:
self.fields['qtype-'+slug_no_num] = my_fields['qtype-'+slug_no_num]
def __init__(self, *args, **kwargs):
grouped = 'grouping' in kwargs # for the grouping of mult widgets
grouping = kwargs.pop(
'grouping',
None) # this is how you pass kwargs to the form class, yay!
super(SearchForm, self).__init__(*args, **kwargs)
for slug, values in args[0].items():
if slug.startswith('qtype-'):
continue
param_info = get_param_info_by_slug(slug, 'search')
if not param_info:
log.error(
"SearchForm: Could not find param_info entry for slug %s",
str(slug))
continue # todo this should raise end user error
try:
form_type = param_info.form_type
except ParamInfo.DoesNotExist:
continue # this is not a query param, probably a qtype, move along
(form_type, form_type_func,
form_type_format) = parse_form_type(param_info.form_type)
if form_type == 'STRING':
choices = ((x, x) for x in settings.STRING_QTYPES)
self.fields[slug] = forms.CharField(widget=forms.TextInput(
attrs={
'class': 'STRING',
'size': '50',
'tabindex': 0,
'data-slugname': slug
}),
required=False,
label='')
self.fields['qtype-' + slug] = forms.CharField(
required=False,
label='',
widget=forms.Select(choices=choices,
attrs={
'tabindex': 0,
'class': 'STRING'
}),
)
if form_type in settings.RANGE_FORM_TYPES:
choices = ((x, x) for x in settings.RANGE_QTYPES)
slug_no_num = strip_numeric_suffix(slug)
num = get_numeric_suffix(slug)
if not num:
slug = slug + '1'
label = 'max' if num == '2' else 'min'
pi = get_param_info_by_slug(slug, 'search')
# placeholder for input hints (only apply to Min input for now)
if num == '2':
# Get the hints for slug2 from slug1 field in database
pi_slug1 = get_param_info_by_slug(slug[:-1] + '1',
'search')
hints = pi_slug1.field_hints2 if pi_slug1.field_hints2 else ''
else:
hints = pi.field_hints1 if pi.field_hints1 else ''
# dropdown only available when ranges info is available
ranges = pi.get_ranges_info()
dropdown_class = 'op-ranges-dropdown-menu dropdown-toggle' if ranges else ''
data_toggle = 'dropdown' if ranges else ''
self.fields[slug] = MultiFloatField(
required=False,
label=label.capitalize(),
widget=forms.TextInput(
attrs={
'class': 'op-range-input-' + label + ' RANGE ' +
dropdown_class,
'placeholder': hints,
'autocomplete': 'off',
'data-slugname': slug_no_num,
'data-toggle': data_toggle,
'aria-haspopup': 'true',
'aria-expanded': 'false'
}),
)
if not is_single_column_range(pi.param_qualified_name()):
self.fields['qtype-' + slug_no_num] = forms.CharField(
required=False,
label='',
widget=forms.Select(choices=choices,
attrs={
'tabindex': 0,
'class': "RANGE"
}),
)
self.fields.keyOrder = [
slug_no_num + '1', slug_no_num + '2',
'qtype-' + slug_no_num
] # makes sure min is first! boo ya!
else:
self.fields.keyOrder = [
slug_no_num + '1', slug_no_num + '2'
] # makes sure min is first! boo ya!
elif form_type in settings.MULT_FORM_TYPES:
#self.fields[slug]= MultiStringField(forms.Field)
try:
param_qualified_name = ParamInfo.objects.get(
slug=slug).param_qualified_name()
except ParamInfo.DoesNotExist:
param_qualified_name = ParamInfo.objects.get(
old_slug=slug).param_qualified_name()
except ParamInfo.DoesNotExist:
continue # XXX
mult_param = get_mult_name(param_qualified_name)
model = apps.get_model('search',
mult_param.title().replace('_', ''))
#grouped mult fields:
if grouped:
choices = [(mult.label, mult.label)
for mult in model.objects.filter(
grouping=grouping, display='Y').order_by(
'disp_order')]
else:
choices = [(mult.label, mult.label)
for mult in model.objects.filter(
display='Y').order_by('disp_order')]
if param_qualified_name == 'obs_surface_geometry_name.target_name':
self.fields[slug] = forms.CharField(
# label = ParamInfo.objects.get(slug=slug).label,
label='',
widget=forms.RadioSelect(
attrs={'class': 'singlechoice'}, choices=choices),
required=False)
else:
self.fields[slug] = forms.CharField(
# label = ParamInfo.objects.get(slug=slug).label,
label='',
widget=forms.CheckboxSelectMultiple(
attrs={'class': 'multichoice'}, choices=choices),
required=False)
# XXX RF - This is awful. It takes the last form_type from the above loop, but
# is it possible the loop went more than once??
# hack to get range fields into the right orde since after Django 1.7 this is deprecated:
# self.fields.keyOrder = [slug_no_num+'1', slug_no_num+'2', 'qtype-'+slug_no_num] # makes sure min is first! boo ya!
if form_type in settings.RANGE_FORM_TYPES:
my_fields = self.fields
self.fields = OrderedDict()
self.fields[slug_no_num + '1'] = my_fields[slug_no_num + '1']
self.fields[slug_no_num + '2'] = my_fields[slug_no_num + '2']
if 'qtype-' + slug_no_num in my_fields:
self.fields['qtype-' + slug_no_num] = my_fields['qtype-' +
slug_no_num]
def api_get_range_endpoints(request, slug, fmt):
"""Compute and return range widget endpoints (min, max, nulls)
This is a PUBLIC API.
Compute and return range widget endpoints (min, max, nulls) for the
widget defined by [slug] based on current search defined in request.
Format: [__]api/meta/range/endpoints/(?P<slug>[-\w]+)
.(?P<fmt>json|html|csv)
Arguments: Normal search arguments
reqno=<N> (Optional)
Can return JSON, HTML, or CSV.
Returned JSON:
{"min": 63.592, "max": 88.637, "nulls": 2365}
or
{"min": 63.592, "max": 88.637, "nulls": 2365, "reqno": 123}
Note that min and max can be strings, not just real numbers. This
happens, for example, with spacecraft clock counts, and may also happen
with floating point values when we want to force a particular display
format (such as full-length numbers instead of exponential notation).
{"min": "0.0000", "max": "50000.0000", "nulls": 11}
Returned HTML:
<body>
<dl>
<dt>min</dt><dd>0.0000</dd>
<dt>max</dt><dd>50000.0000</dd>
<dt>nulls</dt><dd>11</dd>
</dl>
</body>
Returned CSV:
min,max,nulls
0.0000,50000.0000,11
"""
api_code = enter_api_call('api_get_range_endpoints', request)
if not request or request.GET is None:
ret = Http404(settings.HTTP404_NO_REQUEST)
exit_api_call(api_code, ret)
raise ret
param_info = get_param_info_by_slug(slug, from_ui=True)
if not param_info:
log.error('get_range_endpoints: Could not find param_info entry for '+
'slug %s', str(slug))
ret = Http404(settings.HTTP404_UNKNOWN_SLUG)
exit_api_call(api_code, ret)
raise ret
param_name = param_info.name # Just name
param_qualified_name = param_info.param_qualified_name() # category.name
(form_type, form_type_func,
form_type_format) = parse_form_type(param_info.form_type)
table_name = param_info.category_name
try:
table_model = apps.get_model('search',
table_name.title().replace('_',''))
except LookupError:
log.error('api_get_range_endpoints: Could not get_model for %s',
table_name.title().replace('_',''))
ret = HttpResponseServerError(settings.HTTP500_INTERNAL_ERROR)
exit_api_call(api_code, ret)
return ret
param_no_num = strip_numeric_suffix(param_name)
param1 = param_no_num + '1'
param2 = param_no_num + '2'
if (form_type in settings.RANGE_FORM_TYPES and
param_info.slug[-1] not in '12'):
param1 = param2 = param_no_num # single column range query
(selections, extras) = url_to_search_params(request.GET)
if selections is None:
log.error('api_get_range_endpoints: Could not find selections for '
+'request %s', str(request.GET))
ret = Http404(settings.HTTP404_SEARCH_PARAMS_INVALID)
exit_api_call(api_code, ret)
raise ret
# Remove this param from the user's query if it is constrained.
# This keeps the green hinting numbers from reacting to changes to its
# own field.
qualified_param_name_no_num = strip_numeric_suffix(param_qualified_name)
for to_remove in [qualified_param_name_no_num,
qualified_param_name_no_num + '1',
qualified_param_name_no_num + '2']:
if to_remove in selections:
del selections[to_remove]
if selections:
user_table = get_user_query_table(selections, extras, api_code=api_code)
if user_table is None:
log.error('api_get_range_endpoints: Count not retrieve query table'
+' for *** Selections %s *** Extras %s',
str(selections), str(extras))
ret = HttpResponseServerError(settings.HTTP500_SEARCH_FAILED)
exit_api_call(api_code, ret)
return ret
else:
user_table = None
# Is this result already cached?
cache_key = (settings.CACHE_KEY_PREFIX + ':rangeep:'
+ qualified_param_name_no_num)
if user_table:
cache_num, cache_new_flag = set_user_search_number(selections, extras)
if cache_num is None:
log.error('api_get_range_endpoints: Failed to create cache table '
+'for *** Selections %s *** Extras %s',
str(selections), str(extras))
exit_api_call(api_code, Http404)
raise Http404
# We're guaranteed the table actually exists here, since
# get_user_query_table has already returned for the same search.
cache_key += ':' + str(cache_num)
range_endpoints = cache.get(cache_key)
if range_endpoints is None:
# We didn't find a cache entry, so calculate the endpoints
results = table_model.objects
if selections:
# There are selections, so tie the query to user_table
if table_name == 'obs_general':
where = (connection.ops.quote_name(table_name)+'.id='
+connection.ops.quote_name(user_table)+'.id')
else:
where = (connection.ops.quote_name(table_name)
+'.'+connection.ops.quote_name('obs_general_id')+'='
+connection.ops.quote_name(user_table)+'.id')
range_endpoints = (results.extra(where=[where],
tables=[user_table]).
aggregate(min=Min(param1), max=Max(param2)))
where += ' AND ' + param1 + ' IS NULL AND ' + param2 + ' IS NULL'
range_endpoints['nulls'] = (results.extra(where=[where],
tables=[user_table])
.count())
else:
# There are no selections, so hit the whole table
range_endpoints = results.all().aggregate(min=Min(param1),
max=Max(param2))
where = param1 + ' IS NULL AND ' + param2 + ' IS NULL'
range_endpoints['nulls'] = (results.all().extra(where=[where])
.count())
range_endpoints['min'] = format_metadata_number_or_func(
range_endpoints['min'],
form_type_func,
form_type_format)
range_endpoints['max'] = format_metadata_number_or_func(
range_endpoints['max'],
form_type_func,
form_type_format)
cache.set(cache_key, range_endpoints)
reqno = get_reqno(request)
if reqno is not None and fmt == 'json':
range_endpoints['reqno'] = reqno
if fmt == 'json':
ret = json_response(range_endpoints)
elif fmt == 'html':
ret = render_to_response('metadata/endpoints.html',
{'data': range_endpoints})
elif fmt == 'csv':
ret = csv_response(slug, [[range_endpoints['min'],
range_endpoints['max'],
range_endpoints['nulls']]],
['min', 'max', 'nulls'])
else:
log.error('api_get_range_endpoints: Unknown format "%s"', fmt)
ret = Http404(settings.HTTP404_UNKNOWN_FORMAT)
exit_api_call(api_code, ret)
raise ret
exit_api_call(api_code, ret)
return ret
def api_get_mult_counts(request, slug, fmt):
"""Return the mults for a given slug along with result counts.
This is a PUBLIC API.
Format: [__]api/meta/mults/(?P<slug>[-\w]+).(?P<fmt>json|html|csv)
Arguments: Normal search arguments
reqno=<N> (Optional)
Can return JSON, HTML, or CSV.
Returned JSON:
{'field': slug, 'mults': mults}
or:
{'field': slug, 'mults': mults, 'reqno': reqno}
mult is a list of entries pairing mult name and result count.
Returned HTML:
<body>
<dl>
<dt>Atlas</dt><dd>2</dd>
<dt>Daphnis</dt><dd>4</dd>
</dl>
</body>
Returned CSV:
name1,name2,name3
number1,number2,number3
"""
api_code = enter_api_call('api_get_mult_counts', request)
if not request or request.GET is None:
ret = Http404(settings.HTTP404_NO_REQUEST)
exit_api_call(api_code, ret)
raise ret
(selections, extras) = url_to_search_params(request.GET)
if selections is None:
log.error('api_get_mult_counts: Failed to get selections for slug %s, '
+'URL %s', str(slug), request.GET)
ret = Http404(settings.HTTP404_SEARCH_PARAMS_INVALID)
exit_api_call(api_code, ret)
raise ret
param_info = get_param_info_by_slug(slug)
if not param_info:
log.error('api_get_mult_counts: Could not find param_info entry for '
+'slug %s *** Selections %s *** Extras %s', str(slug),
str(selections), str(extras))
ret = Http404(settings.HTTP404_UNKNOWN_SLUG)
exit_api_call(api_code, ret)
raise ret
table_name = param_info.category_name
param_qualified_name = param_info.param_qualified_name()
# If this param is in selections already we want to remove it
# We want mults for a param as they would be without itself
if param_qualified_name in selections:
del selections[param_qualified_name]
cache_num, cache_new_flag = set_user_search_number(selections, extras)
if cache_num is None:
log.error('api_get_mult_counts: Failed to create user_selections entry'
+' for *** Selections %s *** Extras %s',
str(selections), str(extras))
ret = HttpResponseServerError(settings.HTTP500_DATABASE_ERROR)
exit_api_call(api_code, ret)
return ret
# Note we don't actually care here if the cache table even exists, because
# if it's in the cache, it must exist, and if it's not in the cache, it
# will be created if necessary by get_user_query_table below.
cache_key = (settings.CACHE_KEY_PREFIX + ':mults_' + param_qualified_name
+ ':' + str(cache_num))
cached_val = cache.get(cache_key)
if cached_val is not None:
mults = cached_val
else:
mult_name = get_mult_name(param_qualified_name)
try:
mult_model = apps.get_model('search',
mult_name.title().replace('_',''))
except LookupError:
log.error('api_get_mult_counts: Could not get_model for %s',
mult_name.title().replace('_',''))
ret = HttpResponseServerError(settings.HTTP500_INTERNAL_ERROR)
exit_api_call(api_code, ret)
return ret
try:
table_model = apps.get_model('search',
table_name.title().replace('_',''))
except LookupError:
log.error('api_get_mult_counts: Could not get_model for %s',
table_name.title().replace('_',''))
ret = HttpResponseServerError(settings.HTTP500_INTERNAL_ERROR)
exit_api_call(api_code, ret)
return ret
results = (table_model.objects.values(mult_name)
.annotate(Count(mult_name)))
user_table = get_user_query_table(selections, extras, api_code=api_code)
if selections and not user_table:
log.error('api_get_mult_counts: has selections but no user_table '
+'found *** Selections %s *** Extras %s',
str(selections), str(extras))
ret = HttpResponseServerError(settings.HTTP500_SEARCH_FAILED)
exit_api_call(api_code, ret)
return ret
if selections:
# selections are constrained so join in the user_table
if table_name == 'obs_general':
where = [connection.ops.quote_name(table_name) + '.id='
+ connection.ops.quote_name(user_table) + '.id']
else:
where = [connection.ops.quote_name(table_name)
+ '.obs_general_id='
+ connection.ops.quote_name(user_table) + '.id']
results = results.extra(where=where, tables=[user_table])
mult_result_list = []
for row in results:
mult_id = row[mult_name]
try:
mult = mult_model.objects.get(id=mult_id)
mult_disp_order = mult.disp_order
mult_label = mult.label
except ObjectDoesNotExist:
log.error('api_get_mult_counts: Could not find mult entry for '
+'mult_model %s id %s', str(mult_model), str(mult_id))
ret = HttpResponseServerError(settings.HTTP500_INTERNAL_ERROR)
exit_api_call(api_code, ret)
return ret
mult_result_list.append((mult_disp_order,
(mult_label,
row[mult_name + '__count'])))
mult_result_list.sort()
mults = OrderedDict() # info to return
for _, mult_info in mult_result_list:
mults[mult_info[0]] = mult_info[1]
cache.set(cache_key, mults)
data = {'field': slug,
'mults': mults}
reqno = get_reqno(request)
if reqno is not None and fmt == 'json':
data['reqno'] = reqno
if fmt == 'json':
ret = json_response(data)
elif fmt == 'html':
ret = render_to_response('metadata/mults.html', data)
elif fmt == 'csv':
ret = csv_response(slug, [list(mults.values())],
column_names=list(mults.keys()))
else:
log.error('api_get_mult_counts: Unknown format "%s"', fmt)
ret = Http404(settings.HTTP404_UNKNOWN_FORMAT)
exit_api_call(api_code, ret)
raise ret
exit_api_call(api_code, ret)
return ret