def get_table_headers(request, template='table_headers.html'):
update_metrics(request)
slugs = request.GET.get('cols', settings.DEFAULT_COLUMNS)
order = request.GET.get('order', None)
if order:
sort_icon = 'fa-sort-desc' if order[0] == '-' else 'fa-sort-asc'
order = order[1:] if order[0] == '-' else order
if not slugs: slugs = settings.DEFAULT_COLUMNS
slugs = slugs.split(',')
columns = []
# if this is an ajax call it means it's from our app, so append the
# checkbox column for adding to selections
if (request.is_ajax()):
columns.append(["collection", "Collect"])
param_info = ParamInfo.objects
for slug in slugs:
if slug and slug != 'ring_obs_id':
try:
columns.append([slug, param_info.get(slug=slug).label_results])
except ParamInfo.DoesNotExist:
pass
return render(request, template, locals())
def guide(request):
update_metrics(request)
base_url = 'http://' + request.META['HTTP_HOST'] + '/opus/'
groups = Group.objects.all()
resources = Resource.objects.filter(
display=True).select_related().order_by('disp_order')
return render(request, 'guide.html', locals())
def edit_collection_range(request, **kwargs):
update_metrics(request)
"""
the request will come in not as a single ring_obs_id
but as min and max ring_obs_ids + a range in a search query
"""
if not request.session.get('has_session'):
request.session['has_session'] = True
session_id = request.session.session_key
colls_table_name = get_collection_table(session_id)
(action, ring_obs_id, request_no,
expected_request_no) = check_collection_args(request, **kwargs)
id_range = request.GET.get('addrange', False)
if not id_range:
return False
# "invalid ringobsid pair"
(min_id, max_id) = id_range.split(',')
(selections, extras) = urlToSearchParams(request.GET)
from results.views import *
data = getData(request, "raw")
selected_range = []
in_range = False # loop has reached the range selected
column_slugs = request.GET.get('cols', settings.DEFAULT_COLUMNS)
ring_obs_id_key = column_slugs.split(',').index('ringobsid')
# return HttpResponse(json.dumps(data['page']));
for row in data['page']:
ring_obs_id = row[ring_obs_id_key]
if ring_obs_id == min_id:
in_range = True
if in_range:
selected_range.append(ring_obs_id)
if in_range & (ring_obs_id == max_id):
# this is the last one, update the collection
if action == 'addrange':
bulk_add_to_collection([rid for rid in selected_range],
session_id)
if not selected_range:
log.error("edit_collection_range failed to find range " + id_range)
log.error(selections)
return get_collection_count(session_id)
def getRangeEndpoints(request, slug, fmt='json'):
"""
returns valid range endpoints for field given selections and extras
"""
# if this param is in selections we want to remove it,
# want results for param as they would be without itself constrained
# extras['qtypes']['']
update_metrics(request)
param_info = search.views.get_param_info_by_slug(slug)
param_name = param_info.param_name()
form_type = param_info.form_type
table_name = param_info.category_name
param_name1 = stripNumericSuffix(param_name.split('.')[1]) + '1'
param_name2 = stripNumericSuffix(param_name.split('.')[1]) + '2'
param_name_no_num = stripNumericSuffix(param_name1)
table_model = apps.get_model('search', table_name.title().replace('_', ''))
if form_type == 'RANGE' and '1' not in param_info.slug and '2' not in param_info.slug:
param_name1 = param_name2 = param_name_no_num
try:
(selections, extras) = search.views.urlToSearchParams(request.GET)
user_table = search.views.getUserQueryTable(selections, extras)
has_selections = True
except TypeError:
selections = {}
has_selections = False
user_table = False
# we remove this param from the user's query if it's there, because
# otherwise it will just return it's own values
if param_name1 in selections:
del selections[param_name1]
if param_name2 in selections:
del selections[param_name2]
# cached already?
cache_key = "rangeep" + param_name_no_num
if user_table:
cache_key += str(search.views.setUserSearchNo(selections, extras))
if cache.get(cache_key) is not None:
range_endpoints = cache.get(cache_key)
return responseFormats(range_endpoints, fmt, template='mults.html')
try:
results = table_model.objects # this is a count(*), group_by query
except AttributeError, e:
log.error(e)
log.error("could not find table model for table_name: %s" % table_name)
raise Http404("Does Not Exist")
def getResultCount(request, fmt='json'):
"""
result count for a search
"""
update_metrics(request)
if request.GET is None:
return HttpResponse(json.dumps({'result_count': '0'}),
content_type='application/json')
try:
(selections, extras) = search.views.urlToSearchParams(request.GET)
except TypeError:
log.error("could not find selections for request")
log.error(request.GET)
raise Http404
reqno = request.GET.get('reqno', '')
if selections is False:
count = 'not found'
return HttpResponse(json.dumps({'result_count': count}),
content_type='application/json')
table = search.views.getUserQueryTable(selections, extras)
if table is False:
count = 0
else:
cache_key = "resultcount:" + table
if (cache.get(cache_key)):
count = cache.get(cache_key)
else:
cursor = connection.cursor()
cursor.execute("select count(*) from " +
connection.ops.quote_name(table))
try:
count = cursor.fetchone()
count = count[0]
except:
count = 0
cache.set(cache_key, count, 0)
data = {'result_count': count}
if (request.is_ajax()):
data['reqno'] = request.GET['reqno']
return responseFormats({'data': [data]}, fmt, template='result_count.html')
def init_detail_page(request, **kwargs):
"""
this loads the initial parts of the detail tab on first loads
these are the things that are fast to compute while other parts of the page
are handled with ajax calls because they are slower
the detail page calls other views via ajax:
results.get_metadata
"""
update_metrics(request)
template = "detail.html"
slugs = request.GET.get('cols', False)
ring_obs_id = kwargs['ring_obs_id']
# get the preview image and some general info
try:
img = Image.objects.get(ring_obs_id=ring_obs_id)
except Image.DoesNotExist:
img = None
base_vol_path = get_base_path_previews(ring_obs_id)
path = settings.IMAGE_HTTP_PATH + base_vol_path
if 'CIRS' in base_vol_path:
path = path.replace('previews', 'diagrams')
instrument_id = ObsGeneral.objects.filter(
ring_obs_id=ring_obs_id).values('instrument_id')[0]['instrument_id']
# get the preview guide url
preview_guide_url = ''
if instrument_id == 'COCIRS':
preview_guide_url = 'http://pds-rings.seti.org/cassini/cirs/COCIRS_previews.txt'
if instrument_id == 'COUVIS':
preview_guide_url = 'http://pds-rings.seti.org/cassini/uvis/UVIS_previews.txt'
if instrument_id == 'COVIMS':
preview_guide_url = 'http://pds-rings.seti.org/cassini/vims/COVIMS_previews.txt'
# get the list of files for this observation
files = getFiles(ring_obs_id, fmt='raw')[ring_obs_id]
file_list = {}
for product_type in files:
if product_type not in file_list:
file_list[product_type] = []
for f in files[product_type]:
ext = f.split('.').pop()
file_list[product_type].append({'ext': ext, 'link': f})
return render(request, template, locals())
def getColumnChooser(request, **kwargs):
update_metrics(request)
slugs = request.GET.get('cols', settings.DEFAULT_COLUMNS).split(',')
slugs = filter(
None, slugs) # sometimes 'cols' is in url but is blank, so fails above
if not slugs:
slugs = settings.DEFAULT_COLUMNS.split(',')
all_slugs_info = getColumnInfo(slugs)
namespace = 'column_chooser_input'
menu = getMenuLabels(request, 'results')['menu']
return render(request, "choose_columns.html", locals())
def edit_collection_addall(request, **kwargs):
"""
add the entire result set to the collection cart
This may be turned off. The way to turn this off is:
- comment out html link in apps/ui/templates/browse_headers.html
- add these lines below:
# turn off this functionality
log.debug("edit_collection_addall is currently turned off. see apps/user_collections.edit_collection_addall")
return # this thing is turned off for now
The reason is it needs more testing, but this branch makes a big
efficiency improvements to the way downloads are handled, and fixes
some things, so I wanted to merge it into master
Things that needs further exploration:
This functionality provides no checks on how large a cart can be.
There needs to be some limit.
It doesn't hide the menu link when the result count is too high.
And what happens when it bumps against the MAX_CUM_DOWNLOAD_SIZE.
The functionality is there but these are questions!
To bring this functionality back for testing do the folloing:
- uncomment the "add all to cart" link in apps/ui/templates/browse_headers.html
- comment out the 2 lines below in this function
"""
# turn off this functionality
log.error(
"edit_collection_addall is currently unavailable. see user_collections.edit_collection_addall()"
)
return # this thing is turned off for now
update_metrics(request)
session_id = request.session.session_key
colls_table_name = get_collection_table(session_id)
(selections, extras) = urlToSearchParams(request.GET)
query_table_name = getUserQueryTable(selections, extras)
cursor = connection.cursor()
coll_table_name = get_collection_table(session_id)
sql = "replace into " + connection.ops.quote_name(coll_table_name) + \
" (id, ring_obs_id) select o.id, o.ring_obs_id from obs_general o, " + connection.ops.quote_name(query_table_name) + \
" s where o.id = s.id"
cursor.execute(sql)
return get_collection_count(session_id)
def get_download_info_API(request):
"""
this serves get_download_info as an api endpoint
but takes a request object as input
and inspects the request for product type / preview image filters
"""
update_metrics(request)
from user_collections.views import get_collection_table # circulosity
session_id = request.session.session_key
colls_table_name = get_collection_table(session_id)
product_types = []
product_types_str = request.GET.get('types', None)
if product_types_str:
product_types = product_types_str.split(',')
previews = []
previews_str = request.GET.get('previews', None)
if previews_str:
previews = previews_str.split(',')
# since we are assuming this is coming from user interaction
# if no filters exist then none of this product type is wanted
if product_types == ['none'] and previews == ['none']:
# ie this happens when all product types are unchecked in the interface
return HttpResponse(json.dumps({
'size': '0',
'count': '0'
}),
content_type='application/json')
if previews == ['all']:
previews = [i[0] for i in settings.image_sizes]
# now get the files and download size / count for this cart
urls = []
from results.views import *
download_size, count = get_download_info(product_types, previews,
colls_table_name)
# make pretty size string
download_size = nice_file_size(download_size)
return HttpResponse(json.dumps({
'size': download_size,
'count': count
}),
content_type='application/json')
def getImage(request,size='med', ring_obs_id='',fmt='mouse'): # mouse?
"""
size = thumb, small, med, full
return ring_obs_id + ' ' + size
return HttpResponse(img + "<br>" + ring_obs_id + ' ' + size +' '+ fmt)
"""
update_metrics(request)
try:
img = Image.objects.filter(ring_obs_id=ring_obs_id).values(size)[0][size]
except IndexError:
return
path = settings.IMAGE_HTTP_PATH + get_base_path_previews(ring_obs_id)
return responseFormats({'data':[{'img':img, 'path':path}]}, fmt, size=size, path=path, template='image_list.html')
def getData(request,fmt):
update_metrics(request)
"""
a page of results for a given search
"""
if not request.session.get('has_session'):
request.session['has_session'] = True
session_id = request.session.session_key
[page_no, limit, page, page_ids, order] = getPage(request)
checkboxes = True if (request.is_ajax()) else False
slugs = request.GET.get('cols',settings.DEFAULT_COLUMNS)
if not slugs: slugs = settings.DEFAULT_COLUMNS
is_column_chooser = request.GET.get('col_chooser', False)
labels = []
id_index = 0
for slug in slugs.split(','):
if slug == 'ringobsid':
id_index = slugs.split(',').index(slug)
try:
labels += [ParamInfo.objects.get(slug=slug).label_results]
except ParamInfo.DoesNotExist:
# this slug doens't match anything in param info, nix it
log.error('could not find param_info for ' + slug)
continue
if is_column_chooser:
labels.insert(0, "add") # adds a column for checkbox add-to-collections
collection = ''
if request.is_ajax():
# find the members of user collection in this page
# for pre-filling checkboxes
collection = get_collection_in_page(page, session_id)
data = {'page_no':page_no, 'limit':limit, 'page':page, 'count':len(page), 'labels': labels}
if fmt == 'raw':
return data
else:
return responseFormats(data,fmt,template='data.html', id_index=id_index, labels=labels,checkboxes=checkboxes, collection=collection, order=order)
def getImage(request,size='med', ring_obs_id='',fmt='mouse'): # mouse?
"""
size = thumb, small, med, full
return ring_obs_id + ' ' + size
return HttpResponse(img + "<br>" + ring_obs_id + ' ' + size +' '+ fmt)
"""
update_metrics(request)
try:
img = Image.objects.filter(ring_obs_id=ring_obs_id).values(size)[0][size]
except IndexError:
log.error('getImage: IndexError - Could not find ring_obs_id %s, size %s', str(ring_obs_id), str(size))
return
path = settings.IMAGE_HTTP_PATH + get_base_path_previews(ring_obs_id)
if 'CIRS' in ring_obs_id:
path = path.replace('previews','diagrams')
return responseFormats({'data':[{'img':img, 'path':path}]}, fmt, size=size, path=path, template='image_list.html')
def get_metadata_by_slugs(request, ring_obs_id, slugs, fmt):
"""
returns results for specified slugs
"""
update_metrics(request)
params_by_table = {} # params by table_name
data = []
all_info = {}
for slug in slugs:
param_info = get_param_info_by_slug(slug)
if not param_info:
continue # todo this should raise end user error
table_name = param_info.category_name
params_by_table.setdefault(table_name, []).append(param_info.param_name().split('.')[1])
all_info[slug] = param_info # to get things like dictionary entries for interface
if slugs and not all_info:
# none of the slugs were valid slugs
# can't ignore them and return all metadata because can lead to infinite recursion here
raise Http404
for table_name, param_list in params_by_table.items():
model_name = ''.join(table_name.title().split('_'))
table_model = apps.get_model('search', model_name)
results = table_model.objects.filter(ring_obs_id=ring_obs_id).values(*param_list)
# are not ring_obs_id unique in all obs tables so why is this not a .get query
if not results:
# this ring_obs_id doesn't exist in this table, log this..
log.error('Could not find ring_obs_id %s in table %s', ring_obs_id, table_name)
for param,value in results[0].items():
data.append({param: value})
if fmt == 'html':
return render(request, 'detail_metadata_slugs.html',locals())
if fmt == 'json':
return HttpResponse(json.dumps(data), content_type="application/json")
if fmt == 'raw':
return data, all_info # includes definitions for opus interface
def getValidMults(request, slug, fmt='json'):
"""
fetch mult widget hinting data for widget defined by slug
based on current search defined in request
this is the widget hinting numbers that appear next to each
possible checkbox value in a mult/group widget (green numbers)
pass request, slug, and response format
returns valid mults for a given field (slug) like so:
{ 'field':slug,'mults':mults }
"""
update_metrics(request)
try:
(selections, extras) = search.views.urlToSearchParams(request.GET)
except Exception, e:
log.error('Failed to get selections for slug %s, URL %s', str(slug),
request.GET)
log.error('.. %s', str(e))
selections = {}
def getFilesAPI(request, ring_obs_id=None, fmt=None, loc_type=None):
if not ring_obs_id:
ring_obs_id = ''
if not fmt:
fmt = 'raw' # the format this function returns
if not loc_type:
loc_type = 'url'
update_metrics(request)
product_types = request.GET.get('types',[])
previews = request.GET.get('previews',[])
if product_types:
product_types = product_types.split(',')
if previews:
previews = previews.split(',')
# we want the api to return all possible files unless otherwise described
if not product_types:
product_types = 'all'
if not previews:
previews = 'all'
if previews == ['none']:
previews = []
if request and request.GET and not ring_obs_id:
# no ring_obs_id passed, get files from search results
(selections,extras) = urlToSearchParams(request.GET)
page = getData(request,'raw')['page']
if not len(page):
return False
ring_obs_id = [p[0] for p in page]
return getFiles(ring_obs_id=ring_obs_id, fmt=fmt, loc_type=loc_type, product_types=product_types, previews=previews)
def collection_status(request, collection_name='default'):
"""
#todo this method needs tests
"""
update_metrics(request)
expected_request_no = 1
if not request.session.get('has_session'):
count = 0
else:
session_id = request.session.session_key
count = get_collection_count(session_id)
try:
expected_request_no = request.session['expected_request_no']
except KeyError:
pass # leave xpected_request_no = 1
return HttpResponse(
json.dumps({
"count": count,
"expected_request_no": expected_request_no
}))
def getMenu(request):
""" hack, need to get menu sometimes without rendering,
ie from another view.. so this is for column chooser
couldn't get template include/block.super to heed GET vars """
update_metrics(request)
return getMenuLabels(request, 'search')
def create_download(request,
collection_name=None,
ring_obs_ids=None,
fmt=None):
"""
feeds request to getFiles and zips up all files it finds into zip file
and adds a manifest file and md5 checksums
"""
update_metrics(request)
fmt = request.GET.get('fmt', "raw")
product_types = request.GET.get('types', 'none')
product_types = product_types.split(',')
previews = request.GET.get('previews', 'none')
previews = previews.split(',')
if not ring_obs_ids:
ring_obs_ids = []
from user_collections.views import *
ring_obs_ids = get_all_in_collection(request)
if type(ring_obs_ids) is unicode or type(ring_obs_ids).__name__ == 'str':
ring_obs_ids = [ring_obs_id]
if not ring_obs_ids:
raise Http404
# create some file names
zip_file_name = create_zip_filename()
chksum_file_name = settings.TAR_FILE_PATH + "checksum_" + zip_file_name.split(
".")[0] + ".txt"
manifest_file_name = settings.TAR_FILE_PATH + "manifest_" + zip_file_name.split(
".")[0] + ".txt"
csv_file_name = settings.TAR_FILE_PATH + "csv_" + zip_file_name.split(
".")[0] + ".txt"
create_csv_file(request, csv_file_name)
# fetch the full file paths we'll be zipping up
import results
files = results.views.getFiles(ring_obs_ids,
fmt="raw",
loc_type="path",
product_types=product_types,
previews=previews)
if not files:
log.error(
"no files found from results.views.getFiles in downloads.create_download"
)
raise Http404
# zip each file into tarball and create a manifest too
zip_file = zipfile.ZipFile(settings.TAR_FILE_PATH + zip_file_name,
mode='w')
chksum = open(chksum_file_name, "w")
manifest = open(manifest_file_name, "w")
size, download_count = get_download_info(files)
# don't keep creating downloads after user has reached their size limit
cum_download_size = request.session.get('cum_download_size', 0)
if cum_download_size > settings.MAX_CUM_DOWNLOAD_SIZE:
# user is trying to download > MAX_CUM_DOWNLOAD_SIZE
return HttpResponse("Sorry, Max cumulative download size reached " +
str(cum_download_size) + ' > ' +
str(settings.MAX_CUM_DOWNLOAD_SIZE))
else:
cum_download_size = cum_download_size + size
request.session['cum_download_size'] = int(cum_download_size)
errors = []
added = []
for ring_obs_id in files:
for product_type in files[ring_obs_id]:
for f in files[ring_obs_id][product_type]:
if 'FMT' in f or 'fmt' in f:
pretty_name = '/'.join(f.split("/")[-3:]).upper()
digest = "%s:%s" % (pretty_name, md5(f))
mdigest = "%s:%s" % (ring_obs_id, pretty_name)
else:
pretty_name = f.split("/")[-1]
if product_type != 'preview_image':
pretty_name = pretty_name.upper()
digest = "%s:%s" % (pretty_name, md5(f))
mdigest = "%s:%s" % (ring_obs_id, pretty_name)
if pretty_name not in added:
chksum.write(digest + "\n")
manifest.write(mdigest + "\n")
try:
zip_file.write(f, arcname=f.split(
"/")[-1]) # arcname = fielname only, not full path
added.append(pretty_name)
except Exception, e:
log.error(e)
errors.append("could not find: " + pretty_name)
def get_browse_headers(request, template='browse_headers.html'):
update_metrics(request)
return render(request, template, locals())
def getData(request,fmt):
update_metrics(request)
"""
return sa page of data for a given search and page_no like so:
data = {
'page_no':page_no,
'limit':limit,
'page':page, # tabular page data
'count':len(page),
'labels': labels
}
can return raw or http response
"""
if not request.session.get('has_session'):
request.session['has_session'] = True
session_id = request.session.session_key
[page_no, limit, page, page_ids, order] = getPage(request)
checkboxes = request.is_ajax()
slugs = request.GET.get('cols',settings.DEFAULT_COLUMNS)
if not slugs: slugs = settings.DEFAULT_COLUMNS
is_column_chooser = request.GET.get('col_chooser', False)
labels = []
id_index = 0
for slug in slugs.split(','):
if slug == 'ringobsid':
id_index = slugs.split(',').index(slug)
try:
labels += [ParamInfo.objects.get(slug=slug).label_results]
except ParamInfo.DoesNotExist:
# this slug doens't match anything in param info, nix it
if '1' in slug:
# single column range slugs will not have the index, but
# will come in with it because of how ui is designed, so
# look for the slug without the index
temp_slug = slug[:-1]
try:
labels += [ParamInfo.objects.get(slug=temp_slug).label_results]
except ParamInfo.DoesNotExist:
log.error('Could not find param_info for %s', slug)
continue
if is_column_chooser:
labels.insert(0, "add") # adds a column for checkbox add-to-collections
collection = ''
if request.is_ajax():
# find the members of user collection in this page
# for pre-filling checkboxes
collection = get_collection_in_page(page, session_id)
data = {'page_no':page_no, 'limit':limit, 'page':page, 'count':len(page), 'labels': labels}
if fmt == 'raw':
return data
else:
return responseFormats(data,fmt,template='data.html', id_index=id_index, labels=labels,checkboxes=checkboxes, collection=collection, order=order)
def get_metadata(request, ring_obs_id, fmt):
"""
results for a single observation
all the data, in categories
pass cols to narrow by particular fields
pass cat to list value of all fields in named category(s)
accepts 'cols' as a GET var which is a list of columns by slug
however the response does not return field names as slugs but as field name
so this is strange
TODO:
make it return cols by slug as field name if cols are slugs (no underscores)
if cols does has underscores make it return field names as column name
(this is a fix for backward compatablility )
and om make 'field' an alias for 'cols' plz omg what is 'cols' even
if cat is passed returns all in the named category(s) and ignores cols
you will have to add a column to table_names "slug" to get a
url-worthy representation of the category name
"""
update_metrics(request)
if not ring_obs_id: raise Http404
try:
slugs = request.GET.get('cols', False)
if slugs:
return get_metadata_by_slugs(request, ring_obs_id, slugs.split(','), fmt)
except AttributeError:
pass # no request was sent
try:
cats = request.GET.get('cats',False)
except AttributeError:
cats = False # no request was send, legacy requirement?
data = SortedDict({}) # will hold data struct to be returned
all_info = {} # holds all the param info objects
# find all the tables (categories) this observation belongs to,
if not cats:
all_tables = TableName.objects.filter(display='Y').order_by('disp_order')
else:
# restrict table to those found in cats
all_tables = TableName.objects.filter(table_name__in=cats.split(','), display='Y').order_by('disp_order')
# now find all params and their values in each of these tables:
for table in all_tables:
table_label = table.label
table_name = table.table_name
model_name = ''.join(table_name.title().split('_'))
try:
table_model = apps.get_model('search', model_name)
except LookupError:
log.error("could not find data model for category %s " % model_name)
continue
all_slugs = [param.slug for param in ParamInfo.objects.filter(category_name=table_name, display_results=1).order_by('disp_order')]
all_params = [param.name for param in ParamInfo.objects.filter(category_name=table_name, display_results=1).order_by('disp_order')]
for k, slug in enumerate(all_slugs):
param_info = get_param_info_by_slug(slug)
name = param_info.name
all_info[name] = param_info
if all_params:
try:
results = table_model.objects.filter(ring_obs_id=ring_obs_id).values(*all_params)[0]
# results is an ordinary dict so here to make sure we have the correct ordering:
ordered_results = SortedDict({})
for param in all_params:
ordered_results[param] = results[param]
data[table_label] = ordered_results
except AttributeError: pass # no results found in this table, move along
except IndexError: pass # no results found in this table, move along
except FieldError: pass # this ring_obs_id not found in this table
if fmt == 'html':
return render(request, 'detail_metadata.html',locals())
if fmt == 'json':
return HttpResponse(json.dumps(data), content_type="application/json")
if fmt == 'raw':
return data, all_info # includes definitions for opus interface
def getWidget(request, **kwargs):
""" search form widget as string, http response"""
update_metrics(request)
slug = kwargs['slug']
fmt = kwargs['fmt']
form = ''
param_info = get_param_info_by_slug(slug)
form_type = param_info.form_type
param_name = param_info.param_name()
dictionary = param_info.get_dictionary_info()
form_vals = {slug: None}
auto_id = True
selections = {}
if (request.GET):
try:
(selections, extras) = urlToSearchParams(request.GET)
except TypeError:
pass
addlink = request.GET.get('addlink', True) # suppresses the add_str link
remove_str = '<a class = "remove_input" href = "">-</a>'
add_str = '<a class = "add_input" href = "">add</a> '
append_to_label = '' # text to append to a widget label
search_form = param_info.search_form
if 'obs_surface_geometry__' in search_form:
# append the target name to surface geo widget labels
try:
append_to_label = " - %s" % search_form.split('__')[1].title()
except KeyError:
pass
if form_type in settings.RANGE_FIELDS:
auto_id = False
slug_no_num = stripNumericSuffix(slug)
param_name_no_num = stripNumericSuffix(param_name)
slug1 = slug_no_num + '1'
slug2 = slug_no_num + '2'
param1 = param_name_no_num + '1'
param2 = param_name_no_num + '2'
form_vals = {slug1: None, slug2: None}
# find length of longest list of selections for either param1 or param2,
# tells us how many times to go through loop below
try:
len1 = len(selections[param1])
except:
len1 = 0
try:
len2 = len(selections[param2])
except:
len2 = 0
lngth = len1 if len1 > len2 else len2
if not lngth: # param is not constrained
form = str(SearchForm(form_vals, auto_id=auto_id).as_ul())
if addlink == 'false':
form = '<ul>' + form + '<li>' + remove_str + '</li></ul>' # remove input is last list item in form
else:
form = '<span>' + add_str + '</span><ul>' + form + '</ul>' # add input link comes before form
else: # param is constrained
key = 0
while key < lngth:
try:
form_vals[slug1] = selections[param1][key]
except (IndexError, KeyError) as e:
form_vals[slug1] = None
try:
form_vals[slug2] = selections[param2][key]
except (IndexError, KeyError) as e:
form_vals[slug2] = None
qtypes = request.GET.get('qtype-' + slug, False)
if qtypes:
try:
form_vals['qtype-' + slug] = qtypes.split(',')[key]
except KeyError:
form_vals['qtype-' + slug] = False
form = form + str(
SearchForm(form_vals, auto_id=auto_id).as_ul())
if key > 0:
form = '<ul>' + form + '<li>' + remove_str + '</li></ul>' # remove input is last list item in form
else:
form = '<span>' + add_str + '</span><ul>' + form + '</ul>' # add input link comes before form
if lngth > 1:
form = form + '</span><div style = "clear: both;"></div></section><section><span class="widget_form">'
key = key + 1
elif form_type == 'STRING':
auto_id = False
if param_name in selections:
key = 0
for value in selections[param_name]:
form_vals[slug] = value
qtypes = request.GET.get('qtype-' + slug, False)
if qtypes:
try:
form_vals['qtype-' + slug] = qtypes.split(',')[key]
except KeyError:
form_vals['qtype-' + slug] = False
form = form + str(
SearchForm(form_vals, auto_id=auto_id).as_ul())
if key > 0:
form = form + '<li>' + remove_str + '</li>'
else:
form = form + '<li>' + add_str + '</li>'
key = key + 1
else:
form = str(SearchForm(form_vals, auto_id=auto_id).as_ul())
if addlink == 'false':
form = form + '<li>' + remove_str + '<li>'
else:
form = form + '<li>' + add_str + '<li>'
# MULT form types
elif form_type in settings.MULT_FORM_TYPES:
if param_name in selections:
form_vals = {slug: selections[param_name]}
# determine if this mult param has a grouping field (see doc/group_widgets.md for howto on grouping fields)
mult_param = getMultName(param_name)
model = apps.get_model('search', mult_param.title().replace('_', ''))
try:
grouping = model.objects.distinct().values('grouping')
grouping_table = 'grouping_' + param_name.split('.')[1]
grouping_model = apps.get_model(
'metadata',
grouping_table.title().replace('_', ''))
for group_info in grouping_model.objects.order_by('disp_order'):
gvalue = group_info.value
glabel = group_info.label if group_info.label else 'Other'
if glabel == 'NULL': glabel = 'Other'
if model.objects.filter(grouping=gvalue)[0:1]:
form += "\n\n" + \
'<div class = "mult_group_label_container" id = "mult_group_' + str(glabel) + '">' + \
'<span class = "indicator fa fa-plus"></span>' + \
'<span class = "mult_group_label">' + str(glabel) + '</span></div>' + \
'<ul class = "mult_group">' + \
SearchForm(form_vals, auto_id = '%s_' + str(gvalue), grouping=gvalue).as_ul() + \
'</ul>'
except FieldError:
# this model does not have grouping
form = SearchForm(form_vals, auto_id=auto_id).as_ul()
else: # all other form types
if param_name in selections:
form_vals = {slug: selections[param_name]}
form = SearchForm(form_vals, auto_id=auto_id).as_ul()
param_info = get_param_info_by_slug(slug)
label = param_info.label
intro = param_info.intro
range_fields = settings.RANGE_FIELDS
if fmt == 'raw':
return str(form)
else:
template = "widget.html"
return render(request, template, locals())
def view_collection(request, collection_name, template="collections.html"):
""" the collection tab http endpoint
the information it returns about product types and files does not
relect user filters such as product types and preview images
"""
update_metrics(request)
# nav stuff - page | limit | columns | offset
page_no = int(request.GET.get('page', 1))
limit = int(request.GET.get('limit', 100))
column_slugs = request.GET.get('cols', settings.DEFAULT_COLUMNS)
from results.views import * # circulosity
column_slugs = column_slugs.split(',')
columns = []
for slug in column_slugs:
columns += [ParamInfo.objects.get(slug=slug).param_name()]
offset = (page_no - 1) * limit
# collection
if not request.session.get('has_session'):
request.session['has_session'] = True
session_id = request.session.session_key
colls_table_name = get_collection_table(session_id)
files = getFiles(fmt="raw",
collection=True,
previews='none',
session_id=session_id)
all_product_types = Files.objects.all().values('product_type').distinct()
# download_info
from downloads.views import get_download_info
download_size, download_count = get_download_info(files)
download_size = nice_file_size(download_size) # pretty display it
# images and join with the collections table
where = "images.ring_obs_id = " + connection.ops.quote_name(
colls_table_name) + ".ring_obs_id"
images = Image.objects
images = images.extra(where=[where], tables=[colls_table_name])
image_types = settings.IMAGE_TYPES
image_count = len(images)
# product files
# for this we want the list of all possible product types for this dataset
# todo: this is really inefficient, another place that large carts are going to be unhappy
product_counts = dict(
[(i, 0) for i in [p['product_type'] for p in all_product_types]]
) # a dictionary with product_Type names as keys and all values set to zero
ring_obs_ids = [f for f in files]
product_counts_query = Files.objects.filter(
ring_obs_id__in=ring_obs_ids).values('product_type').annotate(
Count('product_type'))
product_counts_nonzero = {
i['product_type']: i['product_type__count']
for i in product_counts_query
}
# update the product_count array with the non-zero counts
for product_type, pcount in product_counts_nonzero.items():
product_counts[product_type] = pcount
column_values = []
for param_name in columns:
table_name = param_name.split('.')[0]
if table_name == 'obs_general':
column_values.append(param_name.split('.')[1])
else:
column_values.append(
param_name.split('.')[0].lower().replace('_', '') + '__' +
param_name.split('.')[1])
# figure out what tables do we need to join in and build query
triggered_tables = list(set([t.split('.')[0] for t in columns]))
try:
triggered_tables.remove('obs_general')
except ValueError:
pass # obs_general table wasn't in there for whatever reason
# set up the where clause to join with the rest of the tables
where = "obs_general.ring_obs_id = " + connection.ops.quote_name(
colls_table_name) + ".ring_obs_id"
triggered_tables.append(colls_table_name)
# bring in the triggered_tables
results = ObsGeneral.objects.extra(where=[where], tables=triggered_tables)
results = results.values(*column_values)[offset:offset + limit]
page_ids = [o['ring_obs_id'] for o in results]
return render(request, template, locals())
def getImages(request,size,fmt):
update_metrics(request)
"""
this returns rows from images table that correspond to request
some rows will not have images, this function doesn't return 'image_not_found' information
if a row doesn't have an image you get nothing. you lose. good day sir. #fixme #todo
"""
if not request.session.get('has_session'):
request.session['has_session'] = True
session_id = request.session.session_key
alt_size = request.GET.get('alt_size','')
columns = request.GET.get('cols',settings.DEFAULT_COLUMNS)
try:
[page_no, limit, page, page_ids, order] = getPage(request)
except TypeError: # getPage returns False
raise Http404('could not find page')
image_links = Image.objects.filter(ring_obs_id__in=page_ids)
if not image_links:
log.error('no image found for:')
log.error(page_ids[:50])
# page_ids
if alt_size:
image_links = image_links.values('ring_obs_id',size,alt_size)
else:
image_links = image_links.values('ring_obs_id',size)
# add the base_path to each image
all_sizes = ['small','thumb','med','full']
for k, im in enumerate(image_links):
for s in all_sizes:
if s in im:
image_links[k][s] = get_base_path_previews(im['ring_obs_id']) + im[s]
# to preserve the order of page_ids as lamely as possible :P
ordered_image_links = []
for ring_obs_id in page_ids:
found = False
for link in image_links:
if ring_obs_id == link['ring_obs_id']:
found = True
ordered_image_links.append(link)
if not found:
# return the thumbnail not found link
ordered_image_links.append({size:settings.THUMBNAIL_NOT_FOUND, 'ring_obs_id':ring_obs_id})
image_links = ordered_image_links
collection_members = get_collection_in_page(page, session_id)
# find which are in collections, mark unfound images 'not found'
for image in image_links:
image['img'] = image[size] if image[size] else 'not found'
if collection_members:
from user_collections.views import *
if image['ring_obs_id'] in collection_members:
image['in_collection'] = True
path = settings.IMAGE_HTTP_PATH
if (request.is_ajax()):
template = 'gallery.html'
else: template = 'image_list.html'
# image_links
return responseFormats({'data':[i for i in image_links]},fmt, size=size, path=path, alt_size=alt_size, columns_str=columns.split(','), template=template, order=order)
def edit_collection(request, **kwargs):
update_metrics(request)
"""
edits a single ring_obs_id in a user collection (user "selections")
# edits?
"""
if not request.session.get('has_session'):
request.session['has_session'] = True
session_id = request.session.session_key
checkArgs = check_collection_args(request, **kwargs)
if type(checkArgs).__name__ == 'list' and checkArgs:
(action, ring_obs_id, request_no, expected_request_no) = checkArgs
else:
return HttpResponse(json.dumps({"err": checkArgs}))
# just add this request to the queue, every request gets queued
add_to_queue(request, request_no, action, ring_obs_id)
"""
# todo
turning this off for now, we will get the queue of whatever request_no is passed
to us, without checking against what is expected_request_no
by turning this off we are at risk of ajax race conditions
but it's breaking something and no time fo dat right now
Issue is here:
https://bitbucket.org/ringsnode/opus2/issue/75/collections-downloads-majorly-broken
# todo:
# now look for the next expected request in the queue
if get_queued(request, expected_request_no):
# found the next expected request in the queue
(request,action,ring_obs_id) = get_queued(request, expected_request_no)
else:
# the expected request has not yet arrived, do nothing
return HttpResponse(json.dumps({"err":"waiting"}))
"""
# instead of the above we are doing this:
(action, ring_obs_id) = get_queued(request, request_no)
# redux: these will be not so simple saddly, they will insert or delete from
# the colleciton table..
if action == 'add':
add_to_collection(ring_obs_id, session_id)
elif (action == 'remove'):
remove_from_collection(ring_obs_id, session_id)
elif (action in ['addrange', 'removerange']):
collection_count = edit_collection_range(request, **kwargs)
remove_from_queue(request, request_no)
"""
try:
json = {"msg":"yay!","count":len(collection), "collection": ', '.join(collection)}
except:
json = collection
"""
collection_count = get_collection_count(session_id)
json_data = {
"err": False,
"count": collection_count,
"request_no": request_no
}
return HttpResponse(json.dumps(json_data))
def getValidMults(request, slug, fmt='json'):
"""
returns list of valid mult ids or labels for field in GET param "field"
based on current search defined in request
field_format = 'ids' or 'labels' depending on how you want your data back
(OPUS UI will use ids but they aren't very readable for everyone else)
"""
update_metrics(request)
try:
(selections, extras) = search.views.urlToSearchParams(request.GET)
except:
selections = {}
param_info = search.views.get_param_info_by_slug(slug)
table_name = param_info.category_name
param_name = param_info.param_name()
# if this param is in selections we want to remove it,
# want mults for a param as they would be without itself
if param_name in selections:
del selections[param_name]
has_selections = False if len(selections.keys()) < 1 else True
cache_key = "mults" + param_name + str(
search.views.setUserSearchNo(selections))
if (cache.get(cache_key) is not None):
mults = cache.get(cache_key)
else:
mult_name = getMultName(param_name) # the name of the field to query
mult_model = apps.get_model('search',
mult_name.title().replace('_', ''))
table_model = apps.get_model('search',
table_name.title().replace('_', ''))
mults = {} # info to return
results = table_model.objects.values(mult_name).annotate(
Count(mult_name)) # this is a count(*), group_by query!
user_table = search.views.getUserQueryTable(selections, extras)
if has_selections and not user_table:
# selections are constrained so join in the user_table
log.error(
'getValidMults has_selections = true but no user_table found:')
log.error(selections)
log.error(extras)
raise Http404
if table_name == 'obs_general':
where = table_name + ".id = " + user_table + ".id"
else:
where = table_name + ".obs_general_id = " + user_table + ".id"
results = results.extra(where=[where], tables=[user_table])
for row in results:
mult_id = row[mult_name]
try:
try:
mult = mult_model.objects.get(id=mult_id).label
except:
log.debug('could not find mult label for ')
log.debug('id: ' + str(mult_id))
log.debug(mult_model)
mult = mult_id # fall back to id if there is no label
mults[mult] = row[mult_name + '__count']
except:
pass # a none object may be found in the data but if it doesn't have a table row we don't handle it
cache.set(cache_key, mults, 0)
multdata = {'field': slug, 'mults': mults}
if (request.is_ajax()):
reqno = request.GET.get('reqno', '')
multdata['reqno'] = reqno
return responseFormats(multdata, fmt, template='mults.html')
def getPage(request, colls=None, colls_page=None, page=None):
update_metrics(request)
"""
the gets the metadata and images to build a page of results
"""
# get some stuff from the url or fall back to defaults
if not request.session.get('has_session'):
request.session['has_session'] = True
session_id = request.session.session_key
if not colls:
collection_page = request.GET.get('colls',False)
else:
collection_page = colls
limit = request.GET.get('limit',100)
limit = int(limit)
slugs = request.GET.get('cols', settings.DEFAULT_COLUMNS)
if not slugs:
slugs = settings.DEFAULT_COLUMNS # i dunno why the above doesn't suffice
columns = []
for slug in slugs.split(','):
try:
columns += [ParamInfo.objects.get(slug=slug).param_name()]
except ParamInfo.DoesNotExist:
pass
triggered_tables = list(set([param_name.split('.')[0] for param_name in columns]))
try:
triggered_tables.remove('obs_general') # we remove it because it is the primary
# model so don't need to add it to extra tables
except ValueError:
pass # obs_general isn't in there
if not collection_page:
# this is for a search query
order = request.GET.get('order','time1')
# figure out column order in table
if order:
try:
descending = '-' if order[0] == '-' else None
order_slug = order.strip('-') # strip off any minus sign to look up param name
order = ParamInfo.objects.get(slug=order_slug).param_name()
if descending:
order = '-' + order
except DoesNotExist:
order = False
# figure out page we are asking for
if not page:
page_no = request.GET.get('page',1)
if page_no != 'all':
page_no = int(page_no)
else:
page_no == page
# ok now that we have everything from the url get stuff from db
(selections,extras) = urlToSearchParams(request.GET)
user_query_table = getUserQueryTable(selections,extras)
# figure out what tables do we need to join in and build query
triggered_tables.append(user_query_table)
where = "obs_general.id = " + connection.ops.quote_name(user_query_table) + ".id"
results = ObsGeneral.objects.extra(where=[where], tables=triggered_tables)
else:
# this is for a collection
# find the ordering
order = request.GET.get('colls_order', False)
if not order:
# get regular order if collections doesn't have a special order
order = request.GET.get('order',False)
if order:
try:
order_param = order.strip('-') # strip off any minus sign to look up param name
descending = order[0] if (order[0] == '-') else None
order = ParamInfo.objects.get(slug=order_param).name
if descending:
order = '-' + order
except DoesNotExist:
order = False
if not colls_page:
page_no = request.GET.get('colls_page',1)
if page_no != 'all':
page_no = int(page_no)
else:
page_no = colls_page
# figure out what tables do we need to join in and build query
triggered_tables = list(set([t.split('.')[0] for t in columns]))
try:
triggered_tables.remove('obs_general')
except ValueError:
pass # obs_general table wasn't in there for whatever reason
# join in the collections table
colls_table_name = get_collection_table(session_id)
triggered_tables.append(colls_table_name)
where = "obs_general.ring_obs_id = " + connection.ops.quote_name(colls_table_name) + ".ring_obs_id"
results = ObsGeneral.objects.extra(where=[where], tables=triggered_tables)
# now we have results object (either search or collections)
if order:
results = results.order_by(order)
# this is the thing you pass to django model via values()
# so we have the table names a bit to get what django wants:
column_values = []
for param_name in columns:
table_name = param_name.split('.')[0]
if table_name == 'obs_general':
column_values.append(param_name.split('.')[1])
else:
column_values.append(param_name.split('.')[0].lower().replace('_','') + '__' + param_name.split('.')[1])
"""
the limit is pretty much always 100, the user cannot change it in the interface
but as an aide to finding the right chunk of a result set to search for
for the 'add range' click functinality, the front end may send a large limit, like say
page_no = 42 and limit = 400
that means start the page at 42 and go 4 pages, and somewhere in there is our range
this is how 'add range' works accross multiple pages
so the way of computing starting offset here should always use the base_limit of 100
using the passed limit will result inthe wront offset because of way offset is computed here
this may be an aweful hack.
"""
# todo: redux: look at line 559 you are essentially doing this query twice? in the same method and peforming the query each time cuz it has changed!
if page_no != 'all':
base_limit = 100 # explainer of sorts is above
offset = (page_no-1)*base_limit # we don't use Django's pagination because of that count(*) that it does.
results = results.values_list(*column_values)[offset:offset+int(limit)]
else:
results = results.values_list(*column_values)
# return a simple list of ring_obs_ids
ring_obs_id_index = column_values.index('ring_obs_id')
page_ids = [o[ring_obs_id_index] for o in results]
if not len(page_ids):
return False
return [page_no, limit, list(results), page_ids, order]
def getRangeEndpoints(request, slug, fmt='json'):
"""
fetch range widget hinting data for widget defined by slug
based on current search defined in request
this is the valid range endpoints that appear in
range widgets (green numbers)
returns a dictionary like:
{ min: 63.592, max: 88.637, nulls: 2365}
"""
# if this param is in selections we want to remove it,
# want results for param as they would be without itself constrained
# extras['qtypes']['']
update_metrics(request)
param_info = search.views.get_param_info_by_slug(slug)
param_name = param_info.param_name()
form_type = param_info.form_type
table_name = param_info.category_name
# "param" is the field name, the param_name with the table_name stripped
param1 = stripNumericSuffix(param_name.split('.')[1]) + '1'
param2 = stripNumericSuffix(param_name.split('.')[1]) + '2'
param_no_num = stripNumericSuffix(param1)
table_model = apps.get_model('search', table_name.title().replace('_', ''))
if form_type == 'RANGE' and '1' not in param_info.slug and '2' not in param_info.slug:
param1 = param2 = param_no_num # single column range query
try:
(selections, extras) = search.views.urlToSearchParams(request.GET)
user_table = search.views.getUserQueryTable(selections, extras)
has_selections = True
except TypeError:
selections = {}
has_selections = False
user_table = False
# remove this param from the user's query if it is constrained
# this keeps the green hinting numbers from reacting
# to changes to its own field
param_name_no_num = stripNumericSuffix(param_name)
to_remove = [
param_name_no_num, param_name_no_num + '1', param_name_no_num + '2'
]
for p in to_remove:
if p in selections:
del selections[p]
if not bool(selections):
has_selections = False
user_table = False
# cached already?
cache_key = "rangeep" + param_no_num
if user_table:
cache_key += str(search.views.setUserSearchNo(selections, extras))
if cache.get(cache_key) is not None:
range_endpoints = cache.get(cache_key)
return responseFormats(range_endpoints, fmt, template='mults.html')
# no cache found, calculating..
try:
results = table_model.objects # this is a count(*), group_by query
except AttributeError, e:
log.error("getRangeEndpoints threw: %s", str(e))
log.error("Could not find table model for table_name: %s", table_name)
raise Http404("Does Not Exist")
