def get_table_headers(request, template='table_headers.html'): update_metrics(request) slugs = request.GET.get('cols', settings.DEFAULT_COLUMNS) order = request.GET.get('order', None) if order: sort_icon = 'fa-sort-desc' if order[0] == '-' else 'fa-sort-asc' order = order[1:] if order[0] == '-' else order if not slugs: slugs = settings.DEFAULT_COLUMNS slugs = slugs.split(',') columns = [] # if this is an ajax call it means it's from our app, so append the # checkbox column for adding to selections if (request.is_ajax()): columns.append(["collection", "Collect"]) param_info = ParamInfo.objects for slug in slugs: if slug and slug != 'ring_obs_id': try: columns.append([slug, param_info.get(slug=slug).label_results]) except ParamInfo.DoesNotExist: pass return render(request, template, locals())
def guide(request): update_metrics(request) base_url = 'http://' + request.META['HTTP_HOST'] + '/opus/' groups = Group.objects.all() resources = Resource.objects.filter( display=True).select_related().order_by('disp_order') return render(request, 'guide.html', locals())
def edit_collection_range(request, **kwargs): update_metrics(request) """ the request will come in not as a single ring_obs_id but as min and max ring_obs_ids + a range in a search query """ if not request.session.get('has_session'): request.session['has_session'] = True session_id = request.session.session_key colls_table_name = get_collection_table(session_id) (action, ring_obs_id, request_no, expected_request_no) = check_collection_args(request, **kwargs) id_range = request.GET.get('addrange', False) if not id_range: return False # "invalid ringobsid pair" (min_id, max_id) = id_range.split(',') (selections, extras) = urlToSearchParams(request.GET) from results.views import * data = getData(request, "raw") selected_range = [] in_range = False # loop has reached the range selected column_slugs = request.GET.get('cols', settings.DEFAULT_COLUMNS) ring_obs_id_key = column_slugs.split(',').index('ringobsid') # return HttpResponse(json.dumps(data['page'])); for row in data['page']: ring_obs_id = row[ring_obs_id_key] if ring_obs_id == min_id: in_range = True if in_range: selected_range.append(ring_obs_id) if in_range & (ring_obs_id == max_id): # this is the last one, update the collection if action == 'addrange': bulk_add_to_collection([rid for rid in selected_range], session_id) if not selected_range: log.error("edit_collection_range failed to find range " + id_range) log.error(selections) return get_collection_count(session_id)
def getRangeEndpoints(request, slug, fmt='json'): """ returns valid range endpoints for field given selections and extras """ # if this param is in selections we want to remove it, # want results for param as they would be without itself constrained # extras['qtypes'][''] update_metrics(request) param_info = search.views.get_param_info_by_slug(slug) param_name = param_info.param_name() form_type = param_info.form_type table_name = param_info.category_name param_name1 = stripNumericSuffix(param_name.split('.')[1]) + '1' param_name2 = stripNumericSuffix(param_name.split('.')[1]) + '2' param_name_no_num = stripNumericSuffix(param_name1) table_model = apps.get_model('search', table_name.title().replace('_', '')) if form_type == 'RANGE' and '1' not in param_info.slug and '2' not in param_info.slug: param_name1 = param_name2 = param_name_no_num try: (selections, extras) = search.views.urlToSearchParams(request.GET) user_table = search.views.getUserQueryTable(selections, extras) has_selections = True except TypeError: selections = {} has_selections = False user_table = False # we remove this param from the user's query if it's there, because # otherwise it will just return it's own values if param_name1 in selections: del selections[param_name1] if param_name2 in selections: del selections[param_name2] # cached already? cache_key = "rangeep" + param_name_no_num if user_table: cache_key += str(search.views.setUserSearchNo(selections, extras)) if cache.get(cache_key) is not None: range_endpoints = cache.get(cache_key) return responseFormats(range_endpoints, fmt, template='mults.html') try: results = table_model.objects # this is a count(*), group_by query except AttributeError, e: log.error(e) log.error("could not find table model for table_name: %s" % table_name) raise Http404("Does Not Exist")
def getResultCount(request, fmt='json'): """ result count for a search """ update_metrics(request) if request.GET is None: return HttpResponse(json.dumps({'result_count': '0'}), content_type='application/json') try: (selections, extras) = search.views.urlToSearchParams(request.GET) except TypeError: log.error("could not find selections for request") log.error(request.GET) raise Http404 reqno = request.GET.get('reqno', '') if selections is False: count = 'not found' return HttpResponse(json.dumps({'result_count': count}), content_type='application/json') table = search.views.getUserQueryTable(selections, extras) if table is False: count = 0 else: cache_key = "resultcount:" + table if (cache.get(cache_key)): count = cache.get(cache_key) else: cursor = connection.cursor() cursor.execute("select count(*) from " + connection.ops.quote_name(table)) try: count = cursor.fetchone() count = count[0] except: count = 0 cache.set(cache_key, count, 0) data = {'result_count': count} if (request.is_ajax()): data['reqno'] = request.GET['reqno'] return responseFormats({'data': [data]}, fmt, template='result_count.html')
def init_detail_page(request, **kwargs): """ this loads the initial parts of the detail tab on first loads these are the things that are fast to compute while other parts of the page are handled with ajax calls because they are slower the detail page calls other views via ajax: results.get_metadata """ update_metrics(request) template = "detail.html" slugs = request.GET.get('cols', False) ring_obs_id = kwargs['ring_obs_id'] # get the preview image and some general info try: img = Image.objects.get(ring_obs_id=ring_obs_id) except Image.DoesNotExist: img = None base_vol_path = get_base_path_previews(ring_obs_id) path = settings.IMAGE_HTTP_PATH + base_vol_path if 'CIRS' in base_vol_path: path = path.replace('previews', 'diagrams') instrument_id = ObsGeneral.objects.filter( ring_obs_id=ring_obs_id).values('instrument_id')[0]['instrument_id'] # get the preview guide url preview_guide_url = '' if instrument_id == 'COCIRS': preview_guide_url = 'http://pds-rings.seti.org/cassini/cirs/COCIRS_previews.txt' if instrument_id == 'COUVIS': preview_guide_url = 'http://pds-rings.seti.org/cassini/uvis/UVIS_previews.txt' if instrument_id == 'COVIMS': preview_guide_url = 'http://pds-rings.seti.org/cassini/vims/COVIMS_previews.txt' # get the list of files for this observation files = getFiles(ring_obs_id, fmt='raw')[ring_obs_id] file_list = {} for product_type in files: if product_type not in file_list: file_list[product_type] = [] for f in files[product_type]: ext = f.split('.').pop() file_list[product_type].append({'ext': ext, 'link': f}) return render(request, template, locals())
def getColumnChooser(request, **kwargs): update_metrics(request) slugs = request.GET.get('cols', settings.DEFAULT_COLUMNS).split(',') slugs = filter( None, slugs) # sometimes 'cols' is in url but is blank, so fails above if not slugs: slugs = settings.DEFAULT_COLUMNS.split(',') all_slugs_info = getColumnInfo(slugs) namespace = 'column_chooser_input' menu = getMenuLabels(request, 'results')['menu'] return render(request, "choose_columns.html", locals())
def edit_collection_addall(request, **kwargs): """ add the entire result set to the collection cart This may be turned off. The way to turn this off is: - comment out html link in apps/ui/templates/browse_headers.html - add these lines below: # turn off this functionality log.debug("edit_collection_addall is currently turned off. see apps/user_collections.edit_collection_addall") return # this thing is turned off for now The reason is it needs more testing, but this branch makes a big efficiency improvements to the way downloads are handled, and fixes some things, so I wanted to merge it into master Things that needs further exploration: This functionality provides no checks on how large a cart can be. There needs to be some limit. It doesn't hide the menu link when the result count is too high. And what happens when it bumps against the MAX_CUM_DOWNLOAD_SIZE. The functionality is there but these are questions! To bring this functionality back for testing do the folloing: - uncomment the "add all to cart" link in apps/ui/templates/browse_headers.html - comment out the 2 lines below in this function """ # turn off this functionality log.error( "edit_collection_addall is currently unavailable. see user_collections.edit_collection_addall()" ) return # this thing is turned off for now update_metrics(request) session_id = request.session.session_key colls_table_name = get_collection_table(session_id) (selections, extras) = urlToSearchParams(request.GET) query_table_name = getUserQueryTable(selections, extras) cursor = connection.cursor() coll_table_name = get_collection_table(session_id) sql = "replace into " + connection.ops.quote_name(coll_table_name) + \ " (id, ring_obs_id) select o.id, o.ring_obs_id from obs_general o, " + connection.ops.quote_name(query_table_name) + \ " s where o.id = s.id" cursor.execute(sql) return get_collection_count(session_id)
def get_download_info_API(request): """ this serves get_download_info as an api endpoint but takes a request object as input and inspects the request for product type / preview image filters """ update_metrics(request) from user_collections.views import get_collection_table # circulosity session_id = request.session.session_key colls_table_name = get_collection_table(session_id) product_types = [] product_types_str = request.GET.get('types', None) if product_types_str: product_types = product_types_str.split(',') previews = [] previews_str = request.GET.get('previews', None) if previews_str: previews = previews_str.split(',') # since we are assuming this is coming from user interaction # if no filters exist then none of this product type is wanted if product_types == ['none'] and previews == ['none']: # ie this happens when all product types are unchecked in the interface return HttpResponse(json.dumps({ 'size': '0', 'count': '0' }), content_type='application/json') if previews == ['all']: previews = [i[0] for i in settings.image_sizes] # now get the files and download size / count for this cart urls = [] from results.views import * download_size, count = get_download_info(product_types, previews, colls_table_name) # make pretty size string download_size = nice_file_size(download_size) return HttpResponse(json.dumps({ 'size': download_size, 'count': count }), content_type='application/json')
def getImage(request,size='med', ring_obs_id='',fmt='mouse'): # mouse? """ size = thumb, small, med, full return ring_obs_id + ' ' + size return HttpResponse(img + "<br>" + ring_obs_id + ' ' + size +' '+ fmt) """ update_metrics(request) try: img = Image.objects.filter(ring_obs_id=ring_obs_id).values(size)[0][size] except IndexError: return path = settings.IMAGE_HTTP_PATH + get_base_path_previews(ring_obs_id) return responseFormats({'data':[{'img':img, 'path':path}]}, fmt, size=size, path=path, template='image_list.html')
def getData(request,fmt): update_metrics(request) """ a page of results for a given search """ if not request.session.get('has_session'): request.session['has_session'] = True session_id = request.session.session_key [page_no, limit, page, page_ids, order] = getPage(request) checkboxes = True if (request.is_ajax()) else False slugs = request.GET.get('cols',settings.DEFAULT_COLUMNS) if not slugs: slugs = settings.DEFAULT_COLUMNS is_column_chooser = request.GET.get('col_chooser', False) labels = [] id_index = 0 for slug in slugs.split(','): if slug == 'ringobsid': id_index = slugs.split(',').index(slug) try: labels += [ParamInfo.objects.get(slug=slug).label_results] except ParamInfo.DoesNotExist: # this slug doens't match anything in param info, nix it log.error('could not find param_info for ' + slug) continue if is_column_chooser: labels.insert(0, "add") # adds a column for checkbox add-to-collections collection = '' if request.is_ajax(): # find the members of user collection in this page # for pre-filling checkboxes collection = get_collection_in_page(page, session_id) data = {'page_no':page_no, 'limit':limit, 'page':page, 'count':len(page), 'labels': labels} if fmt == 'raw': return data else: return responseFormats(data,fmt,template='data.html', id_index=id_index, labels=labels,checkboxes=checkboxes, collection=collection, order=order)
def getImage(request,size='med', ring_obs_id='',fmt='mouse'): # mouse? """ size = thumb, small, med, full return ring_obs_id + ' ' + size return HttpResponse(img + "<br>" + ring_obs_id + ' ' + size +' '+ fmt) """ update_metrics(request) try: img = Image.objects.filter(ring_obs_id=ring_obs_id).values(size)[0][size] except IndexError: log.error('getImage: IndexError - Could not find ring_obs_id %s, size %s', str(ring_obs_id), str(size)) return path = settings.IMAGE_HTTP_PATH + get_base_path_previews(ring_obs_id) if 'CIRS' in ring_obs_id: path = path.replace('previews','diagrams') return responseFormats({'data':[{'img':img, 'path':path}]}, fmt, size=size, path=path, template='image_list.html')
def get_metadata_by_slugs(request, ring_obs_id, slugs, fmt): """ returns results for specified slugs """ update_metrics(request) params_by_table = {} # params by table_name data = [] all_info = {} for slug in slugs: param_info = get_param_info_by_slug(slug) if not param_info: continue # todo this should raise end user error table_name = param_info.category_name params_by_table.setdefault(table_name, []).append(param_info.param_name().split('.')[1]) all_info[slug] = param_info # to get things like dictionary entries for interface if slugs and not all_info: # none of the slugs were valid slugs # can't ignore them and return all metadata because can lead to infinite recursion here raise Http404 for table_name, param_list in params_by_table.items(): model_name = ''.join(table_name.title().split('_')) table_model = apps.get_model('search', model_name) results = table_model.objects.filter(ring_obs_id=ring_obs_id).values(*param_list) # are not ring_obs_id unique in all obs tables so why is this not a .get query if not results: # this ring_obs_id doesn't exist in this table, log this.. log.error('Could not find ring_obs_id %s in table %s', ring_obs_id, table_name) for param,value in results[0].items(): data.append({param: value}) if fmt == 'html': return render(request, 'detail_metadata_slugs.html',locals()) if fmt == 'json': return HttpResponse(json.dumps(data), content_type="application/json") if fmt == 'raw': return data, all_info # includes definitions for opus interface
def getValidMults(request, slug, fmt='json'): """ fetch mult widget hinting data for widget defined by slug based on current search defined in request this is the widget hinting numbers that appear next to each possible checkbox value in a mult/group widget (green numbers) pass request, slug, and response format returns valid mults for a given field (slug) like so: { 'field':slug,'mults':mults } """ update_metrics(request) try: (selections, extras) = search.views.urlToSearchParams(request.GET) except Exception, e: log.error('Failed to get selections for slug %s, URL %s', str(slug), request.GET) log.error('.. %s', str(e)) selections = {}
def getFilesAPI(request, ring_obs_id=None, fmt=None, loc_type=None): if not ring_obs_id: ring_obs_id = '' if not fmt: fmt = 'raw' # the format this function returns if not loc_type: loc_type = 'url' update_metrics(request) product_types = request.GET.get('types',[]) previews = request.GET.get('previews',[]) if product_types: product_types = product_types.split(',') if previews: previews = previews.split(',') # we want the api to return all possible files unless otherwise described if not product_types: product_types = 'all' if not previews: previews = 'all' if previews == ['none']: previews = [] if request and request.GET and not ring_obs_id: # no ring_obs_id passed, get files from search results (selections,extras) = urlToSearchParams(request.GET) page = getData(request,'raw')['page'] if not len(page): return False ring_obs_id = [p[0] for p in page] return getFiles(ring_obs_id=ring_obs_id, fmt=fmt, loc_type=loc_type, product_types=product_types, previews=previews)
def collection_status(request, collection_name='default'): """ #todo this method needs tests """ update_metrics(request) expected_request_no = 1 if not request.session.get('has_session'): count = 0 else: session_id = request.session.session_key count = get_collection_count(session_id) try: expected_request_no = request.session['expected_request_no'] except KeyError: pass # leave xpected_request_no = 1 return HttpResponse( json.dumps({ "count": count, "expected_request_no": expected_request_no }))
def getMenu(request): """ hack, need to get menu sometimes without rendering, ie from another view.. so this is for column chooser couldn't get template include/block.super to heed GET vars """ update_metrics(request) return getMenuLabels(request, 'search')
def create_download(request, collection_name=None, ring_obs_ids=None, fmt=None): """ feeds request to getFiles and zips up all files it finds into zip file and adds a manifest file and md5 checksums """ update_metrics(request) fmt = request.GET.get('fmt', "raw") product_types = request.GET.get('types', 'none') product_types = product_types.split(',') previews = request.GET.get('previews', 'none') previews = previews.split(',') if not ring_obs_ids: ring_obs_ids = [] from user_collections.views import * ring_obs_ids = get_all_in_collection(request) if type(ring_obs_ids) is unicode or type(ring_obs_ids).__name__ == 'str': ring_obs_ids = [ring_obs_id] if not ring_obs_ids: raise Http404 # create some file names zip_file_name = create_zip_filename() chksum_file_name = settings.TAR_FILE_PATH + "checksum_" + zip_file_name.split( ".")[0] + ".txt" manifest_file_name = settings.TAR_FILE_PATH + "manifest_" + zip_file_name.split( ".")[0] + ".txt" csv_file_name = settings.TAR_FILE_PATH + "csv_" + zip_file_name.split( ".")[0] + ".txt" create_csv_file(request, csv_file_name) # fetch the full file paths we'll be zipping up import results files = results.views.getFiles(ring_obs_ids, fmt="raw", loc_type="path", product_types=product_types, previews=previews) if not files: log.error( "no files found from results.views.getFiles in downloads.create_download" ) raise Http404 # zip each file into tarball and create a manifest too zip_file = zipfile.ZipFile(settings.TAR_FILE_PATH + zip_file_name, mode='w') chksum = open(chksum_file_name, "w") manifest = open(manifest_file_name, "w") size, download_count = get_download_info(files) # don't keep creating downloads after user has reached their size limit cum_download_size = request.session.get('cum_download_size', 0) if cum_download_size > settings.MAX_CUM_DOWNLOAD_SIZE: # user is trying to download > MAX_CUM_DOWNLOAD_SIZE return HttpResponse("Sorry, Max cumulative download size reached " + str(cum_download_size) + ' > ' + str(settings.MAX_CUM_DOWNLOAD_SIZE)) else: cum_download_size = cum_download_size + size request.session['cum_download_size'] = int(cum_download_size) errors = [] added = [] for ring_obs_id in files: for product_type in files[ring_obs_id]: for f in files[ring_obs_id][product_type]: if 'FMT' in f or 'fmt' in f: pretty_name = '/'.join(f.split("/")[-3:]).upper() digest = "%s:%s" % (pretty_name, md5(f)) mdigest = "%s:%s" % (ring_obs_id, pretty_name) else: pretty_name = f.split("/")[-1] if product_type != 'preview_image': pretty_name = pretty_name.upper() digest = "%s:%s" % (pretty_name, md5(f)) mdigest = "%s:%s" % (ring_obs_id, pretty_name) if pretty_name not in added: chksum.write(digest + "\n") manifest.write(mdigest + "\n") try: zip_file.write(f, arcname=f.split( "/")[-1]) # arcname = fielname only, not full path added.append(pretty_name) except Exception, e: log.error(e) errors.append("could not find: " + pretty_name)
def get_browse_headers(request, template='browse_headers.html'): update_metrics(request) return render(request, template, locals())
def getData(request,fmt): update_metrics(request) """ return sa page of data for a given search and page_no like so: data = { 'page_no':page_no, 'limit':limit, 'page':page, # tabular page data 'count':len(page), 'labels': labels } can return raw or http response """ if not request.session.get('has_session'): request.session['has_session'] = True session_id = request.session.session_key [page_no, limit, page, page_ids, order] = getPage(request) checkboxes = request.is_ajax() slugs = request.GET.get('cols',settings.DEFAULT_COLUMNS) if not slugs: slugs = settings.DEFAULT_COLUMNS is_column_chooser = request.GET.get('col_chooser', False) labels = [] id_index = 0 for slug in slugs.split(','): if slug == 'ringobsid': id_index = slugs.split(',').index(slug) try: labels += [ParamInfo.objects.get(slug=slug).label_results] except ParamInfo.DoesNotExist: # this slug doens't match anything in param info, nix it if '1' in slug: # single column range slugs will not have the index, but # will come in with it because of how ui is designed, so # look for the slug without the index temp_slug = slug[:-1] try: labels += [ParamInfo.objects.get(slug=temp_slug).label_results] except ParamInfo.DoesNotExist: log.error('Could not find param_info for %s', slug) continue if is_column_chooser: labels.insert(0, "add") # adds a column for checkbox add-to-collections collection = '' if request.is_ajax(): # find the members of user collection in this page # for pre-filling checkboxes collection = get_collection_in_page(page, session_id) data = {'page_no':page_no, 'limit':limit, 'page':page, 'count':len(page), 'labels': labels} if fmt == 'raw': return data else: return responseFormats(data,fmt,template='data.html', id_index=id_index, labels=labels,checkboxes=checkboxes, collection=collection, order=order)
def get_metadata(request, ring_obs_id, fmt): """ results for a single observation all the data, in categories pass cols to narrow by particular fields pass cat to list value of all fields in named category(s) accepts 'cols' as a GET var which is a list of columns by slug however the response does not return field names as slugs but as field name so this is strange TODO: make it return cols by slug as field name if cols are slugs (no underscores) if cols does has underscores make it return field names as column name (this is a fix for backward compatablility ) and om make 'field' an alias for 'cols' plz omg what is 'cols' even if cat is passed returns all in the named category(s) and ignores cols you will have to add a column to table_names "slug" to get a url-worthy representation of the category name """ update_metrics(request) if not ring_obs_id: raise Http404 try: slugs = request.GET.get('cols', False) if slugs: return get_metadata_by_slugs(request, ring_obs_id, slugs.split(','), fmt) except AttributeError: pass # no request was sent try: cats = request.GET.get('cats',False) except AttributeError: cats = False # no request was send, legacy requirement? data = SortedDict({}) # will hold data struct to be returned all_info = {} # holds all the param info objects # find all the tables (categories) this observation belongs to, if not cats: all_tables = TableName.objects.filter(display='Y').order_by('disp_order') else: # restrict table to those found in cats all_tables = TableName.objects.filter(table_name__in=cats.split(','), display='Y').order_by('disp_order') # now find all params and their values in each of these tables: for table in all_tables: table_label = table.label table_name = table.table_name model_name = ''.join(table_name.title().split('_')) try: table_model = apps.get_model('search', model_name) except LookupError: log.error("could not find data model for category %s " % model_name) continue all_slugs = [param.slug for param in ParamInfo.objects.filter(category_name=table_name, display_results=1).order_by('disp_order')] all_params = [param.name for param in ParamInfo.objects.filter(category_name=table_name, display_results=1).order_by('disp_order')] for k, slug in enumerate(all_slugs): param_info = get_param_info_by_slug(slug) name = param_info.name all_info[name] = param_info if all_params: try: results = table_model.objects.filter(ring_obs_id=ring_obs_id).values(*all_params)[0] # results is an ordinary dict so here to make sure we have the correct ordering: ordered_results = SortedDict({}) for param in all_params: ordered_results[param] = results[param] data[table_label] = ordered_results except AttributeError: pass # no results found in this table, move along except IndexError: pass # no results found in this table, move along except FieldError: pass # this ring_obs_id not found in this table if fmt == 'html': return render(request, 'detail_metadata.html',locals()) if fmt == 'json': return HttpResponse(json.dumps(data), content_type="application/json") if fmt == 'raw': return data, all_info # includes definitions for opus interface
def getWidget(request, **kwargs): """ search form widget as string, http response""" update_metrics(request) slug = kwargs['slug'] fmt = kwargs['fmt'] form = '' param_info = get_param_info_by_slug(slug) form_type = param_info.form_type param_name = param_info.param_name() dictionary = param_info.get_dictionary_info() form_vals = {slug: None} auto_id = True selections = {} if (request.GET): try: (selections, extras) = urlToSearchParams(request.GET) except TypeError: pass addlink = request.GET.get('addlink', True) # suppresses the add_str link remove_str = '<a class = "remove_input" href = "">-</a>' add_str = '<a class = "add_input" href = "">add</a> ' append_to_label = '' # text to append to a widget label search_form = param_info.search_form if 'obs_surface_geometry__' in search_form: # append the target name to surface geo widget labels try: append_to_label = " - %s" % search_form.split('__')[1].title() except KeyError: pass if form_type in settings.RANGE_FIELDS: auto_id = False slug_no_num = stripNumericSuffix(slug) param_name_no_num = stripNumericSuffix(param_name) slug1 = slug_no_num + '1' slug2 = slug_no_num + '2' param1 = param_name_no_num + '1' param2 = param_name_no_num + '2' form_vals = {slug1: None, slug2: None} # find length of longest list of selections for either param1 or param2, # tells us how many times to go through loop below try: len1 = len(selections[param1]) except: len1 = 0 try: len2 = len(selections[param2]) except: len2 = 0 lngth = len1 if len1 > len2 else len2 if not lngth: # param is not constrained form = str(SearchForm(form_vals, auto_id=auto_id).as_ul()) if addlink == 'false': form = '<ul>' + form + '<li>' + remove_str + '</li></ul>' # remove input is last list item in form else: form = '<span>' + add_str + '</span><ul>' + form + '</ul>' # add input link comes before form else: # param is constrained key = 0 while key < lngth: try: form_vals[slug1] = selections[param1][key] except (IndexError, KeyError) as e: form_vals[slug1] = None try: form_vals[slug2] = selections[param2][key] except (IndexError, KeyError) as e: form_vals[slug2] = None qtypes = request.GET.get('qtype-' + slug, False) if qtypes: try: form_vals['qtype-' + slug] = qtypes.split(',')[key] except KeyError: form_vals['qtype-' + slug] = False form = form + str( SearchForm(form_vals, auto_id=auto_id).as_ul()) if key > 0: form = '<ul>' + form + '<li>' + remove_str + '</li></ul>' # remove input is last list item in form else: form = '<span>' + add_str + '</span><ul>' + form + '</ul>' # add input link comes before form if lngth > 1: form = form + '</span><div style = "clear: both;"></div></section><section><span class="widget_form">' key = key + 1 elif form_type == 'STRING': auto_id = False if param_name in selections: key = 0 for value in selections[param_name]: form_vals[slug] = value qtypes = request.GET.get('qtype-' + slug, False) if qtypes: try: form_vals['qtype-' + slug] = qtypes.split(',')[key] except KeyError: form_vals['qtype-' + slug] = False form = form + str( SearchForm(form_vals, auto_id=auto_id).as_ul()) if key > 0: form = form + '<li>' + remove_str + '</li>' else: form = form + '<li>' + add_str + '</li>' key = key + 1 else: form = str(SearchForm(form_vals, auto_id=auto_id).as_ul()) if addlink == 'false': form = form + '<li>' + remove_str + '<li>' else: form = form + '<li>' + add_str + '<li>' # MULT form types elif form_type in settings.MULT_FORM_TYPES: if param_name in selections: form_vals = {slug: selections[param_name]} # determine if this mult param has a grouping field (see doc/group_widgets.md for howto on grouping fields) mult_param = getMultName(param_name) model = apps.get_model('search', mult_param.title().replace('_', '')) try: grouping = model.objects.distinct().values('grouping') grouping_table = 'grouping_' + param_name.split('.')[1] grouping_model = apps.get_model( 'metadata', grouping_table.title().replace('_', '')) for group_info in grouping_model.objects.order_by('disp_order'): gvalue = group_info.value glabel = group_info.label if group_info.label else 'Other' if glabel == 'NULL': glabel = 'Other' if model.objects.filter(grouping=gvalue)[0:1]: form += "\n\n" + \ '<div class = "mult_group_label_container" id = "mult_group_' + str(glabel) + '">' + \ '<span class = "indicator fa fa-plus"></span>' + \ '<span class = "mult_group_label">' + str(glabel) + '</span></div>' + \ '<ul class = "mult_group">' + \ SearchForm(form_vals, auto_id = '%s_' + str(gvalue), grouping=gvalue).as_ul() + \ '</ul>' except FieldError: # this model does not have grouping form = SearchForm(form_vals, auto_id=auto_id).as_ul() else: # all other form types if param_name in selections: form_vals = {slug: selections[param_name]} form = SearchForm(form_vals, auto_id=auto_id).as_ul() param_info = get_param_info_by_slug(slug) label = param_info.label intro = param_info.intro range_fields = settings.RANGE_FIELDS if fmt == 'raw': return str(form) else: template = "widget.html" return render(request, template, locals())
def view_collection(request, collection_name, template="collections.html"): """ the collection tab http endpoint the information it returns about product types and files does not relect user filters such as product types and preview images """ update_metrics(request) # nav stuff - page | limit | columns | offset page_no = int(request.GET.get('page', 1)) limit = int(request.GET.get('limit', 100)) column_slugs = request.GET.get('cols', settings.DEFAULT_COLUMNS) from results.views import * # circulosity column_slugs = column_slugs.split(',') columns = [] for slug in column_slugs: columns += [ParamInfo.objects.get(slug=slug).param_name()] offset = (page_no - 1) * limit # collection if not request.session.get('has_session'): request.session['has_session'] = True session_id = request.session.session_key colls_table_name = get_collection_table(session_id) files = getFiles(fmt="raw", collection=True, previews='none', session_id=session_id) all_product_types = Files.objects.all().values('product_type').distinct() # download_info from downloads.views import get_download_info download_size, download_count = get_download_info(files) download_size = nice_file_size(download_size) # pretty display it # images and join with the collections table where = "images.ring_obs_id = " + connection.ops.quote_name( colls_table_name) + ".ring_obs_id" images = Image.objects images = images.extra(where=[where], tables=[colls_table_name]) image_types = settings.IMAGE_TYPES image_count = len(images) # product files # for this we want the list of all possible product types for this dataset # todo: this is really inefficient, another place that large carts are going to be unhappy product_counts = dict( [(i, 0) for i in [p['product_type'] for p in all_product_types]] ) # a dictionary with product_Type names as keys and all values set to zero ring_obs_ids = [f for f in files] product_counts_query = Files.objects.filter( ring_obs_id__in=ring_obs_ids).values('product_type').annotate( Count('product_type')) product_counts_nonzero = { i['product_type']: i['product_type__count'] for i in product_counts_query } # update the product_count array with the non-zero counts for product_type, pcount in product_counts_nonzero.items(): product_counts[product_type] = pcount column_values = [] for param_name in columns: table_name = param_name.split('.')[0] if table_name == 'obs_general': column_values.append(param_name.split('.')[1]) else: column_values.append( param_name.split('.')[0].lower().replace('_', '') + '__' + param_name.split('.')[1]) # figure out what tables do we need to join in and build query triggered_tables = list(set([t.split('.')[0] for t in columns])) try: triggered_tables.remove('obs_general') except ValueError: pass # obs_general table wasn't in there for whatever reason # set up the where clause to join with the rest of the tables where = "obs_general.ring_obs_id = " + connection.ops.quote_name( colls_table_name) + ".ring_obs_id" triggered_tables.append(colls_table_name) # bring in the triggered_tables results = ObsGeneral.objects.extra(where=[where], tables=triggered_tables) results = results.values(*column_values)[offset:offset + limit] page_ids = [o['ring_obs_id'] for o in results] return render(request, template, locals())
def getImages(request,size,fmt): update_metrics(request) """ this returns rows from images table that correspond to request some rows will not have images, this function doesn't return 'image_not_found' information if a row doesn't have an image you get nothing. you lose. good day sir. #fixme #todo """ if not request.session.get('has_session'): request.session['has_session'] = True session_id = request.session.session_key alt_size = request.GET.get('alt_size','') columns = request.GET.get('cols',settings.DEFAULT_COLUMNS) try: [page_no, limit, page, page_ids, order] = getPage(request) except TypeError: # getPage returns False raise Http404('could not find page') image_links = Image.objects.filter(ring_obs_id__in=page_ids) if not image_links: log.error('no image found for:') log.error(page_ids[:50]) # page_ids if alt_size: image_links = image_links.values('ring_obs_id',size,alt_size) else: image_links = image_links.values('ring_obs_id',size) # add the base_path to each image all_sizes = ['small','thumb','med','full'] for k, im in enumerate(image_links): for s in all_sizes: if s in im: image_links[k][s] = get_base_path_previews(im['ring_obs_id']) + im[s] # to preserve the order of page_ids as lamely as possible :P ordered_image_links = [] for ring_obs_id in page_ids: found = False for link in image_links: if ring_obs_id == link['ring_obs_id']: found = True ordered_image_links.append(link) if not found: # return the thumbnail not found link ordered_image_links.append({size:settings.THUMBNAIL_NOT_FOUND, 'ring_obs_id':ring_obs_id}) image_links = ordered_image_links collection_members = get_collection_in_page(page, session_id) # find which are in collections, mark unfound images 'not found' for image in image_links: image['img'] = image[size] if image[size] else 'not found' if collection_members: from user_collections.views import * if image['ring_obs_id'] in collection_members: image['in_collection'] = True path = settings.IMAGE_HTTP_PATH if (request.is_ajax()): template = 'gallery.html' else: template = 'image_list.html' # image_links return responseFormats({'data':[i for i in image_links]},fmt, size=size, path=path, alt_size=alt_size, columns_str=columns.split(','), template=template, order=order)
def edit_collection(request, **kwargs): update_metrics(request) """ edits a single ring_obs_id in a user collection (user "selections") # edits? """ if not request.session.get('has_session'): request.session['has_session'] = True session_id = request.session.session_key checkArgs = check_collection_args(request, **kwargs) if type(checkArgs).__name__ == 'list' and checkArgs: (action, ring_obs_id, request_no, expected_request_no) = checkArgs else: return HttpResponse(json.dumps({"err": checkArgs})) # just add this request to the queue, every request gets queued add_to_queue(request, request_no, action, ring_obs_id) """ # todo turning this off for now, we will get the queue of whatever request_no is passed to us, without checking against what is expected_request_no by turning this off we are at risk of ajax race conditions but it's breaking something and no time fo dat right now Issue is here: https://bitbucket.org/ringsnode/opus2/issue/75/collections-downloads-majorly-broken # todo: # now look for the next expected request in the queue if get_queued(request, expected_request_no): # found the next expected request in the queue (request,action,ring_obs_id) = get_queued(request, expected_request_no) else: # the expected request has not yet arrived, do nothing return HttpResponse(json.dumps({"err":"waiting"})) """ # instead of the above we are doing this: (action, ring_obs_id) = get_queued(request, request_no) # redux: these will be not so simple saddly, they will insert or delete from # the colleciton table.. if action == 'add': add_to_collection(ring_obs_id, session_id) elif (action == 'remove'): remove_from_collection(ring_obs_id, session_id) elif (action in ['addrange', 'removerange']): collection_count = edit_collection_range(request, **kwargs) remove_from_queue(request, request_no) """ try: json = {"msg":"yay!","count":len(collection), "collection": ', '.join(collection)} except: json = collection """ collection_count = get_collection_count(session_id) json_data = { "err": False, "count": collection_count, "request_no": request_no } return HttpResponse(json.dumps(json_data))
def getValidMults(request, slug, fmt='json'): """ returns list of valid mult ids or labels for field in GET param "field" based on current search defined in request field_format = 'ids' or 'labels' depending on how you want your data back (OPUS UI will use ids but they aren't very readable for everyone else) """ update_metrics(request) try: (selections, extras) = search.views.urlToSearchParams(request.GET) except: selections = {} param_info = search.views.get_param_info_by_slug(slug) table_name = param_info.category_name param_name = param_info.param_name() # if this param is in selections we want to remove it, # want mults for a param as they would be without itself if param_name in selections: del selections[param_name] has_selections = False if len(selections.keys()) < 1 else True cache_key = "mults" + param_name + str( search.views.setUserSearchNo(selections)) if (cache.get(cache_key) is not None): mults = cache.get(cache_key) else: mult_name = getMultName(param_name) # the name of the field to query mult_model = apps.get_model('search', mult_name.title().replace('_', '')) table_model = apps.get_model('search', table_name.title().replace('_', '')) mults = {} # info to return results = table_model.objects.values(mult_name).annotate( Count(mult_name)) # this is a count(*), group_by query! user_table = search.views.getUserQueryTable(selections, extras) if has_selections and not user_table: # selections are constrained so join in the user_table log.error( 'getValidMults has_selections = true but no user_table found:') log.error(selections) log.error(extras) raise Http404 if table_name == 'obs_general': where = table_name + ".id = " + user_table + ".id" else: where = table_name + ".obs_general_id = " + user_table + ".id" results = results.extra(where=[where], tables=[user_table]) for row in results: mult_id = row[mult_name] try: try: mult = mult_model.objects.get(id=mult_id).label except: log.debug('could not find mult label for ') log.debug('id: ' + str(mult_id)) log.debug(mult_model) mult = mult_id # fall back to id if there is no label mults[mult] = row[mult_name + '__count'] except: pass # a none object may be found in the data but if it doesn't have a table row we don't handle it cache.set(cache_key, mults, 0) multdata = {'field': slug, 'mults': mults} if (request.is_ajax()): reqno = request.GET.get('reqno', '') multdata['reqno'] = reqno return responseFormats(multdata, fmt, template='mults.html')
def getPage(request, colls=None, colls_page=None, page=None): update_metrics(request) """ the gets the metadata and images to build a page of results """ # get some stuff from the url or fall back to defaults if not request.session.get('has_session'): request.session['has_session'] = True session_id = request.session.session_key if not colls: collection_page = request.GET.get('colls',False) else: collection_page = colls limit = request.GET.get('limit',100) limit = int(limit) slugs = request.GET.get('cols', settings.DEFAULT_COLUMNS) if not slugs: slugs = settings.DEFAULT_COLUMNS # i dunno why the above doesn't suffice columns = [] for slug in slugs.split(','): try: columns += [ParamInfo.objects.get(slug=slug).param_name()] except ParamInfo.DoesNotExist: pass triggered_tables = list(set([param_name.split('.')[0] for param_name in columns])) try: triggered_tables.remove('obs_general') # we remove it because it is the primary # model so don't need to add it to extra tables except ValueError: pass # obs_general isn't in there if not collection_page: # this is for a search query order = request.GET.get('order','time1') # figure out column order in table if order: try: descending = '-' if order[0] == '-' else None order_slug = order.strip('-') # strip off any minus sign to look up param name order = ParamInfo.objects.get(slug=order_slug).param_name() if descending: order = '-' + order except DoesNotExist: order = False # figure out page we are asking for if not page: page_no = request.GET.get('page',1) if page_no != 'all': page_no = int(page_no) else: page_no == page # ok now that we have everything from the url get stuff from db (selections,extras) = urlToSearchParams(request.GET) user_query_table = getUserQueryTable(selections,extras) # figure out what tables do we need to join in and build query triggered_tables.append(user_query_table) where = "obs_general.id = " + connection.ops.quote_name(user_query_table) + ".id" results = ObsGeneral.objects.extra(where=[where], tables=triggered_tables) else: # this is for a collection # find the ordering order = request.GET.get('colls_order', False) if not order: # get regular order if collections doesn't have a special order order = request.GET.get('order',False) if order: try: order_param = order.strip('-') # strip off any minus sign to look up param name descending = order[0] if (order[0] == '-') else None order = ParamInfo.objects.get(slug=order_param).name if descending: order = '-' + order except DoesNotExist: order = False if not colls_page: page_no = request.GET.get('colls_page',1) if page_no != 'all': page_no = int(page_no) else: page_no = colls_page # figure out what tables do we need to join in and build query triggered_tables = list(set([t.split('.')[0] for t in columns])) try: triggered_tables.remove('obs_general') except ValueError: pass # obs_general table wasn't in there for whatever reason # join in the collections table colls_table_name = get_collection_table(session_id) triggered_tables.append(colls_table_name) where = "obs_general.ring_obs_id = " + connection.ops.quote_name(colls_table_name) + ".ring_obs_id" results = ObsGeneral.objects.extra(where=[where], tables=triggered_tables) # now we have results object (either search or collections) if order: results = results.order_by(order) # this is the thing you pass to django model via values() # so we have the table names a bit to get what django wants: column_values = [] for param_name in columns: table_name = param_name.split('.')[0] if table_name == 'obs_general': column_values.append(param_name.split('.')[1]) else: column_values.append(param_name.split('.')[0].lower().replace('_','') + '__' + param_name.split('.')[1]) """ the limit is pretty much always 100, the user cannot change it in the interface but as an aide to finding the right chunk of a result set to search for for the 'add range' click functinality, the front end may send a large limit, like say page_no = 42 and limit = 400 that means start the page at 42 and go 4 pages, and somewhere in there is our range this is how 'add range' works accross multiple pages so the way of computing starting offset here should always use the base_limit of 100 using the passed limit will result inthe wront offset because of way offset is computed here this may be an aweful hack. """ # todo: redux: look at line 559 you are essentially doing this query twice? in the same method and peforming the query each time cuz it has changed! if page_no != 'all': base_limit = 100 # explainer of sorts is above offset = (page_no-1)*base_limit # we don't use Django's pagination because of that count(*) that it does. results = results.values_list(*column_values)[offset:offset+int(limit)] else: results = results.values_list(*column_values) # return a simple list of ring_obs_ids ring_obs_id_index = column_values.index('ring_obs_id') page_ids = [o[ring_obs_id_index] for o in results] if not len(page_ids): return False return [page_no, limit, list(results), page_ids, order]
def getRangeEndpoints(request, slug, fmt='json'): """ fetch range widget hinting data for widget defined by slug based on current search defined in request this is the valid range endpoints that appear in range widgets (green numbers) returns a dictionary like: { min: 63.592, max: 88.637, nulls: 2365} """ # if this param is in selections we want to remove it, # want results for param as they would be without itself constrained # extras['qtypes'][''] update_metrics(request) param_info = search.views.get_param_info_by_slug(slug) param_name = param_info.param_name() form_type = param_info.form_type table_name = param_info.category_name # "param" is the field name, the param_name with the table_name stripped param1 = stripNumericSuffix(param_name.split('.')[1]) + '1' param2 = stripNumericSuffix(param_name.split('.')[1]) + '2' param_no_num = stripNumericSuffix(param1) table_model = apps.get_model('search', table_name.title().replace('_', '')) if form_type == 'RANGE' and '1' not in param_info.slug and '2' not in param_info.slug: param1 = param2 = param_no_num # single column range query try: (selections, extras) = search.views.urlToSearchParams(request.GET) user_table = search.views.getUserQueryTable(selections, extras) has_selections = True except TypeError: selections = {} has_selections = False user_table = False # remove this param from the user's query if it is constrained # this keeps the green hinting numbers from reacting # to changes to its own field param_name_no_num = stripNumericSuffix(param_name) to_remove = [ param_name_no_num, param_name_no_num + '1', param_name_no_num + '2' ] for p in to_remove: if p in selections: del selections[p] if not bool(selections): has_selections = False user_table = False # cached already? cache_key = "rangeep" + param_no_num if user_table: cache_key += str(search.views.setUserSearchNo(selections, extras)) if cache.get(cache_key) is not None: range_endpoints = cache.get(cache_key) return responseFormats(range_endpoints, fmt, template='mults.html') # no cache found, calculating.. try: results = table_model.objects # this is a count(*), group_by query except AttributeError, e: log.error("getRangeEndpoints threw: %s", str(e)) log.error("Could not find table model for table_name: %s", table_name) raise Http404("Does Not Exist")