def make_response_id(self): """Makes the ID for the response JSON-LD""" # NOTE: The order of the URL parameters will be predicable # to help normalize search IDs. if self.id: return self.id sl = SearchLinks(request_dict=self.request_dict, base_search_url=self.base_search_url) urls = sl.make_urls_from_request_dict() self.id = urls['html'] return self.id
def add_text_fields(self): """Adds general and project-specific property text query options """ text_fields = [] # first add a general key-word search option act_request_dict = copy.deepcopy(self.request_dict) sl = SearchLinks(request_dict=act_request_dict, base_search_url=self.base_search_url) # Remove non search related params. sl.remove_non_query_params() raw_fulltext_search = utilities.get_request_param_value( act_request_dict, param='q', default=None, as_list=False, solr_escape=False, ) field = LastUpdatedOrderedDict() field['id'] = '#textfield-keyword-search' field['label'] = configs.FILTER_TEXT_SEARCH_TITLE field['oc-api:search-term'] = raw_fulltext_search if not raw_fulltext_search: # No keyword search found in request, so # make a template for keyword searches. sl.add_param_value('q', configs.URL_TEXT_QUERY_TEMPLATE) urls = sl.make_urls_from_request_dict() field['oc-api:template'] = urls['html'] field['oc-api:template-json'] = urls['json'] else: # A keyword search was found, so make a # template for replacing it with another # keyword search. sl.replace_param_value('q', match_old_value=raw_fulltext_search, new_value=configs.URL_TEXT_QUERY_TEMPLATE) urls = sl.make_urls_from_request_dict() field['oc-api:template'] = urls['html'] field['oc-api:template-json'] = urls['json'] text_fields.append(field) # NOTE This adds project specific property text # query options are listed in the search-filters # Add the text fields if they exist. if len(text_fields): self.result['oc-api:has-text-search'] = text_fields return text_fields
def make_related_media_facets(self, solr_json): """Makes related media facets from a solr_json response""" options = [] for media_config in configs.FACETS_RELATED_MEDIA['oc-api:has-rel-media-options']: facet_val_count_tups = utilities.get_path_facet_value_count_tuples( media_config['facet_path'], solr_json ) media_type_total_count = 0 for facet_val, facet_count in facet_val_count_tups: if facet_val == "0": # Skip, this facet_value is for # items with NO related media of this type continue media_type_total_count += facet_count if media_type_total_count == 0: # No items have related media of this type, # so continue and don't make a facet option # for this. continue sl = SearchLinks( request_dict=copy.deepcopy(self.request_dict), base_search_url=self.base_search_url ) # Remove non search related params. sl.remove_non_query_params() sl.replace_param_value( media_config['param_key'], new_value=1, ) urls = sl.make_urls_from_request_dict() if urls['html'] == self.current_filters_url: # The new URL matches our current filter # url, so don't add this facet option. continue option = LastUpdatedOrderedDict() option['label'] = media_config['label'] option['count'] = media_type_total_count option['id'] = urls['html'] option['json'] = urls['json'] options.append(option) if not len(options): # We found no related media configs, so return None return None # Return the related media facets object. rel_media_facets = LastUpdatedOrderedDict() rel_media_facets['id'] = configs.FACETS_RELATED_MEDIA['id'] rel_media_facets['label'] = configs.FACETS_RELATED_MEDIA['label'] rel_media_facets['oc-api:has-rel-media-options'] = options return rel_media_facets
def _set_current_filters_url(self): """Make a URL for the current filters, removed of all paging and other irrelevant parameters. """ # NOTE: we use the self.current_filters_url to determine if # a given new filter url (say for a facet) is new (meaning if # it actually changes search/query filters). If it is not new, # then the result should not present the new filter url as # a search option. # # NOTE: Since we generate search urls programatically via the # SearchLinks class, we have a predictable order of parameters # and values. This makes it possible to match url strings. sl = SearchLinks(request_dict=copy.deepcopy(self.request_dict), base_search_url=self.base_search_url) # Remove non search related params. sl.remove_non_query_params() urls = sl.make_urls_from_request_dict() self.current_filters_url = urls['html'] return self.current_filters_url
def _make_paging_links(self, start, rows, act_request_dict): """Makes links for paging for start rows from a request dict""" start = str(int(start)) rows = str(rows) # Remove previously set parameters relating to paging. for param in ['start', 'rows']: if param in act_request_dict: act_request_dict.pop(param, None) sl = SearchLinks(request_dict=act_request_dict, base_search_url=self.base_search_url) sl.add_param_value('start', start) sl.add_param_value('rows', rows) urls = sl.make_urls_from_request_dict() return urls
def add_links_to_act_filter( self, param_key, match_old_value, new_value, act_filter, request_dict, make_text_template=False, ): """Adds links to an active filter""" act_request_dict = copy.deepcopy(request_dict) sl = SearchLinks(request_dict=act_request_dict, base_search_url=self.base_search_url) # Remove non search related params. sl.remove_non_query_params() sl.replace_param_value( param_key, match_old_value=match_old_value, new_value=new_value, ) urls = sl.make_urls_from_request_dict() if make_text_template: # Make template for a text search act_filter['oc-api:template'] = urls['html'] act_filter['oc-api:template-json'] = urls['json'] return act_filter if urls['html'] == self.current_filters_url: # The urls don't actually change state, so don't # add new remove or broaden links, and skip out. return act_filter if new_value is None: # If the new_value is None, then we're completely # removing the search filter. act_filter['oc-api:remove'] = urls['html'] act_filter['oc-api:remove-json'] = urls['json'] else: # Make links to broaden a filter to a higher # point in a given filter's hierarchy act_filter['oc-api:broaden'] = urls['html'] act_filter['oc-api:broaden-json'] = urls['json'] return act_filter
def make_item_type_facets(self, solr_json): """Makes item_type facets from a solr_json response""" item_type_path_keys = ( configs.FACETS_SOLR_ROOT_PATH_KEYS + ['item_type'] ) item_type_val_count_list = utilities.get_dict_path_value( item_type_path_keys, solr_json, default=[] ) if not len(item_type_val_count_list): return None options_tuples = utilities.get_facet_value_count_tuples( item_type_val_count_list ) if not len(options_tuples): return None # Iterate through tuples of item_type counts options = [] for facet_value, count in options_tuples: # The get_item_type_dict should return the # type_dict for slugs, full uris, prefixed URIs # or lower-case item types. type_dict = utilities.get_item_type_dict( facet_value ) if not type_dict: # Unrecognized item type. Skip. continue sl = SearchLinks( request_dict=copy.deepcopy(self.request_dict), base_search_url=self.base_search_url ) # Remove non search related params. sl.remove_non_query_params() # Update the request dict for this facet option. sl.replace_param_value( 'type', match_old_value=None, new_value=facet_value, ) urls = sl.make_urls_from_request_dict() if urls['html'] == self.current_filters_url: # The new URL matches our current filter # url, so don't add this facet option. continue option = LastUpdatedOrderedDict() option['id'] = urls['html'] option['json'] = urls['json'] for key, val in type_dict.items(): option[key] = val options.append(option) if not len(options): return None facet = configs.FACETS_ITEM_TYPE.copy() facet['oc-api:has-id-options'] = options return facet
def add_range_options_list(self, param_key, match_old_value, data_type, field_max_value, options_tuples, round_digits=None): """Adds option dict object to a list based on data-type. :param str data_type: Solr data-type to match for inclusion in the output options list. :param list options_tuples: List of (facet_value, count) tuples """ delim = configs.REQUEST_PROP_HIERARCHY_DELIM options = [] options_length = len(options_tuples) for i, option_tup in enumerate(options_tuples): facet_value, count = option_tup if count < 1: # We don't make facet options for facet values with no # records. continue if (i + 1) == options_length: max_value = field_max_value # This indicates a less than or equal range # for the max_value. range_query_end = ']' else: # Get the first element of the next options # tuple. That's the facet value for the # next maximum value. max_value = options_tuples[(i + 1)][0] # This indicates a less than range for the # max_value. We don't want to include # values for the next facet. range_query_end = '}' label = facet_value if round_digits is not None: label = str(round(float(facet_value), round_digits)) if data_type == 'xsd:integer': min_value = int(round(float(facet_value), 0)) max_value = int(round(float(max_value), 0)) elif data_type == 'xsd:double': if round_digits is not None: label = str(round(float(facet_value), round_digits)) min_value = float(facet_value) max_value = float(max_value) elif data_type == 'xsd:date': min_value = facet_value else: # How van we even be here with the wrong data-type? continue # Except for the last range query option, a # range query is greater than or equal to the min_value # and less than the max_value. # # For the last range query option, the range query # is greater than or equal to the min_value and less than # or equal to the maximum value (thus, we include the # max_value for the last, greatest facet option). range_query = '[{min_val} TO {max_val}{q_end}'.format( min_val=min_value, max_val=max_value, q_end=range_query_end, ) new_value = None old_range_seps = [(delim + '['), (delim + '{')] for old_range_sep in old_range_seps: if not old_range_sep in match_old_value: continue if new_value is not None: continue # Remove the part of the match_old_value that # has the range query value. old_parts = match_old_value.split(old_range_sep) # The first part of the old_parts has the # old range removed. new_value = old_parts[0] + delim + range_query if new_value is None: # No old range query to replace. if match_old_value.endswith(delim): new_value = match_old_value + range_query else: new_value = match_old_value + delim + range_query sl = SearchLinks(request_dict=copy.deepcopy(self.request_dict), base_search_url=self.base_search_url) # Remove non search related params. sl.remove_non_query_params() # Update the request dict for this facet option. sl.replace_param_value( param_key, match_old_value=match_old_value, new_value=new_value, ) urls = sl.make_urls_from_request_dict() if urls['html'] == self.current_filters_url: # The new URL matches our current filter # url, so don't add this facet option. continue option = LastUpdatedOrderedDict() option['id'] = urls['html'] option['json'] = urls['json'] option['label'] = label option['count'] = count option['oc-api:min'] = min_value option['oc-api:max'] = max_value options.append(option) return options
def add_options_list_for_data_type(self, param_key, match_old_value, delim, data_type, options_tuples): """Adds option dict object to a list based on data-type. :param str data_type: Solr data-type to match for inclusion in the output options list. :param list options_tuples: List of (facet_value, count) tuples """ options = [] for facet_value, count in options_tuples: if count < 1: # We don't make facet options for facet values with no # records. continue # Parse the solr encoded entity string. Note this does # NOT make a request to the database. parsed_val = utilities.parse_solr_encoded_entity_str( facet_value, base_url=self.base_url) if not parsed_val: # Can't interpret this as a solr value, so skip continue if parsed_val.get('data_type') != data_type: # The data type for this value does not match the # data type for the options list that we are # building. continue # The new_value is generally the slug part of the parsed_val # (derived from the facet_value). However, context (path) # items are different, and we use the label for the new_value. if param_key == 'path': new_value = parsed_val['label'] else: new_value = parsed_val['slug'] if match_old_value is not None: # We have an old value to match. So the new_value # will be the match_old_value + delim + the new_value. if not match_old_value.endswith(delim): new_value = match_old_value + delim + new_value else: new_value = match_old_value + new_value sl = SearchLinks(request_dict=copy.deepcopy(self.request_dict), base_search_url=self.base_search_url) # Remove non search related params. sl.remove_non_query_params() if param_key == 'prop': # Prop can be a list. If the match_old_value is None # then we add to new_value to the existing list of # all prop parameter values. add_to_param_list = True else: # All the other param_key's can only take a single # value. add_to_param_list = False # Update the request dict for this facet option. sl.replace_param_value( param_key, match_old_value=match_old_value, new_value=new_value, add_to_param_list=add_to_param_list, ) urls = sl.make_urls_from_request_dict() if urls['html'] == self.current_filters_url: # The new URL matches our current filter # url, so don't add this facet option. continue option = LastUpdatedOrderedDict() option['id'] = urls['html'] option['json'] = urls['json'] option['rdfs:isDefinedBy'] = parsed_val['uri'] option['slug'] = parsed_val['slug'] option['label'] = parsed_val['label'] option['count'] = count options.append(option) return options
def make_sort_links_list(self, request_dict): """ makes a list of the links for sort options """ sort_links = [] # Make a deep copy, because we're mutating the # request_dict. act_request_dict = copy.deepcopy(request_dict) # Remove a sort or start parameter. for param in ['sort', 'start']: if param in act_request_dict: act_request_dict.pop(param, None) order_opts = [{ 'key': 'asc', 'order': 'ascending' }, { 'key': 'desc', 'order': 'descending' }] for act_sort in configs.SORT_OPTIONS: if not act_sort.get('opt'): continue for order_opt in order_opts: sl = SearchLinks(request_dict=act_request_dict, base_search_url=self.base_search_url) if act_sort.get('value') is None: if order_opt['key'] == 'asc': # Skip the option of adding an ascending order sort # option if the act_sort value is None (the default sorting # option by interest score). continue else: # Make sort links for this sort_option and # order_opt. act_sort_val = (act_sort['value'] + self.request_sort_dir_delim + order_opt['key']) sl.add_param_value('sort', act_sort_val) urls = sl.make_urls_from_request_dict() new_sort_obj = LastUpdatedOrderedDict() new_sort_obj['id'] = urls['html'] new_sort_obj['json'] = urls['json'] new_sort_obj['type'] = act_sort['type'] new_sort_obj['label'] = act_sort['label'] new_sort_obj['oc-api:sort-order'] = order_opt['order'] in_active_list = False for cur_act_sort in self.current_sorting: if (new_sort_obj['type'] == cur_act_sort['type'] and new_sort_obj['oc-api:sort-order'] == cur_act_sort['oc-api:sort-order']): # the new_sort_obj option is ALREADY in use in_active_list = True break if in_active_list is False: sort_links.append(new_sort_obj) # Return the sort option links return sort_links
def make_chronology_facet_options(self, solr_json): """Makes chronology facets from a solr_json response""" chrono_path_keys = ( configs.FACETS_SOLR_ROOT_PATH_KEYS + ['form_use_life_chrono_tile'] ) chrono_val_count_list = utilities.get_dict_path_value( chrono_path_keys, solr_json, default=[] ) if not len(chrono_val_count_list): return None options_tuples = utilities.get_facet_value_count_tuples( chrono_val_count_list ) if not len(options_tuples): return None # Check to see if the client included any request parameters # that limited the chronological range of the request. self._set_client_earliest_latest_limits() valid_tile_dicts = self._make_valid_options_tile_dicts( options_tuples ) if not len(valid_tile_dicts): # None of the chronological tiles are valid # given the query requirements. return None # Determine the aggregation depth needed to group chronological # tiles together into a reasonable number of options. self._get_tile_aggregation_depth(valid_tile_dicts) aggregate_tiles = {} for tile_dict in valid_tile_dicts: # Now aggregate the tiles. trim_tile_key = tile_dict['tile_key'][:self.default_aggregation_depth] if trim_tile_key not in aggregate_tiles: # Make the aggregate tile dictionary # object. chrono_t = ChronoTile() agg_dict = chrono_t.decode_path_dates(trim_tile_key) if (self.min_date is not None and agg_dict['earliest_bce'] < self.min_date): # The aggregated date range looks too early, so # set it to the earliest allowed. agg_dict['earliest_bce'] = self.min_date if (self.max_date is not None and agg_dict['latest_bce'] > self.max_date): # The aggregated date range looks too late, so # set it to the latest date range allowed. agg_dict['latest_bce'] = self.max_date agg_dict['tile_key'] = trim_tile_key agg_dict['count'] = 0 aggregate_tiles[trim_tile_key] = agg_dict aggregate_tiles[trim_tile_key]['count'] += tile_dict['count'] agg_tile_list = [tile_dict for _, tile_dict in aggregate_tiles.items()] # Now sort by earliest bce, then reversed latest bce # this makes puts early dates with longest timespans first sorted_agg_tiles = sorted( agg_tile_list, key=lambda k: (k['earliest_bce'], -k['latest_bce']) ) options = [] for tile_dict in sorted_agg_tiles: sl = SearchLinks( request_dict=copy.deepcopy(self.request_dict), base_search_url=self.base_search_url ) # Remove non search related params. sl.remove_non_query_params() # Update the request dict for this facet option. sl.replace_param_value( 'form-chronotile', match_old_value=None, new_value=tile_dict['tile_key'], ) sl.replace_param_value( 'form-start', match_old_value=None, new_value=tile_dict['earliest_bce'], ) sl.replace_param_value( 'form-stop', match_old_value=None, new_value=tile_dict['latest_bce'], ) urls = sl.make_urls_from_request_dict() if urls['html'] == self.current_filters_url: # The new URL matches our current filter # url, so don't add this facet option. continue option = LastUpdatedOrderedDict() option['id'] = urls['html'] option['json'] = urls['json'] option['count'] = tile_dict['count'] option['category'] = 'oc-api:chrono-facet' option['start'] = ISOyears().make_iso_from_float( tile_dict['earliest_bce'] ) option['stop'] = ISOyears().make_iso_from_float( tile_dict['latest_bce'] ) properties = LastUpdatedOrderedDict() properties['early bce/ce'] = tile_dict['earliest_bce'] properties['late bce/ce'] = tile_dict['latest_bce'] option['properties'] = properties options.append(option) return options
def make_geo_contained_in_facet_options(self, solr_json): """Gets geospace item query set from a list of options tuples""" geosource_path_keys = (configs.FACETS_SOLR_ROOT_PATH_KEYS + ['disc_geosource']) geosource_val_count_list = utilities.get_dict_path_value( geosource_path_keys, solr_json, default=[]) if not len(geosource_val_count_list): return None # Make the list of tile, count tuples. options_tuples = utilities.get_facet_value_count_tuples( geosource_val_count_list) if not len(options_tuples): return None uuids = [] parsed_solr_entities = {} uuid_geo_dict = {} for solr_entity_str, count in options_tuples: parsed_entity = utilities.parse_solr_encoded_entity_str( solr_entity_str, base_url=self.base_url) if not parsed_entity: logger.warn( 'Cannot parse entity from {}'.format(solr_entity_str)) continue if not '/' in parsed_entity['uri']: logger.warn('Invalid uri from {}'.format(solr_entity_str)) continue uri_parts = parsed_entity['uri'].split('/') uuid = uri_parts[-1] parsed_entity['uuid'] = uuid parsed_solr_entities[solr_entity_str] = parsed_entity uuids.append(uuid) # Make a dictionary of geospace objects keyed by uuid. This # will hit the database in one query to get all geospace # objects not present in the cache. uuid_geo_dict = self._make_cache_geospace_obj_dict(uuids) # Make a dict of context paths, keyed by uuid. This will also # hit the database in only 1 query, for all context paths not # already present in the cache. uuid_context_dict = self._get_cache_contexts_dict(uuids) # Now make the final geo_options = [] for solr_entity_str, count in options_tuples: if solr_entity_str not in parsed_solr_entities: # This solr_entity_str did not validate to extract a UUID. continue parsed_entity = parsed_solr_entities[solr_entity_str] uuid = parsed_entity['uuid'] geo_obj = uuid_geo_dict.get(uuid) if geo_obj is None: logger.warn('No geospace object for {}'.format(uuid)) continue context_path = uuid_context_dict.get(uuid) if context_path is None: logger.warn('No context path for {}'.format(uuid)) continue sl = SearchLinks(request_dict=copy.deepcopy(self.request_dict), base_search_url=self.base_search_url) # Remove non search related params. sl.remove_non_query_params() # Update the request dict for this facet option. sl.replace_param_value( 'path', match_old_value=None, new_value=context_path, ) urls = sl.make_urls_from_request_dict() # NOTE: We're not checking if the URLs are the same # as the current search URL, because part of the point # of listing these features is for visualization display # in the front end. option = LastUpdatedOrderedDict() # The fragment id in the URLs are so we don't have an # ID collision with context facets. option['id'] = urls['html'] + '#geo-in' option['json'] = urls['json'] + '#geo-in' option['count'] = count option['type'] = 'Feature' option['category'] = 'oc-api:geo-contained-in-feature' # Add some general chronology information to the # geospatial feature. option = self._add_when_object_to_feature_option( uuid, option, ) # Add the geometry from the geo_obj coordinates. First # check to make sure they are OK with the the GeoJSON # right-hand rule. geometry = LastUpdatedOrderedDict() geometry['id'] = '#geo-in-geom-{}'.format(uuid) geometry['type'] = geo_obj.ftype coord_obj = json.loads(geo_obj.coordinates) v_geojson = ValidateGeoJson() coord_obj = v_geojson.fix_geometry_rings_dir( geo_obj.ftype, coord_obj) geometry['coordinates'] = coord_obj option['geometry'] = geometry properties = LastUpdatedOrderedDict() properties['id'] = '#geo-in-props-{}'.format(uuid) properties['href'] = option['id'] properties['item-href'] = parsed_entity['uri'] properties['label'] = context_path properties['feature-type'] = 'containing-region' properties['count'] = count properties['early bce/ce'] = self.min_date properties['late bce/ce'] = self.max_date option['properties'] = properties geo_options.append(option) return geo_options
def make_geotile_facet_options(self, solr_json): """Makes geographic tile facets from a solr_json response""" geotile_path_keys = (configs.FACETS_SOLR_ROOT_PATH_KEYS + ['discovery_geotile']) geotile_val_count_list = utilities.get_dict_path_value( geotile_path_keys, solr_json, default=[]) if not len(geotile_val_count_list): return None # Make the list of tile, count tuples. options_tuples = utilities.get_facet_value_count_tuples( geotile_val_count_list) if not len(options_tuples): return None valid_tile_tuples = self._make_valid_options_tile_tuples( options_tuples) if not len(valid_tile_tuples): # None of the chronological tiles are valid # given the query requirements. return None # Determine the aggregation depth needed to group geotiles # together into a reasonable number of options. self._get_tile_aggregation_depth(valid_tile_tuples) # Determine the min tile depth. We need to return this to # the client so the client knows not to over-zoom. tile_lens = [len(tile) for tile, _ in valid_tile_tuples] self.min_depth = min(tile_lens) # Get the client's requested feature type for the geotile # facets. feature_type = utilities.get_request_param_value( self.request_dict, param='geo-facet-type', default=self.default_tile_feature_type, as_list=False, solr_escape=False, ) if feature_type not in self.valid_tile_feature_types: # If the requested feature type is not in the # valid list of feature types, just use the default. feature_type = self.default_tile_feature_type aggregate_tiles = {} for tile, count in valid_tile_tuples: # Now aggregate the tiles. trim_tile_key = tile[:self.default_aggregation_depth] if trim_tile_key not in aggregate_tiles: # Make the aggregate tile with a count # of zero aggregate_tiles[trim_tile_key] = 0 aggregate_tiles[trim_tile_key] += count options = [] for tile, count in aggregate_tiles.items(): sl = SearchLinks(request_dict=copy.deepcopy(self.request_dict), base_search_url=self.base_search_url) # Remove non search related params. sl.remove_non_query_params() # Update the request dict for this facet option. sl.replace_param_value( 'disc-geotile', match_old_value=None, new_value=tile, ) urls = sl.make_urls_from_request_dict() if urls['html'] == self.current_filters_url: # The new URL matches our current filter # url, so don't add this facet option. continue option = LastUpdatedOrderedDict() option['id'] = urls['html'] option['json'] = urls['json'] option['count'] = count option['type'] = 'Feature' option['category'] = 'oc-api:geo-facet' # Add some general chronology information to the # geospatial tile. option = self._add_when_object_to_feature_option( tile, option, ) gm = GlobalMercator() if feature_type == 'Polygon': # Get polygon coordinates (a list of lists) geo_coords = gm.quadtree_to_geojson_poly_coords(tile) elif feature_type == 'Point': # Get point coordinates (a list of lon,lat values) geo_coords = gm.quadtree_to_geojson_lon_lat(tile) else: # We shouldn't be here! continue # Add the geometry object to the facet option. geometry = LastUpdatedOrderedDict() geometry['id'] = '#geo-disc-tile-geom-{}'.format(tile) geometry['type'] = feature_type geometry['coordinates'] = geo_coords option['geometry'] = geometry properties = LastUpdatedOrderedDict() properties['id'] = '#geo-disc-tile-{}'.format(tile) properties['href'] = option['id'] properties['label'] = 'Discovery region ({})'.format( (len(options) + 1)) properties['feature-type'] = 'discovery region (facet)' properties['count'] = count properties['early bce/ce'] = self.min_date properties['late bce/ce'] = self.max_date option['properties'] = properties options.append(option) return options