class BaseIndex(indexes.SearchIndex): text = indexes.CharField(document=True, use_template=True, stored=False) url = indexes.CharField(model_attr='get_absolute_url', indexed=False) title = indexes.CharField() description = indexes.CharField(null=True) fullname = indexes.CharField(null=True) author = indexes.CharField(null=True) author_url = indexes.CharField(null=True, indexed=False) created = indexes.DateTimeField(model_attr='created', null=True) modified = indexes.DateTimeField(model_attr='modified', null=True) type = indexes.CharField() icon_name = indexes.CharField(indexed=False) fullname_and_username = indexes.CharField(null=True, stored=False) hits = indexes.IntegerField(model_attr='hits') modified_by = indexes.CharField(null=True) modified_by_url = indexes.CharField(null=True) def get_updated_field(self): return 'modified' def get_boost(self, obj): if obj.hits <= 10: return 1 return math.log(obj.hits) def prepare(self, obj): self.author_obj = None if hasattr(obj, 'get_author'): self.author_obj = obj.get_author() data = super(BaseIndex, self).prepare(obj) data['boost'] = self.get_boost(obj) return data def prepare_author(self, obj): if self.author_obj: return self.author_obj.username return obj.author def prepare_author_url(self, obj): if self.author_obj: return self.author_obj.get_absolute_url() return None def prepare_fullname(self, obj): if hasattr(obj, 'modified_by'): modified_by = obj.get_modified_by() if modified_by: return modified_by.get_full_name() if self.author_obj: return self.author_obj.get_full_name() return obj.author def prepare_fullname_and_username(self, obj): if hasattr(obj, 'modified_by'): modified_by = obj.get_modified_by() if modified_by: return u'{}\n{}'.format( modified_by.get_full_name(), modified_by.username, ) if not self.author_obj: return obj.author return u'{}\n{}'.format( self.author_obj.get_full_name(), self.author_obj.username, ) def prepare_modified_by(self, obj): return self.prepare_fullname(obj) def prepare_modified_by_url(self, obj): if hasattr(obj, 'modified_by'): modified_by = obj.get_modified_by() if modified_by: return modified_by.get_absolute_url() if self.author_obj: return self.author_obj.get_absolute_url() return None
class LayerIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.EdgeNgramField(document=True, use_template=True) oid = indexes.CharField(model_attr='resourcebase_ptr_id') uuid = indexes.CharField(model_attr='uuid') type = indexes.CharField(faceted=True) subtype = indexes.CharField(faceted=True) name = indexes.CharField(model_attr="name") title = indexes.CharField(model_attr="title", boost=2) title_sortable = indexes.CharField(indexed=False) # Necessary for sorting description = indexes.CharField(model_attr="abstract", boost=1.5) owner = indexes.CharField(model_attr="owner", faceted=True, null=True) modified = indexes.DateTimeField(model_attr="date") category = indexes.CharField(model_attr="category__identifier", faceted=True, null=True) detail_url = indexes.CharField(model_attr="get_absolute_url") bbox_left = indexes.FloatField(model_attr="bbox_x0", null=True) bbox_right = indexes.FloatField(model_attr="bbox_x1", null=True) bbox_bottom = indexes.FloatField(model_attr="bbox_y0", null=True) bbox_top = indexes.FloatField(model_attr="bbox_y1", null=True) temporal_extent_start = indexes.DateTimeField( model_attr="temporal_extent_start", null=True) temporal_extent_end = indexes.DateTimeField( model_attr="temporal_extent_end", null=True) keywords = indexes.MultiValueField(model_attr="keyword_slug_list", null=True, faceted=True) regions = indexes.MultiValueField(model_attr="region_slug_list", null=True, faceted=True) popular_count = indexes.IntegerField(model_attr="popular_count", default=0, boost=20) share_count = indexes.IntegerField(model_attr="share_count", default=0) rating = indexes.IntegerField(null=True) num_ratings = indexes.IntegerField() num_comments = indexes.IntegerField() thumbnail_url = indexes.CharField(model_attr="thumbnail_url", null=True) def get_model(self): return Layer def prepare_type(self, obj): return "layer" def prepare_subtype(self, obj): if obj.storeType == "dataStore": return "vector" elif obj.storeType == "coverageStore": return "raster" elif obj.storeType == "remoteStore": return "remote" def prepare_download_links(self, obj): try: links = obj.download_links() prepped = [(ext, name.encode(), extra) for ext, name, extra in links] return prepped except: return None def prepare_metadata_links(self, obj): try: return obj.metadata_links except: return None def prepare_rating(self, obj): ct = ContentType.objects.get_for_model(obj) try: rating = OverallRating.objects.filter( object_id=obj.pk, content_type=ct).aggregate(r=Avg("rating"))["r"] return float(str(rating or "0")) except OverallRating.DoesNotExist: return 0.0 def prepare_num_ratings(self, obj): ct = ContentType.objects.get_for_model(obj) try: return OverallRating.objects.filter(object_id=obj.pk, content_type=ct).all().count() except OverallRating.DoesNotExist: return 0 def prepare_num_comments(self, obj): try: return Comment.objects.filter( object_id=obj.pk, content_type=ContentType.objects.get_for_model( obj)).all().count() except: return 0 def prepare_title_sortable(self, obj): return obj.title.lower()
class ContractIndex(CelerySearchIndex, indexes.Indexable): def get_model(self): return Contract def get_updated_field(self): return 'updated' # needed text = indexes.CharField(document=True, use_template=True) pk = indexes.IntegerField(indexed=True, stored=True, faceted=True) local_custodians = indexes.MultiValueField(indexed=True, stored=True, faceted=True) contacts = indexes.MultiValueField(indexed=True, stored=True, faceted=True) partners = indexes.MultiValueField(indexed=True, stored=True, faceted=True) has_legal_documents = indexes.BooleanField(indexed=True, stored=True, faceted=True) company_roles = indexes.MultiValueField(indexed=True, stored=True, faceted=True) partners_roles = indexes.MultiValueField(indexed=True, stored=True, faceted=True) project = indexes.CharField(indexed=True, stored=True, faceted=True) data_declarations = indexes.MultiValueField(indexed=True, stored=True, faceted=True) def prepare_local_custodians(self, obj): return [u.full_name for u in obj.local_custodians.all()] def prepare_company_roles(self, obj): return [str(r) for r in obj.company_roles.all()] def prepare_partners_roles(self, obj): roles = GDPRRole.objects.filter(partners_roles__contract=obj).distinct() return [str(r) for r in roles] def prepare_contacts(self, obj): contacts = [] for p in obj.partners_roles.all(): for c in p.contacts.all(): contacts.append('%s %s' % (c.first_name, c.last_name)) if contacts: return contacts else: return ['no collaborator'] def prepare_partners(self, obj): return [p.partner.name for p in obj.partners_roles.all()] def prepare_has_legal_documents(self, obj): return obj.legal_documents.exists() def prepare_project(self, obj): try: acronym = obj.project return acronym except AttributeError: return None def prepare_data_declarations(self, obj): return [data_declaration.title for data_declaration in obj.data_declarations.all()]
class ProductIndex(indexes.SearchIndex, indexes.Indexable): # Search text text = indexes.EdgeNgramField( document=True, use_template=True, template_name='search/indexes/product/item_text.txt') upc = indexes.CharField(model_attr="upc", null=True) title = indexes.EdgeNgramField(model_attr='title', null=True) # Fields for faceting product_class = indexes.CharField(null=True, faceted=True) category = indexes.MultiValueField(null=True, faceted=True) price = indexes.DecimalField(null=True, faceted=True) num_in_stock = indexes.IntegerField(null=True, faceted=True) rating = indexes.IntegerField(null=True, faceted=True) # Spelling suggestions suggestions = indexes.FacetCharField() date_created = indexes.DateTimeField(model_attr='date_created') date_updated = indexes.DateTimeField(model_attr='date_updated') def get_model(self): return get_model('catalogue', 'Product') def index_queryset(self, using=None): # Only index browsable products (not each individual child product) return self.get_model().browsable.order_by('-date_updated') def read_queryset(self, using=None): return self.get_model().browsable.base_queryset() def prepare_product_class(self, obj): return obj.get_product_class().name def prepare_category(self, obj): categories = obj.categories.all() if len(categories) > 0: return [category.full_name for category in categories] def prepare_rating(self, obj): if obj.rating is not None: return int(obj.rating) # Pricing and stock is tricky as it can vary per customer. However, the # most common case is for customers to see the same prices and stock levels # and so we implement that case here. def prepare_price(self, obj): result = None if obj.is_parent: result = strategy.fetch_for_parent(obj) elif obj.has_stockrecords: result = strategy.fetch_for_product(obj) if result: if result.price.is_tax_known: return result.price.incl_tax return result.price.excl_tax def prepare_num_in_stock(self, obj): if obj.is_parent: # Don't return a stock level for parent products return None elif obj.has_stockrecords: result = strategy.fetch_for_product(obj) return result.stockrecord.net_stock_level def prepare(self, obj): prepared_data = super(ProductIndex, self).prepare(obj) # We use Haystack's dynamic fields to ensure that the title field used # for sorting is of type "string'. if is_solr_supported(): prepared_data['title_s'] = prepared_data['title'] # Use title to for spelling suggestions prepared_data['suggestions'] = prepared_data['text'] return prepared_data def get_updated_field(self): """ Used to specify the field used to determine if an object has been updated Can be used to filter the query set when updating the index """ return 'date_updated'
class ProductIndex(indexes.SearchIndex, indexes.Indexable): # Search text text = indexes.EdgeNgramField( document=True, use_template=True, template_name='oscar/search/indexes/product/item_text.txt') upc = indexes.CharField(model_attr="upc", null=True) title = indexes.EdgeNgramField(model_attr='title', null=True) # Fields for faceting product_class = indexes.CharField(null=True, faceted=True) category = indexes.CharField(null=True, faceted=True) price = indexes.DecimalField(null=True, faceted=True) num_in_stock = indexes.IntegerField(null=True, faceted=True) rating = indexes.IntegerField(null=True, faceted=True) date_created = indexes.DateTimeField(model_attr='date_created') date_updated = indexes.DateTimeField(model_attr='date_updated') def get_model(self): return get_model('catalogue', 'Product') def index_queryset(self, using=None): # Only index browsable products (not each individual variant) return self.get_model().browsable.order_by('-date_updated') def read_queryset(self, using=None): return self.get_model().browsable.base_queryset() def prepare_product_class(self, obj): return obj.product_class.name def prepare_category(self, obj): categories = obj.categories.all() if len(categories) > 0: return categories[0].full_name def prepare_rating(self, obj): if obj.rating is not None: return int(obj.rating) def prepare_price(self, obj): # Pricing is tricky as products do not necessarily have a single price # (although that is the most common scenario). if obj.has_stockrecords: # Fetch a "default" stockinfo result = strategy.fetch(obj) if result.price.is_tax_known: return result.price.incl_tax return result.price.excl_tax def prepare_num_in_stock(self, obj): if obj.has_stockrecords: result = strategy.fetch(obj) return result.stockrecord.net_stock_level def get_updated_field(self): """ Used to specify the field used to determine if an object has been updated Can be used to filter the query set when updating the index """ return 'date_updated'
class TranscriptPageIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) highlight = indexes.CharField(model_attr='text') material_type = indexes.CharField(default='Transcript', faceted=True) grouping_key = indexes.FacetField(facet_for='grouping_key') # not really a facet, just an exact key slug = indexes.CharField(model_attr='transcript__slug', indexed=False) transcript_id = indexes.CharField(model_attr='transcript__id') title = indexes.CharField(model_attr='transcript__title') total_pages = indexes.IntegerField(model_attr='transcript__total_pages', default=0, null=True) language = indexes.CharField(default='English', faceted=True) source = indexes.CharField(default='Trial Transcript', faceted=True) seq_number = indexes.IntegerField(model_attr='seq_number') volume_number = indexes.CharField(model_attr='volume__volume_number') volume_seq_number = indexes.CharField(model_attr='volume_seq_number') page_label = indexes.CharField(model_attr='page_label', null=True) date = indexes.CharField(faceted=True, null=True) date_year = indexes.CharField(faceted=True, null=True) date_sort = indexes.DateTimeField(model_attr='date', null=True) authors = indexes.MultiValueField(faceted=True, null=True) defendants = indexes.MultiValueField(faceted=True, null=True) case_names = indexes.MultiValueField(model_attr='transcript__case__short_name', faceted=True) case_tags = indexes.MultiValueField(model_attr='transcript__case__tag_name', faceted=True) evidence_codes = indexes.MultiValueField(null=True) exhibit_codes = indexes.MultiValueField(null=True) trial_activities = indexes.MultiValueField(faceted=True, null=True) def get_model(self): return TranscriptPage def get_updated_field(self): return 'updated_at' def prepare_grouping_key(self, page): # This is a hack to group transcripts but not pages in a single query. # Transcripts get a group key, pages get a unique key. # This can be changed to make grouping work on volume or something else. return 'Transcript_{}'.format(page.transcript.id) def prepare_date(self, page): if page.date: return page.date.strftime('%d %B %Y') def prepare_date_year(self, page): if page.date: return page.date.year def prepare_defendants(self, page): return [defendant.full_name() for defendant in page.transcript.case.defendants.all()] def prepare_authors(self, page): # TODO return [] def prepare_evidence_codes(self, page): return page.extract_evidence_codes() def prepare_exhibit_codes(self, page): return page.extract_exhibit_codes() def prepare_trial_activities(self, page): return [activity.short_name for activity in page.transcript.case.activities.all()]
class ProgramIndex(BaseIndex, indexes.Indexable, OrganizationsMixin): model = Program uuid = indexes.CharField(model_attr='uuid') title = indexes.CharField(model_attr='title', boost=TITLE_FIELD_BOOST) title_autocomplete = indexes.NgramField(model_attr='title', boost=TITLE_FIELD_BOOST) subtitle = indexes.CharField(model_attr='subtitle') type = indexes.CharField(model_attr='type__name', faceted=True) marketing_url = indexes.CharField(null=True) organizations = indexes.MultiValueField(faceted=True) authoring_organizations = indexes.MultiValueField(faceted=True) authoring_organizations_autocomplete = indexes.NgramField( boost=ORG_FIELD_BOOST) authoring_organization_uuids = indexes.MultiValueField() subject_uuids = indexes.MultiValueField() staff_uuids = indexes.MultiValueField() authoring_organization_bodies = indexes.MultiValueField() credit_backing_organizations = indexes.MultiValueField(faceted=True) card_image_url = indexes.CharField(model_attr='card_image_url', null=True) status = indexes.CharField(model_attr='status', faceted=True) partner = indexes.CharField(model_attr='partner__short_code', null=True, faceted=True) start = indexes.DateTimeField(model_attr='start', null=True, faceted=True) seat_types = indexes.MultiValueField(model_attr='seat_types', null=True, faceted=True) published = indexes.BooleanField(null=False, faceted=True) min_hours_effort_per_week = indexes.IntegerField( model_attr='min_hours_effort_per_week', null=True) max_hours_effort_per_week = indexes.IntegerField( model_attr='max_hours_effort_per_week', null=True) weeks_to_complete_min = indexes.IntegerField( model_attr='weeks_to_complete_min', null=True) weeks_to_complete_max = indexes.IntegerField( model_attr='weeks_to_complete_max', null=True) language = indexes.MultiValueField(faceted=True) hidden = indexes.BooleanField(model_attr='hidden', faceted=True) is_program_eligible_for_one_click_purchase = indexes.BooleanField( model_attr='is_program_eligible_for_one_click_purchase', null=False) def prepare_aggregation_key(self, obj): return 'program:{}'.format(obj.uuid) def prepare_published(self, obj): return obj.status == ProgramStatus.Active def prepare_organizations(self, obj): return self.prepare_authoring_organizations( obj) + self.prepare_credit_backing_organizations(obj) def prepare_subject_uuids(self, obj): return set([ str(subject.uuid) for course in obj.courses.all() for subject in course.subjects.all() ]) def prepare_staff_uuids(self, obj): return set([ str(staff.uuid) for course_run in obj.course_runs for staff in course_run.staff.all() ]) def prepare_credit_backing_organizations(self, obj): return self._prepare_organizations( obj.credit_backing_organizations.all()) def prepare_marketing_url(self, obj): return obj.marketing_url def prepare_language(self, obj): return [self._prepare_language(language) for language in obj.languages]
class SystemVersionIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.NgramField(document=True, use_template=True) created = indexes.DateTimeField(model_attr='created') letter = indexes.CharField() logo = indexes.CharField(model_attr='logo') name = indexes.CharField(model_attr='system__name') slug = indexes.CharField(model_attr='system__slug') start_year = indexes.IntegerField(model_attr='start_year', null=True) end_year = indexes.IntegerField(model_attr='end_year', null=True) lowercase_name = indexes.NgramField() autocomplete_name = indexes.NgramField(model_attr='system__name') compatible_with = indexes.MultiValueField() countries = indexes.MultiValueField() derived_from = indexes.MultiValueField() embedded = indexes.MultiValueField() inspired_by = indexes.MultiValueField() oses = indexes.MultiValueField() written_langs = indexes.MultiValueField() project_types = indexes.MultiValueField() licenses = indexes.MultiValueField() features = indexes.MultiValueField() feature_options = indexes.MultiValueField() def get_model(self): return SystemVersion def index_queryset(self, using=None): return self.get_model().objects \ .filter(is_current=True) \ .select_related('system') def prepare_compatible_with(self, obj): if obj.meta_id is None: return [] values = list(obj.meta.compatible_with.values_list('slug', flat=True)) return values def prepare_countries(self, obj): values = [c for c in obj.countries] return values def prepare_derived_from(self, obj): if obj.meta_id is None: return [] values = list(obj.meta.derived_from.values_list('slug', flat=True)) return values def prepare_embedded(self, obj): if obj.meta_id is None: return [] values = list(obj.meta.embedded.values_list('slug', flat=True)) return values def prepare_inspired_by(self, obj): if obj.meta_id is None: return [] values = list(obj.meta.inspired_by.values_list('slug', flat=True)) return values def prepare_oses(self, obj): if obj.meta_id is None: return [] values = [pk for pk in obj.meta.oses.values_list('slug', flat=True)] return values def prepare_written_langs(self, obj): if obj.meta_id is None: return [] values = [ pk for pk in obj.meta.written_in.values_list('slug', flat=True) ] return values def prepare_project_types(self, obj): values = list(obj.project_types.values_list('slug', flat=True)) return values def prepare_licenses(self, obj): if obj.meta_id is None: return [] values = list(obj.meta.licenses.values_list('slug', flat=True)) return values def prepare_features(self, obj): values = [ pk for pk in Feature.objects \ .filter(system_features__system_id=obj.id) \ .values_list('id', flat=True) ] return values def prepare_feature_options(self, obj): values = [ pk for pk in FeatureOption.objects \ .filter(system_features__system_id=obj.id) \ .values_list('id', flat=True) ] return values def prepare_letter(self, obj): return obj.system.name[0].lower() def prepare_lowercase_name(self, obj): return obj.system.name.lower().strip() def prepare_autocomplete_name(self, obj): return obj.system.name.strip() pass
class WikiEntryIndex(indexes.SearchIndex): text = indexes.CharField(document=True, use_template=True) project = indexes.IntegerField(model_attr='project', use_template=True) user = indexes.CharField(model_attr='author')
class JobIndex(indexes.SearchIndex, indexes.Indexable): """ All fields that you want stored will be put here, in django model form """ job_source_name = indexes.CharField() all_locations = MultiValueSpecialField(field_type='location_char') buid = indexes.IntegerField(model_attr='buid_id') city = LocationCharField(faceted=True, model_attr='city', null=True) city_ac = indexes.EdgeNgramField(model_attr='city', null=True, stored=False) city_slab = indexes.CharField(faceted=True) city_slug = indexes.CharField(model_attr="citySlug", stored=False) company = indexes.CharField(faceted=True) company_buid_slab = indexes.CharField(faceted=True) company_canonical_microsite = indexes.CharField(faceted=True) company_enhanced = indexes.BooleanField(indexed=True, stored=True) company_member = indexes.BooleanField(indexed=True, stored=True) company_digital_strategies_customer = indexes.BooleanField(indexed=True, stored=True) company_ac = indexes.EdgeNgramField(null=True, stored=False) company_slab = indexes.CharField(faceted=True) country = LocationCharField(model_attr='country', faceted=True, null=True) country_ac = indexes.EdgeNgramField(model_attr='country', null=True) country_short = indexes.CharField(model_attr='country_short', faceted=True) country_slab = indexes.CharField(faceted=True) country_slug = indexes.CharField(model_attr="countrySlug", stored=False) date_new = indexes.DateTimeField(model_attr='date_new', null=True, faceted=True) date_added = indexes.DateTimeField(null=True, faceted=True) date_updated = indexes.DateTimeField(model_attr='date_updated', null=True, faceted=True) description = KeywordSearchField(model_attr="description", stored=True, indexed=True) federal_contractor = indexes.CharField(faceted=False) full_loc = indexes.CharField(faceted=True, stored=False) html_description = indexes.CharField(model_attr="html_description", stored=True, indexed=False) ind = MultiValueSpecialField(field_type='string') link = indexes.CharField(stored=True, indexed=False) location = LocationCharField(model_attr='location', faceted=True, null=True) GeoLocation = indexes.LocationField(model_attr='location') lat_long_buid_slab = indexes.CharField(faceted=True) moc = indexes.MultiValueField(faceted=True, null=True, stored=False) mocid = indexes.MultiValueField(null=True) moc_slab = indexes.MultiValueField(faceted=True, null=True, stored=False) mapped_moc = indexes.MultiValueField(faceted=True, null=True, stored=False) mapped_mocid = indexes.MultiValueField(null=True, stored=False) mapped_moc_slab = indexes.MultiValueField(faceted=True, null=True, stored=False) network = indexes.BooleanField() onet = indexes.MultiValueField(model_attr='onet_id', faceted=True, null=True) reqid = ExactStringField(model_attr='reqid', null=True) salted_date = indexes.DateTimeField(stored=False, null=True) staffing_code = indexes.CharField(faceted=False) state = LocationCharField(model_attr='state', faceted=True, null=True) state_ac = indexes.EdgeNgramField(model_attr='state', null=True, stored=False) state_short = indexes.CharField(model_attr='state_short', faceted=True, null=True) state_slab = indexes.CharField(faceted=True) state_slug = indexes.CharField(model_attr="stateSlug", stored=False) text = TextAndSymbols(document=True, use_template=True, stored=False) title = TextAndSymbols(model_attr='title', faceted=True) title_ac = indexes.EdgeNgramField(model_attr='title', null=True, stored=False) title_slab = indexes.CharField(faceted=True) title_slug = indexes.CharField(model_attr='titleSlug') uid = indexes.IntegerField(model_attr='uid') guid = ExactStringField(model_attr='guid') zipcode = indexes.CharField(model_attr='zipcode', null=True) # Fields for post-a-job is_posted = indexes.BooleanField() on_sites = MultiValueSpecialField(field_type='int') apply_info = ExactStringField(indexed=False) def get_model(self): return jobListing def prepare(self, obj): """ Fetches and adds/alters data before indexing. """ self.prepared_data = { ID: get_identifier(obj), DJANGO_CT: "%s.%s" % (obj._meta.app_label, obj._meta.module_name), DJANGO_ID: force_unicode(obj.pk), } for field_name, field in self.fields.items(): # Use the possibly overridden name, which will default to the # variable name of the field. self.prepared_data[field.index_fieldname] = field.prepare(obj) if hasattr(self, "prepare_%s" % field_name): value = getattr(self, "prepare_%s" % field_name)(obj) self.prepared_data[field.index_fieldname] = value bu = BusinessUnit.objects.get(id=obj.buid_id) co_name = bu.title co_slug = bu.title_slug or slugify(co_name) co_slab = u"{cs}/careers::{cn}".format(cs=co_slug, cn=co_name) self.prepared_data['company'] = co_name self.prepared_data['company_ac'] = co_name self.prepared_data['company_slab'] = co_slab if obj.onet_id: mocs = moc_models.Moc.objects.filter(onets__joblisting=obj) moc_slab = [ "%s/%s/%s/vet-jobs::%s - %s" % (moc.title_slug, moc.code, moc.branch, moc.code, moc.title) for moc in mocs ] moc_set = [moc.code for moc in mocs] mocids = [moc.id for moc in mocs] self.prepared_data['mocid'] = mocids self.prepared_data['moc_slab'] = moc_slab self.prepared_data['moc'] = moc_set return self.prepared_data def prepare_full_loc(self, obj): fields = ['city', 'state', 'location', 'country'] strings = [ '%s::%s' % (f, getattr(obj, f)) for f in fields if getattr(obj, f) ] return '@@'.join(strings) def prepare_country_slab(self, obj): return "%s/jobs::%s" % (obj.country_short.lower(), obj.country) def prepare_state_slab(self, obj): if obj.stateSlug: url = "%s/%s/jobs" % (obj.stateSlug, obj.country_short.lower()) return "%s::%s" % (url, obj.state) def prepare_city_slab(self, obj): """ See prepare_state_slab for the reason being the "none" The rest of this is storing the facet with or with out state data to go along with the city data. """ url = "%s/%s/%s/jobs" % (obj.citySlug, obj.stateSlug, obj.country_short.lower()) return "%s::%s" % (url, obj.location) def prepare_title_slab(self, obj): """ Creates title slab. """ if obj.titleSlug and obj.titleSlug != "none": return "%s/jobs-in::%s" % (obj.titleSlug.strip('-'), obj.title)
from dig.models import Document index_fields = {} for name, field in settings.ALL_FIELDS.items(): if field['index'] == False: continue kwargs = { 'document': field['document'], 'faceted': field['faceted'], 'model_attr': field['name'], } if field['type'] == 'date': index_field = indexes.DateField(null=True, **kwargs) elif field['type'] == 'int': index_field = indexes.IntegerField(null=True, **kwargs) elif field['type'] in ('float', 'latitude', 'longitude'): index_field = indexes.FloatField(null=True, **kwargs) elif field['type'] == 'boolean': index_field = indexes.BooleanField(**kwargs) elif field['type'] == 'null_boolean': index_field = indexes.BooleanField(null=True, **kwargs) elif field['type'] in ('char', 'text'): index_field = indexes.CharField(**kwargs) else: raise Exception("Unrecognized search index field type '%s'." % field['type']) index_fields[field['name']] = index_field # Override metaclass to let us specify fields non-declaratively.
class CredentialIndex(TxnAwareSearchIndex, indexes.Indexable): document = indexes.CharField(document=True, use_template=True) name = indexes.MultiValueField() location = indexes.MultiValueField() category = indexes.MultiValueField() topic_id = indexes.IntegerField(model_attr="topic_id") topic_type = indexes.CharField(model_attr="topic__type") source_id = indexes.CharField(model_attr="topic__source_id") inactive = indexes.BooleanField(model_attr="inactive") revoked = indexes.BooleanField(model_attr="revoked") latest = indexes.BooleanField(model_attr="latest") create_timestamp = indexes.DateTimeField(model_attr="create_timestamp") effective_date = indexes.DateTimeField(model_attr="effective_date") revoked_date = indexes.DateTimeField(model_attr="revoked_date", null=True) credential_set_id = indexes.IntegerField(model_attr="credential_set_id", null=True) credential_type_id = indexes.IntegerField(model_attr="credential_type_id") topic_credential_type_id = indexes.MultiValueField() issuer_id = indexes.IntegerField(model_attr="credential_type__issuer_id") schema_name = indexes.CharField(model_attr="credential_type__schema__name") schema_version = indexes.CharField( model_attr="credential_type__schema__version") credential_id = indexes.CharField(model_attr="credential_id") @staticmethod def prepare_name(obj): return [name.text for name in obj.all_names] @staticmethod def prepare_category(obj): return [ "{}::{}".format(cat.type, cat.value) for cat in obj.all_categories ] @staticmethod def prepare_topic_credential_type_id(obj): return [ct_id for ct_id in obj.all_credential_type_ids] @staticmethod def prepare_location(obj): locations = [] for address in obj.addresses.all(): loc = " ".join( filter( None, ( address.addressee, address.civic_address, address.city, address.province, address.postal_code, address.country, ), )) if loc: locations.append(loc) return locations def get_model(self): return CredentialModel def index_queryset(self, using=None): prefetch = ("addresses", "attributes", "names") select = ( "credential_set", "credential_type", "credential_type__schema", "topic", ) queryset = (super(CredentialIndex, self).index_queryset(using).prefetch_related( *prefetch).select_related(*select)) return queryset def read_queryset(self, using=None): select = ("credential_type__issuer", ) queryset = self.index_queryset(using).select_related(*select) return queryset def get_updated_field(self): return "update_timestamp"
class IncidentIndex(CelerySearchIndex, indexes.Indexable, IncidentPrepMeta): """ This document handles the construction of the Incident Solr document. """ text = indexes.CharField(document=True, use_template=True) title_en = indexes.CharField(model_attr='title_en', null=True) title_ar = indexes.CharField(model_attr='title_ar', null=True) confidence_score = indexes.IntegerField( model_attr='confidence_score', null=True, faceted=True) incident_times = indexes.MultiValueField(faceted=True) locations = indexes.MultiValueField(faceted=True) incident_labels = indexes.MultiValueField(faceted=True) incident_assigned_user = indexes.CharField( default='unassigned', model_attr='assigned_user', faceted=True, null=True ) # prepare functions for these can be found in IncidentPrepMeta count_actors = indexes.IntegerField() count_bulletins = indexes.IntegerField() count_incidents = indexes.IntegerField() assigned_user = indexes.CharField() most_recent_status_incident = indexes.CharField(faceted=True) incident_created = indexes.DateTimeField( model_attr='incident_created', faceted=True, null=True) incident_modified = indexes.DateTimeField( model_attr='incident_modified', faceted=True, null=True) incident_modified_date = indexes.DateField(faceted=True) incident_created_date = indexes.DateField(faceted=True) incident_searchable_locations = indexes.MultiValueField(faceted=True) resource_uri = indexes.CharField() incident_crimes = indexes.MultiValueField(faceted=True) crimes = indexes.MultiValueField() labels = indexes.MultiValueField() incident_confidence_bucket = indexes.CharField(faceted=True, null=True) """ times = indexes.MultiValueField() actors_role = indexes.MultiValueField() actors = indexes.MultiValueField() actor_roles_status = indexes.MultiValueField() incident_comments = indexes.MultiValueField() ref_bulletins = indexes.MultiValueField() ref_incidents = indexes.MultiValueField() incident_details_en = indexes.CharField( model_attr='incident_details_en', null=True) incident_details_ar = indexes.CharField( model_attr='incident_details_ar', null=True) """ def get_model(self): return Incident def prepare_actors(self, object): """ Returns an array of tastypi uris related to the Actor's associated actors """ return ActorPrepMeta().prepare_actors(object) def prepare_actors_role(self, object): """ Returns the correctly formated uri related to this incident instance for the tastypie api """ return ActorPrepMeta().prepare_actors_role(object) def get_updated_field(self): return 'incident_modified'
class VotationIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) # faceting fields act_type = indexes.FacetCharField() is_key = indexes.FacetCharField(model_attr='is_key_yesno') organ = indexes.FacetCharField(model_attr='sitting__institution__lowername') votation_date = indexes.FacetDateField(model_attr='sitting__date') n_presents_range = indexes.FacetCharField() gap_yes_no = indexes.FacetIntegerField() gap_yes_no_range = indexes.FacetCharField() # stored fields, used not to touch DB # while showing results url = indexes.CharField(indexed=False, stored=True) act_rendered = indexes.CharField(use_template=True, indexed=False) act_url = indexes.MultiValueField(indexed=True, stored=True) act_descr = indexes.CharField(indexed=False, stored=True, default='') title = indexes.CharField(indexed=False, stored=True, model_attr='act__title', default='') votation_num = indexes.CharField(indexed=False, stored=True, model_attr='idnum') votation_sitting_num = indexes.IntegerField(indexed=False, stored=True, model_attr='sitting__number') votation_n_presents = indexes.IntegerField(indexed=True, stored=True, model_attr='n_presents') votation_n_yes = indexes.IntegerField(indexed=True, stored=True, model_attr='n_yes') votation_n_no = indexes.IntegerField(indexed=True, stored=True, model_attr='n_no') votation_n_abst = indexes.IntegerField(indexed=True, stored=True, model_attr='n_abst') votation_n_maj = indexes.IntegerField(indexed=True, stored=True, model_attr='n_maj') votation_n_rebels = indexes.IntegerField(indexed=True, stored=True, model_attr='n_rebels', faceted=True) votation_outcome = indexes.IntegerField(indexed=False, stored=True, model_attr='outcome') votation_outcome_display = indexes.FacetCharField(stored=True, default='') is_secret = indexes.FacetCharField(stored=True, default='') month = indexes.FacetCharField() # needed to filter votations by person or charge person = indexes.MultiValueField(indexed=True, stored=False) charge = indexes.MultiValueField(indexed=True, stored=False) charge_present = indexes.MultiValueField(indexed=True, stored=False) charge_absent = indexes.MultiValueField(indexed=True, stored=False) charge_rebel = indexes.MultiValueField(indexed=True, stored=False) def get_model(self): return Votation def prepare_url(self, obj): return obj.get_absolute_url() def prepare_act_type(self, obj): activate(settings.LANGUAGE_CODE) res = _('none') related_act = obj.act if not related_act: transitions = Transition.objects.filter(votation=obj) if len(transitions) > 0: related_act = transitions[0].act if related_act: res = related_act.get_type_name() return res def prepare_act_descr(self, obj): res = obj.act_descr related_act = obj.act if not related_act: transitions = Transition.objects.filter(votation=obj) if len(transitions) > 0: related_act = transitions[0].act if related_act: res = related_act.adj_title or related_act.title return res def prepare_act_url(self, obj): res = set([]) related_act = obj.act if not related_act: transitions = Transition.objects.filter(votation=obj) if len(transitions) > 0: related_act = transitions[0].act if related_act: res = set([ related_act.downcast().get_absolute_url() ]) return res def prepare_month(self, obj): return obj.sitting.date.strftime("%B") def prepare_votation_outcome_display(self, obj): return obj.get_outcome_display() def prepare_person(self, obj): return set( [p['person__slug'] for p in obj.charge_set.values('person__slug').distinct()] ) def prepare_charge(self, obj): return set( [p['id'] for p in obj.charge_set.values('id').distinct()] ) | \ set( [p['original_charge__id'] for p in obj.charge_set.values('original_charge__id').distinct()] ) def prepare_charge_present(self, obj): s = set() for cv in obj.chargevote_set.filter( vote__in=[ChargeVote.VOTES.yes, ChargeVote.VOTES.no, ChargeVote.VOTES.abstained, ChargeVote.VOTES.pres, ChargeVote.VOTES.secret]): s.add(cv.charge.id) if cv.charge.original_charge: s.add(cv.charge.original_charge.id) return s def prepare_charge_absent(self, obj): s = set() for cv in obj.chargevote_set.filter(vote=ChargeVote.VOTES.absent): s.add(cv.charge.id) if cv.charge.original_charge: s.add(cv.charge.original_charge.id) return s def prepare_charge_rebel(self, obj): #return set( # [cv['charge__id'] for cv in obj.chargevote_set.filter(is_rebel=True).values('charge__id').distinct()] #) s = set() for cv in obj.chargevote_set.filter(is_rebel=True): s.add(cv.charge.id) return s def prepare_n_presents_range(self, obj): charges_count = obj.charges_count if charges_count > 0: decimal = 10 * obj.n_presents / charges_count return str(10 * decimal) + '% - ' + str(10 * (decimal + 1) - 1) + '%' \ if decimal < 10 else '100%' def prepare_gap_yes_no(self, obj): return abs(obj.n_yes - obj.n_no) def prepare_gap_yes_no_range(self, obj): v = abs(obj.n_yes - obj.n_no) if v <= 2: return '1 - 2' elif 3 <= v <= 6: return '3 - 6' elif 7 <= v <= 15: return '7 - 15' elif 16 <= v <= 22: return '16 - 22' else: return '23+' def prepare_is_secret(self, obj): activate(settings.LANGUAGE_CODE) if obj.is_secret: return _('yes') else: return _('no') def prepare_properties(self, obj): activate(settings.LANGUAGE_CODE) count = obj.majority_vs_minority may_presents = count['majority'].get('YES', 0) + count['majority'].get('NO', 0) + \ count['majority'].get('ABSTAINED', 0) + count['majority'].get('PRES', 0) + \ count['majority'].get('SECRET', 0) min_presents = count['minority'].get('YES', 0) + count['minority'].get('NO', 0) + \ count['minority'].get('ABSTAINED', 0) + count['minority'].get('PRES', 0) + \ count['minority'].get('SECRET', 0) total_presents = may_presents + min_presents may_partecipants = count['majority'].get('YES', 0) + count['majority'].get('NO', 0) min_partecipants = count['minority'].get('YES', 0) + count['minority'].get('NO', 0) total_partecipants = may_partecipants + min_partecipants quorum = (total_partecipants / 2) + 1 total_yes = count['majority'].get('YES', 0) + count['minority'].get('YES', 0) total_no = count['majority'].get('NO', 0) + count['minority'].get('NO', 0) total_abst = count['majority'].get('ABSTAINED', 0) + count['minority'].get('ABSTAINED', 0) total_pres = count['majority'].get('PRES', 0) + count['minority'].get('PRES', 0) if obj.outcome == 1: outcome = 'YES' elif obj.outcome == 2: outcome = 'NO' properties = [] # no numero legale if (may_presents < 16): properties.append(_("minority decisive for legal number")) if ('SECRET' not in count['majority']): # si e no si equivalgono: outcome NO if total_yes == total_no: quorum = total_yes; # minoranza decisiva if (count['majority'][outcome] < quorum): properties.append(_("minority decisive for outcome")) return set(properties)
class JobPostingIndex(CelerySearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) # Shitty workaround to the content__contains problem contains = indexes.EdgeNgramField(use_template=True, template_name='search/indexes/employer/jobposting_text.txt') company_name = indexes.CharField(model_attr='praxis__company_name') email = indexes.CharField(model_attr='praxis__business__user__email') address = indexes.CharField(model_attr='praxis__address') zip_code = indexes.DecimalField(model_attr='praxis__zip_code') city = indexes.CharField(model_attr='praxis__city') state = indexes.CharField(model_attr='praxis__state') solo_practitioner = indexes.BooleanField( model_attr='praxis__solo_practitioner') multi_practitioner = indexes.BooleanField( model_attr='praxis__multi_practitioner') corporate = indexes.BooleanField(model_attr='praxis__corporate') fee_for_service = indexes.BooleanField( model_attr='praxis__fee_for_service') insurance = indexes.BooleanField(model_attr='praxis__insurance') capitation_medicaid = indexes.BooleanField( model_attr='praxis__capitation_medicaid') position_name = indexes.CharField(model_attr='position_name') posting_title = indexes.CharField(model_attr='posting_title') job_position = indexes.IntegerField(model_attr='job_position') job_position_text = indexes.CharField(model_attr='get_job_position_display') schedule_type = indexes.BooleanField(model_attr='schedule_type') schedule_type_text = indexes.CharField( model_attr='get_schedule_type_display') monday_daytime = indexes.BooleanField(model_attr='monday_daytime') monday_evening = indexes.BooleanField(model_attr='monday_evening') tuesday_daytime = indexes.BooleanField(model_attr='tuesday_daytime') tuesday_evening = indexes.BooleanField(model_attr='tuesday_evening') wednesday_daytime = indexes.BooleanField(model_attr='wednesday_daytime') wednesday_evening = indexes.BooleanField(model_attr='wednesday_evening') thursday_daytime = indexes.BooleanField(model_attr='thursday_daytime') thursday_evening = indexes.BooleanField(model_attr='thursday_evening') friday_daytime = indexes.BooleanField(model_attr='friday_daytime') friday_evening = indexes.BooleanField(model_attr='friday_evening') saturday_daytime = indexes.BooleanField(model_attr='saturday_daytime') saturday_evening = indexes.BooleanField(model_attr='saturday_evening') sunday_daytime = indexes.BooleanField(model_attr='sunday_daytime') sunday_evening = indexes.BooleanField(model_attr='sunday_evening') compensation_type = indexes.BooleanField(model_attr='compensation_type') hourly_wage = indexes.IntegerField(model_attr='hourly_wage', null=True) hourly_wage_text = indexes.CharField( model_attr='get_hourly_wage_display', null=True) annualy_wage = indexes.IntegerField(model_attr='annualy_wage', null=True) annualy_wage_text = indexes.CharField( model_attr='get_annualy_wage_display', null=True) production = indexes.BooleanField(model_attr='production') collection = indexes.BooleanField(model_attr='collection') experience_years = indexes.IntegerField(model_attr='experience_years') benefit_1 = indexes.BooleanField(model_attr='benefit_1') benefit_2 = indexes.BooleanField(model_attr='benefit_2') benefit_3 = indexes.BooleanField(model_attr='benefit_3') benefit_4 = indexes.BooleanField(model_attr='benefit_4') benefit_5 = indexes.BooleanField(model_attr='benefit_5') benefit_6 = indexes.BooleanField(model_attr='benefit_6') benefit_other_text = indexes.CharField(model_attr='benefit_other_text') visa = indexes.BooleanField(model_attr='visa') visa_text = indexes.BooleanField(model_attr='get_visa_display') additional_comments = indexes.CharField(model_attr='additional_comments') matches = indexes.MultiValueField() location = indexes.LocationField() def get_model(self): return JobPosting def _get_default_filter_query(self): return Q(is_posted=True) def index_queryset(self, using=None): return super(JobPostingIndex, self).index_queryset(using=using).filter( self._get_default_filter_query()) def read_queryset(self, using=None): return super(JobPostingIndex, self).read_queryset(using=using).filter( self._get_default_filter_query()) def build_queryset(self, using=None, start_date=None, end_date=None): return super(JobPostingIndex, self).build_queryset( using=using, start_date=start_date, end_date=end_date).filter( self._get_default_filter_query()) def update_object(self, identifier, using=None, **kwargs): instance = get_instance_from_identifier(identifier) if not instance.is_posted: self.remove_object(identifier, using=using, **kwargs) super(JobPostingIndex, self).update_object(instance, using=using, **kwargs) def prepare_matches(self, obj): return [match.user.pk for match in obj.matches.all()] def get_updated_field(self): return 'updated' def prepare_location(self, obj): try: zip_code = ZipCode.objects.get(code=obj.praxis.zip_code) except (ZipCode.DoesNotExist, ZipCode.MultipleObjectsReturned): return '.0,.0' return '%s,%s' % (zip_code.latitude, zip_code.longitude)
class NodeIndex(indexes.SearchIndex, indexes.Indexable): TYPE_PREFIX = "REFINERY_TYPE" NAME_PREFIX = "REFINERY_NAME" WORKFLOW_OUTPUT_PREFIX = "REFINERY_WORKFLOW_OUTPUT" ANALYSIS_UUID_PREFIX = "REFINERY_ANALYSIS_UUID" SUBANALYSIS_PREFIX = "REFINERY_SUBANALYSIS" FILETYPE_PREFIX = "REFINERY_FILETYPE" DOWNLOAD_URL = "REFINERY_DOWNLOAD_URL_s" text = indexes.CharField(document=True, use_template=True) uuid = indexes.CharField(model_attr='uuid') study_uuid = indexes.CharField(model_attr='study__uuid') assay_uuid = indexes.CharField(model_attr='assay__uuid', null=True) data_set_uuid = indexes.CharField(null=True) type = indexes.CharField(model_attr='type') name = indexes.CharField(model_attr='name', null=True) file_uuid = indexes.CharField(model_attr='file_uuid', null=True) species = indexes.IntegerField(model_attr='species', null=True) genome_build = indexes.CharField(model_attr='genome_build', null=True) is_annotation = indexes.BooleanField(model_attr='is_annotation') analysis_uuid = indexes.CharField(model_attr='analysis_uuid', null=True) subanalysis = indexes.IntegerField(model_attr='subanalysis', null=True) workflow_output = indexes.CharField(model_attr='workflow_output', null=True) # TODO: add modification date (based on registry) def get_model(self): return Node def index_queryset(self, using=None): """Used when the entire index for model is updated.""" return self.get_model().objects.all() GENERIC_SUFFIX = "_generic_s" def _assay_data(self, object): data = {} for field in Assay._meta.fields: if field.name in ['id', 'uuid', 'study', 'file_name']: continue key = field.name + '_Characteristics' + NodeIndex.GENERIC_SUFFIX data[key] = set() assay = object.assay if assay is not None: assay_attr = getattr(assay, field.name) if assay_attr is not None: data[key].add(assay_attr) return data # dynamic fields: # https://groups.google.com/forum/?fromgroups#!topic/django-haystack/g39QjTkN-Yg # http://stackoverflow.com/questions/7399871/django-haystack-sort-results-by-title def prepare(self, object): data = super(NodeIndex, self).prepare(object) annotations = AnnotatedNode.objects.filter(node=object) id_suffix = str(object.study.id) try: data_set = object.study.get_dataset() data['data_set_uuid'] = data_set.uuid except RuntimeError as e: logger.warn(e) if object.assay is not None: id_suffix += "_" + str(object.assay.id) id_suffix = "_" + id_suffix + "_s" data['filename_Characteristics' + NodeIndex.GENERIC_SUFFIX] = \ re.sub(r'.*/', '', data['name']) data.update(self._assay_data(object)) # create dynamic fields for each attribute for annotation in annotations: name = annotation.attribute_type if annotation.attribute_subtype is not None: name = annotation.attribute_subtype + "_" + name value = annotation.attribute_value if annotation.attribute_value_unit is not None: value += " " + annotation.attribute_value_unit name = re.sub(r'\W', settings.REFINERY_SOLR_SPACE_DYNAMIC_FIELDS, name) uniq_key = name + id_suffix generic_key = name + NodeIndex.GENERIC_SUFFIX # a node might have multiple parents with different attribute # values for a given attribute # e.g. parentA Characteristic[cell type] = K562 and # parentB Characteristic[cell type] = HeLa # child nodes should inherit all attributes of their parents as a # concatenation of the unique list # old version (only one attribute kept): # data[key] = value for key in (uniq_key, generic_key): if key not in data: data[key] = set() if value != "": data[key].add(value) else: data[key].add("N/A") # iterate over all keys in data and join sets into strings # TODO: This doesn't feel right: facet each separately? for key, value in data.iteritems(): if type(value) is set: data[key] = " + ".join(value) try: file_store_item = FileStoreItem.objects.get( uuid=object.file_uuid) except(FileStoreItem.DoesNotExist, FileStoreItem.MultipleObjectsReturned) as e: logger.error("Couldn't properly fetch FileStoreItem: %s", e) file_store_item = None data.update({ NodeIndex.DOWNLOAD_URL: '' if file_store_item is None else file_store_item.get_datafile_url(), NodeIndex.TYPE_PREFIX + id_suffix: object.type, NodeIndex.NAME_PREFIX + id_suffix: object.name, NodeIndex.FILETYPE_PREFIX + id_suffix: "" if file_store_item is None else file_store_item.get_filetype(), NodeIndex.ANALYSIS_UUID_PREFIX + id_suffix: "N/A" if object.get_analysis() is None else object.get_analysis().name, NodeIndex.SUBANALYSIS_PREFIX + id_suffix: (-1 if object.subanalysis is None # TODO: upgrade flake8 else object.subanalysis), # and remove parentheses NodeIndex.WORKFLOW_OUTPUT_PREFIX + id_suffix: "N/A" if object.workflow_output is None else object.workflow_output }) return data
class TendenciBaseSearchIndex(indexes.SearchIndex): text = indexes.CharField(document=True, use_template=True) # TendenciBaseModel Fields allow_anonymous_view = indexes.BooleanField(model_attr='allow_anonymous_view', null=True) allow_user_view = indexes.BooleanField(model_attr='allow_user_view', null=True) allow_member_view = indexes.BooleanField(model_attr='allow_member_view', null=True) # allow_anonymous_edit = indexes.BooleanField(model_attr='allow_anonymous_edit') allow_user_edit = indexes.BooleanField(model_attr='allow_user_edit', null=True) allow_member_edit = indexes.BooleanField(model_attr='allow_member_edit', null=True) creator = indexes.CharField(model_attr='creator', null=True) creator_username = indexes.CharField(model_attr='creator_username', null=True) owner = indexes.CharField(model_attr='owner', null=True) owner_username = indexes.CharField(model_attr='owner_username', null=True) status = indexes.BooleanField(model_attr='status', null=True) status_detail = indexes.CharField(model_attr='status_detail') create_dt = indexes.DateTimeField(model_attr='create_dt', null=True) update_dt = indexes.DateTimeField(model_attr='update_dt', null=True) # permission fields primary_key = indexes.IntegerField(model_attr='pk') users_can_view = indexes.MultiValueField(null=True) groups_can_view = indexes.MultiValueField(null=True) # add order field for sorting. the subclasses can override # the prepare_order method to sort by a different field order = indexes.DateTimeField() def get_model(self): return None def prepare_allow_anonymous_view(self, obj): if is_whoosh(): try: temp = int(obj.allow_anonymous_view) except TypeError: temp = 0 return temp return obj.allow_anonymous_view def prepare_allow_user_view(self, obj): if is_whoosh(): try: temp = int(obj.allow_user_view) except TypeError: temp = 0 return temp return obj.allow_user_view def prepare_allow_member_view(self, obj): if is_whoosh(): try: temp = int(obj.allow_member_view) except TypeError: temp = 0 return temp return obj.allow_member_view def prepare_allow_user_edit(self, obj): if is_whoosh(): try: temp = int(obj.allow_user_edit) except TypeError: temp = 0 return temp return obj.allow_user_edit def prepare_allow_member_edit(self, obj): if is_whoosh(): try: temp = int(obj.allow_member_edit) except TypeError: temp = 0 return temp return obj.allow_member_edit def prepare_status(self, obj): if is_whoosh(): try: temp = int(obj.status) except TypeError: temp = 0 return temp return obj.status def prepare_order(self, obj): return obj.create_dt def get_updated_field(self): return 'update_dt' def prepare_users_can_view(self, obj): """ This needs to be overwritten if 'view' permission label does not follow the standard convention: (app_label).view_(module_name) """ return ObjectPermission.objects.users_with_perms('%s.view_%s' % (obj._meta.app_label, obj._meta.model_name), obj) def prepare_groups_can_view(self, obj): """ This needs to be overwritten if 'view' permission label does not follow the standard convention: (app_label).view_(module_name) """ return ObjectPermission.objects.groups_with_perms('%s.view_%s' % (obj._meta.app_label, obj._meta.model_name), obj)
class MapStoryIndex(indexes.SearchIndex, indexes.Indexable): id = indexes.IntegerField(model_attr='id') abstract = indexes.CharField(model_attr="abstract", boost=1.5) category__gn_description = indexes.CharField( model_attr="category__gn_description", null=True) distribution_description = indexes.CharField( model_attr="distribution_description", null=True) distribution_url = indexes.CharField(model_attr="distribution_url", null=True) owner__username = indexes.CharField(model_attr="owner", faceted=True, null=True) popular_count = indexes.IntegerField(model_attr="popular_count", default=0, boost=20) share_count = indexes.IntegerField(model_attr="share_count", default=0) rating = indexes.IntegerField(null=True) thumbnail_url = indexes.CharField(model_attr="thumbnail_url", null=True) detail_url = indexes.CharField(model_attr="detail_url", null=True) uuid = indexes.CharField(model_attr="uuid") title = indexes.CharField(model_attr="title", boost=2) date = indexes.DateTimeField(model_attr="date") text = indexes.EdgeNgramField(document=True, use_template=True, stored=False) type = indexes.CharField(faceted=True) title_sortable = indexes.CharField(indexed=False, stored=False) # Necessary for sorting category = indexes.CharField(model_attr="category__identifier", faceted=True, null=True, stored=True) bbox_left = indexes.FloatField(model_attr="bbox_x0", null=True, stored=False) bbox_right = indexes.FloatField(model_attr="bbox_x1", null=True, stored=False) bbox_bottom = indexes.FloatField(model_attr="bbox_y0", null=True, stored=False) bbox_top = indexes.FloatField(model_attr="bbox_y1", null=True, stored=False) temporal_extent_start = indexes.DateTimeField( model_attr="temporal_extent_start", null=True, stored=False) temporal_extent_end = indexes.DateTimeField( model_attr="temporal_extent_end", null=True, stored=False) keywords = indexes.MultiValueField(model_attr="keyword_slug_list", null=True, faceted=True, stored=True) regions = indexes.MultiValueField(model_attr="region_name_list", null=True, faceted=True, stored=True) popular_count = indexes.IntegerField(model_attr="popular_count", default=0, boost=20) share_count = indexes.IntegerField(model_attr="share_count", default=0) rating = indexes.IntegerField(null=True) num_ratings = indexes.IntegerField(stored=False) num_comments = indexes.IntegerField(stored=False) # Need to grab the owner's first and last name as well as published status owner__first_name = indexes.CharField(model_attr="owner__first_name", faceted=True, null=True) owner__last_name = indexes.CharField(model_attr="owner__last_name", faceted=True, null=True) is_published = indexes.BooleanField(model_attr="is_published") # Featured status for front page carousel featured = indexes.BooleanField(model_attr="featured") def get_model(self): return MapStory def prepare_type(self, obj): return "mapstory" def prepare_rating(self, obj): ct = ContentType.objects.get_for_model(obj) try: rating = OverallRating.objects.filter( object_id=obj.pk, content_type=ct).aggregate(r=Avg("rating"))["r"] return float(str(rating or "0")) except OverallRating.DoesNotExist: return 0.0 def prepare_num_ratings(self, obj): ct = ContentType.objects.get_for_model(obj) try: return OverallRating.objects.filter(object_id=obj.pk, content_type=ct).all().count() except OverallRating.DoesNotExist: return 0 def prepare_num_comments(self, obj): try: return Comment.objects.filter( object_id=obj.pk, content_type=ContentType.objects.get_for_model( obj)).all().count() except: return 0 def prepare_title_sortable(self, obj): return obj.title.lower()
class CourseRunIndex(BaseCourseIndex, indexes.Indexable): model = CourseRun course_key = indexes.CharField(model_attr='course__key', stored=True) org = indexes.CharField() number = indexes.CharField() status = indexes.CharField(model_attr='status', faceted=True) start = indexes.DateTimeField(model_attr='start', null=True, faceted=True) end = indexes.DateTimeField(model_attr='end', null=True) enrollment_start = indexes.DateTimeField(model_attr='enrollment_start', null=True) enrollment_end = indexes.DateTimeField(model_attr='enrollment_end', null=True) announcement = indexes.DateTimeField(model_attr='announcement', null=True) min_effort = indexes.IntegerField(model_attr='min_effort', null=True) max_effort = indexes.IntegerField(model_attr='max_effort', null=True) weeks_to_complete = indexes.IntegerField(model_attr='weeks_to_complete', null=True) language = indexes.CharField(null=True, faceted=True) transcript_languages = indexes.MultiValueField(faceted=True) pacing_type = indexes.CharField(model_attr='pacing_type', null=True, faceted=True) marketing_url = indexes.CharField(null=True) slug = indexes.CharField(model_attr='slug', null=True) seat_types = indexes.MultiValueField(model_attr='seat_types', null=True, faceted=True) type = indexes.CharField(model_attr='type', null=True, faceted=True) image_url = indexes.CharField(model_attr='image_url', null=True) partner = indexes.CharField(null=True, faceted=True) program_types = indexes.MultiValueField() published = indexes.BooleanField(null=False, faceted=True) hidden = indexes.BooleanField(model_attr='hidden', faceted=True) mobile_available = indexes.BooleanField(model_attr='mobile_available', faceted=True) authoring_organization_uuids = indexes.MultiValueField() staff_uuids = indexes.MultiValueField() subject_uuids = indexes.MultiValueField() has_enrollable_paid_seats = indexes.BooleanField(null=False) first_enrollable_paid_seat_sku = indexes.CharField(null=True) paid_seat_enrollment_end = indexes.DateTimeField(null=True) license = indexes.MultiValueField(model_attr='license', faceted=True) has_enrollable_seats = indexes.BooleanField( model_attr='has_enrollable_seats', null=False) is_current_and_still_upgradeable = indexes.BooleanField(null=False) def prepare_aggregation_key(self, obj): # Aggregate CourseRuns by Course key since that is how we plan to dedup CourseRuns on the marketing site. return 'courserun:{}'.format(obj.course.key) def prepare_has_enrollable_paid_seats(self, obj): return obj.has_enrollable_paid_seats() def prepare_first_enrollable_paid_seat_sku(self, obj): return obj.first_enrollable_paid_seat_sku() def prepare_is_current_and_still_upgradeable(self, obj): return obj.is_current_and_still_upgradeable() def prepare_paid_seat_enrollment_end(self, obj): return obj.get_paid_seat_enrollment_end() def prepare_partner(self, obj): return obj.course.partner.short_code def prepare_published(self, obj): return obj.status == CourseRunStatus.Published def prepare_language(self, obj): return self._prepare_language(obj.language) def prepare_number(self, obj): course_run_key = CourseKey.from_string(obj.key) return course_run_key.course def prepare_org(self, obj): course_run_key = CourseKey.from_string(obj.key) return course_run_key.org def prepare_transcript_languages(self, obj): return [ self._prepare_language(language) for language in obj.transcript_languages.all() ] def prepare_marketing_url(self, obj): return obj.marketing_url def prepare_program_types(self, obj): return obj.program_types def prepare_staff_uuids(self, obj): return [str(staff.uuid) for staff in obj.staff.all()] def prepare_subject_uuids(self, obj): return [str(subject.uuid) for subject in obj.subjects.all()]
class CourseIndex(BaseCourseIndex, indexes.Indexable): model = Course uuid = indexes.CharField(model_attr='uuid') card_image_url = indexes.CharField(model_attr='card_image_url', null=True) image_url = indexes.CharField(model_attr='image_url', null=True) org = indexes.CharField() status = indexes.CharField(model_attr='course_runs__status') start = indexes.DateTimeField(model_attr='course_runs__start', null=True) end = indexes.DateTimeField(model_attr='course_runs__end', null=True) enrollment_start = indexes.DateTimeField( model_attr='course_runs__enrollment_start', null=True) enrollment_end = indexes.DateTimeField( model_attr='course_runs__enrollment_end', null=True) availability = indexes.CharField(model_attr='course_runs__availability') first_enrollable_paid_seat_price = indexes.IntegerField(null=True) subject_uuids = indexes.MultiValueField() course_runs = indexes.MultiValueField() expected_learning_items = indexes.MultiValueField() prerequisites = indexes.MultiValueField(faceted=True) languages = indexes.MultiValueField() seat_types = indexes.MultiValueField() def read_queryset(self, using=None): # Pre-fetch all fields required by the CourseSearchSerializer. Unfortunately, there's # no way to specify at query time which queryset to use during loading in order to customize # it for the serializer being used qset = super(CourseIndex, self).read_queryset(using=using) return qset.prefetch_related('course_runs__seats__type') def prepare_aggregation_key(self, obj): return 'course:{}'.format(obj.key) def prepare_course_runs(self, obj): return [course_run.key for course_run in obj.course_runs.all()] def prepare_expected_learning_items(self, obj): return [item.value for item in obj.expected_learning_items.all()] def prepare_prerequisites(self, obj): return [prerequisite.name for prerequisite in obj.prerequisites.all()] def prepare_org(self, obj): course_run = obj.course_runs.all().first() if course_run: return CourseKey.from_string(course_run.key).org return None def prepare_first_enrollable_paid_seat_price(self, obj): return obj.first_enrollable_paid_seat_price def prepare_seat_types(self, obj): seat_types = [ seat.slug for course_run in obj.course_runs.all() for seat in course_run.seat_types ] return list(set(seat_types)) def prepare_subject_uuids(self, obj): return [str(subject.uuid) for subject in obj.subjects.all()] def prepare_languages(self, obj): return { self._prepare_language(course_run.language) for course_run in obj.course_runs.all() if course_run.language }
class DataDeclarationIndex(CelerySearchIndex, indexes.Indexable): def get_model(self): return DataDeclaration def get_updated_field(self): return 'updated' text = indexes.CharField(document=True, use_template=True) pk = indexes.IntegerField(indexed=True, stored=True, faceted=True) cohorts = indexes.MultiValueField(indexed=True, stored=True, faceted=True) consent_status = indexes.CharField(model_attr='consent_status', indexed=True, stored=True, faceted=True) data_types = indexes.MultiValueField(indexed=True, stored=True, faceted=True) data_types_generated = indexes.MultiValueField(indexed=True, stored=True, faceted=True) data_types_received = indexes.MultiValueField(indexed=True, stored=True, faceted=True) deidentification_method = indexes.CharField(indexed=True, stored=True, faceted=True) dpia = indexes.CharField(indexed=True, stored=True, faceted=True) embargo_date = indexes.DateField(indexed=True, stored=True) end_of_storage_duration = indexes.DateField(indexed=True, stored=True) has_special_subjects = indexes.BooleanField(indexed=True, stored=True, faceted=True) local_custodians = indexes.MultiValueField(indexed=True, stored=True, faceted=True) other_external_id = indexes.CharField(indexed=True, stored=True, faceted=True) project = indexes.CharField(indexed=True, stored=True, faceted=True) special_subjects_description = indexes.CharField(indexed=True, stored=True, faceted=True) subjects_category = indexes.CharField(indexed=True, stored=True, faceted=True) submission_id = indexes.CharField(indexed=True, stored=True, faceted=True) title = indexes.CharField(indexed=True, stored=True, faceted=True) title_l = indexes.CharField(indexed=False, stored=True) unique_id = indexes.CharField(indexed=True, stored=True, faceted=True) autocomplete = indexes.EdgeNgramField() def prepare_autocomplete(self, obj): text_parts = [ obj.title, str(obj.dataset), " ".join([str(c for c in obj.cohorts.all())]), " ".join([str(l) for l in obj.dataset.local_custodians.all()]) ] if obj.dataset.project: text_parts.append(str(obj.dataset.project)) return " ".join(text_parts) def prepare_title(self, obj): return obj.title def prepare_title_l(self, obj): if obj.title: return obj.title.lower().strip() def prepare_deidentification_method(self, obj): return obj.deidentification_method.name def prepare_data_use_restrictions(self, obj): return [o.restriction_class for o in obj.data_use_restrictions.all()] def prepare_subjects_category(self, obj): return obj.subjects_category.name def prepare_cohorts(self, obj): return [str(c) for c in obj.cohorts.all()] def prepare_project(self, obj): if obj.dataset.project: return str(obj.dataset.project) return None def prepare_local_custodians(self, obj): return [u.full_name for u in obj.dataset.local_custodians.all()]
class CourseRunIndex(BaseCourseIndex, indexes.Indexable): model = CourseRun course_key = indexes.CharField(model_attr='course__key', stored=True) org = indexes.CharField() number = indexes.CharField() status = indexes.CharField(model_attr='status', faceted=True) start = indexes.DateTimeField(model_attr='start', null=True, faceted=True) end = indexes.DateTimeField(model_attr='end', null=True) go_live_date = indexes.DateTimeField(model_attr='go_live_date', null=True) enrollment_start = indexes.DateTimeField(model_attr='enrollment_start', null=True) enrollment_end = indexes.DateTimeField(model_attr='enrollment_end', null=True) availability = indexes.CharField(model_attr='availability') announcement = indexes.DateTimeField(model_attr='announcement', null=True) min_effort = indexes.IntegerField(model_attr='min_effort', null=True) max_effort = indexes.IntegerField(model_attr='max_effort', null=True) weeks_to_complete = indexes.IntegerField(model_attr='weeks_to_complete', null=True) language = indexes.CharField(null=True, faceted=True) transcript_languages = indexes.MultiValueField(faceted=True) pacing_type = indexes.CharField(model_attr='pacing_type', null=True, faceted=True) marketing_url = indexes.CharField(null=True) slug = indexes.CharField(model_attr='slug', null=True) seat_types = indexes.MultiValueField(model_attr='seat_types__slug', null=True, faceted=True) type = indexes.CharField(model_attr='type_legacy', null=True, faceted=True) image_url = indexes.CharField(model_attr='image_url', null=True) partner = indexes.CharField(null=True, faceted=True) program_types = indexes.MultiValueField() published = indexes.BooleanField(null=False, faceted=True) hidden = indexes.BooleanField(model_attr='hidden', faceted=True) mobile_available = indexes.BooleanField(model_attr='mobile_available', faceted=True) authoring_organization_uuids = indexes.MultiValueField() staff_uuids = indexes.MultiValueField() subject_uuids = indexes.MultiValueField() has_enrollable_paid_seats = indexes.BooleanField(null=False) first_enrollable_paid_seat_sku = indexes.CharField(null=True) first_enrollable_paid_seat_price = indexes.IntegerField(null=True) paid_seat_enrollment_end = indexes.DateTimeField(null=True) license = indexes.MultiValueField(model_attr='license', faceted=True) has_enrollable_seats = indexes.BooleanField( model_attr='has_enrollable_seats', null=False) is_current_and_still_upgradeable = indexes.BooleanField(null=False) def read_queryset(self, using=None): # Pre-fetch all fields required by the CourseRunSearchSerializer. Unfortunately, there's # no way to specify at query time which queryset to use during loading in order to customize # it for the serializer being used qset = super(CourseRunIndex, self).read_queryset(using=using) return qset.prefetch_related('seats__type', ) def prepare_aggregation_key(self, obj): # Aggregate CourseRuns by Course key since that is how we plan to dedup CourseRuns on the marketing site. return 'courserun:{}'.format(obj.course.key) def prepare_has_enrollable_paid_seats(self, obj): return obj.has_enrollable_paid_seats() def prepare_first_enrollable_paid_seat_sku(self, obj): return obj.first_enrollable_paid_seat_sku() def prepare_first_enrollable_paid_seat_price(self, obj): return obj.first_enrollable_paid_seat_price def prepare_is_current_and_still_upgradeable(self, obj): return obj.is_current_and_still_upgradeable() def prepare_paid_seat_enrollment_end(self, obj): return obj.get_paid_seat_enrollment_end() def prepare_partner(self, obj): return obj.course.partner.short_code def prepare_published(self, obj): return obj.status == CourseRunStatus.Published def prepare_language(self, obj): return self._prepare_language(obj.language) def prepare_number(self, obj): course_run_key = CourseKey.from_string(obj.key) return course_run_key.course def prepare_org(self, obj): course_run_key = CourseKey.from_string(obj.key) return course_run_key.org def prepare_transcript_languages(self, obj): return [ self._prepare_language(language) for language in obj.transcript_languages.all() ] def prepare_marketing_url(self, obj): return obj.marketing_url def prepare_program_types(self, obj): return obj.program_types def prepare_staff_uuids(self, obj): return [str(staff.uuid) for staff in obj.staff.all()] def prepare_subject_uuids(self, obj): return [str(subject.uuid) for subject in obj.subjects.all()]
class RecordIndex(indexes.SearchIndex): type = 'faads' # sectors sectors = indexes.MultiValueField(null=False) subsectors = indexes.MultiValueField(null=True) # categories cfda_program = indexes.IntegerField(model_attr='cfda_program__id', null=True) budget_function = indexes.MultiValueField(null=True) # many-to-many via cfda program description funding_type = indexes.MultiValueField(null=True) # many-to-many via cfda program description action_type = indexes.IntegerField(model_attr='action_type__id', null=True) recipient_type = indexes.IntegerField(model_attr='recipient_type__id', null=True) record_type = indexes.IntegerField(model_attr='record_type__id', null=True) assistance_type = indexes.IntegerField(model_attr='assistance_type__id', null=True) # chrono fiscal_year = indexes.IntegerField(model_attr='fiscal_year', null=True) obligation_date = indexes.DateField(model_attr='obligation_action_date', null=True) # dollars non_federal_amount = indexes.IntegerField(model_attr='non_federal_funding_amount', null=True) federal_amount = indexes.IntegerField(model_attr='federal_funding_amount', null=True) total_amount = indexes.IntegerField(model_attr='total_funding_amount', null=True) # text text = indexes.CharField(document=True, model_attr='project_description', null=True) recipient = indexes.CharField(model_attr='recipient_name', null=True) # geo recipient_county = indexes.IntegerField(model_attr='recipient_county__id', null=True) recipient_state = indexes.IntegerField(model_attr='recipient_state__id', null=True) principal_place_state = indexes.IntegerField(model_attr='principal_place_state__id', null=True) principal_place_county = indexes.IntegerField(model_attr='principal_place_county__id', null=True) # combo all_text = indexes.CharField(null=True) all_states = indexes.MultiValueField(null=True) def prepare_all_text(self, object): s = "%s %s" % (getattr(object, 'recipient_name', ''), getattr(object, 'project_description', '')) return len(s)>0 and s.strip() or None def prepare_all_states(self, object): r = [] for f in ('principal_place_state', 'recipient_state'): s = getattr(object, f, None) if s is not None: i = getattr(s, 'id', None) if i is not None: r.append(i) return len(r)>0 and r or -1 def prepare_cfda_program(self, object): if object.cfda_program is None: return None return object.cfda_program.id def prepare_budget_function(self, object): if object.cfda_program is None: return None budget_functions = [] for budget_account in object.cfda_program.budget_accounts.all(): if budget_account.budget_function != None: budget_functions.append(budget_account.budget_function.id) return budget_functions def prepare_funding_type(self, object): if object.cfda_program is None: return None funding_types = [] for budget_account in object.cfda_program.budget_accounts.all(): funding_types.append(budget_account.fund_code) return funding_types def prepare_action_type(self, object): if object.action_type != None: return object.action_type.id else: return None def prepare_recipient_type(self, object): if object.recipient_type != None: return object.recipient_type.id else: return None def prepare_record_type(self, object): if object.record_type != None: return object.record_type.id else: return None def prepare_assistance_type(self, object): if object.assistance_type != None: return object.assistance_type.id else: return None def prepare_fiscal_year(self, object): if object.fiscal_year != None: return int(object.fiscal_year) else: return None def prepare_obligation_date(self, object): try: if object.obligation_action_date.year > 1900: return object.obligation_action_date else: return None except: return None def prepare_federal_amount(self, object): if object.federal_funding_amount: return int(object.federal_funding_amount) else: return None def prepare_non_federal_amount(self, object): if object.non_federal_funding_amount: return int(object.non_federal_funding_amount) else: return None def prepare_total_amount(self, object): if object.total_funding_amount: return int(object.total_funding_amount) else: return None def prepare_recipient(self, object): return '%s' % (object.recipient_name) def prepare_recipient_county(self, object): if object.recipient_county: return object.recipient_county.id else: return -1 def prepare_recipient_state(self, object): if object.recipient_state: return object.recipient_state.id else: return -1 def prepare_principal_place_county(self, object): if object.principal_place_county: return object.principal_place_county.id else: return -1 def prepare_principal_place_state(self, object): if object.principal_place_state: return object.principal_place_state.id else: return -1 def prepare_sectors(self, object): return map(lambda x: x.id, object.sectors.all()) def get_queryset(self): "Used when the entire index for model is updated." return Record.objects.all()
class PackageIndex(PackageCelerySearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) name = indexes.CharField(model_attr="name", boost=1.5) summary = indexes.CharField(null=True) description = indexes.CharField(null=True) author = indexes.CharField(null=True) maintainer = indexes.CharField(null=True) downloads = indexes.IntegerField(model_attr="downloads", indexed=False) url = indexes.CharField(model_attr="get_absolute_url", indexed=False) operating_systems = indexes.MultiValueField( null=True, faceted=True, facet_class=indexes.FacetMultiValueField) licenses = indexes.MultiValueField( null=True, faceted=True, facet_class=indexes.FacetMultiValueField) implementations = indexes.MultiValueField( null=True, faceted=True, facet_class=indexes.FacetMultiValueField) python_versions = indexes.MultiValueField( null=True, faceted=True, facet_class=indexes.FacetMultiValueField) versions = indexes.MultiValueField(null=True) release_count = indexes.IntegerField(default=0) def get_model(self): return Package def prepare(self, obj): data = super(PackageIndex, self).prepare(obj) # For ES, because it doesn't tokenize on ``_``, which causes problems # on lots of searches. if '_' in data['name']: data['name'] += ' ' + data['name'].replace('_', '-') if obj.latest: data["summary"] = obj.latest.summary data["author"] = obj.latest.author if obj.latest.author else None data[ "maintainer"] = obj.latest.maintainer if obj.latest.maintainer else None data[ "description"] = obj.latest.description if obj.latest.description else None operating_systems = [] licenses = [] implementations = [] python_versions = [] for classifier in obj.latest.classifiers.all(): if classifier.trove.startswith("License ::"): # We Have a License for This Project licenses.append( classifier.trove.rsplit("::", 1)[1].strip()) elif classifier.trove.startswith("Operating System ::"): operating_systems.append( classifier.trove.rsplit("::", 1)[1].strip()) elif classifier.trove.startswith( "Programming Language :: Python :: Implementation ::"): implementations.append( classifier.trove.rsplit("::", 1)[1].strip()) elif classifier.trove.startswith( "Programming Language :: Python ::"): if classifier.trove == "Programming Language :: Python :: 2 :: Only": python_versions.append("2.x") elif classifier.trove.startswith( "Programming Language :: Python :: 2"): python_versions.append("2.x") elif classifier.trove.startswith( "Programming Language :: Python :: 3"): python_versions.append("3.x") else: python_versions.append( classifier.trove.rsplit("::", 1)[1].strip()) if not licenses: licenses = [_("Unknown")] licenses = [x for x in licenses if x not in ["OSI Approved"]] licenses = [LICENSES.get(x, x) for x in licenses] data["licenses"] = licenses if not operating_systems: operating_systems = [_("Unknown")] data["operating_systems"] = operating_systems if not implementations: implementations = [_("Unknown")] data["implementations"] = implementations if not python_versions: python_versions = [_("Unknown")] data["python_versions"] = python_versions # Pack in all the versions in decending order. releases = obj.releases.all().order_by("-order") data["versions"] = [ release.version for release in releases if release.version ] data["release_count"] = releases.count() # We want to scale the boost for this document based on how many downloads have # been recorded for this package. # @@@ Might want to actually tier these values instead of percentage them. # Cap out downloads at 100k capped_downloads = min(data["downloads"], 10000) boost = capped_downloads / 10000.0 data["_boost"] = 1.0 + boost return data
class BovespaCompanyFileIndex(BaseSearchIndex, indexes.Indexable): ccvm = indexes.CharField(model_attr="ccvm", faceted=True) # The type of document doc_type = indexes.CharField(model_attr="doc_type", faceted=True) # The date of the data for the company fiscal_date = indexes.DateField(model_attr="fiscal_date", faceted=True) version = indexes.CharField(model_attr="version", faceted=True) status = indexes.CharField(model_attr="status", faceted=True) # When was created the entity and the last modification date created_at = indexes.DateTimeField(model_attr="created_at", faceted=True) updated_at = indexes.DateTimeField(model_attr="updated_at", faceted=True) is_deleted = indexes.BooleanField(model_attr="is_deleted", faceted=True) deleted_reason = indexes.CharField(model_attr="deleted_reason") # The url to the file that contains the information protocol = indexes.CharField(model_attr="protocol") # Date when the files where presented delivery_date = indexes.DateTimeField(model_attr="delivery_date", faceted=True) # The motivation of the delivery. delivery_type = indexes.CharField(model_attr="delivery_type", faceted=True) # The name of the company company_name = indexes.CharField(model_attr="company_name", faceted=True) # The cnpj of the company company_cnpj = indexes.CharField(model_attr="company_cnpj", faceted=True) # The Fiscal Date decomposed into year, quarter, month fiscal_date_y = indexes.IntegerField(model_attr="fiscal_date_y", faceted=True) fiscal_date_yd = indexes.IntegerField(model_attr="fiscal_date_yd", faceted=True) fiscal_date_q = indexes.IntegerField(model_attr="fiscal_date_q", faceted=True) fiscal_date_m = indexes.IntegerField(model_attr="fiscal_date_m", faceted=True) fiscal_date_md = indexes.IntegerField(model_attr="fiscal_date_md", faceted=True) fiscal_date_w = indexes.IntegerField(model_attr="fiscal_date_w", faceted=True) fiscal_date_wd = indexes.IntegerField(model_attr="fiscal_date_wd", faceted=True) fiscal_date_yq = indexes.CharField(model_attr="fiscal_date_yq", faceted=True) fiscal_date_ym = indexes.CharField(model_attr="fiscal_date_ym", faceted=True) # The url to the file that contains the information source_url = indexes.CharField(model_attr="source_url") # The url to the file that contains the information file_url = indexes.CharField(model_attr="file_url") # The name of the file file_name = indexes.CharField(model_attr="file_name") # The extension of the filename file_extension = indexes.CharField(model_attr="file_extension", faceted=True) # content = indexes.MultiValueField( # null=True, model_attr="content", faceted=True) class Meta: text_fields = [ "file_name", "source_url", "company_name", "company_cnpj", "delivery_type", "file_url", "deleted_reason", ] # Once the index has been created it cannot be changed # with sync_indexes. Changes should be made by hand. index_settings = { "realtime": "true", "autoCommitTime": "100", "ramBufferSize": "2048" } def get_model(self): return BovespaCompanyFile def index_queryset(self, using=None): return self.get_model().objects.filter(created_at__lte=timezone.now(), is_deleted=False)
class CourseRunIndex(BaseCourseIndex, indexes.Indexable): model = CourseRun course_key = indexes.CharField(model_attr='course__key', stored=True) org = indexes.CharField() number = indexes.CharField() status = indexes.CharField(model_attr='status', faceted=True) start = indexes.DateTimeField(model_attr='start', null=True, faceted=True) end = indexes.DateTimeField(model_attr='end', null=True) enrollment_start = indexes.DateTimeField(model_attr='enrollment_start', null=True) enrollment_end = indexes.DateTimeField(model_attr='enrollment_end', null=True) announcement = indexes.DateTimeField(model_attr='announcement', null=True) min_effort = indexes.IntegerField(model_attr='min_effort', null=True) max_effort = indexes.IntegerField(model_attr='max_effort', null=True) language = indexes.CharField(null=True, faceted=True) transcript_languages = indexes.MultiValueField(faceted=True) pacing_type = indexes.CharField(model_attr='pacing_type', null=True, faceted=True) marketing_url = indexes.CharField(null=True) slug = indexes.CharField(model_attr='slug', null=True) seat_types = indexes.MultiValueField(model_attr='seat_types', null=True, faceted=True) type = indexes.CharField(model_attr='type', null=True, faceted=True) image_url = indexes.CharField(model_attr='card_image_url', null=True) partner = indexes.CharField(null=True, faceted=True) program_types = indexes.MultiValueField() published = indexes.BooleanField(null=False, faceted=True) hidden = indexes.BooleanField(model_attr='hidden', faceted=True) mobile_available = indexes.BooleanField(model_attr='mobile_available', faceted=True) authoring_organization_uuids = indexes.MultiValueField() staff_uuids = indexes.MultiValueField() subject_uuids = indexes.MultiValueField() has_enrollable_paid_seats = indexes.BooleanField(null=False) paid_seat_enrollment_end = indexes.DateTimeField(null=True) def prepare_has_enrollable_paid_seats(self, obj): return obj.has_enrollable_paid_seats() def prepare_paid_seat_enrollment_end(self, obj): return obj.get_paid_seat_enrollment_end() def prepare_partner(self, obj): return obj.course.partner.short_code def prepare_published(self, obj): return obj.status == CourseRunStatus.Published def _prepare_language(self, language): if language: # ECOM-5466: Render the macro language for all languages except Chinese if language.code.startswith('zh'): return language.name else: return language.macrolanguage return None def prepare_language(self, obj): return self._prepare_language(obj.language) def prepare_number(self, obj): course_run_key = CourseKey.from_string(obj.key) return course_run_key.course def prepare_org(self, obj): course_run_key = CourseKey.from_string(obj.key) return course_run_key.org def prepare_transcript_languages(self, obj): return [ self._prepare_language(language) for language in obj.transcript_languages.all() ] def prepare_marketing_url(self, obj): return obj.marketing_url def prepare_program_types(self, obj): return obj.program_types def prepare_staff_uuids(self, obj): return [str(staff.uuid) for staff in obj.staff.all()] def prepare_subject_uuids(self, obj): return [str(subject.uuid) for subject in obj.subjects.all()]
class ReviewRequestIndex(BaseSearchIndex, indexes.Indexable): """A Haystack search index for Review Requests.""" model = ReviewRequest local_site_attr = 'local_site_id' # We shouldn't use 'id' as a field name because it's by default reserved # for Haystack. Hiding it will cause duplicates when updating the index. review_request_id = indexes.IntegerField(model_attr='display_id') summary = indexes.CharField(model_attr='summary') description = indexes.CharField(model_attr='description') testing_done = indexes.CharField(model_attr='testing_done') bug = indexes.CharField(model_attr='bugs_closed') username = indexes.CharField(model_attr='submitter__username') user_display_name = indexes.CharField() author = indexes.CharField(model_attr='submitter__get_full_name') last_updated = indexes.DateTimeField(model_attr='last_updated') url = indexes.CharField(model_attr='get_absolute_url') file = indexes.CharField() # These fields all contain information needed to perform queries about # whether a review request is accessible by a given user. private = indexes.BooleanField() private_repository_id = indexes.IntegerField() private_target_groups = indexes.MultiValueField() target_users = indexes.MultiValueField() def get_model(self): """Returns the Django model for this index.""" return ReviewRequest def get_updated_field(self): return 'last_updated' def index_queryset(self, using=None): """Index only public pending and submitted review requests.""" return ( self.get_model().objects .public(status=None, extra_query=Q(status='P') | Q(status='S'), show_all_local_sites=True, filter_private=False) .select_related('diffset_history', 'local_site', 'repository', 'submitter') .prefetch_related('diffset_history__diffsets__files', 'target_groups', 'target_people') ) def prepare_file(self, obj): return set([ (filediff.source_file, filediff.dest_file) for diffset in obj.diffset_history.diffsets.all() for filediff in diffset.files.all() ]) def prepare_private(self, review_request): """Prepare the private flag for the index. This will be set to true if the review request isn't generally accessible to users. """ return not review_request.is_accessible_by(AnonymousUser(), silent=True) def prepare_private_repository_id(self, review_request): """Prepare the private repository ID, if any, for the index. If there's no repository, or it's public, 0 will be returned instead. """ if review_request.repository and not review_request.repository.public: return review_request.repository_id else: return 0 def prepare_private_target_groups(self, review_request): """Prepare the list of invite-only target groups for the index. If there aren't any invite-only groups associated, ``[0]`` will be returned. This allows queries to be performed that check that none of the groups are private, since we can't query against empty lists. """ return [ group.pk for group in review_request.target_groups.all() if group.invite_only ] or [0] def prepare_target_users(self, review_request): """Prepare the list of target users for the index. If there aren't any target users, ``[0]`` will be returned. This allows queries to be performed that check that there aren't any users in the list, since we can't query against empty lists. """ return [ user.pk for user in review_request.target_people.all() ] or [0] def prepare_user_display_name(self, obj): return user_displayname(obj.submitter)
class LayerIndex(indexes.SearchIndex, indexes.Indexable): id = indexes.IntegerField(model_attr='resourcebase_ptr_id') abstract = indexes.CharField(model_attr="abstract", boost=1.5) category__gn_description = indexes.CharField( model_attr="category__gn_description", null=True) csw_type = indexes.CharField(model_attr="csw_type") csw_wkt_geometry = indexes.CharField(model_attr="csw_wkt_geometry") detail_url = indexes.CharField(model_attr="get_absolute_url") owner__username = indexes.CharField(model_attr="owner", faceted=True, null=True) is_published = indexes.BooleanField(model_attr="is_published") featured = indexes.BooleanField(model_attr="featured") popular_count = indexes.IntegerField(model_attr="popular_count", default=0, boost=20) share_count = indexes.IntegerField(model_attr="share_count", default=0) rating = indexes.IntegerField(null=True) srid = indexes.CharField(model_attr="srid") supplemental_information = indexes.CharField( model_attr="supplemental_information", null=True) thumbnail_url = indexes.CharField(model_attr="thumbnail_url", null=True) uuid = indexes.CharField(model_attr="uuid") title = indexes.CharField(model_attr="title", boost=2) date = indexes.DateTimeField(model_attr="date") text = indexes.EdgeNgramField(document=True, use_template=True, stored=False) type = indexes.CharField(faceted=True) subtype = indexes.CharField(faceted=True) alternate = indexes.CharField(model_attr='alternate') title_sortable = indexes.CharField(indexed=False, stored=False) # Necessary for sorting category = indexes.CharField(model_attr="category__identifier", faceted=True, null=True, stored=True) bbox_left = indexes.FloatField(model_attr="bbox_x0", null=True, stored=False) bbox_right = indexes.FloatField(model_attr="bbox_x1", null=True, stored=False) bbox_bottom = indexes.FloatField(model_attr="bbox_y0", null=True, stored=False) bbox_top = indexes.FloatField(model_attr="bbox_y1", null=True, stored=False) temporal_extent_start = indexes.DateTimeField( model_attr="temporal_extent_start", null=True, stored=False) temporal_extent_end = indexes.DateTimeField( model_attr="temporal_extent_end", null=True, stored=False) keywords = indexes.MultiValueField(model_attr="keyword_slug_list", null=True, faceted=True, stored=True) regions = indexes.MultiValueField(model_attr="region_name_list", null=True, faceted=True, stored=True) popular_count = indexes.IntegerField(model_attr="popular_count", default=0, boost=20) share_count = indexes.IntegerField(model_attr="share_count", default=0) rating = indexes.IntegerField(null=True) num_ratings = indexes.IntegerField(stored=False) num_comments = indexes.IntegerField(stored=False) def get_model(self): return Layer def prepare_type(self, obj): return "layer" def prepare_subtype(self, obj): if obj.storeType == "dataStore": if obj.has_time: return "vector_time" else: return "vector" elif obj.storeType == "coverageStore": return "raster" elif obj.storeType == "remoteStore": return "remote" def prepare_rating(self, obj): ct = ContentType.objects.get_for_model(obj) try: rating = OverallRating.objects.filter( object_id=obj.pk, content_type=ct).aggregate(r=Avg("rating"))["r"] return float(str(rating or "0")) except OverallRating.DoesNotExist: return 0.0 def prepare_num_ratings(self, obj): ct = ContentType.objects.get_for_model(obj) try: return OverallRating.objects.filter(object_id=obj.pk, content_type=ct).all().count() except OverallRating.DoesNotExist: return 0 def prepare_num_comments(self, obj): try: return Comment.objects.filter( object_id=obj.pk, content_type=ContentType.objects.get_for_model( obj)).all().count() except Exception: return 0 def prepare_title_sortable(self, obj): return obj.title.lower()
class ProjectIndex(CelerySearchIndex, indexes.Indexable): def get_model(self): return Project def get_updated_field(self): return 'updated' # needed text = indexes.CharField(document=True, use_template=True) acronym = indexes.CharField(indexed=True, stored=True, faceted=True) acronym_l = indexes.CharField(indexed=False, stored=True) cner_notes = indexes.CharField(indexed=True, stored=True) contacts = indexes.MultiValueField(indexed=True, stored=True, faceted=True) company_personnel = indexes.MultiValueField(indexed=True, stored=True, faceted=True) description = indexes.CharField(indexed=True, stored=True, faceted=True) disease_terms = indexes.MultiValueField(indexed=True, stored=True, faceted=True) end_year = indexes.IntegerField(indexed=True, stored=True, faceted=True) erp_notes = indexes.CharField(indexed=True, stored=True, faceted=True) funding_sources = indexes.MultiValueField(indexed=True, stored=True, faceted=True) gene_terms = indexes.MultiValueField(indexed=True, stored=True, faceted=True) has_cner = indexes.BooleanField(indexed=True, stored=True, faceted=True) has_erp = indexes.BooleanField(indexed=True, stored=True, faceted=True) has_legal_documents = indexes.BooleanField(indexed=True, stored=True, faceted=True) umbrella_project = indexes.CharField(indexed=True, stored=True, faceted=True) phenotype_terms = indexes.MultiValueField(indexed=True, stored=True, faceted=True) project_web_page = indexes.CharField(indexed=True, stored=True, faceted=True) publications = indexes.MultiValueField(indexed=True, stored=True) start_year = indexes.IntegerField(indexed=True, stored=True, faceted=True) study_terms = indexes.MultiValueField(indexed=True, stored=True, faceted=True) title = indexes.CharField(indexed=True, stored=True, faceted=True) title_l = indexes.CharField(indexed=False, stored=True) local_custodians = indexes.MultiValueField(indexed=True, stored=True, faceted=True) def prepare_acronym_l(self, obj): if obj.acronym: return obj.acronym.lower().strip() def prepare_title_l(self, obj): if obj.title: return obj.title.lower().strip() def prepare_contacts(self, obj): return [str(o) for o in obj.contacts.all()] def prepare_company_personnel(self, obj): return [str(o) for o in obj.company_personnel.all()] def prepare_funding_sources(self, obj): return [str(o) for o in obj.funding_sources.all()] def prepare_publications(self, obj): return [str(o) for o in obj.publications.all()] def prepare_local_custodians(self, obj): return [str(o) for o in obj.local_custodians.all()] def prepare_has_legal_documents(self, obj): return obj.legal_documents.exists() def prepare_acronym(self, obj): return obj.acronym def prepare_cner_notes(self, obj): return obj.cner_notes def prepare_description(self, obj): return obj.description def prepare_disease_terms(self, obj): return [str(o) for o in obj.disease_terms.all()] def prepare_end_year(self, obj): if obj.end_date: return obj.end_date.year else: return None def prepare_erp_notes(self, obj): return obj.erp_notes def prepare_gene_terms(self, obj): return [str(o) for o in obj.gene_terms.all()] def prepare_has_cner(self, obj): return obj.has_cner def prepare_has_erp(self, obj): return obj.has_erp def prepare_umbrella_project(self, obj): return obj.umbrella_project def prepare_phenotype_terms(self, obj): return [str(o) for o in obj.phenotype_terms.all()] def prepare_project_web_page(self, obj): return obj.project_web_page def prepare_start_year(self, obj): if obj.start_date: return obj.start_date.year else: return None def prepare_study_terms(self, obj): return [str(o) for o in obj.study_terms.all()] def prepare_title(self, obj): return obj.title
class StoryIndex(ObjectIndex, indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=True) title = indexes.CharField(model_attr="title", boost=DEFAULT_BOOST) facet_model_name = indexes.CharField(faceted=True) sort_name = indexes.CharField(model_attr='issue__series__sort_name', indexed=False) key_date = indexes.CharField(model_attr='issue__key_date', indexed=False) sort_code = indexes.IntegerField(model_attr='issue__sort_code', indexed=False) sequence_number = indexes.IntegerField(model_attr='sequence_number', indexed=False) type = indexes.CharField(model_attr='type__name', indexed=False) year = indexes.IntegerField() date = indexes.DateField(faceted=True) country = indexes.CharField(model_attr='issue__series__country__name', faceted=True, indexed=False) language = indexes.CharField(model_attr='issue__series__language__code', faceted=True, indexed=False) publisher = indexes.CharField(model_attr='issue__series__publisher__name', faceted=True, indexed=False) feature = indexes.MultiValueField(faceted=True, indexed=False, null=True) script = indexes.MultiValueField(faceted=True, null=True) pencils = indexes.MultiValueField(faceted=True, null=True) inks = indexes.MultiValueField(faceted=True, null=True) colors = indexes.MultiValueField(faceted=True, null=True) letters = indexes.MultiValueField(faceted=True, null=True) editing = indexes.MultiValueField(faceted=True, null=True) def get_model(self): return Story def prepare_facet_model_name(self, obj): return "story" def prepare_year(self, obj): if obj.issue.key_date: return int(obj.issue.key_date[:4]) else: return 9999 def prepare_date(self, obj): if obj.issue.key_date: year, month, day = on_sale_date_fields(obj.issue.key_date) if month < 1 or month > 12: month = 1 if day < 1 or day > 31: day = 1 try: return date(year, month, day) except ValueError: return date(year, month, 1) else: return None def prepare_key_date(self, obj): if obj.issue.key_date: return obj.issue.key_date else: return "9999-99-99" def _prepare_credit(self, obj, field): return_val = [(val.strip()) for val in getattr(obj, field).split(';')] credits = obj.active_credits.filter(credit_type__name=field) if credits: if return_val == ['']: return_val = [ val.creator.display_credit(val, url=False) for val in credits ] else: return_val.extend([ val.creator.display_credit(val, url=False) for val in credits ]) if return_val == ['']: return None else: return return_val def prepare_script(self, obj): return self._prepare_credit(obj, 'script') def prepare_pencils(self, obj): return self._prepare_credit(obj, 'pencils') def prepare_inks(self, obj): return self._prepare_credit(obj, 'inks') def prepare_colors(self, obj): return self._prepare_credit(obj, 'colors') def prepare_letters(self, obj): return self._prepare_credit(obj, 'letters') def prepare_editing(self, obj): return_val = [(val.strip()) for val in getattr(obj, 'editing').split(';')] return_val.extend([ (val.strip()) for val in getattr(obj.issue, 'editing').split(';') ]) credits = obj.active_credits.filter(credit_type__name='editing') if credits: if return_val == ['']: return_val = [ val.creator.display_credit(val, url=False) for val in credits ] else: return_val.extend([ val.creator.display_credit(val, url=False) for val in credits ]) credits = obj.issue.active_credits.filter(credit_type__name='editing') if credits: if return_val == ['']: return_val = [ val.creator.display_credit(val, url=False) for val in credits ] else: return_val.extend([ val.creator.display_credit(val, url=False) for val in credits ]) if return_val == ['']: return None else: return return_val def prepare_feature(self, obj): return_val = [(val.strip()) for val in getattr(obj, 'feature').split(';')] features = obj.feature_object.all() if features: if return_val == ['']: return_val = [val.name for val in features] else: return_val.extend([val.name for val in features]) if return_val == ['']: return None else: return return_val def index_queryset(self, using=None): """Used when the entire index for model is updated.""" return super(ObjectIndex, self).index_queryset(using).exclude( type=STORY_TYPES['blank']).filter(deleted=False)