class MappingVersionIndex(OCLSearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
external_id = SortOrFilterField(model_attr='external_id',
indexed=True,
stored=True,
null=True)
lastUpdate = indexes.DateTimeField(model_attr='updated_at',
indexed=True,
stored=True)
retired = indexes.BooleanField(model_attr='retired',
indexed=True,
stored=True,
faceted=True)
mapType = SortOrFilterField(model_attr='map_type',
indexed=True,
stored=True,
faceted=True)
source = SortOrFilterField(model_attr='source',
indexed=True,
stored=True,
faceted=True)
owner = SortOrFilterField(model_attr='owner',
indexed=True,
stored=True,
faceted=True)
ownerType = SortOrFilterField(model_attr='owner_type',
indexed=True,
stored=True,
faceted=True)
concept = FilterField(indexed=True, stored=True, faceted=True)
fromConcept = FilterField(indexed=True, stored=True, faceted=True)
toConcept = FilterField(indexed=True, stored=True, null=True, faceted=True)
conceptSource = FilterField(indexed=True, stored=True, faceted=True)
fromConceptSource = SortOrFilterField(indexed=True,
stored=True,
faceted=True)
toConceptSource = SortOrFilterField(indexed=True,
stored=True,
faceted=True)
conceptOwner = FilterField(faceted=True)
fromConceptOwner = SortOrFilterField(indexed=True,
stored=True,
faceted=True)
toConceptOwner = SortOrFilterField(indexed=True, stored=True, faceted=True)
conceptOwnerType = FilterField(faceted=True)
fromConceptOwnerType = SortOrFilterField(indexed=True,
stored=True,
faceted=True)
toConceptOwnerType = SortOrFilterField(indexed=True,
stored=True,
faceted=True)
source_version = FilterField()
collection = FilterField()
collection_version = FilterField()
public_can_view = indexes.BooleanField(model_attr='public_can_view',
indexed=True,
stored=True)
is_active = indexes.BooleanField(model_attr='is_active',
indexed=True,
stored=True)
is_latest_version = indexes.BooleanField(model_attr='is_latest_version',
indexed=True,
stored=True)
def get_model(self):
return MappingVersion
def prepare(self, obj):
self.prepared_data = super(MappingVersionIndex, self).prepare(obj)
self.prepared_data['fromConcept'] = [
obj.from_concept_url, obj.from_concept_code, obj.from_concept_name
]
self.prepared_data['toConcept'] = [
obj.get_to_concept_code(),
obj.get_to_concept_name()
]
self.prepared_data['concept'] = self.prepared_data[
'fromConcept'] + self.prepared_data['toConcept']
self.prepared_data['fromConceptSource'] = obj.from_source_name
self.prepared_data['toConceptSource'] = obj.to_source_name
self.prepared_data['conceptSource'] = [
obj.from_source_name, obj.to_source_name
]
self.prepared_data['fromConceptOwner'] = obj.from_source_owner
self.prepared_data['toConceptOwner'] = obj.to_source_owner
self.prepared_data['conceptOwner'] = [
obj.from_source_owner, obj.to_source_owner
]
self.prepared_data['fromConceptOwnerType'] = obj.from_source_owner_type
self.prepared_data['toConceptOwnerType'] = obj.to_source_owner_type
self.prepared_data['conceptOwnerType'] = [
obj.from_source_owner_type, obj.to_source_owner_type
]
source_version_ids = []
source = obj.parent
source_versions = SourceVersion.objects.filter(
versioned_object_id=source.id, )
for sv in source_versions:
if obj.id in sv.mappings:
source_version_ids.append(sv.id)
self.prepared_data['source_version'] = source_version_ids
return self.prepared_data
def prepare_collection_version(self, obj):
return obj.collection_version_ids
def prepare_collection(self, obj):
return obj.collection_ids
class MediaItemIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
item_acces = indexes.CharField(model_attr='collection__public_access',
faceted=True)
item_status = indexes.CharField(model_attr='collection__document_status',
faceted=True)
digitized = indexes.BooleanField(default=False, faceted=True)
media_type = indexes.CharField(model_attr='media_type',
null='None',
faceted=True)
recording_context = indexes.CharField(
model_attr='collection__recording_context', default='', faceted=True)
physical_format = indexes.CharField(
model_attr='collection__physical_format', default='', faceted=True)
#content_auto = indexes.EdgeNgramField(model_attr='content')
#advance search
title = indexes.CharField(model_attr='title')
code = KeywordField(model_attr='code', default='')
location_principal = indexes.CharField(null='None', boost=1.05)
location_relation = indexes.CharField()
ethnic_group = indexes.CharField(model_attr='ethnic_group', default='')
instruments = indexes.CharField(default='')
collectors = indexes.CharField(model_attr='collector', default='')
recorded_from_date = indexes.DateField(model_attr='recorded_from_date',
null='None')
recorded_to_date = indexes.DateField(model_attr='recorded_to_date',
null='None')
year_published = indexes.IntegerField(
model_attr='collection__year_published', default='')
def prepare_digitized(self, obj):
if obj.file.name:
return True
elif '/' in obj.url:
return True
else:
return False
def get_model(self):
return MediaItem
def prepare_location_principal(self, obj):
if obj.location is not None:
return u"".join(obj.location.name)
else:
return None
def prepare_location_relation(self, obj):
location = []
if obj.location is not None:
location_alias = LocationAlias.objects.filter(
location__name=obj.location)
location_rela = LocationRelation.objects.filter(
location__name=obj.location)
for rela in location_rela:
location.append(rela.ancestor_location.name)
for alias in location_alias:
location.append(alias.alias)
if obj.location.current_location is not None:
location.append(obj.location.current_location.name)
#print u"".join(' ' + local for local in location).encode("utf-8")
#print u"%s".encode("utf-8") % location
#print [local for local in location]
return u"".join('|' + local for local in location)
def prepare_instruments(self, obj):
item = MediaItemPerformance.objects.all().filter(media_item__exact=obj)
instruments = []
for material in item:
if material.instrument is not None:
instruments.append(material.instrument.name)
if material.alias is not None:
instruments.append(material.alias.name)
return u"".join('|' + instru for instru in instruments)
def prepare_collectors(self, obj):
collectors = []
collectors.append(obj.collection.collector)
collectors.append(obj.collector)
return u"".join('; ' + collector for collector in collectors)
class CourseRunIndex(BaseCourseIndex, indexes.Indexable):
model = CourseRun
course_key = indexes.CharField(model_attr='course__key', stored=True)
org = indexes.CharField()
number = indexes.CharField()
status = indexes.CharField(model_attr='status', faceted=True)
start = indexes.DateTimeField(model_attr='start', null=True, faceted=True)
end = indexes.DateTimeField(model_attr='end', null=True)
enrollment_start = indexes.DateTimeField(model_attr='enrollment_start', null=True)
enrollment_end = indexes.DateTimeField(model_attr='enrollment_end', null=True)
announcement = indexes.DateTimeField(model_attr='announcement', null=True)
min_effort = indexes.IntegerField(model_attr='min_effort', null=True)
max_effort = indexes.IntegerField(model_attr='max_effort', null=True)
weeks_to_complete = indexes.IntegerField(model_attr='weeks_to_complete', null=True)
language = indexes.CharField(null=True, faceted=True)
transcript_languages = indexes.MultiValueField(faceted=True)
pacing_type = indexes.CharField(model_attr='pacing_type', null=True, faceted=True)
marketing_url = indexes.CharField(null=True)
slug = indexes.CharField(model_attr='slug', null=True)
seat_types = indexes.MultiValueField(model_attr='seat_types', null=True, faceted=True)
type = indexes.CharField(model_attr='type', null=True, faceted=True)
image_url = indexes.CharField(model_attr='image_url', null=True)
partner = indexes.CharField(null=True, faceted=True)
program_types = indexes.MultiValueField()
published = indexes.BooleanField(null=False, faceted=True)
hidden = indexes.BooleanField(model_attr='hidden', faceted=True)
mobile_available = indexes.BooleanField(model_attr='mobile_available', faceted=True)
authoring_organization_uuids = indexes.MultiValueField()
staff_uuids = indexes.MultiValueField()
subject_uuids = indexes.MultiValueField()
has_enrollable_paid_seats = indexes.BooleanField(null=False)
first_enrollable_paid_seat_sku = indexes.CharField(null=True)
paid_seat_enrollment_end = indexes.DateTimeField(null=True)
license = indexes.MultiValueField(model_attr='license', faceted=True)
has_enrollable_seats = indexes.BooleanField(model_attr='has_enrollable_seats', null=False)
is_current_and_still_upgradeable = indexes.BooleanField(null=False)
def prepare_aggregation_key(self, obj):
# Aggregate CourseRuns by Course key since that is how we plan to dedup CourseRuns on the marketing site.
return 'courserun:{}'.format(obj.course.key)
def prepare_has_enrollable_paid_seats(self, obj):
return obj.has_enrollable_paid_seats()
def prepare_first_enrollable_paid_seat_sku(self, obj):
return obj.first_enrollable_paid_seat_sku()
def prepare_is_current_and_still_upgradeable(self, obj):
return obj.is_current_and_still_upgradeable()
def prepare_paid_seat_enrollment_end(self, obj):
return obj.get_paid_seat_enrollment_end()
def prepare_partner(self, obj):
return obj.course.partner.short_code
def prepare_published(self, obj):
return obj.status == CourseRunStatus.Published
def prepare_language(self, obj):
return self._prepare_language(obj.language)
def prepare_number(self, obj):
course_run_key = CourseKey.from_string(obj.key)
return course_run_key.course
def prepare_org(self, obj):
course_run_key = CourseKey.from_string(obj.key)
return course_run_key.org
def prepare_transcript_languages(self, obj):
return [self._prepare_language(language) for language in obj.transcript_languages.all()]
def prepare_marketing_url(self, obj):
return obj.marketing_url
def prepare_program_types(self, obj):
return obj.program_types
def prepare_staff_uuids(self, obj):
return [str(staff.uuid) for staff in obj.staff.all()]
def prepare_subject_uuids(self, obj):
return [str(subject.uuid) for subject in obj.subjects.all()]
class ProgramIndex(BaseIndex, indexes.Indexable, OrganizationsMixin):
model = Program
uuid = indexes.CharField(model_attr='uuid')
title = indexes.CharField(model_attr='title', boost=TITLE_FIELD_BOOST)
title_autocomplete = indexes.NgramField(model_attr='title',
boost=TITLE_FIELD_BOOST)
subtitle = indexes.CharField(model_attr='subtitle')
type = indexes.CharField(model_attr='type__name', faceted=True)
marketing_url = indexes.CharField(null=True)
search_card_display = indexes.MultiValueField()
organizations = indexes.MultiValueField(faceted=True)
authoring_organizations = indexes.MultiValueField(faceted=True)
authoring_organizations_autocomplete = indexes.NgramField(
boost=ORG_FIELD_BOOST)
authoring_organization_uuids = indexes.MultiValueField()
subject_uuids = indexes.MultiValueField()
staff_uuids = indexes.MultiValueField()
authoring_organization_bodies = indexes.MultiValueField()
credit_backing_organizations = indexes.MultiValueField(faceted=True)
card_image_url = indexes.CharField(model_attr='card_image_url', null=True)
status = indexes.CharField(model_attr='status', faceted=True)
partner = indexes.CharField(model_attr='partner__short_code',
null=True,
faceted=True)
start = indexes.DateTimeField(model_attr='start', null=True, faceted=True)
seat_types = indexes.MultiValueField(model_attr='seat_types__slug',
null=True,
faceted=True)
published = indexes.BooleanField(null=False, faceted=True)
min_hours_effort_per_week = indexes.IntegerField(
model_attr='min_hours_effort_per_week', null=True)
max_hours_effort_per_week = indexes.IntegerField(
model_attr='max_hours_effort_per_week', null=True)
weeks_to_complete_min = indexes.IntegerField(
model_attr='weeks_to_complete_min', null=True)
weeks_to_complete_max = indexes.IntegerField(
model_attr='weeks_to_complete_max', null=True)
language = indexes.MultiValueField(faceted=True)
hidden = indexes.BooleanField(model_attr='hidden', faceted=True)
is_program_eligible_for_one_click_purchase = indexes.BooleanField(
model_attr='is_program_eligible_for_one_click_purchase', null=False)
def prepare_aggregation_key(self, obj):
return 'program:{}'.format(obj.uuid)
def prepare_published(self, obj):
return obj.status == ProgramStatus.Active
def prepare_organizations(self, obj):
return self.prepare_authoring_organizations(
obj) + self.prepare_credit_backing_organizations(obj)
def prepare_subject_uuids(self, obj):
return [str(subject.uuid) for subject in obj.subjects]
def prepare_staff_uuids(self, obj):
return {
str(staff.uuid)
for course_run in obj.course_runs
for staff in course_run.staff.all()
}
def prepare_credit_backing_organizations(self, obj):
return self._prepare_organizations(
obj.credit_backing_organizations.all())
def prepare_marketing_url(self, obj):
return obj.marketing_url
def prepare_language(self, obj):
return [self._prepare_language(language) for language in obj.languages]
def prepare_search_card_display(self, obj):
try:
degree = Degree.objects.get(uuid=obj.uuid)
except Degree.DoesNotExist:
return []
return [
degree.search_card_ranking, degree.search_card_cost,
degree.search_card_courses
]
class _PageIndex(_get_index_base()):
_language = language_code
language = indexes.CharField()
text = indexes.CharField(document=True, use_template=False)
pub_date = indexes.DateTimeField(model_attr='publication_date',
null=True)
login_required = indexes.BooleanField(model_attr='login_required')
url = indexes.CharField(stored=True,
indexed=False,
model_attr='get_absolute_url')
title = indexes.CharField(stored=True,
indexed=False,
model_attr='get_title')
site_id = indexes.IntegerField(stored=True,
indexed=True,
model_attr='site_id')
def prepare(self, obj):
current_languge = get_language()
try:
if current_languge != self._language:
activate(self._language)
request = rf.get("/")
request.session = {}
request.LANGUAGE_CODE = self._language
self.prepared_data = super(_PageIndex, self).prepare(obj)
plugins = CMSPlugin.objects.filter(
language=language_code,
placeholder__in=obj.placeholders.all())
text = u''
for base_plugin in plugins:
instance, plugin_type = base_plugin.get_plugin_instance()
if instance is None:
# this is an empty plugin
continue
if hasattr(instance, 'search_fields'):
text += u' '.join(
force_unicode(
_strip_tags(getattr(instance, field, '')))
for field in instance.search_fields)
if getattr(instance, 'search_fulltext', False) or getattr(
plugin_type, 'search_fulltext', False):
text += _strip_tags(
instance.render_plugin(
context=RequestContext(request))) + u' '
text += obj.get_meta_description() or u''
text += u' '
text += obj.get_meta_keywords() or u''
self.prepared_data['text'] = text
self.prepared_data['language'] = self._language
return self.prepared_data
finally:
if get_language() != current_languge:
activate(current_languge)
def index_queryset(self):
# get the correct language and exclude pages that have a redirect
base_qs = super(_PageIndex, self).index_queryset()
result_qs = EmptyQuerySet()
for site_obj in Site.objects.all():
qs = base_qs.published(site=site_obj.id).filter(
Q(title_set__language=language_code)
& (Q(title_set__redirect__exact='')
| Q(title_set__redirect__isnull=True)))
if 'publisher' in settings.INSTALLED_APPS:
qs = qs.filter(publisher_is_draft=True)
qs = qs.distinct()
result_qs |= qs
return result_qs
class TendenciBaseSearchIndex(indexes.SearchIndex):
text = indexes.CharField(document=True, use_template=True)
# TendenciBaseModel Fields
allow_anonymous_view = indexes.BooleanField(
model_attr='allow_anonymous_view', null=True)
allow_user_view = indexes.BooleanField(model_attr='allow_user_view',
null=True)
allow_member_view = indexes.BooleanField(model_attr='allow_member_view',
null=True)
# allow_anonymous_edit = indexes.BooleanField(model_attr='allow_anonymous_edit')
allow_user_edit = indexes.BooleanField(model_attr='allow_user_edit',
null=True)
allow_member_edit = indexes.BooleanField(model_attr='allow_member_edit',
null=True)
creator = indexes.CharField(model_attr='creator', null=True)
creator_username = indexes.CharField(model_attr='creator_username',
null=True)
owner = indexes.CharField(model_attr='owner', null=True)
owner_username = indexes.CharField(model_attr='owner_username', null=True)
status = indexes.BooleanField(model_attr='status', null=True)
status_detail = indexes.CharField(model_attr='status_detail')
create_dt = indexes.DateTimeField(model_attr='create_dt', null=True)
update_dt = indexes.DateTimeField(model_attr='update_dt', null=True)
# permission fields
users_can_view = indexes.MultiValueField(null=True)
groups_can_view = indexes.MultiValueField(null=True)
# PK: needed for exclude list_tags
primary_key = indexes.CharField(model_attr='pk')
# add order field for sorting. the subclasses can override
# the prepare_order method to sort by a different field
order = indexes.DateTimeField()
def get_model(self):
return None
def prepare_allow_anonymous_view(self, obj):
if is_whoosh():
try:
temp = int(obj.allow_anonymous_view)
except TypeError:
temp = 0
return temp
return obj.allow_anonymous_view
def prepare_allow_user_view(self, obj):
if is_whoosh():
try:
temp = int(obj.allow_user_view)
except TypeError:
temp = 0
return temp
return obj.allow_user_view
def prepare_allow_member_view(self, obj):
if is_whoosh():
try:
temp = int(obj.allow_member_view)
except TypeError:
temp = 0
return temp
return obj.allow_member_view
def prepare_allow_user_edit(self, obj):
if is_whoosh():
try:
temp = int(obj.allow_user_edit)
except TypeError:
temp = 0
return temp
return obj.allow_user_edit
def prepare_allow_member_edit(self, obj):
if is_whoosh():
try:
temp = int(obj.allow_member_edit)
except TypeError:
temp = 0
return temp
return obj.allow_member_edit
def prepare_status(self, obj):
if is_whoosh():
try:
temp = int(obj.status)
except TypeError:
temp = 0
return temp
return obj.status
def prepare_order(self, obj):
return obj.create_dt
def get_updated_field(self):
return 'update_dt'
def prepare_users_can_view(self, obj):
"""
This needs to be overwritten if 'view' permission label does not follow the standard convention:
(app_label).view_(module_name)
"""
return ObjectPermission.objects.users_with_perms(
'%s.view_%s' % (obj._meta.app_label, obj._meta.model_name), obj)
def prepare_groups_can_view(self, obj):
"""
This needs to be overwritten if 'view' permission label does not follow the standard convention:
(app_label).view_(module_name)
"""
return ObjectPermission.objects.groups_with_perms(
'%s.view_%s' % (obj._meta.app_label, obj._meta.model_name), obj)
class OrganizationIndex(indexes.ModelSearchIndex, indexes.Indexable):
content_type = indexes.CharField(default='Organization')
type_of_needs = indexes.MultiValueField(indexed=True, stored=True)
type_of_supports = indexes.MultiValueField(indexed=True, stored=True)
topics = indexes.MultiValueField(indexed=True, stored=True)
organization_roles = indexes.MultiValueField(indexed=True, stored=True)
country = indexes.CharField(indexed=True, stored=True)
organization_primary_role = indexes.CharField(indexed=True, stored=True)
organization_types = indexes.MultiValueField(indexed=True, stored=True)
investor_types = indexes.MultiValueField(indexed=True, stored=True)
product_launch = indexes.CharField(indexed=True, stored=True)
funding = indexes.CharField(indexed=True, stored=True)
request_funding = indexes.CharField(indexed=True, stored=True)
growth_stage = indexes.MultiValueField(indexed=True, stored=True)
has_jobs = indexes.BooleanField(indexed=True, stored=True)
jobs_role = indexes.MultiValueField(indexed=True, stored=True)
jobs_position = indexes.MultiValueField(indexed=True, stored=True)
jobs_salary_min = indexes.MultiValueField(indexed=True, stored=True)
jobs_salary_max = indexes.MultiValueField(indexed=True, stored=True)
jobs_skill_set = indexes.MultiValueField(indexed=True, stored=True)
has_relation = indexes.BooleanField(indexed=True, stored=True, default=True)
is_published = indexes.BooleanField(indexed=True, stored=True)
popular = indexes.DecimalField(indexed=True, stored=True)
class Meta:
model = Organization
def index_queryset(self, using=None):
"Used when the entire index for model is updated."
return self.get_model().objects.filter()
def prepare_is_published(self, object):
# is_published = indexes.BooleanField(indexed=True, stored=True)
return bool(object.status > 0) and not object.is_deleted
def prepare_type_of_needs(self, object):
return [inst.permalink for inst in object.type_of_needs.all()]
def prepare_type_of_supports(self, object):
return [inst.permalink for inst in object.type_of_supports.all()]
def prepare_topics(self, object):
return [inst.permalink for inst in object.topics.all()]
def prepare_organization_roles(self, object):
return [inst.permalink for inst in object.organization_roles.all()]
def prepare_country(self, object):
if object.country:
return object.country.permalink
else:
return ''
def prepare_organization_primary_role(self, object):
if object.organization_primary_role:
return object.organization_primary_role.permalink
else:
return ''
def prepare_organization_types(self, object):
return [inst.permalink for inst in object.organization_types.all()]
def prepare_investor_types(self, object):
return [inst.permalink for inst in object.investor_types.all()]
def prepare_product_launch(self, object):
if object.product_launch:
return object.product_launch.permalink
else:
return ''
def prepare_funding(self, object):
if object.funding:
return object.funding.permalink
else:
return ''
def prepare_request_funding(self, object):
if object.request_funding:
return object.request_funding.permalink
else:
return ''
def prepare_growth_stage(self, object):
return [inst.permalink for inst in object.growth_stage.all()]
def prepare_has_jobs(self, object):
return bool(object.jobs.all().count())
def prepare_jobs_role(self, object):
return [inst.role for inst in object.jobs.all()]
def prepare_jobs_position(self, object):
return [inst.position for inst in object.jobs.all()]
def prepare_jobs_salary_min(self, object):
return [inst.salary_min for inst in object.jobs.all()]
def prepare_jobs_salary_max(self, object):
return [inst.salary_max for inst in object.jobs.all()]
def prepare_jobs_skill_set(self, object):
skill_set = set([])
for inst in object.jobs.all():
for skill in inst.skill_set.all():
skill_set |= set([skill.name])
return list(skill_set)
def prepare_popular(self, object):
return object.popular
class PersonIndex(BaseIndex, indexes.Indexable):
name_auto = indexes.EdgeNgramField(model_attr='name')
hidden = indexes.BooleanField(model_attr='hidden')
def get_model(self):
return core_models.Person
class ProductIndex(CelerySearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
condition = indexes.CharField(model_attr='condition', faceted=True)
category = indexes.MultiValueField(faceted=True)
location = indexes.MultiValueField(faceted=True)
variants = indexes.MultiValueField(faceted=True)
price = indexes.FloatField(model_attr='price', null=True)
brand = indexes.CharField(model_attr='brand', faceted=True, null=True)
negotiable = indexes.BooleanField(model_attr='negotiable')
exchangeable = indexes.BooleanField(model_attr='exchangeable')
status = indexes.CharField(model_attr='status')
postedDate = indexes.DateTimeField(model_attr='postedDate')
title = indexes.CharField(model_attr='title', indexed=False)
description = indexes.CharField(model_attr='description', indexed=False)
previewImageURL = indexes.CharField(indexed=False)
thumbnailImageURL = indexes.CharField(indexed=False)
geoLocation = indexes.LocationField(null=True)
def get_model(self):
return Product
def prepare_brand(self, obj):
return obj.brand.name
def prepare_geoLocation(self, obj):
locationName = obj.location.name
@cached_as(Location, extra=locationName)
def __getCorrdinates():
payload = {
'address': locationName,
'components':
'country:%s' % settings.GOOGLE_MAP_API_SEARCH_COUNTRY_CODE,
'key': settings.GOOGLE_MAP_API_KEY
}
request = requests.get(settings.GOOGLE_MAP_API_RESUORCE_URL,
params=payload)
result = request.json()
if (result["status"] == "OK"):
lat = request.json(
)["results"][0]['geometry']['location']["lat"]
lng = request.json(
)["results"][0]['geometry']['location']["lng"]
return "%s,%s" % (lat, lng)
else:
return None
result = __getCorrdinates()
return result
def prepare_condition(self, obj):
return dict(Product.PRODUCT_CONDITIONS).get(obj.condition)
def prepare_previewImageURL(self, obj):
productImages = ProductImage.objects.filter(product__pk=obj.pk)
if productImages:
firstImage = productImages[0]
imageToSend = ProductImage.objects.get(pk=firstImage.pk)
url = imageToSend.image.url
pathWithoutExtension = url[0:url.find('.') - 1]
fileExtension = url[url.find('.') + 1:len(url)]
url = "%s_%s.%s" % (pathWithoutExtension, 'preview', fileExtension)
return url
else:
return 'NONE'
def prepare_thumbnailImageURL(self, obj):
productImages = ProductImage.objects.filter(product__pk=obj.pk)
if productImages:
firstImage = productImages[0]
imageToSend = ProductImage.objects.get(pk=firstImage.pk)
url = imageToSend.image.url
pathWithoutExtension = url[0:url.find('.') - 1]
fileExtension = url[url.find('.') + 1:len(url)]
url = "%s_%s.%s" % (pathWithoutExtension, 'thumbnail',
fileExtension)
return url
else:
return 'NONE'
def prepare_postedDate(self, obj):
return obj.postedDate.strftime('%Y-%m-%dT%H:%M:%SZ')
def prepare_category(self, obj):
output = []
currentCategory = obj.category
categoryArray = []
while (currentCategory is not None):
categoryArray.append(currentCategory.pk)
currentCategory = currentCategory.parentCategory
return categoryArray + [-1]
def prepare_location(self, obj):
output = []
currentLocation = obj.location
locationArray = []
while (currentLocation is not None):
locationArray.append(currentLocation.pk)
currentLocation = currentLocation.parentLocation
return locationArray + [-1]
def prepare_variants(self, obj):
output = []
productDataList = ProductData.objects.filter(product__pk=long(obj.pk))
for productDataItem in productDataList:
if productDataItem.productAttribute.type in [
'CHECKBOX', 'SELECT', 'RADIO'
]:
productDataSelectValues = ProductDataSelectValue.objects.filter(
productData__pk=long(productDataItem.pk))
for productDataSelectItem in productDataSelectValues:
output.append('>'.join([
str(productDataItem.productAttribute.pk),
str(productDataSelectItem.selectValue.pk)
]))
return output
specialties = indexes.MultiValueField(
null=True, model_attr="specialties", faceted=True)
websites = indexes.MultiValueField(
null=True, model_attr="websites")
extra_data = indexes.CharField(
model_attr="extra_data")
# When was created the entity and the last modification date
created_at = indexes.DateTimeField(
model_attr="created_at", faceted=True)
updated_at = indexes.DateTimeField(
model_attr="updated_at", faceted=True)
is_deleted = indexes.BooleanField(
model_attr="is_deleted", faceted=True)
deleted_reason = indexes.CharField(
model_attr="deleted_reason")
class Meta:
text_fields = ["short_description", "long_description", "extra_data"]
# Once the index has been created it cannot be changed
# with sync_indexes. Changes should be made by hand.
index_settings = {
"realtime": "true",
"autoCommitTime": "100",
"ramBufferSize": "2048"
}
class TopicIndex(TxnAwareSearchIndex, indexes.Indexable):
document = indexes.CharField(document=True)
topic_source_id = indexes.CharField(model_attr="source_id")
topic_issuer_id = indexes.IntegerField()
topic_type_id = indexes.IntegerField()
topic_inactive = indexes.BooleanField()
topic_revoked = indexes.BooleanField()
topic_name = indexes.MultiValueField()
topic_address = indexes.MultiValueField()
topic_category = indexes.MultiValueField()
topic_credential_type_id = indexes.MultiValueField()
topic_all_credentials_inactive = indexes.BooleanField()
topic_all_credentials_revoked = indexes.BooleanField()
def get_model(self):
return TopicModel
@staticmethod
def prepare_topic_issuer_id(obj):
if obj.foundational_credential:
return obj.foundational_credential.credential_type.issuer_id
return None
@staticmethod
def prepare_topic_type_id(obj):
if obj.foundational_credential:
return obj.foundational_credential.credential_type_id
return None
@staticmethod
def prepare_topic_inactive(obj):
if obj.foundational_credential:
return obj.foundational_credential.inactive
return None
@staticmethod
def prepare_topic_revoked(obj):
if obj.foundational_credential:
return obj.foundational_credential.revoked
return None
@staticmethod
def prepare_topic_category(obj):
if obj.foundational_credential:
return [
f"{cat.type}::{cat.value}"
for cat in obj.foundational_credential.all_categories
]
return []
@staticmethod
def prepare_topic_name(obj):
# May need to expand this to inactive credentials
return [name.text for name in obj.get_active_names()]
@staticmethod
def prepare_topic_address(obj):
# May need to expand this to inactive credentials
return [
address.civic_address for address in obj.get_active_addresses()
]
@staticmethod
def prepare_topic_credential_type_id(obj):
# May need to expand this to inactive credentials
credential_type_ids = obj.get_active_credential_type_ids()
if credential_type_ids:
return list(credential_type_ids)
return []
@staticmethod
def prepare_topic_all_credentials_inactive(obj):
all_creds_inactive = True
for credential in obj.credentials.all():
if not credential.inactive:
all_creds_inactive = False
return all_creds_inactive
@staticmethod
def prepare_topic_all_credentials_revoked(obj):
all_creds_revoked = True
for credential in obj.credentials.all():
if not credential.revoked:
all_creds_revoked = False
return all_creds_revoked
def get_updated_field(self):
return "update_timestamp"
class MediaIndex(indexes.SearchIndex):
is_public = indexes.BooleanField(default=True)
datatype = indexes.CharField(model_attr='datatype')
author = indexes.MultiValueField(faceted=True)
source = indexes.MultiValueField(faceted=True)
tag = indexes.MultiValueField(faceted=True)
taxon = indexes.MultiValueField(faceted=True)
tour = indexes.MultiValueField()
reference = indexes.MultiValueField()
size = indexes.IntegerField(model_attr='size__id', default=0, faceted=True)
sublocation = indexes.CharField(model_attr='sublocation__id',
default=0,
faceted=True)
city = indexes.CharField(model_attr='city__id', default=0, faceted=True)
state = indexes.CharField(model_attr='state__id', default=0, faceted=True)
country = indexes.CharField(model_attr='country__id',
default=0,
faceted=True)
stats__pageviews = indexes.IntegerField(model_attr='stats__pageviews',
default=0)
timestamp = indexes.DateField(model_attr='timestamp', null=True)
date = indexes.DateField(model_attr='date', null=True)
pub_date = indexes.DateField(model_attr='pub_date', null=True)
id = indexes.IntegerField(model_attr='id')
highlight = indexes.BooleanField(model_attr='highlight', default=False)
title_en = indexes.CharField(model_attr='title_en', default='')
title = indexes.CharField(model_attr='title_pt')
thumb = indexes.CharField(model_attr='thumb_filepath')
url = indexes.CharField(model_attr='get_absolute_url')
def prepare_author(self, media):
return [author.id for author in media.author_set.all()]
# "%s##%s" % (author.slug, author.name,) for author in media.author_set.all()]
# Author.objects.filter(images__pk = object.pk)]
def prepare_source(self, media):
return [source.id for source in media.source_set.all()]
# "%s##%s" % (source.slug, source.name,) for source in media.source_set.all()]
# Source.objects.filter(images__pk = object.pk)]
def prepare_tag(self, media):
return [tag.id for tag in media.tag_set.all()]
#Tag.objects.filter(images__pk = object.pk)]
def prepare_taxon(self, media):
return [taxon.id for taxon in media.taxon_set.all()]
#Taxon.objects.filter(images__pk = object.pk)]
def prepare_tour(self, media):
return [tour.id for tour in media.tour_set.all()]
#Taxon.objects.filter(images__pk = object.pk)]
def prepare_reference(self, media):
return [reference.id for reference in media.reference_set.all()]
#Taxon.objects.filter(images__pk = object.pk)]
def index_queryset(self):
'''Used when the entire index for model is updated.'''
return self.get_model().objects.filter(is_public=True)
#select_related('author', 'tag', 'taxon', 'size', 'sublocation', 'city', 'state', 'country', 'rights')
def prepare(self, obj):
'''Fetches and adds/alters data before indexing.'''
self.prepared_data = super(MediaIndex, self).prepare(obj)
u = type(unicode())
for key, data in self.prepared_data.items():
if type(data) == u:
self.prepared_data[key] = strip_accents(data)
return self.prepared_data
class EventIndex(indexes.SearchIndex, indexes.Indexable):
# text: multiple fields use to do full-text search
text = indexes.MultiValueField(document=True, stored=False)
title = indexes.CharField(model_attr='title')
description = indexes.CharField(model_attr='description', null=True)
slug = indexes.CharField(model_attr='slug', null=False)
start = indexes.DateTimeField(model_attr='start', faceted=True)
speakers = indexes.MultiValueField(faceted=True, null=True)
department = indexes.CharField(faceted=True, null=True)
location = indexes.CharField(faceted=True, null=True)
topics = indexes.MultiValueField(faceted=True, null=True)
is_published = indexes.BooleanField(null=False)
is_cancelled = indexes.BooleanField(null=False)
group = indexes.CharField(faceted=True, null=True)
group_slug = indexes.CharField(null=True)
lists = indexes.MultiValueField(faceted=True, null=True)
def get_model(self):
return Event
def index_queryset(self, using=None):
"""Used when the entire index for model is updated."""
#return self.get_model().objects.filter(pub_date__lte=datetime.datetime.now())
return self.get_model().objects.all()
def prepare(self, obj):
"""Overriding the prepare() method of SearchIndex in order to add our most complicated fields
It is done in prepare() rather than the individual prepare_FIELD() to avoid having to do
multiple calls to the APIs...
"""
self.prepared_data = super(EventIndex, self).prepare(obj)
topics = obj.api_topics
topics_pref_labels = []
topic_alt_labels = []
if topics:
for topic in topics:
topics_pref_labels.append(topic.get('prefLabel', ''))
topic_alt_labels.extend(topic.get('altLabels', []))
if obj.department_organiser:
api_dept = obj.api_organisation
else:
api_dept = None
if obj.location:
api_loc = obj.api_location
else:
api_loc = None
speakers_names = [speaker.name for speaker in obj.speakers.all()]
# Speakers
self.prepared_data[self.speakers.index_fieldname] = speakers_names
# Department organiser
if api_dept:
self.prepared_data[self.department.index_fieldname] = api_dept.get(
'name', '')
# Location
if api_loc:
self.prepared_data[self.location.index_fieldname] = api_loc.get(
'name', '')
# Topics
if topics_pref_labels:
self.prepared_data[
self.topics.index_fieldname] = topics_pref_labels
# Published status
self.prepared_data[
self.is_published.index_fieldname] = obj.is_published
self.prepared_data[
self.is_cancelled.index_fieldname] = obj.is_cancelled
# Series name
if obj.group:
self.prepared_data[self.group.index_fieldname] = obj.group.title
self.prepared_data[
self.group_slug.index_fieldname] = obj.group.slug
# lists
lists_names = [
list.title
for list in obj.public_collections_containing_this_event.all()
]
self.prepared_data[self.lists.index_fieldname] = lists_names
full_text_content = [] # used when searching full text
if obj.title:
full_text_content.append(obj.title)
if obj.description:
full_text_content.append(obj.description)
if topics_pref_labels:
full_text_content.extend(topics_pref_labels)
if topic_alt_labels:
full_text_content.extend(topic_alt_labels)
if obj.group:
full_text_content.append(obj.group.title)
full_text_content.extend(speakers_names)
full_text_content.extend(lists_names)
self.prepared_data[self.text.index_fieldname] = full_text_content
return self.prepared_data
class VoyageIndex(indexes.SearchIndex, indexes.Indexable):
"""
Index method for class Voyage.
"""
# NOTE: To add support for a new language, a simple regular expression seach-and replace
# can be used to create a variable with the additional language. For instance, the following
# regular expression adds a language "es" for each variable with language "pt":
#
# Search: (.*)lang_pt(.*)
# Replace: $0\n$1lang_es$2
#
# A similar substitution can be used in schema.xml and managed-schema.xml
# There you should first create a fieldType for the language (e.g. with
# stopwords in the matching language).
text = indexes.CharField(document=True, use_template=True)
var_imp_voyage_began = indexes.IntegerField(null=True)
var_voyage_id = indexes.IntegerField(null=True, model_attr='voyage_id')
var_voyage_in_cd_rom = indexes.BooleanField(null=True, model_attr="voyage_in_cd_rom")
var_ship_name = indexes.NgramField(null=True, model_attr='voyage_ship__ship_name')
var_ship_name_plaintext = indexes.CharField(null=True, faceted=True, indexed=True, model_attr='voyage_ship__ship_name')
var_nationality = indexes.CharField(null=True, model_attr='voyage_ship__nationality_ship__label')
var_nationality_plaintext = indexes.CharField(null=True, faceted=True, model_attr='voyage_ship__nationality_ship__label')
var_imputed_nationality = indexes.CharField(null=True, model_attr='voyage_ship__imputed_nationality__label')
var_imputed_nationality_plaintext = indexes.CharField(null=True, faceted=True, model_attr='voyage_ship__imputed_nationality__label')
var_vessel_construction_place = indexes.CharField(null=True, model_attr='voyage_ship__vessel_construction_place__place')
var_vessel_construction_place_lang_en = TranslatedTextField(null=True, model_attr='voyage_ship__vessel_construction_place__place')
var_vessel_construction_place_lang_pt = TranslatedTextField(null=True, model_attr='voyage_ship__vessel_construction_place__place')
var_vessel_construction_place_lang_es = TranslatedTextField(null=True, model_attr='voyage_ship__vessel_construction_place__place')
var_year_of_construction = indexes.IntegerField(null=True, model_attr='voyage_ship__year_of_construction')
var_registered_place = indexes.CharField(null=True, model_attr='voyage_ship__registered_place__place')
var_registered_place_lang_en = TranslatedTextField(null=True, model_attr='voyage_ship__registered_place__place')
var_registered_place_lang_pt = TranslatedTextField(null=True, model_attr='voyage_ship__registered_place__place')
var_registered_place_lang_es = TranslatedTextField(null=True, model_attr='voyage_ship__registered_place__place')
var_registered_year = indexes.IntegerField(null=True, model_attr='voyage_ship__registered_year')
var_rig_of_vessel = indexes.CharField(null=True, model_attr='voyage_ship__rig_of_vessel__label')
var_rig_of_vessel_plaintext = indexes.CharField(null=True, faceted=True, model_attr='voyage_ship__rig_of_vessel__label')
var_tonnage = indexes.FloatField(null=True, faceted=True, model_attr='voyage_ship__tonnage')
var_tonnage_mod = indexes.FloatField(null=True, model_attr='voyage_ship__tonnage_mod')
var_guns_mounted = indexes.IntegerField(null=True, model_attr='voyage_ship__guns_mounted')
var_owner = indexes.NgramField(null=True)
var_owner_plaintext = indexes.CharField(null=True, faceted=True)
var_nationality_idnum = indexes.IntegerField(null=True, model_attr='voyage_ship__nationality_ship__value')
var_imputed_nationality_idnum = indexes.IntegerField(null=True, model_attr='voyage_ship__imputed_nationality__value')
var_vessel_construction_place_idnum = indexes.IntegerField(null=True, model_attr='voyage_ship__vessel_construction_place__value')
var_registered_place_idnum = indexes.IntegerField(null=True, model_attr='voyage_ship__registered_place__value')
var_rig_of_vessel_idnum = indexes.IntegerField(null=True, model_attr='voyage_ship__rig_of_vessel__value')
# Voyage Outcome
var_outcome_voyage = RelatedCharField(null=True, related_model=VoyageOutcome, model_attr='particular_outcome__label')
var_outcome_voyage_lang_en = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='particular_outcome__label')
var_outcome_voyage_lang_pt = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='particular_outcome__label')
var_outcome_voyage_lang_es = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='particular_outcome__label')
var_outcome_slaves = RelatedCharField(null=True, related_model=VoyageOutcome, model_attr='outcome_slaves__label')
var_outcome_slaves_lang_en = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='outcome_slaves__label')
var_outcome_slaves_lang_pt = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='outcome_slaves__label')
var_outcome_slaves_lang_es = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='outcome_slaves__label')
var_outcome_ship_captured = RelatedCharField(null=True, related_model=VoyageOutcome, model_attr='vessel_captured_outcome__label')
var_outcome_ship_captured_lang_en = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='vessel_captured_outcome__label')
var_outcome_ship_captured_lang_pt = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='vessel_captured_outcome__label')
var_outcome_ship_captured_lang_es = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='vessel_captured_outcome__label')
var_outcome_owner = RelatedCharField(null=True, related_model=VoyageOutcome, model_attr='outcome_owner__label')
var_outcome_owner_lang_en = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='outcome_owner__label')
var_outcome_owner_lang_pt = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='outcome_owner__label')
var_outcome_owner_lang_es = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='outcome_owner__label')
var_resistance = RelatedCharField(null=True, related_model=VoyageOutcome, model_attr='resistance__label')
var_resistance_lang_en = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='resistance__label')
var_resistance_lang_pt = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='resistance__label')
var_resistance_lang_es = RelatedTranslatedTextField(null=True, related_model=VoyageOutcome, model_attr='resistance__label')
var_outcome_voyage_idnum = RelatedIntegerField(null=True, related_model=VoyageOutcome, model_attr='particular_outcome__value')
var_outcome_slaves_idnum = RelatedIntegerField(null=True, related_model=VoyageOutcome, model_attr='outcome_slaves__value')
var_outcome_ship_captured_idnum = RelatedIntegerField(null=True, related_model=VoyageOutcome, model_attr='vessel_captured_outcome__value')
var_outcome_owner_idnum = RelatedIntegerField(null=True, related_model=VoyageOutcome, model_attr='outcome_owner__value')
var_resistance_idnum = RelatedIntegerField(null=True, related_model=VoyageOutcome, model_attr='resistance__value')
# Voyage itinerary
var_imp_port_voyage_begin = indexes.CharField(null=True, model_attr='voyage_itinerary__imp_port_voyage_begin__place')
var_imp_port_voyage_begin_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_port_voyage_begin__place')
var_imp_port_voyage_begin_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_port_voyage_begin__place')
var_imp_port_voyage_begin_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_port_voyage_begin__place')
var_first_place_slave_purchase = indexes.CharField(null=True, model_attr='voyage_itinerary__first_place_slave_purchase__place')
var_first_place_slave_purchase_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_place_slave_purchase__place')
var_first_place_slave_purchase_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_place_slave_purchase__place')
var_first_place_slave_purchase_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_place_slave_purchase__place')
var_second_place_slave_purchase = indexes.CharField(null=True, model_attr='voyage_itinerary__second_place_slave_purchase__place')
var_second_place_slave_purchase_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_place_slave_purchase__place')
var_second_place_slave_purchase_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_place_slave_purchase__place')
var_second_place_slave_purchase_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_place_slave_purchase__place')
var_third_place_slave_purchase = indexes.CharField(null=True, model_attr='voyage_itinerary__third_place_slave_purchase__place')
var_third_place_slave_purchase_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_place_slave_purchase__place')
var_third_place_slave_purchase_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_place_slave_purchase__place')
var_third_place_slave_purchase_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_place_slave_purchase__place')
var_imp_principal_place_of_slave_purchase = indexes.CharField(null=True, model_attr='voyage_itinerary__imp_principal_place_of_slave_purchase__place')
var_imp_principal_place_of_slave_purchase_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_place_of_slave_purchase__place')
var_imp_principal_place_of_slave_purchase_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_place_of_slave_purchase__place')
var_imp_principal_place_of_slave_purchase_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_place_of_slave_purchase__place')
var_port_of_call_before_atl_crossing = indexes.NgramField(null=True, model_attr='voyage_itinerary__port_of_call_before_atl_crossing__place')
var_port_of_call_before_atl_crossing_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__port_of_call_before_atl_crossing__place')
var_port_of_call_before_atl_crossing_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__port_of_call_before_atl_crossing__place')
var_port_of_call_before_atl_crossing_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__port_of_call_before_atl_crossing__place')
var_first_landing_place = indexes.CharField(null=True, model_attr='voyage_itinerary__first_landing_place__place')
var_first_landing_place_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_landing_place__place')
var_first_landing_place_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_landing_place__place')
var_first_landing_place_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_landing_place__place')
var_second_landing_place = indexes.CharField(null=True, model_attr='voyage_itinerary__second_landing_place__place')
var_second_landing_place_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_landing_place__place')
var_second_landing_place_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_landing_place__place')
var_second_landing_place_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_landing_place__place')
var_third_landing_place = indexes.CharField(null=True, model_attr='voyage_itinerary__third_landing_place__place')
var_third_landing_place_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_landing_place__place')
var_third_landing_place_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_landing_place__place')
var_third_landing_place_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_landing_place__place')
var_imp_principal_port_slave_dis = indexes.NgramField(null=True, model_attr='voyage_itinerary__imp_principal_port_slave_dis__place')
var_imp_principal_port_slave_dis_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_port_slave_dis__place')
var_imp_principal_port_slave_dis_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_port_slave_dis__place')
var_imp_principal_port_slave_dis_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_port_slave_dis__place')
var_place_voyage_ended = indexes.CharField(null=True, model_attr='voyage_itinerary__place_voyage_ended__place')
var_place_voyage_ended_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__place_voyage_ended__place')
var_place_voyage_ended_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__place_voyage_ended__place')
var_place_voyage_ended_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__place_voyage_ended__place')
var_imp_port_voyage_begin_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__imp_port_voyage_begin__value')
var_first_place_slave_purchase_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__first_place_slave_purchase__value')
var_second_place_slave_purchase_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__second_place_slave_purchase__value')
var_third_place_slave_purchase_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__third_place_slave_purchase__value')
var_imp_principal_place_of_slave_purchase_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__imp_principal_place_of_slave_purchase__value')
var_port_of_call_before_atl_crossing_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__port_of_call_before_atl_crossing__value')
var_first_landing_place_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__first_landing_place__value')
var_second_landing_place_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__second_landing_place__value')
var_third_landing_place_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__third_landing_place__value')
var_imp_principal_port_slave_dis_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__imp_principal_port_slave_dis__value')
var_place_voyage_ended_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__place_voyage_ended__value')
## Region variables
var_imp_region_voyage_begin = indexes.CharField(null=True, model_attr='voyage_itinerary__imp_region_voyage_begin__region')
var_imp_region_voyage_begin_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_region_voyage_begin__region')
var_imp_region_voyage_begin_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_region_voyage_begin__region')
var_imp_region_voyage_begin_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_region_voyage_begin__region')
var_first_region_slave_emb = indexes.CharField(null=True, model_attr='voyage_itinerary__first_region_slave_emb__region')
var_first_region_slave_emb_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_region_slave_emb__region')
var_first_region_slave_emb_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_region_slave_emb__region')
var_first_region_slave_emb_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_region_slave_emb__region')
var_second_region_slave_emb = indexes.CharField(null=True, model_attr='voyage_itinerary__second_region_slave_emb__region')
var_second_region_slave_emb_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_region_slave_emb__region')
var_second_region_slave_emb_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_region_slave_emb__region')
var_second_region_slave_emb_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_region_slave_emb__region')
var_third_region_slave_emb = indexes.CharField(null=True, model_attr='voyage_itinerary__third_region_slave_emb__region')
var_third_region_slave_emb_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_region_slave_emb__region')
var_third_region_slave_emb_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_region_slave_emb__region')
var_third_region_slave_emb_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_region_slave_emb__region')
var_imp_principal_region_of_slave_purchase = indexes.CharField(null=True, model_attr='voyage_itinerary__imp_principal_region_of_slave_purchase__region')
var_imp_principal_region_of_slave_purchase_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_region_of_slave_purchase__region')
var_imp_principal_region_of_slave_purchase_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_region_of_slave_purchase__region')
var_imp_principal_region_of_slave_purchase_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_region_of_slave_purchase__region')
var_first_landing_region = indexes.CharField(null=True, model_attr='voyage_itinerary__first_landing_region__region')
var_first_landing_region_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_landing_region__region')
var_first_landing_region_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_landing_region__region')
var_first_landing_region_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__first_landing_region__region')
var_second_landing_region = indexes.CharField(null=True, model_attr='voyage_itinerary__second_landing_region__region')
var_second_landing_region_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_landing_region__region')
var_second_landing_region_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_landing_region__region')
var_second_landing_region_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__second_landing_region__region')
var_third_landing_region = indexes.CharField(null=True, model_attr='voyage_itinerary__third_landing_region__region')
var_third_landing_region_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_landing_region__region')
var_third_landing_region_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_landing_region__region')
var_third_landing_region_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__third_landing_region__region')
var_imp_principal_region_slave_dis = indexes.CharField(null=True, model_attr='voyage_itinerary__imp_principal_region_slave_dis__region')
var_imp_principal_region_slave_dis_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_region_slave_dis__region')
var_imp_principal_region_slave_dis_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_region_slave_dis__region')
var_imp_principal_region_slave_dis_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__imp_principal_region_slave_dis__region')
var_region_voyage_ended = indexes.CharField(null=True, model_attr='voyage_itinerary__place_voyage_ended__region')
var_region_voyage_ended_lang_en = TranslatedTextField(null=True, model_attr='voyage_itinerary__place_voyage_ended__region')
var_region_voyage_ended_lang_pt = TranslatedTextField(null=True, model_attr='voyage_itinerary__place_voyage_ended__region')
var_region_voyage_ended_lang_es = TranslatedTextField(null=True, model_attr='voyage_itinerary__place_voyage_ended__region')
var_imp_region_voyage_begin_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__imp_region_voyage_begin__value')
var_first_region_slave_emb_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__first_region_slave_emb__value')
var_second_region_slave_emb_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__second_region_slave_emb__value')
var_third_region_slave_emb_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__third_region_slave_emb__value')
var_imp_principal_region_of_slave_purchase_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__imp_principal_region_of_slave_purchase__value')
var_first_landing_region_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__first_landing_region__value')
var_second_landing_region_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__second_landing_region__value')
var_imp_principal_region_slave_dis_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__imp_principal_region_slave_dis__value')
var_region_voyage_ended_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__place_voyage_ended__region__value')
# Broad Region variables
var_imp_principal_broad_region_disembark_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__imp_broad_region_slave_dis__value')
var_imp_broad_region_voyage_begin_idnum = indexes.IntegerField(null=True, model_attr='voyage_itinerary__imp_broad_region_voyage_begin__value')
# Voyage captain and crew
var_captain = indexes.NgramField(null=True)
var_captain_plaintext = indexes.CharField(null=True, faceted=True, indexed=True)
var_crew_voyage_outset = indexes.IntegerField(null=True, model_attr='voyage_crew__crew_voyage_outset')
var_crew_first_landing = indexes.IntegerField(null=True, model_attr='voyage_crew__crew_first_landing')
var_crew_died_complete_voyage = indexes.IntegerField(null=True, model_attr='voyage_crew__crew_died_complete_voyage')
# Voyage dates
# Month field is used for filtering by month
var_imp_arrival_at_port_of_dis = indexes.IntegerField(null=True, faceted=True)
var_voyage_began = indexes.DateField(null=True)
var_voyage_began_partial = indexes.CharField(null=True, model_attr='voyage_dates__voyage_began', indexed=True)
var_voyage_began_month = indexes.IntegerField(null=True)
var_slave_purchase_began = indexes.DateField(null=True)
var_slave_purchase_began_partial = indexes.CharField(null=True, model_attr='voyage_dates__slave_purchase_began', indexed=True)
var_slave_purchase_began_month = indexes.IntegerField(null=True)
var_date_departed_africa = indexes.DateField(null=True)
var_date_departed_africa_partial = indexes.CharField(null=True, model_attr='voyage_dates__date_departed_africa', indexed=True)
var_date_departed_africa_month = indexes.IntegerField(null=True)
var_first_dis_of_slaves = indexes.DateField(null=True)
var_first_dis_of_slaves_partial = indexes.CharField(null=True, model_attr='voyage_dates__first_dis_of_slaves', indexed=True)
var_first_dis_of_slaves_month = indexes.IntegerField(null=True)
var_departure_last_place_of_landing = indexes.DateField(null=True)
var_departure_last_place_of_landing_partial = indexes.CharField(null=True, model_attr='voyage_dates__departure_last_place_of_landing', indexed=True)
var_departure_last_place_of_landing_month = indexes.IntegerField(null=True)
var_voyage_completed = indexes.DateField(null=True)
var_voyage_completed_partial = indexes.CharField(null=True, model_attr='voyage_dates__voyage_completed', indexed=True)
var_voyage_completed_month = indexes.IntegerField(null=True)
var_imp_length_home_to_disembark = indexes.IntegerField(null=True)
var_length_middle_passage_days = indexes.IntegerField(null=True, faceted=True)
# Voyage numbers
var_num_slaves_intended_first_port = indexes.IntegerField(null=True, model_attr='voyage_slaves_numbers__num_slaves_intended_first_port')
var_num_slaves_carried_first_port = indexes.IntegerField(null=True, model_attr='voyage_slaves_numbers__num_slaves_carried_first_port')
var_num_slaves_carried_second_port = indexes.IntegerField(null=True, model_attr='voyage_slaves_numbers__num_slaves_carried_second_port')
var_num_slaves_carried_third_port = indexes.IntegerField(null=True, model_attr='voyage_slaves_numbers__num_slaves_carried_third_port')
# To be corrected (copied from previous comment)
var_total_num_slaves_purchased = indexes.IntegerField(null=True, faceted=True, model_attr='voyage_slaves_numbers__total_num_slaves_dep_last_slaving_port')
# To be corrected (copied from previous comment)
var_imp_total_num_slaves_purchased = indexes.IntegerField(null=True, faceted=True, model_attr='voyage_slaves_numbers__imp_total_num_slaves_embarked')
var_total_num_slaves_arr_first_port_embark = indexes.IntegerField(null=True, model_attr='voyage_slaves_numbers__total_num_slaves_arr_first_port_embark')
var_num_slaves_disembark_first_place = indexes.IntegerField(null=True, model_attr='voyage_slaves_numbers__num_slaves_disembark_first_place')
var_num_slaves_disembark_second_place = indexes.IntegerField(null=True, model_attr='voyage_slaves_numbers__num_slaves_disembark_second_place')
var_num_slaves_disembark_third_place = indexes.IntegerField(null=True, model_attr='voyage_slaves_numbers__num_slaves_disembark_third_place')
var_imp_total_slaves_disembarked = indexes.IntegerField(null=True, faceted=True, model_attr='voyage_slaves_numbers__imp_total_num_slaves_disembarked')
# Voyage characteristics
var_imputed_percentage_men = indexes.FloatField(null=True, model_attr='voyage_slaves_numbers__percentage_men')
var_imputed_percentage_women = indexes.FloatField(null=True, model_attr='voyage_slaves_numbers__percentage_women')
var_imputed_percentage_boys = indexes.FloatField(null=True, model_attr='voyage_slaves_numbers__percentage_boy')
var_imputed_percentage_girls = indexes.FloatField(null=True, model_attr='voyage_slaves_numbers__percentage_girl')
var_imputed_percentage_female = indexes.FloatField(null=True, model_attr='voyage_slaves_numbers__percentage_female')
var_imputed_percentage_male = indexes.FloatField(null=True, faceted=True, model_attr='voyage_slaves_numbers__percentage_male')
var_imputed_percentage_child = indexes.FloatField(null=True, faceted=True, model_attr='voyage_slaves_numbers__percentage_child')
var_imputed_sterling_cash = indexes.FloatField(null=True, model_attr='voyage_slaves_numbers__imp_jamaican_cash_price')
var_imputed_death_middle_passage = indexes.IntegerField(null=True, model_attr='voyage_slaves_numbers__imp_mortality_during_voyage')
var_imputed_mortality = indexes.FloatField(null=True, faceted=True, model_attr='voyage_slaves_numbers__imp_mortality_ratio')
# Sources
var_sources = indexes.MultiValueField(indexed=True, stored=True, null=True)
var_sources_plaintext = indexes.CharField(null=True, faceted=True, indexed=True)
var_sources_plaintext_search = indexes.NgramField(null=True, faceted=False, indexed=True)
var_short_ref = indexes.MultiValueField()
var_long_ref = indexes.CharField(null=True)
# Links
var_voyage_links = indexes.MultiValueField(indexed=True, stored=True, null=True)
# Intra-American vs Trans-Atlantic.
var_intra_american_voyage = indexes.BooleanField(null=False, indexed=True, model_attr='is_intra_american')
def get_model(self):
return Voyage
def get_updated_field(self):
return 'last_update'
def index_queryset(self, using=None):
"""Used when the entire index for model is updated."""
return Voyage.both_objects.all()
def prepare_var_imp_voyage_began(self, obj):
try:
return getYear(obj.voyage_dates.imp_voyage_began)
except AttributeError, TypeError:
return None
class MapIndex(indexes.SearchIndex, indexes.Indexable):
id = indexes.IntegerField(model_attr='id')
abstract = indexes.CharField(model_attr="abstract", boost=1.5)
category__gn_description = indexes.CharField(
model_attr="category__gn_description", null=True)
csw_type = indexes.CharField(model_attr="csw_type")
csw_wkt_geometry = indexes.CharField(model_attr="csw_wkt_geometry")
detail_url = indexes.CharField(model_attr="get_absolute_url")
distribution_description = indexes.CharField(
model_attr="distribution_description", null=True)
distribution_url = indexes.CharField(model_attr="distribution_url",
null=True)
owner__username = indexes.CharField(model_attr="owner",
faceted=True,
null=True)
popular_count = indexes.IntegerField(model_attr="popular_count",
default=0,
boost=20)
share_count = indexes.IntegerField(model_attr="share_count", default=0)
rating = indexes.IntegerField(null=True)
srid = indexes.CharField(model_attr="srid")
supplemental_information = indexes.CharField(
model_attr="supplemental_information", null=True)
thumbnail_url = indexes.CharField(model_attr="thumbnail_url", null=True)
uuid = indexes.CharField(model_attr="uuid")
title = indexes.CharField(model_attr="title", boost=2)
date = indexes.DateTimeField(model_attr="date")
text = indexes.EdgeNgramField(document=True,
use_template=True,
stored=False)
type = indexes.CharField(faceted=True)
title_sortable = indexes.CharField(indexed=False,
stored=False) # Necessary for sorting
category = indexes.CharField(model_attr="category__identifier",
faceted=True,
null=True,
stored=True)
bbox_left = indexes.FloatField(model_attr="bbox_x0",
null=True,
stored=False)
bbox_right = indexes.FloatField(model_attr="bbox_x1",
null=True,
stored=False)
bbox_bottom = indexes.FloatField(model_attr="bbox_y0",
null=True,
stored=False)
bbox_top = indexes.FloatField(model_attr="bbox_y1",
null=True,
stored=False)
temporal_extent_start = indexes.DateTimeField(
model_attr="temporal_extent_start", null=True, stored=False)
temporal_extent_end = indexes.DateTimeField(
model_attr="temporal_extent_end", null=True, stored=False)
keywords = indexes.MultiValueField(model_attr="keyword_slug_list",
null=True,
faceted=True,
stored=True)
regions = indexes.MultiValueField(model_attr="region_name_list",
null=True,
faceted=True,
stored=True)
popular_count = indexes.IntegerField(model_attr="popular_count",
default=0,
boost=20)
share_count = indexes.IntegerField(model_attr="share_count", default=0)
rating = indexes.IntegerField(null=True)
num_ratings = indexes.IntegerField(stored=False)
num_comments = indexes.IntegerField(stored=False)
# Need to grab the owner's first and last name as well as published status
owner__first_name = indexes.CharField(model_attr="owner__first_name",
faceted=True,
null=True)
owner__last_name = indexes.CharField(model_attr="owner__last_name",
faceted=True,
null=True)
is_published = indexes.BooleanField(model_attr="is_published")
# Featured status for front page carousel
featured = indexes.BooleanField(model_attr="featured")
def get_model(self):
return Map
def prepare_type(self, obj):
return "map"
def prepare_rating(self, obj):
ct = ContentType.objects.get_for_model(obj)
try:
rating = OverallRating.objects.filter(
object_id=obj.pk,
content_type=ct).aggregate(r=Avg("rating"))["r"]
return float(str(rating or "0"))
except OverallRating.DoesNotExist:
return 0.0
def prepare_num_ratings(self, obj):
ct = ContentType.objects.get_for_model(obj)
try:
return OverallRating.objects.filter(object_id=obj.pk,
content_type=ct).all().count()
except OverallRating.DoesNotExist:
return 0
def prepare_num_comments(self, obj):
try:
return Comment.objects.filter(
object_id=obj.pk,
content_type=ContentType.objects.get_for_model(
obj)).all().count()
except:
return 0
def prepare_title_sortable(self, obj):
return obj.title.lower()
class BiologicalCollectionIndex(indexes.SearchIndex, indexes.Indexable):
model_pk = indexes.IntegerField(model_attr='pk')
id = indexes.CharField()
text = indexes.CharField(document=True, use_template=True)
original_species_name = indexes.NgramField(
model_attr='original_species_name',
indexed=True
)
original_species_name_exact = indexes.CharField(
model_attr='original_species_name',
indexed=True
)
vernacular_names = indexes.CharField(
indexed=True
)
collector = indexes.NgramField(
model_attr='collector',
indexed=True
)
collection_date_year = indexes.IntegerField(
model_attr='collection_date__year',
indexed=True
)
collection_date_month = indexes.IntegerField(
model_attr='collection_date__month',
indexed=True
)
category = indexes.CharField(
model_attr='category'
)
present = indexes.BooleanField(
model_attr='present'
)
absent = indexes.BooleanField(
model_attr='absent'
)
validated = indexes.BooleanField(
model_attr='is_validated'
)
collection_date = indexes.DateField(
model_attr='collection_date'
)
owner = indexes.CharField(
model_attr='owner__username'
)
notes = indexes.CharField(
model_attr='notes'
)
location_site_name = indexes.CharField(
model_attr='site__name',
indexed=True
)
location_site_id = indexes.CharField(
model_attr='site__id'
)
location_center = indexes.LocationField()
location_coordinates = indexes.CharField()
taxonomy = indexes.IntegerField(indexed=True)
taxonomy_not_null = indexes.BooleanField(indexed=True)
taxon_canonical_name = indexes.NgramField(
model_attr='taxonomy__canonical_name',
indexed=True
)
taxon_scientific_name = indexes.NgramField(
model_attr='taxonomy__scientific_name',
indexed=True
)
taxon_canonical_name_exact = indexes.CharField(
model_attr='taxonomy__canonical_name',
indexed=True
)
taxon_scientific_name_exact = indexes.CharField(
model_attr='taxonomy__scientific_name',
indexed=True
)
taxon_class = indexes.CharField(
indexed=True
)
reference_category = indexes.CharField(
model_attr='reference_category',
indexed=True
)
reference = indexes.CharField(
model_attr='reference',
indexed=True
)
site_id_indexed = indexes.IntegerField(
indexed=True
)
endemism = indexes.CharField(
indexed=True
)
iucn_status = indexes.CharField(
indexed=True
)
river_catchments = indexes.CharField(
indexed=True
)
boundary = indexes.CharField(
indexed=True
)
def prepare_taxon_class(self, obj):
if obj.taxonomy:
return obj.taxonomy.class_name
return ''
def prepare_site_id_indexed(self, obj):
if obj.site:
return obj.site.id
return 0
def prepare_taxonomy(self, obj):
if obj.taxonomy:
return obj.taxonomy.id
return 0
def prepare_taxonomy_not_null(self, obj):
if obj.taxonomy:
return True
return False
def prepare_location_center(self, obj):
if obj.site:
return '%s,%s' % obj.site.get_centroid().coords
return '0,0'
def prepare_location_coordinates(self, obj):
if obj.site:
return '%s,%s' % obj.site.get_centroid().coords
return '0,0'
def prepare_boundary(self, obj):
if not obj.site:
return ''
if not obj.site.boundary:
return ''
ids = []
boundary = obj.site.boundary
while True:
try:
ids.append(boundary.id)
boundary = boundary.top_level_boundary
except AttributeError:
break
return '_' + '_'.join([str(i) for i in ids]) + '_'
def prepare_endemism(self, obj):
if not obj.taxonomy:
return ''
if not obj.taxonomy.endemism:
return ''
return obj.taxonomy.endemism.name
def prepare_iucn_status(self, obj):
if not obj.taxonomy:
return ''
if not obj.taxonomy.iucn_status:
return ''
return obj.taxonomy.iucn_status.category
def prepare_vernacular_names(self, obj):
if not obj.taxonomy:
return ''
return ','.join(obj.taxonomy.vernacular_names.filter(
language='eng'
).values_list('name', flat=True))
def prepare_river_catchments(self, obj):
if not obj.site:
return ''
if not obj.site.location_context_document:
return ''
river_catchment_array = get_river_catchment_site(obj.site)
river_catchment_string = ''
for river_catchment in river_catchment_array:
river_catchment_string += '_'
river_catchment_string += river_catchment.replace(' ', '_')
river_catchment_string += '_'
return river_catchment_string
class Meta:
app_label = 'bims'
def index_queryset(self, using=None):
"""Used to reindex model"""
return self.get_model().objects.all()
def get_model(self):
return BiologicalCollectionRecord
class NodeIndex(indexes.SearchIndex, indexes.Indexable):
TYPE_PREFIX = "REFINERY_TYPE"
NAME_PREFIX = "REFINERY_NAME"
WORKFLOW_OUTPUT_PREFIX = "REFINERY_WORKFLOW_OUTPUT"
ANALYSIS_UUID_PREFIX = "REFINERY_ANALYSIS_UUID"
SUBANALYSIS_PREFIX = "REFINERY_SUBANALYSIS"
FILETYPE_PREFIX = "REFINERY_FILETYPE"
DOWNLOAD_URL = "REFINERY_DOWNLOAD_URL_s"
text = indexes.CharField(document=True, use_template=True)
uuid = indexes.CharField(model_attr='uuid')
study_uuid = indexes.CharField(model_attr='study__uuid')
assay_uuid = indexes.CharField(model_attr='assay__uuid', null=True)
data_set_uuid = indexes.CharField(null=True)
type = indexes.CharField(model_attr='type')
name = indexes.CharField(model_attr='name', null=True)
file_uuid = indexes.CharField(model_attr='file_uuid', null=True)
species = indexes.IntegerField(model_attr='species', null=True)
genome_build = indexes.CharField(model_attr='genome_build', null=True)
is_annotation = indexes.BooleanField(model_attr='is_annotation')
analysis_uuid = indexes.CharField(model_attr='analysis_uuid', null=True)
subanalysis = indexes.IntegerField(model_attr='subanalysis', null=True)
workflow_output = indexes.CharField(model_attr='workflow_output',
null=True)
# TODO: add modification date (based on registry)
def get_model(self):
return Node
def index_queryset(self, using=None):
"""Used when the entire index for model is updated."""
return self.get_model().objects.all()
GENERIC_SUFFIX = "_generic_s"
def _assay_data(self, object):
data = {}
for field in Assay._meta.fields:
if field.name in ['id', 'uuid', 'study', 'file_name']:
continue
key = field.name + '_Characteristics' + NodeIndex.GENERIC_SUFFIX
data[key] = set()
assay = object.assay
if assay is not None:
assay_attr = getattr(assay, field.name)
if assay_attr is not None:
data[key].add(assay_attr)
return data
# dynamic fields:
# https://groups.google.com/forum/?fromgroups#!topic/django-haystack/g39QjTkN-Yg
# http://stackoverflow.com/questions/7399871/django-haystack-sort-results-by-title
def prepare(self, object):
data = super(NodeIndex, self).prepare(object)
annotations = AnnotatedNode.objects.filter(node=object)
id_suffix = str(object.study.id)
try:
data_set = object.study.get_dataset()
data['data_set_uuid'] = data_set.uuid
except RuntimeError as e:
logger.warn(e)
if object.assay is not None:
id_suffix += "_" + str(object.assay.id)
id_suffix = "_" + id_suffix + "_s"
data['filename_Characteristics' + NodeIndex.GENERIC_SUFFIX] = \
re.sub(r'.*/', '', data['name'])
data.update(self._assay_data(object))
# create dynamic fields for each attribute
for annotation in annotations:
name = annotation.attribute_type
if annotation.attribute_subtype is not None:
name = annotation.attribute_subtype + "_" + name
value = annotation.attribute_value
if annotation.attribute_value_unit is not None:
value += " " + annotation.attribute_value_unit
name = re.sub(r'\W', settings.REFINERY_SOLR_SPACE_DYNAMIC_FIELDS,
name)
uniq_key = name + id_suffix
generic_key = name + NodeIndex.GENERIC_SUFFIX
# a node might have multiple parents with different attribute
# values for a given attribute
# e.g. parentA Characteristic[cell type] = K562 and
# parentB Characteristic[cell type] = HeLa
# child nodes should inherit all attributes of their parents as a
# concatenation of the unique list
# old version (only one attribute kept):
# data[key] = value
for key in (uniq_key, generic_key):
if key not in data:
data[key] = set()
if value != "":
data[key].add(value)
else:
data[key].add("N/A")
# iterate over all keys in data and join sets into strings
for key, value in data.iteritems():
if type(value) is set:
data[key] = " + ".join(sorted(value))
try:
file_store_item = FileStoreItem.objects.get(uuid=object.file_uuid)
except (FileStoreItem.DoesNotExist,
FileStoreItem.MultipleObjectsReturned) as e:
logger.error("Couldn't properly fetch FileStoreItem: %s", e)
file_store_item = None
data.update({
NodeIndex.DOWNLOAD_URL:
''
if file_store_item is None else file_store_item.get_datafile_url(),
NodeIndex.TYPE_PREFIX + id_suffix:
object.type,
NodeIndex.NAME_PREFIX + id_suffix:
object.name,
NodeIndex.FILETYPE_PREFIX + id_suffix:
"" if file_store_item is None else file_store_item.get_filetype(),
NodeIndex.ANALYSIS_UUID_PREFIX + id_suffix:
"N/A"
if object.get_analysis() is None else object.get_analysis().name,
NodeIndex.SUBANALYSIS_PREFIX + id_suffix: (
-1 if object.subanalysis is None # TODO: upgrade flake8
else object.subanalysis), # and remove parentheses
NodeIndex.WORKFLOW_OUTPUT_PREFIX + id_suffix:
"N/A" if object.workflow_output is None else object.workflow_output
})
return data
class WareAppPrefixInfoIndex(indexes.ModelSearchIndex, indexes.Indexable):
id = indexes.IntegerField(model_attr='wareApp_key__id')
name = indexes.CharField(model_attr='wareApp_key__name')
money = indexes.DecimalField(model_attr='wareApp_key__money')
image_400x400 = indexes.CharField(model_attr='wareApp_key__image_400x400')
unix = indexes.CharField(model_attr='wareApp_key__unix')
stock = indexes.BooleanField(model_attr='wareApp_key__stock')
release = indexes.BooleanField(model_attr='wareApp_key__release')
category = indexes.CharField(model_attr='classify_key__name')
subcategory = indexes.CharField(model_attr='classifythere_key__name')
brand = indexes.CharField(model_attr='brand_key__PrefixKey__filter_id')
scene = indexes.CharField(model_attr='scene_key__PrefixKey__filter_id')
technology = indexes.CharField(
model_attr='technology_key__PrefixKey__filter_id')
producttype = indexes.CharField(
model_attr='producttype_key__PrefixKey__filter_id')
pricerange = indexes.CharField(
model_attr='pricerange_key__PrefixKey__filter_id')
'''
分离筛选器参数
'''
def get_sid_prefix(self, obj, sid):
name = ''
if obj.brand_key and int(obj.brand_key.PrefixKey.filter_id) == sid:
name = '{}.{}'.format(obj.brand_key.PrefixKey.t1,
obj.brand_key.name)
pass
if obj.producttype_key and int(
obj.producttype_key.PrefixKey.filter_id) == sid:
name = '{}.{}'.format(obj.producttype_key.PrefixKey.t2,
obj.producttype_key.name)
pass
if obj.technology_key and int(
obj.technology_key.PrefixKey.filter_id) == sid:
name = '{}.{}'.format(obj.technology_key.PrefixKey.t3,
obj.technology_key.name)
pass
if obj.scene_key and int(obj.scene_key.PrefixKey.filter_id) == sid:
name = '{}.{}'.format(obj.scene_key.PrefixKey.t4,
obj.scene_key.name)
pass
if obj.pricerange_key and int(
obj.pricerange_key.PrefixKey.filter_id) == sid:
name = '{}.{}'.format(obj.pricerange_key.PrefixKey.t5,
obj.pricerange_key.name)
pass
return name
def prepare_image_400x400(self, obj):
return '{}{}{}'.format(settings.HTTP_HOST, settings.MEDIA,
obj.wareApp_key.image_400x400)
def prepare_brand(self, obj):
return self.get_sid_prefix(obj, 0)
def prepare_producttype(self, obj):
return self.get_sid_prefix(obj, 1)
def prepare_technology(self, obj):
return self.get_sid_prefix(obj, 2)
def prepare_scene(self, obj):
return self.get_sid_prefix(obj, 3)
def prepare_pricerange(self, obj):
return self.get_sid_prefix(obj, 4)
class Meta:
model = models.WareAppPrefix
def index_queryset(self, using=None):
return self.get_model().objects.filter(wareApp_key__release=True)
class BaseResourceIndex(indexes.SearchIndex, indexes.Indexable):
"""Define base class for resource indexes."""
text = indexes.CharField(document=True, use_template=True)
short_id = indexes.CharField(model_attr='short_id')
doi = indexes.CharField(model_attr='doi', null=True)
author = indexes.CharField(
faceted=True) # normalized to last, first, middle
author_raw = indexes.CharField(indexed=False) # not normalized
author_url = indexes.CharField(indexed=False, null=True)
title = indexes.CharField()
abstract = indexes.CharField()
creator = indexes.MultiValueField(faceted=True)
contributor = indexes.MultiValueField(faceted=True)
subject = indexes.MultiValueField(faceted=True)
availability = indexes.MultiValueField(faceted=True)
# TODO: We might need more information than a bool in the future
replaced = indexes.BooleanField()
created = indexes.DateTimeField(model_attr='created')
modified = indexes.DateTimeField(model_attr='last_updated')
organization = indexes.MultiValueField(faceted=True)
creator_email = indexes.MultiValueField()
publisher = indexes.CharField(faceted=True)
rating = indexes.IntegerField(model_attr='rating_sum')
coverage = indexes.MultiValueField()
coverage_type = indexes.MultiValueField()
east = indexes.FloatField(null=True)
north = indexes.FloatField(null=True)
northlimit = indexes.FloatField(null=True)
eastlimit = indexes.FloatField(null=True)
southlimit = indexes.FloatField(null=True)
westlimit = indexes.FloatField(null=True)
start_date = indexes.DateField(null=True)
end_date = indexes.DateField(null=True)
storage_type = indexes.CharField()
# # TODO: SOLR extension needs to be installed for these to work
# coverage_point = indexes.LocationField(null=True)
# coverage_southwest = indexes.LocationField(null=True)
# coverage_northeast = indexes.LocationField(null=True)
format = indexes.MultiValueField()
identifier = indexes.MultiValueField()
language = indexes.CharField(faceted=True)
source = indexes.MultiValueField()
relation = indexes.MultiValueField()
resource_type = indexes.CharField(faceted=True)
content_type = indexes.MultiValueField(faceted=True)
comment = indexes.MultiValueField()
comments_count = indexes.IntegerField(faceted=True)
owner_login = indexes.MultiValueField(faceted=True)
owner = indexes.MultiValueField(faceted=True)
owners_count = indexes.IntegerField(faceted=True)
# # TODO: We might need these later for social discovery
# viewer_login = indexes.MultiValueField(faceted=True)
# viewer = indexes.MultiValueField(faceted=True)
# viewers_count = indexes.IntegerField(faceted=True)
# editor_login = indexes.MultiValueField(faceted=True)
# editor = indexes.MultiValueField(faceted=True)
# editors_count = indexes.IntegerField(faceted=True)
person = indexes.MultiValueField(faceted=True)
# non-core metadata
geometry_type = indexes.CharField(faceted=True)
field_name = indexes.CharField()
field_type = indexes.CharField()
field_type_code = indexes.CharField()
variable = indexes.MultiValueField(faceted=True)
variable_type = indexes.MultiValueField(faceted=True)
variable_shape = indexes.MultiValueField()
variable_descriptive_name = indexes.MultiValueField()
variable_speciation = indexes.MultiValueField()
site = indexes.MultiValueField()
method = indexes.MultiValueField()
quality_level = indexes.MultiValueField()
data_source = indexes.MultiValueField()
sample_medium = indexes.MultiValueField(faceted=True)
units = indexes.MultiValueField(faceted=True)
units_type = indexes.MultiValueField(faceted=True)
aggregation_statistics = indexes.MultiValueField()
absolute_url = indexes.CharField(indexed=False)
# extra metadata
extra = indexes.MultiValueField()
def get_model(self):
"""Return BaseResource model."""
return BaseResource
def index_queryset(self, using=None):
"""Return queryset including discoverable and public resources."""
return self.get_model().objects.filter(
Q(raccess__discoverable=True) | Q(raccess__public=True))
def prepare_created(self, obj):
return obj.created.strftime('%Y-%m-%dT%H:%M:%SZ')
def prepare_modified(self, obj):
return obj.last_updated.strftime('%Y-%m-%dT%H:%M:%SZ')
def prepare_title(self, obj):
"""Return metadata title if exists, otherwise return 'none'."""
if hasattr(obj, 'metadata') and obj.metadata.title.value is not None:
return obj.metadata.title.value.lstrip()
else:
return 'none'
def prepare_abstract(self, obj):
"""Return metadata abstract if exists, otherwise return None."""
if hasattr(obj, 'metadata') and obj.metadata.description is not None and \
obj.metadata.description.abstract is not None:
return obj.metadata.description.abstract.lstrip()
else:
return None
def prepare_author_raw(self, obj):
"""
Return metadata author if exists, otherwise return None.
This must be represented as a single-value field to enable sorting.
"""
if hasattr(obj, 'metadata'):
first_creator = obj.metadata.creators.filter(order=1).first()
if first_creator.name:
return first_creator.name.lstrip()
elif first_creator.organization:
return first_creator.organization.strip()
else:
return 'none'
else:
return 'none'
# TODO: it is confusing that the "author" is the first "creator"
def prepare_author(self, obj):
"""
Return first creator if exists, otherwise return empty list.
This must be represented as a single-value field to enable sorting.
"""
if hasattr(obj, 'metadata'):
first_creator = obj.metadata.creators.filter(order=1).first()
if first_creator.name:
normalized = normalize_name(first_creator.name)
return normalized
elif first_creator.organization:
return first_creator.organization.strip()
else:
return 'none'
else:
return 'none'
def prepare_author_url(self, obj):
"""
Return metadata author description url if exists, otherwise return None.
This field is stored but not indexed, to avoid hitting the Django database during response.
"""
if hasattr(obj, 'metadata'):
first_creator = obj.metadata.creators.filter(order=1).first()
if first_creator.description is not None:
return first_creator.description
else:
return None
else:
return None
def prepare_creator(self, obj):
"""
Return metadata creators if they exist, otherwise return empty array.
This field can have multiple values
"""
if hasattr(obj, 'metadata'):
return [
normalize_name(creator.name)
for creator in obj.metadata.creators.all().exclude(
name__isnull=True).exclude(name='')
]
else:
return []
def prepare_contributor(self, obj):
"""
Return metadata contributors if they exist, otherwise return empty array.
This field can have multiple values. Contributors include creators.
"""
if hasattr(obj, 'metadata'):
output1 = [
normalize_name(contributor.name)
for contributor in obj.metadata.contributors.all().exclude(
name__isnull=True).exclude(name='')
]
return list(set(output1)) # eliminate duplicates
else:
return []
def prepare_subject(self, obj):
"""
Return metadata subjects if they exist, otherwise return empty array.
This field can have multiple values.
"""
if hasattr(obj, 'metadata'):
return [
subject.value.strip()
for subject in obj.metadata.subjects.all().exclude(
value__isnull=True)
]
else:
return []
def prepare_organization(self, obj):
"""
Return metadata organization if it exists, otherwise return empty array.
"""
organizations = []
if hasattr(obj, 'metadata'):
for creator in obj.metadata.creators.all():
if (creator.organization is not None):
organizations.append(creator.organization.strip())
return organizations
def prepare_publisher(self, obj):
"""
Return metadata publisher if it exists; otherwise return empty array.
"""
if hasattr(obj, 'metadata'):
publisher = obj.metadata.publisher
if publisher is not None:
return unicode(publisher).lstrip()
else:
return None
else:
return None
def prepare_creator_email(self, obj):
"""Return metadata emails if exists, otherwise return empty array."""
if hasattr(obj, 'metadata'):
return [
creator.email.strip()
for creator in obj.metadata.creators.all().exclude(
email__isnull=True).exclude(email='')
]
else:
return []
def prepare_availability(self, obj):
"""
availability is published, public, or discoverable
To make faceting work properly, all flags that are True are represented.
"""
options = []
if hasattr(obj, 'raccess'):
if obj.raccess.published:
options.append('published')
elif obj.raccess.public:
options.append('public')
elif obj.raccess.discoverable:
options.append('discoverable')
else:
options.append('private')
else:
options.append('private')
return options
def prepare_replaced(self, obj):
"""Return True if 'isReplacedBy' attribute exists, otherwise return False."""
if hasattr(obj, 'metadata'):
return obj.metadata.relations.all().filter(
type='isReplacedBy').exists()
else:
return False
def prepare_coverage(self, obj):
"""Return resource coverage if exists, otherwise return empty array."""
# TODO: reject empty coverages
if hasattr(obj, 'metadata'):
return [
coverage._value.strip()
for coverage in obj.metadata.coverages.all()
]
else:
return []
def prepare_coverage_type(self, obj):
"""
Return resource coverage types if exists, otherwise return empty array.
This field can have multiple values.
"""
if hasattr(obj, 'metadata'):
return [
coverage.type.strip()
for coverage in obj.metadata.coverages.all()
]
else:
return []
# TODO: THIS IS SIMPLY THE WRONG WAY TO DO THINGS.
# Should use geopy Point and Haystack LocationField throughout,
# instead of encoding limits literally.
# See http://django-haystack.readthedocs.io/en/v2.6.0/spatial.html
# TODO: If there are multiple coverage objects with the same type, only first is returned.
def prepare_east(self, obj):
"""Return resource coverage east bound if exists, otherwise return None."""
if hasattr(obj, 'metadata'):
for coverage in obj.metadata.coverages.all():
if coverage.type == 'point':
return float(coverage.value["east"])
# TODO: this returns the box center, not the extent
# TODO: probably better to call this something different.
elif coverage.type == 'box':
return (float(coverage.value["eastlimit"]) +
float(coverage.value["westlimit"])) / 2
else:
return None
# TODO: If there are multiple coverage objects with the same type, only first is returned.
def prepare_north(self, obj):
"""Return resource coverage north bound if exists, otherwise return None."""
if hasattr(obj, 'metadata'):
for coverage in obj.metadata.coverages.all():
if coverage.type == 'point':
return float(coverage.value["north"])
# TODO: This returns the box center, not the extent
elif coverage.type == 'box':
return (float(coverage.value["northlimit"]) +
float(coverage.value["southlimit"])) / 2
else:
return None
# TODO: If there are multiple coverage objects with the same type, only first is returned.
def prepare_northlimit(self, obj):
"""Return resource coverage north limit if exists, otherwise return None."""
if hasattr(obj, 'metadata'):
# TODO: does not index properly if there are multiple coverages of the same type.
for coverage in obj.metadata.coverages.all():
if coverage.type == 'box':
return coverage.value["northlimit"]
else:
return None
# TODO: If there are multiple coverage objects with the same type, only first is returned.
def prepare_eastlimit(self, obj):
"""Return resource coverage east limit if exists, otherwise return None."""
if hasattr(obj, 'metadata'):
# TODO: does not index properly if there are multiple coverages of the same type.
for coverage in obj.metadata.coverages.all():
if coverage.type == 'box':
return coverage.value["eastlimit"]
else:
return None
# TODO: If there are multiple coverage objects with the same type, only first is returned.
def prepare_southlimit(self, obj):
"""Return resource coverage south limit if exists, otherwise return None."""
if hasattr(obj, 'metadata'):
# TODO: does not index properly if there are multiple coverages of the same type.
for coverage in obj.metadata.coverages.all():
if coverage.type == 'box':
return coverage.value["southlimit"]
else:
return None
# TODO: If there are multiple coverage objects with the same type, only first is returned.
def prepare_westlimit(self, obj):
"""Return resource coverage west limit if exists, otherwise return None."""
if hasattr(obj, 'metadata'):
# TODO: does not index properly if there are multiple coverages of the same type.
for coverage in obj.metadata.coverages.all():
if coverage.type == 'box':
return coverage.value["westlimit"]
else:
return None
# TODO: time coverages do not specify timezone, and timezone support is active.
# TODO: Why aren't time coverages specified as Django DateTime objects?
# TODO: If there are multiple coverage objects with the same type, only first is returned.
def prepare_start_date(self, obj):
"""Return resource coverage start date if exists, otherwise return None."""
if hasattr(obj, 'metadata'):
for coverage in obj.metadata.coverages.all():
if coverage.type == 'period':
clean_date = coverage.value["start"][:10]
if "/" in clean_date:
parsed_date = clean_date.split("/")
start_date = parsed_date[2] + '-' + parsed_date[
0] + '-' + parsed_date[1]
else:
parsed_date = clean_date.split("-")
start_date = parsed_date[0] + '-' + parsed_date[
1] + '-' + parsed_date[2]
start_date = remove_whitespace(
start_date) # no embedded spaces
try:
start_date_object = datetime.strptime(
start_date, '%Y-%m-%d')
except ValueError:
logger = logging.getLogger(__name__)
logger.error(
"invalid start date {} in resource {}".format(
obj.short_id, start_date))
return None
return start_date_object
else:
return None
# TODO: time coverages do not specify timezone, and timezone support is active.
# TODO: Why aren't time coverages specified as Django DateTime objects?
# TODO: If there are multiple coverage objects with the same type, only first is returned.
def prepare_end_date(self, obj):
"""Return resource coverage end date if exists, otherwise return None."""
if hasattr(obj, 'metadata'):
for coverage in obj.metadata.coverages.all():
if coverage.type == 'period' and 'end' in coverage.value:
clean_date = coverage.value["end"][:10]
if "/" in clean_date:
parsed_date = clean_date.split("/")
end_date = parsed_date[2] + '-' + parsed_date[
0] + '-' + parsed_date[1]
else:
parsed_date = clean_date.split("-")
end_date = parsed_date[0] + '-' + parsed_date[
1] + '-' + parsed_date[2]
end_date = remove_whitespace(
end_date) # no embedded spaces
try:
end_date_object = datetime.strptime(
end_date, '%Y-%m-%d')
except ValueError:
logger = logging.getLogger(__name__)
logger.error(
"invalid end date {} in resource {}".format(
end_date, obj.short_id))
return None
return end_date_object
else:
return None
def prepare_storage_type(self, obj):
return obj.storage_type
# # TODO: SOLR extension needs to be installed for these to work
# def prepare_coverage_point(self, obj):
# """ Return Point object associated with coverage, or None """
# if hasattr(obj, 'metadata'):
# for coverage in obj.metadata.coverages.all():
# if coverage.type == 'point':
# return Point(float(coverage.value["east"]),
# float(coverage.value["north"]))
# return None
# def prepare_coverage_southwest(self, obj):
# """ Return southwest limit of bounding box, or None """
# if hasattr(obj, 'metadata'):
# for coverage in obj.metadata.coverages.all():
# if coverage.type == 'box':
# return Point(float(coverage.value["westlimit"]),
# float(coverage.value["southlimit"]))
# return None
# def prepare_coverage_northeast(self, obj):
# """ Return northeast limit of bounding box, or None """
# if hasattr(obj, 'metadata'):
# for coverage in obj.metadata.coverages.all():
# if coverage.type == 'box':
# return Point(float(coverage.value["eastlimit"]),
# float(coverage.value["northlimit"]))
# return None
def prepare_format(self, obj):
"""Return metadata formats if metadata exists, otherwise return empty array."""
if hasattr(obj, 'metadata'):
return [
format.value.strip() for format in obj.metadata.formats.all()
]
else:
return []
def prepare_identifier(self, obj):
"""Return metadata identifiers if metadata exists, otherwise return empty array."""
if hasattr(obj, 'metadata'):
return [
identifier.name.strip()
for identifier in obj.metadata.identifiers.all()
]
else:
return []
def prepare_language(self, obj):
"""Return resource language if exists, otherwise return None."""
if hasattr(obj, 'metadata'):
return obj.metadata.language.code.strip()
else:
return None
def prepare_source(self, obj):
"""Return resource sources if exists, otherwise return empty array."""
if hasattr(obj, 'metadata'):
return [
source.derived_from.strip()
for source in obj.metadata.sources.all()
]
else:
return []
def prepare_relation(self, obj):
"""Return resource relations if exists, otherwise return empty array."""
if hasattr(obj, 'metadata'):
return [
relation.value.strip()
for relation in obj.metadata.relations.all()
]
else:
return []
def prepare_resource_type(self, obj):
"""Resource type is verbose_name attribute of obj argument."""
return obj.verbose_name
def prepare_content_type(self, obj):
if obj.verbose_name != 'Composite Resource':
return [obj.discovery_content_type]
else:
output = []
for f in obj.logical_files:
output.append(f.get_discovery_content_type())
return output
def prepare_comment(self, obj):
"""Return list of all comments on resource."""
return [comment.comment.strip() for comment in obj.comments.all()]
def prepare_comments_count(self, obj):
"""Return count of resource comments."""
return obj.comments_count
def prepare_owner_login(self, obj):
"""Return list of usernames that have ownership access to resource."""
if hasattr(obj, 'raccess'):
return [owner.username for owner in obj.raccess.owners.all()]
else:
return []
# TODO: should utilize name from user profile rather than from User field
def prepare_owner(self, obj):
"""Return list of names of resource owners."""
names = []
if hasattr(obj, 'raccess'):
for owner in obj.raccess.owners.all():
name = normalize_name(owner.first_name.capitalize() + ' ' +
owner.last_name.capitalize())
names.append(name)
return names
# TODO: should utilize name from user profile rather than from User field
def prepare_person(self, obj):
"""Return list of normalized names of resource contributors and owners."""
output0 = []
output1 = []
output2 = []
if hasattr(obj, 'raccess'):
for owner in obj.raccess.owners.all():
name = normalize_name(owner.first_name.capitalize() + ' ' +
owner.last_name.capitalize())
output0.append(name)
if hasattr(obj, 'metadata'):
output1 = [
normalize_name(creator.name)
for creator in obj.metadata.creators.all().exclude(
name__isnull=True).exclude(name='')
]
output2 = [
normalize_name(contributor.name)
for contributor in obj.metadata.contributors.all().exclude(
name__isnull=True).exclude(name='')
]
return list(set(output0 + output1 + output2)) # eliminate duplicates
def prepare_owners_count(self, obj):
"""Return count of resource owners if 'raccess' attribute exists, othrerwise return 0."""
if hasattr(obj, 'raccess'):
return obj.raccess.owners.all().count()
else:
return 0
# # TODO: We might need these later for social discovery
# def prepare_viewer_login(self, obj):
# """Return usernames of users that can view resource, otherwise return empty array."""
# if hasattr(obj, 'raccess'):
# return [viewer.username for viewer in obj.raccess.view_users.all()]
# else:
# return []
# def prepare_viewer(self, obj):
# """Return full names of users that can view resource, otherwise return empty array."""
# names = []
# if hasattr(obj, 'raccess'):
# for viewer in obj.raccess.view_users.all():
# name = viewer.last_name + ', ' + viewer.first_name
# names.append(name)
# return names
# def prepare_viewers_count(self, obj):
# """Return count of users who can view resource, otherwise return 0."""
# if hasattr(obj, 'raccess'):
# return obj.raccess.view_users.all().count()
# else:
# return 0
# def prepare_editor_login(self, obj):
# """Return usernames of editors of a resource, otherwise return 0."""
# if hasattr(obj, 'raccess'):
# return [editor.username for editor in obj.raccess.edit_users.all()]
# else:
# return 0
# def prepare_editor(self, obj):
# """Return full names of editors of a resource, otherwise return empty array."""
# names = []
# if hasattr(obj, 'raccess'):
# for editor in obj.raccess.edit_users.all():
# name = editor.last_name + ', ' + editor.first_name
# names.append(name)
# return names
# def prepare_editors_count(self, obj):
# """Return count of editors of a resource, otherwise return 0."""
# if hasattr(obj, 'raccess'):
# return obj.raccess.edit_users.all().count()
# else:
# return 0
# TODO: These should probably be multi-value fields and pick up all types.
def prepare_geometry_type(self, obj):
"""
Return geometry type if metadata exists, otherwise return [].
"""
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, GeographicFeatureMetaData):
geometry_info = obj.metadata.geometryinformation
if geometry_info is not None:
return geometry_info.geometryType
else:
return None
else:
return None
else:
return None
def prepare_field_name(self, obj):
"""
Return metadata field name if exists, otherwise return [].
"""
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, GeographicFeatureMetaData):
field_info = obj.metadata.fieldinformations.all().first()
if field_info is not None and field_info.fieldName is not None:
return field_info.fieldName.strip()
else:
return None
else:
return None
else:
return None
def prepare_field_type(self, obj):
"""
Return metadata field type if exists, otherwise return None.
"""
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, GeographicFeatureMetaData):
field_info = obj.metadata.fieldinformations.all().first()
if field_info is not None and field_info.fieldType is not None:
return field_info.fieldType.strip()
else:
return None
else:
return None
else:
return None
def prepare_field_type_code(self, obj):
"""
Return metadata field type code if exists, otherwise return [].
"""
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, GeographicFeatureMetaData):
field_info = obj.metadata.fieldinformations.all().first()
if field_info is not None and field_info.fieldTypeCode is not None:
return field_info.fieldTypeCode.strip()
else:
return None
else:
return None
else:
return None
def prepare_variable(self, obj):
"""
Return metadata variable names if exists, otherwise return empty array.
"""
variable_names = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, NetcdfMetaData):
for variable in obj.metadata.variables.all():
variable_names.append(variable.name.strip())
elif isinstance(obj.metadata, RefTSMetadata):
for variable in obj.metadata.variables.all():
variable_names.append(variable.name.strip())
elif isinstance(obj.metadata, TimeSeriesMetaData):
for variable in obj.metadata.variables:
variable_names.append(variable.variable_name.strip())
return variable_names
def prepare_variable_type(self, obj):
"""
Return metadata variable types if exists, otherwise return empty array.
"""
variable_types = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, NetcdfMetaData):
for variable in obj.metadata.variables.all():
variable_types.append(variable.type.strip())
elif isinstance(obj.metadata, RefTSMetadata):
for variable in obj.metadata.variables.all():
variable_types.append(variable.data_type.strip())
elif isinstance(obj.metadata, TimeSeriesMetaData):
for variable in obj.metadata.variables:
variable_types.append(variable.variable_type.strip())
return variable_types
def prepare_variable_shape(self, obj):
"""
Return metadata variable shapes if exists, otherwise return empty array.
"""
variable_shapes = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, NetcdfMetaData):
for variable in obj.metadata.variables.all():
if variable.shape is not None:
variable_shapes.append(variable.shape.strip())
return variable_shapes
def prepare_variable_descriptive_name(self, obj):
"""
Return metadata variable descriptive names if exists, otherwise return empty array.
"""
variable_descriptive_names = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, NetcdfMetaData):
for variable in obj.metadata.variables.all():
if variable.descriptive_name is not None:
variable_descriptive_names.append(
variable.descriptive_name.strip())
return variable_descriptive_names
def prepare_variable_speciation(self, obj):
"""
Return metadata variable speciations if exists, otherwise return empty array.
"""
variable_speciations = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, TimeSeriesMetaData):
for variable in obj.metadata.variables:
if variable.speciation is not None:
variable_speciations.append(
variable.speciation.strip())
return variable_speciations
def prepare_site(self, obj):
"""
Return list of sites if exists, otherwise return empty array.
"""
sites = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, RefTSMetadata):
for site in obj.metadata.sites.all():
if site.name is not None and site.name != '':
sites.append(site.name.strip())
elif isinstance(obj.metadata, TimeSeriesMetaData):
for site in obj.metadata.sites:
if site.site_name is not None and site.site_name != '':
sites.append(site.site_name.strip())
return sites
def prepare_method(self, obj):
"""
Return list of methods if exists, otherwise return empty array.
"""
methods = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, RefTSMetadata):
for method in obj.metadata.methods.all():
if method.description is not None:
methods.append(method.description.strip())
elif isinstance(obj.metadata, TimeSeriesMetaData):
for method in obj.metadata.methods:
if method.method_description is not None:
methods.append(method.method_description.strip())
return methods
def prepare_quality_level(self, obj):
"""
Return list of quality levels if exists, otherwise return empty array.
"""
quality_levels = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, RefTSMetadata):
for quality_level in obj.metadata.quality_levels.all():
if quality_level.code is not None:
quality_levels.append(quality_level.code.strip())
return quality_levels
def prepare_data_source(self, obj):
"""
Return list of data sources if exists, otherwise return empty array.
"""
data_sources = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, RefTSMetadata):
for data_source in obj.metadata.datasources.all():
if data_source.code is not None:
data_sources.append(data_source.code.strip())
return data_sources
def prepare_sample_medium(self, obj):
"""
Return list of sample mediums if exists, otherwise return empty array.
"""
sample_mediums = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, TimeSeriesMetaData):
for time_series_result in obj.metadata.time_series_results:
if time_series_result.sample_medium is not None:
sample_mediums.append(time_series_result.sample_medium)
elif isinstance(obj.metadata, RefTSMetadata):
for variable in obj.metadata.variables.all():
if variable.sample_medium is not None:
sample_mediums.append(variable.sample_medium)
return list(set(sample_mediums))
def prepare_units(self, obj):
"""
Return list of units names if exists, otherwise return empty array.
"""
units_names = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, TimeSeriesMetaData):
for time_series_result in obj.metadata.time_series_results:
if time_series_result.units_name is not None:
units_names.append(time_series_result.units_name)
return units_names
def prepare_units_type(self, obj):
"""
Return list of units types if exists, otherwise return empty array.
"""
units_types = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, TimeSeriesMetaData):
for time_series_result in obj.metadata.time_series_results:
if time_series_result.units_type is not None:
units_types.append(
time_series_result.units_type.strip())
return units_types
def prepare_aggregation_statistics(self, obj):
"""
Return list of aggregation statistics if exists, otherwise return empty array.
"""
aggregation_statistics = []
if hasattr(obj, 'metadata'):
if isinstance(obj.metadata, TimeSeriesMetaData):
for time_series_result in obj.metadata.time_series_results:
if time_series_result.aggregation_statistics is not None:
aggregation_statistics.append(
time_series_result.aggregation_statistics)
return aggregation_statistics
def prepare_absolute_url(self, obj):
"""Return absolute URL of object."""
return obj.get_absolute_url()
def prepare_extra(self, obj):
""" For extra metadata, include both key and value """
extra = []
for key, value in obj.extra_metadata.items():
extra.append(key + ': ' + value)
return extra
class BiologicalCollectionIndex(indexes.SearchIndex, indexes.Indexable):
model_pk = indexes.IntegerField(model_attr='pk')
id = indexes.CharField()
text = indexes.CharField(document=True, use_template=True)
original_species_name = indexes.NgramField(
model_attr='original_species_name',
indexed=True
)
collector = indexes.NgramField(
model_attr='collector',
indexed=True
)
collection_date_year = indexes.IntegerField(
model_attr='collection_date__year',
indexed=True
)
collection_date_month = indexes.IntegerField(
model_attr='collection_date__month',
indexed=True
)
category = indexes.CharField(
model_attr='category'
)
present = indexes.BooleanField(
model_attr='present'
)
absent = indexes.BooleanField(
model_attr='absent'
)
validated = indexes.BooleanField(
model_attr='validated'
)
collection_date = indexes.DateField(
model_attr='collection_date'
)
owner = indexes.CharField(
model_attr='owner__username'
)
notes = indexes.CharField(
model_attr='notes'
)
location_site_name = indexes.CharField(
model_attr='site__name'
)
location_site_id = indexes.CharField(
model_attr='site__id'
)
location_center = indexes.LocationField()
taxon_gbif = indexes.IntegerField(indexed=True)
taxon_gbif_not_null = indexes.BooleanField(indexed=True)
def prepare_taxon_gbif(self, obj):
if obj.taxon_gbif_id:
return obj.taxon_gbif_id.id
return 0
def prepare_taxon_gbif_not_null(self, obj):
if obj.taxon_gbif_id:
return True
return False
def prepare_location_center(self, obj):
if obj.site:
return '%s,%s' % obj.site.get_centroid().coords
return '0,0'
class Meta:
app_label = 'bims'
def index_queryset(self, using=None):
"""Used to reindex model"""
return self.get_model().objects.all()
def get_model(self):
return BiologicalCollectionRecord
class CourseRunIndex(BaseCourseIndex, indexes.Indexable):
model = CourseRun
course_key = indexes.CharField(model_attr='course__key', stored=True)
org = indexes.CharField()
number = indexes.CharField()
status = indexes.CharField(model_attr='status', faceted=True)
start = indexes.DateTimeField(model_attr='start', null=True, faceted=True)
end = indexes.DateTimeField(model_attr='end', null=True)
go_live_date = indexes.DateTimeField(model_attr='go_live_date', null=True)
enrollment_start = indexes.DateTimeField(model_attr='enrollment_start',
null=True)
enrollment_end = indexes.DateTimeField(model_attr='enrollment_end',
null=True)
availability = indexes.CharField(model_attr='availability')
announcement = indexes.DateTimeField(model_attr='announcement', null=True)
min_effort = indexes.IntegerField(model_attr='min_effort', null=True)
max_effort = indexes.IntegerField(model_attr='max_effort', null=True)
weeks_to_complete = indexes.IntegerField(model_attr='weeks_to_complete',
null=True)
language = indexes.CharField(null=True, faceted=True)
transcript_languages = indexes.MultiValueField(faceted=True)
pacing_type = indexes.CharField(model_attr='pacing_type',
null=True,
faceted=True)
is_marketable = indexes.BooleanField(model_attr='type__is_marketable',
faceted=True)
marketing_url = indexes.CharField(null=True)
slug = indexes.CharField(model_attr='slug', null=True)
seat_types = indexes.MultiValueField(model_attr='seat_types__slug',
null=True,
faceted=True)
type = indexes.CharField(model_attr='type_legacy', null=True, faceted=True)
image_url = indexes.CharField(model_attr='image_url', null=True)
partner = indexes.CharField(null=True, faceted=True)
program_types = indexes.MultiValueField()
published = indexes.BooleanField(null=False, faceted=True)
hidden = indexes.BooleanField(model_attr='hidden', faceted=True)
mobile_available = indexes.BooleanField(model_attr='mobile_available',
faceted=True)
authoring_organization_uuids = indexes.MultiValueField()
staff_uuids = indexes.MultiValueField()
subject_uuids = indexes.MultiValueField()
has_enrollable_paid_seats = indexes.BooleanField(null=False)
first_enrollable_paid_seat_sku = indexes.CharField(null=True)
first_enrollable_paid_seat_price = indexes.IntegerField(null=True)
paid_seat_enrollment_end = indexes.DateTimeField(null=True)
license = indexes.MultiValueField(model_attr='license', faceted=True)
has_enrollable_seats = indexes.BooleanField(
model_attr='has_enrollable_seats', null=False)
is_current_and_still_upgradeable = indexes.BooleanField(null=False)
def read_queryset(self, using=None):
# Pre-fetch all fields required by the CourseRunSearchSerializer. Unfortunately, there's
# no way to specify at query time which queryset to use during loading in order to customize
# it for the serializer being used
qset = super(CourseRunIndex, self).read_queryset(using=using)
return qset.prefetch_related('seats__type', )
def prepare_aggregation_key(self, obj):
# Aggregate CourseRuns by Course key since that is how we plan to dedup CourseRuns on the marketing site.
return 'courserun:{}'.format(obj.course.key)
def prepare_has_enrollable_paid_seats(self, obj):
return obj.has_enrollable_paid_seats()
def prepare_first_enrollable_paid_seat_sku(self, obj):
return obj.first_enrollable_paid_seat_sku()
def prepare_first_enrollable_paid_seat_price(self, obj):
return obj.first_enrollable_paid_seat_price
def prepare_is_current_and_still_upgradeable(self, obj):
return obj.is_current_and_still_upgradeable()
def prepare_paid_seat_enrollment_end(self, obj):
return obj.get_paid_seat_enrollment_end()
def prepare_partner(self, obj):
return obj.course.partner.short_code
def prepare_published(self, obj):
return obj.status == CourseRunStatus.Published
def prepare_language(self, obj):
return self._prepare_language(obj.language)
def prepare_number(self, obj):
course_run_key = CourseKey.from_string(obj.key)
return course_run_key.course
def prepare_org(self, obj):
course_run_key = CourseKey.from_string(obj.key)
return course_run_key.org
def prepare_transcript_languages(self, obj):
return [
self._prepare_language(language)
for language in obj.transcript_languages.all()
]
def prepare_marketing_url(self, obj):
return obj.marketing_url
def prepare_program_types(self, obj):
return obj.program_types
def prepare_staff_uuids(self, obj):
return [str(staff.uuid) for staff in obj.staff.all()]
def prepare_subject_uuids(self, obj):
return [str(subject.uuid) for subject in obj.subjects.all()]
class LayerIndex(indexes.SearchIndex, indexes.Indexable):
id = indexes.IntegerField()
abstract = indexes.CharField(model_attr="abstract", boost=1.5)
category__gn_description = indexes.CharField(
model_attr="category__gn_description", null=True)
csw_type = indexes.CharField(model_attr="csw_type")
csw_wkt_geometry = indexes.CharField(model_attr="csw_wkt_geometry")
detail_url = indexes.CharField(model_attr="get_absolute_url")
owner__username = indexes.CharField(model_attr="owner",
faceted=True,
null=True)
owner__first_name = indexes.CharField(model_attr="owner__first_name",
faceted=True,
null=True)
owner__last_name = indexes.CharField(model_attr="owner__last_name",
faceted=True,
null=True)
is_published = indexes.BooleanField(model_attr="is_published")
featured = indexes.BooleanField(model_attr="featured")
popular_count = indexes.IntegerField(model_attr="popular_count",
default=0,
boost=20)
share_count = indexes.IntegerField(model_attr="share_count", default=0)
rating = indexes.IntegerField(null=True)
srid = indexes.CharField(model_attr="srid")
supplemental_information = indexes.CharField(
model_attr="supplemental_information", null=True)
thumbnail_url = indexes.CharField(model_attr="thumbnail_url", null=True)
uuid = indexes.CharField(model_attr="uuid")
title = indexes.CharField(model_attr="title", boost=2)
date = indexes.DateTimeField(model_attr="date")
text = indexes.EdgeNgramField(document=True,
use_template=True,
stored=False)
type = indexes.CharField(faceted=True)
subtype = indexes.CharField(faceted=True)
typename = indexes.CharField(model_attr='typename')
title_sortable = indexes.CharField(indexed=False,
stored=False) # Necessary for sorting
category = indexes.CharField(model_attr="category__identifier",
faceted=True,
null=True,
stored=True)
bbox_left = indexes.FloatField(model_attr="bbox_x0",
null=True,
stored=False)
bbox_right = indexes.FloatField(model_attr="bbox_x1",
null=True,
stored=False)
bbox_bottom = indexes.FloatField(model_attr="bbox_y0",
null=True,
stored=False)
bbox_top = indexes.FloatField(model_attr="bbox_y1",
null=True,
stored=False)
temporal_extent_start = indexes.DateTimeField(
model_attr="temporal_extent_start", null=True, stored=False)
temporal_extent_end = indexes.DateTimeField(
model_attr="temporal_extent_end", null=True, stored=False)
keywords = indexes.MultiValueField(model_attr="keyword_slug_list",
null=True,
faceted=True,
stored=True)
regions = indexes.MultiValueField(model_attr="region_name_list",
null=True,
faceted=True,
stored=True)
popular_count = indexes.IntegerField(model_attr="popular_count",
default=0,
boost=20)
share_count = indexes.IntegerField(model_attr="share_count", default=0)
rating = indexes.IntegerField(null=True)
num_ratings = indexes.IntegerField(stored=False)
num_comments = indexes.IntegerField(stored=False)
geogig_link = indexes.CharField(null=True)
has_time = indexes.BooleanField(faceted=True, null=True)
def get_model(self):
return Layer
def prepare_id(self, obj):
# this is awful I'm so sorry
if obj.uuid is None or obj.uuid == '':
return obj.id
else:
return int(uuid.UUID(obj.uuid).time_low)
def prepare_type(self, obj):
return "layer"
def prepare_subtype(self, obj):
if obj.storeType == "dataStore":
return "vector"
elif obj.storeType == "coverageStore":
return "raster"
elif obj.storeType == "remoteStore":
return "remote"
def prepare_rating(self, obj):
ct = ContentType.objects.get_for_model(obj)
try:
rating = OverallRating.objects.filter(
object_id=obj.pk,
content_type=ct).aggregate(r=Avg("rating"))["r"]
return float(str(rating or "0"))
except OverallRating.DoesNotExist:
return 0.0
def prepare_num_ratings(self, obj):
ct = ContentType.objects.get_for_model(obj)
try:
return OverallRating.objects.filter(object_id=obj.pk,
content_type=ct).all().count()
except OverallRating.DoesNotExist:
return 0
def prepare_num_comments(self, obj):
try:
return Comment.objects.filter(
object_id=obj.pk,
content_type=ContentType.objects.get_for_model(
obj)).all().count()
except:
return 0
def prepare_title_sortable(self, obj):
return obj.title.lower()
def prepare_geogig_link(self, obj):
try:
return obj.geogig_link
except:
return None
# Check to see if either time extent is set on the object,
# if so, then it is time enabled.
def prepare_has_time(self, obj):
try:
# if either time field is set to a value then time is enabled.
if (obj.temporal_extent_start is not None
or obj.temporal_extent_end is not None):
return True
except:
# when in doubt, it's false.
return False
class LayerIndex(indexes.SearchIndex, indexes.Indexable):
id = indexes.IntegerField(model_attr='resourcebase_ptr_id')
abstract = indexes.CharField(model_attr="abstract", boost=1.5)
category__gn_description = indexes.CharField(
model_attr="category__gn_description", null=True)
csw_type = indexes.CharField(model_attr="csw_type")
csw_wkt_geometry = indexes.CharField(model_attr="csw_wkt_geometry")
detail_url = indexes.CharField(model_attr="get_absolute_url")
owner__username = indexes.CharField(model_attr="owner",
faceted=True,
null=True)
is_published = indexes.BooleanField(model_attr="is_published")
featured = indexes.BooleanField(model_attr="featured")
srid = indexes.CharField(model_attr="srid")
supplemental_information = indexes.CharField(
model_attr="supplemental_information", null=True)
thumbnail_url = indexes.CharField(model_attr="thumbnail_url", null=True)
uuid = indexes.CharField(model_attr="uuid")
title = indexes.CharField(model_attr="title", boost=2)
date = indexes.DateTimeField(model_attr="date")
text = indexes.EdgeNgramField(document=True,
use_template=True,
stored=False)
type = indexes.CharField(faceted=True)
subtype = indexes.CharField(faceted=True)
alternate = indexes.CharField(model_attr='alternate')
title_sortable = indexes.CharField(indexed=False,
stored=False) # Necessary for sorting
category = indexes.CharField(model_attr="category__identifier",
faceted=True,
null=True,
stored=True)
bbox_left = indexes.FloatField(model_attr="bbox_x0",
null=True,
stored=False)
bbox_right = indexes.FloatField(model_attr="bbox_x1",
null=True,
stored=False)
bbox_bottom = indexes.FloatField(model_attr="bbox_y0",
null=True,
stored=False)
bbox_top = indexes.FloatField(model_attr="bbox_y1",
null=True,
stored=False)
temporal_extent_start = indexes.DateTimeField(
model_attr="temporal_extent_start", null=True, stored=False)
temporal_extent_end = indexes.DateTimeField(
model_attr="temporal_extent_end", null=True, stored=False)
keywords = indexes.MultiValueField(model_attr="keyword_slug_list",
null=True,
faceted=True,
stored=True)
regions = indexes.MultiValueField(model_attr="region_name_list",
null=True,
faceted=True,
stored=True)
popular_count = indexes.IntegerField(model_attr="popular_count",
default=0,
boost=20)
share_count = indexes.IntegerField(model_attr="share_count", default=0)
rating = indexes.IntegerField(null=True)
num_ratings = indexes.IntegerField(stored=False)
num_comments = indexes.IntegerField(stored=False)
def get_model(self):
return Dataset
def prepare_type(self, obj):
return "layer"
def prepare_subtype(self, obj):
if obj.subtype == "vector":
if obj.has_time:
return "vector_time"
else:
return "vector"
elif obj.subtype == "raster":
return "raster"
elif obj.subtype in ['tileStore', 'remote']:
return "remote"
def prepare_rating(self, obj):
ct = ContentType.objects.get_for_model(obj)
try:
rating = OverallRating.objects.filter(
object_id=obj.pk,
content_type=ct).aggregate(r=Avg("rating"))["r"]
return float(str(rating or "0"))
except OverallRating.DoesNotExist:
return 0.0
def prepare_num_ratings(self, obj):
ct = ContentType.objects.get_for_model(obj)
try:
return OverallRating.objects.filter(object_id=obj.pk,
content_type=ct).all().count()
except OverallRating.DoesNotExist:
return 0
def prepare_num_comments(self, obj):
try:
return Comment.objects.filter(
object_id=obj.pk,
content_type=ContentType.objects.get_for_model(
obj)).all().count()
except Exception:
return 0
def prepare_title_sortable(self, obj):
return obj.title.lower()
class CompanyIndex(BaseSearchIndex, indexes.Indexable):
_id = indexes.CharField(model_attr="_id")
user = indexes.CharField(model_attr="user")
created_at = indexes.DateTimeField(model_attr="created_at")
updated_at = indexes.DateTimeField(model_attr="updated_at")
is_deleted = indexes.BooleanField(model_attr="is_deleted")
name = indexes.CharField(model_attr="name")
short_description = indexes.CharField(model_attr="short_description")
round_notes = indexes.CharField(model_attr="round_notes")
domain = indexes.CharField(model_attr="domain", faceted=True)
foundation_date = indexes.DateField(model_attr="foundation_date",
faceted=True)
last_round = indexes.DateField(model_attr="last_round", faceted=True)
country_code = indexes.CharField(model_attr="country_code", faceted=True)
stock_symbol = indexes.CharField(model_attr="stock_symbol", faceted=True)
contact_email = indexes.CharField(model_attr="stock_symbol")
# Address UDT fields
address_street_type = indexes.CharField(model_attr="address.street_type")
address_street_name = indexes.CharField(model_attr="address.street_name")
address_street_number = \
indexes.IntegerField(model_attr="address.street_number")
address_state = indexes.CharField(model_attr="address.state", faceted=True)
address_region = indexes.CharField(model_attr="address.region",
faceted=True)
address_city = indexes.CharField(model_attr="address.city", faceted=True)
address_country_code = indexes.CharField(model_attr="address.country_code",
faceted=True)
address_zipcode = indexes.CharField(model_attr="address.zipcode",
faceted=True)
coordinates = indexes.LocationField(model_attr="coordinates")
founders = indexes.MultiValueField(null=True,
model_attr="founders",
faceted=True)
specialties = indexes.MultiValueField(null=True,
model_attr="specialties",
faceted=True)
latest_twitter_followers = indexes.MultiValueField(
null=True, model_attr="latest_twitter_followers")
websites = indexes.MultiValueField(null=True, model_attr="websites")
crawler_config = indexes.CharField(model_attr="crawler_config")
extra_data = indexes.CharField(model_attr="extra_data")
class Meta:
text_fields = ["short_description", "extra_data", "round_notes"]
# Once the index has been created it cannot be changed
# with sync_indexes. Changes should be made by hand.
index_settings = {
"realtime": "true",
"autoCommitTime": "100",
"ramBufferSize": "2048"
}
def get_model(self):
return Company
def index_queryset(self, using=None):
return self.get_model().objects.filter(created_at__lte=timezone.now(),
is_deleted=False)
@staticmethod
def prepare_autocomplete(obj):
return " ".join((obj.name, obj.address_city, obj.address_zipcode))
class MediaCollectionIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
item_acces = indexes.CharField(model_attr='public_access', faceted=True)
item_status = indexes.CharField(model_attr='document_status', faceted=True)
digitized = indexes.BooleanField(default=False, faceted=True)
media_type = indexes.CharField(model_attr='media_type',
null='None',
faceted=True)
recording_context = indexes.CharField(model_attr='recording_context',
default='',
faceted=True)
physical_format = indexes.CharField(model_attr='physical_format',
default='',
faceted=True)
#content_auto = indexes.EdgeNgramField(model_attr='content')
#advance search
title = indexes.CharField(model_attr='title')
code = KeywordField(model_attr='code', default='')
location_principal = indexes.CharField(default='', boost=1.05)
location_relation = indexes.CharField()
ethnic_group = indexes.CharField(default='')
instruments = indexes.CharField(default='')
collectors = indexes.CharField(model_attr='collector', default='')
recorded_from_date = indexes.DateField(model_attr='recorded_from_year',
null=True)
recorded_to_date = indexes.DateField(model_attr='recorded_to_year',
null=True)
year_published = indexes.IntegerField(model_attr='year_published',
default='')
def prepare_digitized(self, obj):
return obj.has_mediafile
def get_model(self):
return MediaCollection
def prepare_location_principal(self, obj):
collec_location = []
for item in obj.items.all():
location = []
if item.location is not None:
collec_location.append(item.location.name)
return u"".join('|' + location for location in collec_location)
def prepare_location_relation(self, obj):
collec_location = []
for item in obj.items.all():
location = []
if item.location is not None:
location_alias = LocationAlias.objects.filter(
location__name=item.location)
location_rela = LocationRelation.objects.filter(
location__name=item.location)
for rela in location_rela:
location.append(rela.ancestor_location.name)
for alias in location_alias:
location.append(alias.alias)
if item.location.current_location is not None:
location.append(item.location.current_location.name)
for name in location:
if name and not name in collec_location:
collec_location.append(name)
return u"".join('|' + location for location in collec_location)
def prepare_ethnic_group(self, obj):
return "%s" % obj.ethnic_groups()
def prepare_instruments(self, obj):
instruments = []
items = obj.items.all()
for item in items:
materials = MediaItemPerformance.objects.all().filter(
media_item__exact=item)
for material in materials:
if material.instrument and not material.instrument in instruments:
instruments.append(material.instrument.name)
if material.alias and not material.alias in instruments:
instruments.append(material.alias.name)
return u"".join('|' + instru for instru in instruments)
def prepare_recorded_from_date(self, obj):
if obj.recorded_from_year != 0:
return datetime.date(int(obj.recorded_from_year), 01, 01)
else:
return None
def prepare_recorded_to_date(self, obj):
if obj.recorded_to_year != 0:
return datetime.date(int(obj.recorded_to_year), 01, 01)
else:
return None
class DefaultTagObjectIndex(indexes.SearchIndex):
#mt_location_place = indexes.CharField(model_attr='media_tag__location_place', null=True)
#mt_people_name = indexes.CharField(model_attr='media_tag__people_name', null=True)
mt_public = indexes.BooleanField(model_attr='media_tag__public', null=True)
class ActivityPageIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(
document=True, use_template=True
)
date = indexes.DateTimeField(
model_attr='date'
)
summary = indexes.CharField(
model_attr='summary'
)
live = indexes.BooleanField(
model_attr='live'
)
big_idea = indexes.CharField(
model_attr='big_idea'
)
essential_questions = indexes.CharField(
model_attr='essential_questions'
)
objectives = indexes.CharField(
model_attr='objectives'
)
what_students_will_do = indexes.CharField(
model_attr='what_students_will_do'
)
building_block = indexes.MultiValueField(
model_attr='building_block', faceted=True
)
school_subject = indexes.MultiValueField(
model_attr='school_subject', faceted=True
)
topic = indexes.MultiValueField(
model_attr='topic', faceted=True
)
# Audience
grade_level = indexes.MultiValueField(
model_attr='grade_level', faceted=True
)
age_range = indexes.MultiValueField(
model_attr='age_range', faceted=True
)
student_characteristics = indexes.MultiValueField(
model_attr='student_characteristics', faceted=True
)
# Activity Characteristics
activity_type = indexes.MultiValueField(
model_attr='activity_type', faceted=True
)
teaching_strategy = indexes.MultiValueField(
model_attr='teaching_strategy', faceted=True
)
blooms_taxonomy_level = indexes.MultiValueField(
model_attr='blooms_taxonomy_level', faceted=True
)
activity_duration = indexes.CharField(
model_attr='activity_duration', faceted=True
)
# Standards taught
jump_start_coalition = indexes.MultiValueField(
model_attr='jump_start_coalition', faceted=True
)
council_for_economic_education = indexes.MultiValueField(
model_attr='council_for_economic_education', faceted=True
)
def prepare_building_block(self, obj):
return [item.id for item in obj.building_block.all()]
def prepare_school_subject(self, obj):
return [item.id for item in obj.school_subject.all()]
def prepare_topic(self, obj):
return [item.id for item in obj.topic.all()]
def prepare_grade_level(self, obj):
return [item.id for item in obj.grade_level.all()]
def prepare_age_range(self, obj):
return [item.id for item in obj.age_range.all()]
def prepare_student_characteristics(self, obj):
return [item.id for item in obj.student_characteristics.all()]
def prepare_activity_type(self, obj):
return [item.id for item in obj.activity_type.all()]
def prepare_teaching_strategy(self, obj):
return [item.id for item in obj.teaching_strategy.all()]
def prepare_blooms_taxonomy_level(self, obj):
return [item.id for item in obj.blooms_taxonomy_level.all()]
def prepare_activity_duration(self, obj):
return obj.activity_duration.id
def prepare_jump_start_coalition(self, obj):
return [item.id for item in obj.jump_start_coalition.all()]
def prepare_council_for_economic_education(self, obj):
return [item.id for item in obj.council_for_economic_education.all()]
def get_model(self):
return ActivityPage
def index_queryset(self, using=None):
"""Only index descendants of the Activity Search page"""
# This is safe because ActivityIndexPage is a singleton
search_page = ActivityIndexPage.objects.get()
return self.get_model().objects.live().descendant_of(search_page)
class SentencesIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(document=True)
id = indexes.IntegerField(model_attr='id')
sentence_text = indexes.CharField()
sentence_text_stemmed = indexes.CharField()
lang = indexes.CharField(default='und')
owner = indexes.CharField(default='')
owner_is_native = indexes.BooleanField(default=False)
is_orphan = indexes.BooleanField(default=True)
tags = indexes.CharField(default='')
is_tagged = indexes.BooleanField(default=False)
created = indexes.DateTimeField(model_attr='created',
default=datetime(1, 1, 1))
modified = indexes.DateTimeField(model_attr='modified',
default=datetime(1, 1, 1))
has_audio = indexes.BooleanField(default=False)
is_unapproved = indexes.BooleanField(default=False)
trans_langs = indexes.CharField(default='')
trans_owners = indexes.CharField(default='')
trans_has_orphan = indexes.BooleanField(default=False)
trans_has_audio = indexes.BooleanField(default=False)
trans_is_unapproved = indexes.BooleanField(default=False)
def get_model(self):
return Sentences
def get_updated_field(self):
return 'modified'
def index_queryset(self, using=None):
return self.get_model().objects.all()
def prepare(self, object):
self.prepared_data = super(SentencesIndex, self).prepare(object)
text = uclean(object.text)
lang = object.lang
user = Users.objects.filter(id=object.user_id)
user = user[0] if user else None
owner = user.username if user else ''
owner = uclean(owner)
is_orphan = not bool(owner)
owner_is_native = bool(
UsersLanguages.objects.filter(of_user_id=object.user_id,
by_user_id=object.user_id,
language_code=lang,
level=5).values_list('language_code',
flat=True))
tags = Tags.objects.filter(
id__in=TagsSentences.objects\
.filter(sentence_id=object.id)\
.values_list('tag_id', flat=True)\
).values_list('name', flat=True)
tags = list(set(tags))
tags = ' | '.join(tags) if tags else ''
tags = uclean(tags)
is_tagged = bool(tags)
is_unapproved = bool(object.correctness == -1)
has_audio = bool(object.hasaudio == 'shtooka'
or object.hasaudio == 'from_users')
direct_props = list(
Sentences.objects.raw(
"""
SELECT `s`.`id`, `s`.`user_id`, `s`.`lang`, `s`.`correctness`, `s`.`hasaudio`
FROM `sentences` as `s`
JOIN `sentences_translations` as `t`
ON `t`.`translation_id` = `s`.id
WHERE `t`.`sentence_id` = %s ;
""", [object.id]))
if direct_props:
props = defaultdict(set)
for prop in direct_props:
for key in ('user_id', 'lang', 'correctness', 'hasaudio'):
props[key].add(getattr(prop, key))
if None in props['lang']:
props['lang'].remove(None)
props['lang'].add('und')
if '' in props['lang']:
props['lang'].remove('')
props['lang'].add('und')
trans_langs = ' | '.join(list(props['lang']))
trans_is_unapproved = -1 in props['correctness']
trans_has_orphan = None in props['user_id']
if trans_has_orphan: props['user_id'].remove(None)
users = list(props['user_id'])
users = Users.objects.filter(id__in=users).values_list('username',
flat=True)
trans_owners = ' | '.join(users)
trans_has_audio = 'shtooka' in props['hasaudio'] or \
'from_users' in props['hasaudio']
self.prepared_data['trans_langs'] = trans_langs
self.prepared_data['trans_owners'] = trans_owners
self.prepared_data['trans_has_audio'] = trans_has_audio
self.prepared_data['trans_is_unapproved'] = trans_is_unapproved
self.prepared_data['trans_has_orphan'] = trans_has_orphan
self.prepared_data['sentence_text'] = text
self.prepared_data['sentence_text_stemmed'] = stemmer.stem(text, lang)
self.prepared_data['owner'] = owner
self.prepared_data['is_orphan'] = is_orphan
self.prepared_data['owner_is_native'] = owner_is_native
self.prepared_data['tags'] = tags
self.prepared_data['is_tagged'] = is_tagged
self.prepared_data['is_unapproved'] = is_unapproved
self.prepared_data['has_audio'] = has_audio
return self.prepared_data
class dataroomdirectoryorfileIndex(indexes.SearchIndex, indexes.Indexable):
"""
dataroomdirectoryorfile索引类
"""
text = indexes.CharField(document=True, use_template=True)
# # 保存在索引库中的字段
id = indexes.IntegerField(model_attr='id')
dataroom = indexes.IntegerField(model_attr='dataroom_id', null=True)
bucket = indexes.CharField(model_attr='bucket', null=True)
parent = indexes.IntegerField(model_attr='parent_id', null=True)
orderNO = indexes.IntegerField(model_attr='orderNO', null=True)
size = indexes.IntegerField(model_attr='size', null=True)
filename = indexes.CharField(model_attr='filename', null=True)
realfilekey = indexes.CharField(model_attr='realfilekey', null=True)
key = indexes.CharField(model_attr='key', null=True)
isFile = indexes.BooleanField(null=True)
fileContent = indexes.CharField(null=True)
createuser = indexes.IntegerField(model_attr='createuser_id', null=True)
def get_model(self):
"""返回建立索引的模型类"""
return dataroomdirectoryorfile
def get_updated_field(self):
return 'lastmodifytime'
def prepare_fileContent(self, obj):
"""obj 是django里的model实例"""
filecontent = None
if obj.isFile and obj.key:
dataroomPath = os.path.join(APILOG_PATH['es_dataroomPDFPath'],
'dataroom_{}'.format(obj.dataroom_id))
file_path = os.path.join(dataroomPath, obj.key)
try:
if os.path.exists(file_path):
filename, type = os.path.splitext(file_path)
if type == '.docx':
doc = docx.Document(file_path)
filecontent = '\n'.join(
[paragraph.text for paragraph in doc.paragraphs])
elif type == '.doc':
filecontent = subprocess.check_output(
["antiword", file_path])
elif type == '.pdf':
filecontent = getPdfWordContent(file_path)
elif type in ['.png', '.jpg', '.jpeg']:
filecontent = BaiDuAipGetImageWord(file_path)
elif type == '.txt':
with open(file_path, "r") as f:
text = f.read()
type = chardet.detect(text)
filecontent = text.decode(type["encoding"],
'ignore')
except Exception:
logexcption(msg='dataroom文件pdf内容提取失败')
return filecontent
def index_queryset(self, using=None):
"""返回要建立索引的数据查询集"""
return self.get_model().objects.filter(is_deleted=False,
dataroom__is_deleted=False,
isFile=True)
class BaseFileIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.NgramField(document=True, use_template=True)
title = indexes.CharField(model_attr='title') # 标题u
pubDate = indexes.DateTimeField(model_attr='pubDate', null=True) # 出版时间
creator = indexes.CharField(model_attr='creator', null=True) # 作者/编者
keywords = indexes.CharField(model_attr='keywords', null=True) # 关键字
check = indexes.BooleanField(model_attr='check', null=True)
updateDate = indexes.DateTimeField(model_attr='updateDate', null=True)
# 用于面搜索和精确搜索
fileType = indexes.MultiValueField()
fileFormat = indexes.MultiValueField()
spatial = indexes.MultiValueField()
discipline = indexes.MultiValueField()
language = indexes.MultiValueField() # 需要prepare_language函数做预处理
subjecttype = indexes.IntegerField(model_attr='subjecttype',
null=True) # 专题类型
def get_model(self):
return FileBaseInfo
def index_queryset(self, using=None):
"""Used when the entire index for model is updated."""
return self.get_model().objects.order_by(
'-updateDate') # check=True才检索出来
def prepare(self, obj):
data = super(BaseFileIndex, self).prepare(obj)
if obj.attachment.path.split('/')[-1].split('.')[-1] == 'pdf':
try:
extracted_data = search_utils.parse_to_string(
obj.attachment.path) # 文件路径
except:
print(obj.filecode)
extracted_data = ''
else:
extracted_data = ''
t = loader.select_template(
('search/indexes/himalaya/filebaseinfo_text.txt', ))
data['text'] = t.render({'object': obj, 'extracted': extracted_data})
ext = SubjectTheme.objects.filter(subjectId=obj.subjecttype)
for item in ext:
if (int(item.fieldType) == 5 or int(item.fieldType) == 6):
ex = []
for key in FileExtendInfo.objects.filter(fieldId=item.id,
fileId=obj.id):
tmp = Category.objects.get(id=int(key.filedValue))
val = []
while tmp.id != int(item.corrAttri):
val.append(tmp.attrName)
tmp = Category.objects.get(id=tmp.pid_id)
for i in range(1, len(val) + 1):
ss = str(i) + '/'
for j in range(0, i):
if j < i - 1:
ss = ss + str(
val[len(val) - 1 - j]).strip() + '/'
else:
ss = ss + str(val[len(val) - 1 - j]).strip()
ex.append(ss)
cer = ex[len(ex) - 1].split('/', 1)
path = str(int(cer[0]) + 1) + '/' + cer[1] + '/' + val[0]
ex.append(path)
else:
ex = []
for key in FileExtendInfo.objects.filter(fieldId=item.id,
fileId=obj.id):
ex.append(key.filedValue)
if (len(ex) != 0):
strs = str(item.id) + '_ext'
data[strs] = ex
return data
def prepare_language(self, obj):
return [lang.lanTypeName for lang in obj.language.all()]
def prepare_spatial(self, obj):
return [spa.spcaeTypeName for spa in obj.spatial.all()]
def prepare_discipline(self, obj):
return [dis.disciplineTypeName for dis in obj.discipline.all()]
def prepare_fileFormat(self, obj):
return [filemat.formatTypeName for filemat in obj.fileFormat.all()]
def prepare_fileType(self, obj):
return [file_type.fileTypeName for file_type in obj.fileType.all()]
