class CompanyDocument(DocType):
# ID
id = fields.IntegerField(attr='id')
avatar = fields.TextField()
slug = fields.StringField()
name = StringField(analyzer=html_strip, fields={
'raw': KeywordField(),
})
about = fields.TextField()
headquarter = fields.TextField()
is_address_public = fields.BooleanField()
website = fields.StringField()
since = fields.StringField()
size_from = fields.IntegerField()
size_to = fields.IntegerField()
creator = fields.ObjectField(attr='create_field_indexing',
properties={'id': fields.IntegerField()})
tags = fields.ObjectField(attr='tag_field_indexing',
properties={
'id': fields.IntegerField(),
'name': fields.StringField()
})
class Django:
model = Company
class ElasticFileDocument(Document):
office_id = IntegerField()
first_crawl_epoch = IntegerField()
web_domains = KeywordField()
class Django:
model = Source_Document
fields = [
'id', 'intersection_status', 'min_income_year', 'max_income_year',
'section_count', 'sha256'
]
class ElasticSectionDocument(Document):
default_field_name = "person_name"
source_document_id = IntegerField()
office_id = IntegerField()
position_and_department = TextField()
income_size = IntegerField()
spouse_income_size = IntegerField()
person_id = IntegerField()
region_id = IntegerField()
car_brands = KeywordField()
class Django:
model = Section
fields = ['id', 'person_name', 'income_year', 'rubric_id', 'gender']
@property
def rubric_str(self):
if self.rubric_id is None:
return "unknown"
else:
return get_russian_rubric_str(self.rubric_id)
class JobDocument(DocType):
# ID
id = fields.IntegerField(attr='id')
title = StringField(analyzer=html_strip, fields={
'raw': KeywordField(),
})
description = fields.TextField()
have_daily_perks = fields.BooleanField()
daily_perks_budget = fields.DoubleField()
have_transportation = fields.BooleanField()
transportation_budget = fields.DoubleField()
have_meal = fields.BooleanField()
meal_budget = fields.DoubleField()
have_space_rest = fields.BooleanField()
space_rest_budget = fields.DoubleField()
is_male = fields.BooleanField()
is_female = fields.BooleanField()
age = fields.IntegerField()
hide_company = fields.BooleanField()
latitude = fields.GeoPointField()
longitude = fields.GeoPointField()
slug = fields.StringField()
publish_date = fields.DateField()
tags = fields.ObjectField(attr='tag_field_indexing',
properties={
'id': fields.IntegerField(),
'name': fields.StringField()
})
company = fields.ObjectField(attr='company_indexing',
properties={
'name': fields.StringField(),
'avatar': fields.StringField(),
'slug': fields.StringField(),
'pk': fields.IntegerField()
})
class Django:
model = Job
class ProfileDocument(DocType):
class Meta:
model = Profile
queryset_pagination = 1000
doc_type = 'profiles.profile'
autocomplete = fields.TextField(
analyzer=edge_ngram_analyzer,
search_analyzer=edge_ngram_search_analyzer,
)
url = fields.KeywordField(attr='get_absolute_url')
api_url = fields.KeywordField(attr='get_api_url')
created = fields.DateField()
updated = fields.DateField()
# time/date properties from user model
date_joined = fields.DateField(attr='user.date_joined')
last_login = fields.DateField(attr='user.last_login')
name = fields.TextField(fielddata=True)
# exact_name = fields.KeywordField(attr='name')
tags = KeywordField()
expertise = KeywordField()
groups = KeywordField()
# labelcode = KeywordField()
#
# # tags = fields.NestedField(properties={
# # 'name': fields.KeywordField(),
# # 'id': fields.IntegerField(),
# # })
#
# creator = fields.KeywordField(attr='creator.username')
#
#
image = KeywordField()
#
# type = fields.KeywordField(attr='get_type_display')
#
# year_start = fields.IntegerField()
# year_end = fields.IntegerField()
#
# description = fields.TextField(attr='description')
country = KeywordField(attr='country.printable_name')
country_code = KeywordField(attr='country.iso2_code')
# recent_activity = fields.IntegerField()
###################################################################
# field preparation
###################################################################
def prepare_autocomplete(self, instance):
text = [instance.name.strip()]
# TODO: check what exact fields needed
# if instance.labelcode:
# text += [instance.labelcode.strip()]
# if instance.get_root():
# text += [instance.get_root().name.strip()]
# if instance.parent:
# text += [instance.parent.name.strip()]
if instance.country:
text += [instance.country.iso2_code]
return text
#
def prepare_name(self, instance):
return instance.get_display_name().strip()
def prepare_tags(self, instance):
return [i.strip() for i in instance.d_tags.split(',') if len(i) > 2]
def prepare_groups(self, instance):
return [g.name.strip() for g in instance.user.groups.all()]
def prepare_expertise(self, instance):
return [e.name.strip() for e in instance.expertise.all()]
def prepare_image(self, instance):
if hasattr(instance, 'main_image') and instance.main_image:
try:
return get_thumbnailer(
instance.main_image).get_thumbnail(THUMBNAIL_OPT).url
except InvalidImageFormatError as e:
pass
# def prepare_recent_activity(self, instance):
# return actor_stream(instance.user).filter(
# timestamp__gte=timezone.now() - timedelta(days=365)
# ).count()
# def prepare_year_start(self, instance):
# if instance.date_start:
# return instance.date_start.year
#
# def prepare_year_end(self, instance):
# if instance.date_end:
# return instance.date_end.year
###################################################################
# custom queryset
###################################################################
def get_queryset(self):
return super(ProfileDocument,
self).get_queryset().select_related('user', 'mentor')
class PlaylistDocument(DocType):
class Meta:
model = Playlist
queryset_pagination = 1000
doc_type = 'alibrary.playlist'
autocomplete = fields.TextField(
analyzer=edge_ngram_analyzer,
search_analyzer=edge_ngram_search_analyzer,
)
url = fields.KeywordField(attr='get_absolute_url')
api_url = fields.KeywordField(attr='get_api_url')
created = fields.DateField()
updated = fields.DateField()
# 'fielddata' is needed for sorting on the filed
name = fields.TextField(fielddata=True)
tags = KeywordField()
user = fields.KeywordField(attr='user.username')
image = KeywordField()
type = fields.KeywordField(attr='get_type_display')
status = fields.KeywordField(attr='get_status_display')
target_duration = fields.KeywordField()
weather = fields.KeywordField()
seasons = fields.KeywordField()
series = fields.KeywordField()
daypart_days = fields.KeywordField()
daypart_slots = fields.KeywordField()
description = fields.TextField()
last_emission = fields.DateField()
next_emission = fields.DateField()
num_emissions = fields.IntegerField()
state_flags = fields.KeywordField()
###################################################################
# field preparation
###################################################################
def prepare_autocomplete(self, instance):
# if instance.type == 'basket':
# return
text = [instance.name.strip()]
if instance.series and instance.series_number:
text += ['{} #{}'.format(instance.series, instance.series_number)]
elif instance.series:
text += [instance.series.name]
return text
def prepare_name(self, instance):
return instance.name.strip()
def prepare_tags(self, instance):
return [i.strip() for i in instance.d_tags.split(',') if len(i) > 2]
#return [{'id': i.pk, 'name': i.name} for i in instance.tags.all()]
def prepare_image(self, instance):
if hasattr(instance, 'main_image') and instance.main_image:
try:
return get_thumbnailer(
instance.main_image).get_thumbnail(THUMBNAIL_OPT).url
except (InvalidImageFormatError, AttributeError):
pass
def prepare_target_duration(self, instance):
if instance.target_duration:
return '{} Minutes'.format(instance.get_target_duration_display())
return
def prepare_series(self, instance):
if instance.series and instance.series_number:
return '{} #{}'.format(instance.series, instance.series_number)
elif instance.series:
return instance.series.name
return
def prepare_last_emission(self, instance):
if instance.last_emission:
return instance.last_emission.time_start
def prepare_next_emission(self, instance):
if instance.next_emission:
return instance.next_emission.time_start
def prepare_num_emissions(self, instance):
return instance.get_emissions().count()
def prepare_weather(self, instance):
return [w.name.strip() for w in instance.weather.all()]
def prepare_seasons(self, instance):
return [s.name.strip() for s in instance.seasons.all()]
def prepare_daypart_days(self, instance):
return [d.get_day_display() for d in instance.dayparts.all()]
def prepare_daypart_slots(self, instance):
return [
'{:%H} - {:%H}h'.format(d.time_start, d.time_end)
for d in instance.dayparts.all()
]
def prepare_state_flags(self, instance):
flags = []
if instance.is_archived:
flags += ['Archived - Yes']
elif instance.type == 'broadcast':
flags += ['Archived - No']
if instance.rotation:
flags += ['In Rotation - Yes']
elif instance.type == 'broadcast':
flags += ['In Rotation - No']
return flags
class MediaDocument(DocType):
class Meta:
model = Media
queryset_pagination = 1000
doc_type = 'alibrary.media'
autocomplete = fields.TextField(
analyzer=edge_ngram_analyzer,
search_analyzer=edge_ngram_search_analyzer,
)
url = fields.KeywordField(attr='get_absolute_url')
api_url = fields.KeywordField(attr='get_api_url')
created = fields.DateField()
updated = fields.DateField()
# 'fielddata' is needed for sorting on the filed
name = fields.TextField(fielddata=True,
fields={'raw': {
'type': 'keyword'
}})
artist_display = fields.KeywordField(attr='get_artist_display',
fields={'raw': {
'type': 'keyword'
}})
release_display = fields.KeywordField(attr='release.name',
fields={'raw': {
'type': 'keyword'
}})
tags = KeywordField()
# id = fields.IntegerField()
creator = fields.KeywordField(attr='creator.username')
image = KeywordField()
type = fields.KeywordField(attr='get_mediatype_display')
version = fields.KeywordField(attr='get_version_display')
#barcode = fields.KeywordField()
description = fields.TextField()
lyrics = fields.TextField()
lyrics_language = fields.KeywordField(attr='get_lyrics_language_display')
#country = KeywordField(attr='release_country.iso2_code')
# audio-properties
duration = fields.IntegerField(attr='master_duration')
bitrate = fields.IntegerField(attr='master_bitrate')
samplerate = fields.IntegerField(attr='master_samplerate')
encoding = fields.KeywordField(attr='master_encoding')
tempo = fields.FloatField(attr='tempo')
# license = fields.NestedField(properties={
# 'name': fields.KeywordField(),
# 'id': fields.IntegerField(),
# })
license = fields.KeywordField()
last_emission = fields.DateField()
num_emissions = fields.IntegerField()
artist_ids = fields.KeywordField()
release_ids = fields.KeywordField()
import_ids = fields.KeywordField()
###################################################################
# field preparation
###################################################################
def prepare_autocomplete(self, instance):
text = [instance.name.strip()]
text += [instance.get_artist_display()]
if instance.release:
text += [instance.release.name]
return text
def prepare_name(self, instance):
return instance.name.strip()
def prepare_tags(self, instance):
return [i.strip() for i in instance.d_tags.split(',') if len(i) > 2]
#return [{'id': i.pk, 'name': i.name} for i in instance.tags.all()]
def prepare_image(self, instance):
if hasattr(instance, 'release') and hasattr(instance.release,
'main_image'):
try:
return get_thumbnailer(
instance.release.main_image).get_thumbnail(
THUMBNAIL_OPT).url
except (InvalidImageFormatError, AttributeError):
pass
def prepare_license(self, instance):
if instance.license:
return instance.license.title
# return {
# 'id': instance.license.pk,
# 'name': instance.license.name,
# }
def prepare_last_emission(self, instance):
if instance.last_emission:
return instance.last_emission.time_start
def prepare_num_emissions(self, instance):
qs = instance.emissions
return qs.count()
def prepare_encoding(self, instance):
if instance.master_encoding:
return instance.master_encoding.upper()
# add all related (appearing artist, extra artist) (uu)ids to the document
def prepare_artist_ids(self, instance):
ids = []
if instance.artist:
ids += [str(instance.artist.uuid)]
for artist in instance.media_artists.all():
ids += [str(artist.uuid)]
for artist in instance.extra_artists.all():
ids += [str(artist.uuid)]
return list(set(ids))
# add related release ids (in this case only one, but keep it as a list for consistency)
def prepare_release_ids(self, instance):
ids = []
if instance.release:
ids += [str(instance.release.uuid)]
return list(set(ids))
def prepare_import_ids(self, instance):
ids = [str(i.uuid) for i in get_import_sessions_for_obj(instance)]
return list(set(ids))
###################################################################
# custom queryset
###################################################################
def get_queryset(self):
return super(MediaDocument, self).get_queryset().all().select_related(
'release', 'artist',
'license').prefetch_related('media_artists', 'extra_artists')
class LabelDocument(DocType):
class Meta:
model = Label
queryset_pagination = 1000
doc_type = 'alibrary.label'
autocomplete = fields.TextField(
analyzer=edge_ngram_analyzer,
search_analyzer=edge_ngram_search_analyzer,
)
# id = fields.IntegerField()
url = fields.KeywordField(attr='get_absolute_url')
api_url = fields.KeywordField(attr='get_api_url')
created = fields.DateField()
updated = fields.DateField()
name = fields.TextField(fielddata=True,
fields={'raw': {
'type': 'keyword'
}})
# TODO: remove 'exact_name' from index/document
exact_name = fields.KeywordField(attr='name')
tags = KeywordField()
labelcode = KeywordField()
# tags = fields.NestedField(properties={
# 'name': fields.KeywordField(),
# 'id': fields.IntegerField(),
# })
creator = fields.KeywordField(attr='creator.username')
image = KeywordField()
type = fields.KeywordField(attr='get_type_display')
year_start = fields.IntegerField()
year_end = fields.IntegerField()
description = fields.TextField(attr='description')
country = KeywordField(attr='country.printable_name')
country_code = KeywordField(attr='country.iso2_code')
import_ids = fields.KeywordField()
###################################################################
# field preparation
###################################################################
def prepare_autocomplete(self, instance):
text = [instance.name.strip()]
# TODO: check what exact fields needed
# if instance.labelcode:
# text += [instance.labelcode.strip()]
# if instance.get_root():
# text += [instance.get_root().name.strip()]
# if instance.parent:
# text += [instance.parent.name.strip()]
if instance.country:
text += [instance.country.iso2_code]
return text
def prepare_name(self, instance):
return instance.name.strip()
def prepare_tags(self, instance):
return [i.strip() for i in instance.d_tags.split(',') if len(i) > 2]
#return [{'id': i.pk, 'name': i.name} for i in instance.tags.all()]
def prepare_image(self, instance):
if hasattr(instance, 'main_image') and instance.main_image:
try:
return get_thumbnailer(
instance.main_image).get_thumbnail(THUMBNAIL_OPT).url
except (InvalidImageFormatError, AttributeError):
pass
def prepare_year_start(self, instance):
if instance.date_start:
return instance.date_start.year
def prepare_year_end(self, instance):
if instance.date_end:
return instance.date_end.year
def prepare_import_ids(self, instance):
ids = [str(i.uuid) for i in get_import_sessions_for_obj(instance)]
return list(set(ids))
###################################################################
# custom queryset
###################################################################
def get_queryset(self):
return super(LabelDocument,
self).get_queryset().select_related('country', 'parent')
class ReleaseDocument(DocType):
class Meta:
model = Release
queryset_pagination = 1000
doc_type = 'alibrary.release'
autocomplete = fields.TextField(
analyzer=edge_ngram_analyzer,
search_analyzer=edge_ngram_search_analyzer,
)
# id = fields.IntegerField()
url = fields.KeywordField(attr='get_absolute_url')
api_url = fields.KeywordField(attr='get_api_url')
created = fields.DateField()
updated = fields.DateField()
# 'fielddata' is needed for sorting on the filed
name = fields.TextField(fielddata=True,
fields={'raw': {
'type': 'keyword'
}})
artist_display = fields.KeywordField(attr='get_artist_display',
fields={'raw': {
'type': 'keyword'
}})
label_display = fields.KeywordField(attr='label.name',
fields={'raw': {
'type': 'keyword'
}})
# name = fields.TextField(
# analyzer=asciifolding_analyzer,
# fielddata=True
# )
tags = KeywordField()
creator = fields.KeywordField(attr='creator.username')
image = KeywordField()
type = fields.KeywordField(attr='get_releasetype_display')
label_type = fields.KeywordField(attr='label.get_type_display')
barcode = fields.KeywordField()
releasedate_year = fields.IntegerField()
catalognumber = fields.KeywordField()
description = fields.TextField()
country = KeywordField(attr='release_country.printable_name')
country_code = KeywordField(attr='release_country.iso2_code')
num_media = fields.IntegerField()
artist_ids = fields.KeywordField()
label_ids = fields.KeywordField()
import_ids = fields.KeywordField()
###################################################################
# field preparation
###################################################################
def prepare_autocomplete(self, instance):
text = [instance.name.strip()]
return text
def prepare_name(self, instance):
return instance.name.strip()
def prepare_tags(self, instance):
return [i.strip() for i in instance.d_tags.split(',') if len(i) > 2]
#return [{'id': i.pk, 'name': i.name} for i in instance.tags.all()]
def prepare_image(self, instance):
if hasattr(instance, 'main_image') and instance.main_image:
try:
return get_thumbnailer(
instance.main_image).get_thumbnail(THUMBNAIL_OPT).url
except (InvalidImageFormatError, AttributeError):
pass
def prepare_num_media(self, instance):
return instance.media_release.count()
def prepare_releasedate_year(self, instance):
if instance.releasedate:
return instance.releasedate.year
# add all related (appearing artist, extra artist) (uu)ids to the document
def prepare_artist_ids(self, instance):
ids = []
for media in instance.get_media():
ids += [str(media.artist.uuid)]
for artist in instance.album_artists.all():
ids += [str(artist.uuid)]
for artist in instance.extra_artists.all():
ids += [str(artist.uuid)]
return list(set(ids))
# add related label ids (in this case only one, but keep it as a list for consistency)
def prepare_label_ids(self, instance):
ids = []
if instance.label:
ids += [str(instance.label.uuid)]
return list(set(ids))
def prepare_import_ids(self, instance):
ids = [str(i.uuid) for i in get_import_sessions_for_obj(instance)]
return list(set(ids))
###################################################################
# custom queryset
###################################################################
def get_queryset(self):
return super(ReleaseDocument,
self).get_queryset().all().select_related(
'release_country').prefetch_related(
'media_release',
'extra_artists',
'album_artists',
)
class ArtistDocument(DocType):
class Meta:
model = Artist
queryset_pagination = 1000
doc_type = 'alibrary.artist'
autocomplete = fields.TextField(
analyzer=edge_ngram_analyzer,
search_analyzer=edge_ngram_search_analyzer,
)
# id = fields.IntegerField()
url = fields.KeywordField(attr='get_absolute_url')
api_url = fields.KeywordField(attr='get_api_url')
created = fields.DateField()
updated = fields.DateField()
# 'fielddata' is needed for sorting on the filed
name = fields.TextField(fielddata=True,
fields={'raw': {
'type': 'keyword'
}})
real_name = fields.TextField(fields={'raw': {'type': 'keyword'}})
namevariations = fields.TextField()
tags = KeywordField()
creator = fields.KeywordField(attr='creator.username')
image = KeywordField()
type = fields.KeywordField(attr='get_type_display')
ipi_code = fields.KeywordField()
isni_code = fields.KeywordField()
year_start = fields.IntegerField()
year_end = fields.IntegerField()
description = fields.TextField(attr='biography')
country = KeywordField(attr='country.printable_name')
country_code = KeywordField(attr='country.iso2_code')
###################################################################
# relation fields
###################################################################
aliases = fields.NestedField(
properties={
'name': fields.TextField(),
'real_name': fields.TextField(),
'pk': fields.IntegerField(),
})
members = fields.NestedField(
properties={
'name': fields.TextField(),
'real_name': fields.TextField(),
'pk': fields.IntegerField(),
})
import_ids = fields.KeywordField()
###################################################################
# field preparation
###################################################################
def prepare_autocomplete(self, instance):
text = [instance.name.strip()]
text += [
i.name.strip() for i in instance.namevariations.nocache().all()
]
return text
def prepare_name(self, instance):
return instance.name.strip()
def prepare_namevariations(self, instance):
return [
i.name.strip() for i in instance.namevariations.nocache().all()
]
def prepare_tags(self, instance):
return [i.strip() for i in instance.d_tags.split(',') if len(i) > 2]
#return [{'id': i.pk, 'name': i.name} for i in instance.tags.all()]
def prepare_image(self, instance):
if hasattr(instance, 'main_image') and instance.main_image:
try:
return get_thumbnailer(
instance.main_image).get_thumbnail(THUMBNAIL_OPT).url
except (InvalidImageFormatError, AttributeError):
pass
def prepare_year_start(self, instance):
if instance.date_start:
return instance.date_start.year
def prepare_year_end(self, instance):
if instance.date_end:
return instance.date_end.year
def prepare_import_ids(self, instance):
ids = [str(i.uuid) for i in get_import_sessions_for_obj(instance)]
return list(set(ids))
###################################################################
# custom queryset
###################################################################
def get_queryset(self):
return super(ArtistDocument, self).get_queryset().filter(
listed=True).select_related('country').prefetch_related(
'aliases', 'members')
class ProfileDocument(DocType):
# ID
id = fields.IntegerField(attr='id')
# ********************************************************************
# *********************** Main data fields for search ****************
# ********************************************************************
gender = StringField(analyzer=html_strip, fields={
'raw': KeywordField(),
})
location = StringField(analyzer=html_strip, fields={'raw': KeywordField()})
avatar = fields.TextField()
about = fields.TextField()
phone = fields.StringField()
slug = fields.StringField()
# age = fields.IntegerField()
height = fields.ObjectField(attr='height_field_indexing',
properties={
'name': StringField(analyzer=html_strip),
'id': fields.IntegerField()
})
weight = fields.ObjectField(attr='weight_field_indexing',
properties={
'name': StringField(analyzer=html_strip),
'id': fields.IntegerField()
})
build = fields.ObjectField(attr='build_field_indexing',
properties={
'name': StringField(analyzer=html_strip),
'id': fields.IntegerField()
})
hair = fields.ObjectField(attr='hair_field_indexing',
properties={
'name': StringField(analyzer=html_strip),
'id': fields.IntegerField()
})
eye = fields.ObjectField(attr='eye_field_indexing',
properties={
'name': StringField(analyzer=html_strip),
'id': fields.IntegerField()
})
ethnicity = fields.ObjectField(attr='ethnicity_field_indexing',
properties={
'name':
StringField(analyzer=html_strip),
'id': fields.IntegerField()
})
auth_user_nested = fields.NestedField(
attr='auth_user_field_indexing',
properties={
'first_name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'last_name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'username':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
})
})
auth_user = fields.ObjectField(
attr='auth_user_field_indexing',
properties={
'first_name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'last_name':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
}),
'username':
StringField(analyzer=html_strip,
fields={
'raw': KeywordField(),
'suggest': fields.CompletionField(),
})
})
media = fields.ObjectField(attr='media_field_indexing',
properties={
'has_photo': fields.BooleanField(),
'has_video': fields.BooleanField(),
'has_audio': fields.BooleanField(),
})
tags = fields.ObjectField(attr='tag_field_indexing',
properties={
'id': fields.IntegerField(),
'name': fields.StringField()
})
class Django:
model = UsersProfile