class ApplicationsDoc(DocType):
id = fields.IntegerField()
slug = TranslatedKeywordField('slug')
title = TranslatedTextField(
'title', common_params={'suggest': fields.CompletionField()})
notes = TranslatedTextField('notes')
author = fields.KeywordField()
url = fields.KeywordField()
image_url = fields.KeywordField(attr='image_url')
image_thumb_url = fields.KeywordField(attr='image_thumb_url')
datasets = datasets_field(attr='published_datasets')
users_following = fields.KeywordField(attr='users_following_list',
multi=True)
tags = TranslatedKeywordsList(attr='tags_list')
views_count = fields.IntegerField()
status = fields.KeywordField()
modified = fields.DateField()
created = fields.DateField()
class Meta:
doc_type = 'application'
model = Application
def get_queryset(self):
return self._doc_type.model.objects.filter(status='published')
class LabEventDoc(Document):
id = fields.IntegerField()
title = TranslatedTextField('title')
notes = TranslatedTextField('notes')
event_type = fields.KeywordField()
execution_date = fields.DateField()
reports = fields.NestedField(
properties={
'type': fields.KeywordField(attr='report_type'),
'download_url': fields.TextField(),
'link': fields.StringField(),
}
)
class Index:
name = settings.ELASTICSEARCH_INDEX_NAMES['lab_events']
settings = settings.ELASTICSEARCH_DSL_INDEX_SETTINGS
class Django:
model = LabEvent
related_models = [LabReport, ]
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, LabReport):
return related_instance.lab_event
def get_queryset(self):
return super().get_queryset().filter(status='published')
def datasets_field(**kwargs):
return fields.NestedField(properties={
'id':
fields.IntegerField(),
'title':
TranslatedTextField('title'),
'notes':
TranslatedTextField('notes'),
'category':
fields.KeywordField(attr='category.title'),
'formats':
fields.KeywordField(attr='formats', multi=True),
'downloads_count':
fields.IntegerField(attr='computed_downloads_count'),
'views_count':
fields.IntegerField(attr='computed_views_count'),
'openness_scores':
fields.IntegerField(attr='openness_scores'),
'modified':
fields.DateField(),
'slug':
TranslatedKeywordField('slug'),
'verified':
fields.DateField(),
},
**kwargs)
class ApplicationDocument(ExtendedDocument):
image_alt = TranslatedTextField('image_alt')
has_image_thumb = fields.BooleanField()
url = fields.KeywordField()
illustrative_graphics_url = fields.KeywordField()
illustrative_graphics_alt = TranslatedTextField(
'illustrative_graphics_alt')
image_url = fields.TextField()
image_thumb_url = fields.KeywordField()
author = fields.KeywordField()
datasets = datasets_field(attr='published_datasets')
external_datasets = fields.NestedField(properties={
'title': fields.KeywordField(),
'url': fields.KeywordField(),
})
main_page_position = fields.IntegerField()
class Index:
name = mcs.ELASTICSEARCH_INDEX_NAMES['applications']
settings = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_SETTINGS
aliases = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_ALIAS
class Django:
model = Application
def article_category_field(**kwargs):
return fields.NestedField(
properties={
'id': fields.IntegerField(),
'image_url': fields.KeywordField(),
'title': TranslatedTextField('title', attr='name'),
'name': TranslatedTextField('name'),
'description': TranslatedTextField('description')
})
class AcceptedDatasetSubmissionDoc(Document):
id = fields.IntegerField()
is_active = fields.BooleanField()
title = TranslatedTextField('title')
notes = TranslatedTextField('notes')
organization_name = TranslatedTextField('organization_name')
potential_possibilities = TranslatedTextField('potential_possibilities')
data_link = fields.TextField()
comment = fields.TextField()
submission_date = fields.DateField()
decision = fields.TextField()
decision_date = fields.DateField()
published_at = fields.DateField()
is_published_for_all = fields.BooleanField()
feedback = fields.NestedField(properties={
'user_id': fields.IntegerField(),
'opinion': fields.StringField()
})
feedback_counters = fields.NestedField(properties={
'plus': fields.IntegerField(),
'minus': fields.IntegerField(),
})
status = fields.TextField()
def prepare_feedback(self, instance):
return [{
'user_id': fb.user.id,
'opinion': fb.opinion
} for fb in instance.feedback.all()]
def prepare_feedback_counters(self, instance):
return instance.feedback_counters
class Index:
name = mcs.ELASTICSEARCH_INDEX_NAMES['accepted_dataset_submissions']
settings = mcs.ELASTICSEARCH_DSL_INDEX_SETTINGS
class Django:
model = AcceptedDatasetSubmission
related_models = [
SubmissionFeedback,
]
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, SubmissionFeedback):
return related_instance.submission
def get_queryset(self):
return super().get_queryset().filter(
status__in=AcceptedDatasetSubmission.PUBLISHED_STATUSES)
def regions_field(**kwargs):
return fields.NestedField(properties={
'region_id':
fields.IntegerField(),
'name':
TranslatedTextField('name'),
'hierarchy_label':
TranslatedTextField('hierarchy_label'),
'bbox':
fields.GeoShapeField('wkt_bbox'),
'coords':
fields.GeoPointField()
},
**kwargs)
def article_category_field(**kwargs):
return fields.NestedField(properties={
'id': fields.IntegerField(),
'name': TranslatedKeywordField('name'),
'description': TranslatedTextField('description')
},
**kwargs)
class ResourceDoc(DocType):
id = fields.IntegerField()
slug = fields.TextField()
uuid = fields.TextField()
title = TranslatedTextField('title', common_params={'suggest': fields.CompletionField()})
description = TranslatedTextField('description')
file_url = fields.TextField(
attr='file_url'
)
download_url = fields.TextField(
attr='download_url'
)
link = fields.TextField()
format = fields.KeywordField()
file_size = fields.LongField()
type = fields.KeywordField()
openness_score = fields.IntegerField()
dataset = fields.NestedField(properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title'),
'slug': TranslatedTextField('slug')
})
views_count = fields.IntegerField()
downloads_count = fields.IntegerField()
status = fields.TextField()
modified = fields.DateField()
created = fields.DateField()
verified = fields.DateField()
data_date = fields.DateField()
class Meta:
doc_type = 'resource'
model = Resource
related_models = [Dataset, ]
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, Dataset):
return related_instance.resources.all()
def get_queryset(self):
return self._doc_type.model.objects.filter(status='published')
class InstitutionDoc(DocType):
id = fields.IntegerField()
slug = TranslatedKeywordField('slug')
title = TranslatedTextField('title', common_params={'suggest': fields.CompletionField()})
description = TranslatedTextField('description')
image_url = fields.TextField(
attr='image_url'
)
postal_code = fields.KeywordField()
city = fields.KeywordField()
street_type = fields.KeywordField()
street = fields.KeywordField()
street_number = fields.KeywordField()
flat_number = fields.KeywordField()
email = fields.KeywordField()
epuap = fields.KeywordField()
institution_type = fields.KeywordField()
regon = fields.KeywordField()
tel = fields.KeywordField(attr='phone_display')
fax = fields.KeywordField(attr='fax_display')
website = fields.KeywordField()
datasets = datasets_field(attr='published_datasets')
status = fields.TextField()
modified = fields.DateField()
created = fields.DateField()
class Meta:
doc_type = 'institution'
model = Organization
related_models = [Dataset]
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, Dataset):
return related_instance.organization
def get_queryset(self):
return self._doc_type.model.objects.filter(status='published')
class ArticleDoc(DocType):
id = fields.IntegerField()
slug = TranslatedKeywordField('slug')
# FIXME jeden obiekt fields.CompletionField() dla wielu pól podrzędnych?
title = TranslatedTextField(
'title', common_params={'suggest': fields.CompletionField()})
notes = TranslatedTextField('notes',
common_params={'raw': fields.TextField()},
std_params=False)
author = fields.KeywordField()
datasets = datasets_field(attr='published_datasets')
category = article_category_field()
license = fields.NestedField(
properties={
'id': fields.IntegerField(),
'name': fields.TextField(),
'title': fields.TextField(),
'url': fields.TextField()
})
tags = TranslatedKeywordsList(attr='tags_list')
views_count = fields.IntegerField()
users_following = fields.KeywordField(attr='users_following_list',
multi=True)
status = fields.TextField()
modified = fields.DateField()
created = fields.DateField()
class Meta:
doc_type = 'article'
model = Article
def get_queryset(self):
return self._doc_type.model.objects.filter(
status__in=['published', 'draft'])
class ShowcaseDocument(ExtendedDocument):
image_alt = TranslatedTextField('image_alt')
has_image_thumb = fields.BooleanField()
url = fields.KeywordField()
illustrative_graphics_url = fields.KeywordField()
illustrative_graphics_alt = TranslatedTextField(
'illustrative_graphics_alt')
image_url = fields.TextField()
image_thumb_url = fields.KeywordField()
author = fields.KeywordField()
datasets = datasets_field(attr='published_datasets')
external_datasets = fields.NestedField(properties={
'title': fields.KeywordField(),
'url': fields.KeywordField(),
})
main_page_position = fields.IntegerField()
showcase_category = fields.KeywordField()
showcase_types = fields.KeywordField(multi=True)
showcase_platforms = fields.KeywordField(multi=True)
license_type = fields.KeywordField()
file_url = fields.KeywordField()
is_desktop_app = fields.BooleanField()
is_mobile_app = fields.BooleanField()
mobile_apple_url = fields.KeywordField()
mobile_google_url = fields.KeywordField()
desktop_linux_url = fields.KeywordField()
desktop_macos_url = fields.KeywordField()
desktop_windows_url = fields.KeywordField()
class Index:
name = mcs.ELASTICSEARCH_INDEX_NAMES['showcases']
settings = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_SETTINGS
aliases = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_ALIAS
class Django:
model = Showcase
class InstitutionDocument(ExtendedDocument):
NOTES_FIELD_NAME = 'description'
image_url = fields.TextField()
abbreviation = fields.KeywordField()
postal_code = fields.KeywordField()
city = fields.KeywordField()
street_type = fields.KeywordField()
street = fields.KeywordField()
street_number = fields.KeywordField()
flat_number = fields.KeywordField()
email = fields.KeywordField()
epuap = fields.KeywordField()
fax = fields.KeywordField(attr='fax_display')
tel = fields.KeywordField(attr='phone_display')
regon = fields.KeywordField()
website = fields.KeywordField()
institution_type = fields.KeywordField()
published_datasets_count = fields.IntegerField()
published_resources_count = fields.IntegerField()
sources = fields.NestedField(
properties={
'title': fields.TextField(),
'url': fields.TextField(),
'source_type': fields.TextField(),
}
)
description = TranslatedTextField('description')
datasets = datasets_field(attr='published_datasets')
class Index:
name = mcs.ELASTICSEARCH_INDEX_NAMES['institutions']
settings = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_SETTINGS
aliases = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_ALIAS
class Django:
model = Organization
related_models = [Dataset, ]
def prepare_model(self, instance):
return 'institution'
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, Dataset):
return related_instance.organization
class RegionDocument(Document):
region_id = fields.IntegerField()
title = TranslatedSuggestField('name')
hierarchy_label = TranslatedTextField('hierarchy_label')
model = fields.KeywordField()
created = fields.DateField()
bbox = fields.GeoShapeField('envelope')
class Index:
name = mcs.ELASTICSEARCH_INDEX_NAMES['regions']
settings = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_SETTINGS
aliases = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_ALIAS
class Django:
model = Region
def prepare_model(self, instance):
return instance._meta.model_name
class DatasetsDoc(DocType):
id = fields.IntegerField()
slug = TranslatedKeywordField('slug')
title = TranslatedTextField('title', common_params={'suggest': fields.CompletionField()})
version = fields.KeywordField()
url = fields.KeywordField()
notes = TranslatedTextField('notes')
institution = fields.NestedField(attr='organization',
properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title'),
'slug': TranslatedTextField('slug'),
})
category = fields.NestedField(
attr='category',
properties={
'id': fields.IntegerField(attr='id'),
'image_url': fields.KeywordField(),
'title': TranslatedTextField('title'),
'description': TranslatedTextField('description')
}
)
resources = fields.NestedField(
properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title')
}
)
applications = fields.NestedField(
properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title')
}
)
articles = fields.NestedField(
properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title')
}
)
tags = TranslatedKeywordsList(attr='tags_list')
# customfields = fields.TextField()
formats = fields.KeywordField(attr='formats', multi=True)
license_condition_db_or_copyrighted = fields.TextField()
license_condition_modification = fields.BooleanField()
license_condition_original = fields.BooleanField()
license_condition_responsibilities = fields.TextField()
license_condition_source = fields.BooleanField()
license_condition_timestamp = fields.BooleanField()
license_name = fields.StringField(attr='license_name')
license_description = fields.StringField(attr='license_description')
update_frequency = fields.KeywordField()
openness_scores = fields.IntegerField(attr='openness_scores', multi=True)
users_following = fields.KeywordField(attr='users_following_list', multi=True)
views_count = fields.IntegerField()
downloads_count = fields.IntegerField()
status = fields.KeywordField()
modified = fields.DateField()
last_modified_resource = fields.DateField(attr='last_modified_resource')
created = fields.DateField()
verified = fields.DateField()
class Meta:
doc_type = 'dataset'
model = Dataset
related_models = [Organization, Category, Application, Article, Resource, UserFollowingDataset]
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, UserFollowingDataset):
return related_instance.follower.followed_applications.all()
if isinstance(related_instance, Application):
return related_instance.datasets.all()
if isinstance(related_instance, Resource):
return related_instance.dataset
if isinstance(related_instance, Category):
return related_instance.dataset_set.filter(status='published')
if isinstance(related_instance, Organization):
return related_instance.datasets.filter(status='published')
def get_queryset(self):
return self._doc_type.model.objects.filter(status='published')
class ResourceDocument(ExtendedDocument):
NOTES_FIELD_NAME = 'description'
format = fields.TextField()
formats = fields.KeywordField(attr='formats_list', multi=True)
openness_score = fields.IntegerField()
openness_scores = fields.IntegerField(multi=True)
media_type = fields.TextField()
downloads_count = fields.IntegerField()
data_date = fields.DateField()
file_url = fields.TextField()
download_url = fields.TextField()
link = fields.TextField()
file_size = fields.IntegerField()
types = fields.KeywordField(multi=True)
dataset = fields.NestedField(
properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title'),
'slug': TranslatedTextField('slug')
})
institution = fields.NestedField(
properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title'),
'slug': TranslatedTextField('slug')
})
source = fields.NestedField(
properties={
'title': fields.TextField(),
'source_type': fields.TextField(),
'url': fields.TextField(),
'update_frequency': TranslatedTextField('update_frequency'),
'last_import_timestamp': fields.DateField(),
})
# ResourceDoc
uuid = fields.TextField()
description = TranslatedTextField('description')
csv_file_url = fields.TextField()
csv_file_size = fields.LongField()
csv_download_url = fields.TextField()
jsonld_file_url = fields.TextField()
jsonld_file_size = fields.LongField()
jsonld_download_url = fields.TextField()
type = fields.KeywordField()
geo_data = fields.NestedField(properties={
'id': fields.IntegerField(),
})
tabular_data = fields.NestedField(properties={
'id': fields.IntegerField(),
})
chartable = fields.NestedField(properties={
'id': fields.IntegerField(),
})
data_special_signs = fields.NestedField(
properties={
'id': fields.IntegerField(),
'symbol': fields.KeywordField(),
'name': TranslatedTextField('name'),
'description': TranslatedTextField('description')
})
is_chart_creation_blocked = fields.BooleanField()
license_code = fields.IntegerField()
update_frequency = fields.KeywordField()
computed_downloads_count = fields.IntegerField()
computed_views_count = fields.IntegerField()
has_high_value_data = fields.BooleanField()
if is_enabled('S37_resources_admin_region_data.be'):
regions = regions_field(attr='all_regions')
if is_enabled('S40_new_file_model.be'):
files = files_field(attr='all_files')
class Index:
name = mcs.ELASTICSEARCH_INDEX_NAMES['resources']
settings = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_SETTINGS
aliases = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_ALIAS
class Django:
model = Resource
related_models = [Dataset, SpecialSign]
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, Dataset):
return related_instance.resources.filter(status='published')
elif isinstance(related_instance, SpecialSign):
return related_instance.special_signs_resources.filter(
status='published')
def prepare_model_name(self, instance):
return instance.category.type
def prepare_openness_scores(self, instance):
return [instance.openness_score]
def prepare_source(self, instance):
serializer = DataSourceSerializer()
if not instance.dataset.source:
return {}
return serializer.dump(instance.dataset.source)
class ExtendedDocument(Document):
NOTES_FIELD_NAME = 'notes'
id = fields.IntegerField()
model = fields.KeywordField()
slug = TranslatedTextField('slug')
title = TranslatedSuggestField('title')
title_synonyms = TranslatedTextField('title',
analyzers=lang_synonyms_analyzers)
title_exact = TranslatedTextField('title', analyzers=lang_exact_analyzers)
notes = TranslatedTextField(NOTES_FIELD_NAME)
notes_synonyms = TranslatedTextField(NOTES_FIELD_NAME,
analyzers=lang_synonyms_analyzers)
notes_exact = TranslatedTextField(NOTES_FIELD_NAME,
analyzers=lang_exact_analyzers)
keywords = fields.NestedField(properties={
'name': fields.KeywordField(),
'language': fields.KeywordField()
})
modified = fields.DateField()
created = fields.DateField()
verified = fields.DateField()
search_date = fields.DateField()
search_type = fields.KeywordField()
status = fields.KeywordField()
visualization_types = fields.KeywordField(multi=True)
subscriptions = fields.NestedField(properties={
'user_id': fields.IntegerField(),
'subscription_id': fields.IntegerField()
})
views_count = fields.IntegerField()
def prepare_notes(self, instance):
notes = getattr(instance, f'{self.NOTES_FIELD_NAME}_translated')
return {
lang_code: getattr(notes, lang_code)
for lang_code in settings.MODELTRANS_AVAILABLE_LANGUAGES
}
prepare_notes_synonyms = prepare_notes
prepare_notes_exact = prepare_notes
def prepare_model(self, instance):
return instance._meta.model_name
def prepare_search_date(self, instance):
return instance.created
def prepare_keywords(self, instance):
return getattr(instance, 'keywords_list', NonIndexableValue)
def prepare_verified(self, instance):
return getattr(instance, 'verified', NonIndexableValue)
def prepare_search_type(self, instance):
return getattr(instance, 'search_type', NonIndexableValue)
def prepare_visualization_types(self, instance):
visualization_types = getattr(instance, 'visualization_types',
NonIndexableValue)
if isinstance(visualization_types,
(tuple, list)) and len(visualization_types) == 0:
visualization_types = ['none']
return visualization_types
def prepare_subscriptions(self, instance):
try:
watcher = ModelWatcher.objects.get_from_instance(instance)
return [{
'user_id': subscription.user_id,
'subscription_id': subscription.id
} for subscription in watcher.subscriptions.all()]
except ModelWatcher.DoesNotExist:
return []
def get_queryset(self):
return super().get_queryset().filter(status='published')
class DatasetDocument(ExtendedDocument):
license_chosen = fields.IntegerField()
license_condition_db_or_copyrighted = fields.TextField()
license_condition_personal_data = fields.TextField()
license_condition_modification = fields.BooleanField()
license_condition_original = fields.BooleanField()
license_condition_responsibilities = fields.TextField()
license_condition_source = fields.BooleanField()
license_condition_timestamp = fields.BooleanField()
license_name = fields.TextField()
license_description = fields.TextField()
resource_modified = fields.DateField(attr='last_modified_resource')
url = fields.KeywordField()
source = fields.NestedField(
properties={
'title': fields.TextField(),
'source_type': fields.TextField(),
'url': fields.TextField(),
'update_frequency': TranslatedTextField('update_frequency'),
'last_import_timestamp': fields.DateField(),
})
formats = fields.KeywordField(multi=True)
types = fields.KeywordField(multi=True)
openness_scores = fields.IntegerField(multi=True)
institution = fields.NestedField(attr='organization',
properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title'),
'slug': TranslatedTextField('slug'),
})
category = fields.NestedField(
properties={
'id': fields.IntegerField(attr='id'),
'image_url': fields.KeywordField(),
'title': TranslatedTextField('title'),
'description': TranslatedTextField('description')
})
categories = fields.NestedField(
properties={
'id': fields.IntegerField(attr='id'),
'image_url': fields.KeywordField(),
'code': fields.KeywordField(),
'title': TranslatedTextField('title'),
'description': TranslatedTextField('description')
})
downloads_count = fields.IntegerField()
image_url = fields.TextField()
image_alt = TranslatedTextField('image_alt')
version = fields.KeywordField()
source_title = fields.TextField()
source_type = fields.TextField()
source_url = fields.TextField()
resources = fields.NestedField(properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title')
})
applications = fields.NestedField(properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title')
})
articles = fields.NestedField(properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title')
})
showcases = fields.NestedField(attr='showcases_published',
properties={
'id': fields.IntegerField(),
'title': TranslatedTextField('title')
})
update_frequency = fields.KeywordField()
users_following = fields.KeywordField(attr='users_following_list',
multi=True)
last_modified_resource = fields.DateField(attr='last_modified_resource')
license_code = fields.IntegerField()
computed_downloads_count = fields.IntegerField()
computed_views_count = fields.IntegerField()
has_high_value_data = fields.BooleanField()
if is_enabled('S37_resources_admin_region_data.be'):
regions = regions_field()
class Index:
name = mcs.ELASTICSEARCH_INDEX_NAMES['datasets']
settings = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_SETTINGS
aliases = mcs.ELASTICSEARCH_DSL_SEARCH_INDEX_ALIAS
class Django:
model = Dataset
related_models = [
Application,
Article,
Category,
DataSource,
Organization,
Resource,
Showcase,
UserFollowingDataset,
]
def get_instances_from_related(self, related_instance):
if isinstance(related_instance, UserFollowingDataset):
return related_instance.follower.followed_applications.all()
if isinstance(related_instance, Application):
return related_instance.datasets.filter(status='published')
if isinstance(related_instance, Article):
return related_instance.datasets.filter(status='published')
if isinstance(related_instance, Resource):
return related_instance.dataset
if isinstance(related_instance, Category):
return related_instance.dataset_set.filter(status='published')
if isinstance(related_instance, Organization):
return related_instance.datasets.filter(status='published')
if isinstance(related_instance, DataSource):
return related_instance.datasource_datasets.filter(
status='published')
if isinstance(related_instance, Showcase):
return related_instance.datasets.filter(status='published')
def prepare_search_date(self, instance):
return instance.verified
def prepare_source(self, instance):
serializer = DataSourceSerializer()
if not instance.source:
return {}
return serializer.dump(instance.source)