class InstitutionXMLSerializer(ExtSchema):
id = fields.Integer()
url = fields.Url(attribute='frontend_absolute_url')
type = fields.Function(
lambda organization: organization.get_institution_type_display())
title = TranslatedStr()
abbreviation = fields.Str()
epuap = fields.Str()
website = fields.Url()
created = fields.DateTime(format='iso8601')
modified = fields.DateTime(format='iso8601')
postal_code = fields.Str()
city = fields.Str()
street = fields.Str()
street_number = fields.Str()
street_type = fields.Str()
flat_number = fields.Str()
email = fields.Str()
tel = fields.Str(data_key='phone_number')
regon = fields.Str()
published_datasets_count = fields.Method('get_published_datasets_count')
published_resources_count = fields.Method('get_published_resources_count')
def get_published_datasets_count(self, organization):
return self.context['published_datasets_count']
def get_published_resources_count(self, organization):
return self.context['published_resources_count']
class InstitutionCSVMetadataSerializer(ExtSchema):
organization_url = fields.Url(attribute='frontend_absolute_url',
data_key=_('Organization URL'))
organization_type = fields.Function(
lambda obj: obj.get_institution_type_display(),
data_key=_('Institution type'))
organization_title = TranslatedStr(attribute='title', data_key=_('Name'))
organization_abbr_title = TranslatedStr(attribute='abbreviation',
data_key=_('Abbreviation'),
default='')
organization_regon = fields.Str(data_key=_('REGON'), attribute='regon')
organization_epuap = fields.Str(attribute='epuap',
data_key=_('EPUAP'),
default='')
organization_website = fields.Url(attribute='website',
data_key=_('Website'))
organization_created = fields.DateTime(attribute='created',
data_key=_('Organization created'),
format='iso8601')
organization_modified = fields.DateTime(
attribute='modified',
data_key=_('Organization modified'),
format='iso8601')
organization_datasets_count = fields.Method(
'get_published_datasets_count', data_key=_('Number of datasets'))
organization_resources_count = fields.Method(
'get_published_resources_count',
data_key=_('Number of organization resources'))
organization_postal_code = fields.Str(attribute='postal_code',
data_key=_('Postal code'))
organization_city = fields.Str(attribute='city', data_key=_('City'))
organization_street_type = fields.Str(attribute='street_type',
data_key=_('Street type'))
organization_street = fields.Str(attribute='street', data_key=_('Street'))
organization_street_number = fields.Str(attribute='street_number',
data_key=_('Street number'))
organization_flat_number = fields.Str(attribute='flat_number',
data_key=_('Flat number'))
organization_email = fields.Email(attribute='email', data_key=_('Email'))
organization_phone_number = fields.Str(attribute='tel',
data_key=_('Phone'))
class Meta:
ordered = True
def get_published_datasets_count(self, organization):
return self.context['published_datasets_count']
def get_published_resources_count(self, organization):
return self.context['published_resources_count']
class ResourceCSVMetadataSerializer(schemas.ExtSchema):
frontend_absolute_url = fields.Url(data_key=_('Resource URL'))
title = TranslatedStr(data_key=_('Resource title'), default='')
description = TranslatedStr(data_key=_('Resource description'))
created = fields.DateTime(data_key=_('Resource created'), format='iso8601')
data_date = fields.Date(data_key=_('Data date'))
openness_score = fields.Int(data_key=_('Openness score'))
resource_type = fields.Function(lambda obj: obj.get_type_display(),
data_key=_('Type'))
format = fields.Str(data_key=_('File format'), default='')
file_size = fields.Function(lambda obj: sizeof_fmt(obj.file_size)
if obj.file_size else '',
data_key=_('File size'))
views_count = fields.Int(attribute='computed_views_count',
data_key=_("Resource views count"))
downloads_count = fields.Int(attribute='computed_downloads_count',
data_key=_("Resource downloads count"))
has_table = fields.Function(lambda obj: _('YES')
if obj.has_table else _('NO'),
data_key=_('Table'))
has_chart = fields.Function(lambda obj: _('YES')
if obj.has_chart else _('NO'),
data_key=_('Map'))
has_map = fields.Function(lambda obj: _('YES') if obj.has_map else _('NO'),
data_key=_('Chart'))
download_url = fields.Url(data_key=_('Download URL'))
data_special_signs = fields.Nested(SpecialSignSchema,
data_key=_('special signs'),
many=True)
@ma.post_dump(pass_many=False)
def prepare_nested_data(self, data, **kwargs):
special_signs = data.get(_('special signs'))
signs_str = '\n'.join([
'{name_label}: {name}, {symbol_label}: "{symbol}", {desc_label}: {desc}'
.format(name=sign['name'],
name_label=_('name'),
symbol=sign['symbol'],
symbol_label=_('symbol'),
desc=sign['description'],
desc_label=_('description')) for sign in special_signs
])
data[_('special signs')] = signs_str
values_with_html = [_('Resource title'), _('Resource description')]
for attribute in values_with_html:
data[attribute] = strip_tags(data[attribute])
return data
class Meta:
ordered = True
class CreateSubmissionAttrs(ObjectAttrs):
applicant_email = core_fields.Email(required=False, default=None)
author = core_fields.Str(required=False, default=None)
title = core_fields.Str(required=True,
faker_type='application title',
example='Some App')
url = core_fields.Url(required=True)
notes = core_fields.Str(required=True)
image = core_fields.Base64String(required=False,
default=None,
max_size=settings.IMAGE_UPLOAD_MAX_SIZE)
datasets = core_fields.List(core_fields.Int(), required=False, default=[])
external_datasets = core_fields.Nested(ExternalResourceSchema,
required=False,
default={},
many=True)
keywords = core_fields.List(core_fields.Str(), default='', required=False)
comment = core_fields.String(required=False,
description='Comment body',
example='Looks unpretty',
default='')
class Meta:
strict = True
ordered = True
object_type = 'application-submission'
class SparqlApiAttrs(ObjectAttrs):
result = fields.Str()
has_previous = fields.Bool()
has_next = fields.Bool()
content_type = fields.Str()
download_url = fields.Url()
class Meta:
object_type = 'sparql'
ordered = True
class DatasetXMLSerializer(ExtSchema):
id = fields.Integer()
url = fields.Url(attribute='frontend_absolute_url')
title = TranslatedStr()
notes = TranslatedStr()
keywords = fields.Function(lambda dataset: (
tag.name for tag in getattr(dataset, f'tags_{get_language()}')))
categories = fields.Nested(DatasetCategoryAttr, many=True)
update_frequency = TransUpdateFreqField()
created = fields.DateTime()
verified = fields.DateTime()
views_count = fields.Int(attribute='computed_views_count')
downloads_count = fields.Int(attribute='computed_downloads_count')
published_resources_count = fields.Int(
attribute='published_resources__count')
license = fields.Str(attribute='license_name')
conditions = fields.Method('get_conditions')
organization = fields.Method('get_organization')
resources = fields.Method('get_resources')
source = fields.Nested(SourceXMLSchema)
def get_conditions(self, dataset):
conditions = _(
'This dataset is public information, it can be reused under the following conditions: '
)
terms = [
str(
dataset._meta.get_field(
'license_condition_modification').verbose_name)
if dataset.license_condition_modification else '',
str(
dataset._meta.get_field('license_condition_source').
verbose_name) if dataset.license_condition_source else '',
dataset.license_condition_db_or_copyrighted,
dataset.license_condition_personal_data
]
return conditions + '\n'.join([term for term in terms if term])
def get_organization(self, dataset):
context = {
'published_datasets_count':
dataset.organization_published_datasets__count,
'published_resources_count':
dataset.organization_published_resources__count,
}
return InstitutionXMLSerializer(many=False, context=context).dump(
dataset.organization)
def get_resources(self, dataset):
return ResourceXMLSerializer(many=True).dump(
dataset.published_resources)
class CreateSubmissionAttrs(ObjectAttrs):
applicant_email = core_fields.Email(required=False, default=None)
author = core_fields.Str(required=False, default=None)
title = core_fields.Str(required=True,
faker_type='application title',
example='Some App')
url = core_fields.Url(required=True)
notes = core_fields.Str(required=True)
image = core_fields.Base64String(required=False,
default=None,
max_size=settings.IMAGE_UPLOAD_MAX_SIZE)
illustrative_graphics = core_fields.Base64String(
required=False, default=None, max_size=settings.IMAGE_UPLOAD_MAX_SIZE)
image_alt = core_fields.Str(required=False, default=None)
datasets = core_fields.List(core_fields.Str(), required=False, default=[])
external_datasets = core_fields.Nested(ExternalResourceSchema,
required=False,
default={},
many=True)
keywords = core_fields.List(core_fields.Str(), default='', required=False)
comment = core_fields.String(required=False,
description='Comment body',
example='Looks unpretty',
default='')
is_personal_data_processing_accepted = core_fields.Boolean(required=True)
is_terms_of_service_accepted = core_fields.Boolean(required=True)
class Meta:
strict = True
ordered = True
object_type = 'application-submission'
@pre_load
def prepare_data(self, data, **kwargs):
data['datasets'] = [
x.replace('dataset-', '') for x in data.get('datasets', []) if x
]
return data
@validates('is_personal_data_processing_accepted')
def validate_is_personal_data_processing_accepted(self, value):
if not value:
raise ValidationError(_('This field is required'))
@validates('is_terms_of_service_accepted')
def validate_is_terms_of_service_accepted(self, value):
if not value:
raise ValidationError(_('This field is required'))
class ScheduleApiAttrs(ObjectAttrs):
start_date = fields.Date()
period_name = fields.Str()
end_date = fields.Date()
new_end_date = fields.Date()
link = fields.Url()
state = fields.Str()
is_blocked = fields.Bool()
name = fields.Str()
total_agents_count = fields.Int()
class Meta:
relationships_schema = ScheduleApiRelationships
object_type = 'schedule'
url_template = '{api_url}/auth/schedules/{ident}'
ordered = True
model = 'schedules.Schedule'
class SubmissionAttrs(ObjectAttrs):
title = fields.Str(required=True, example='Very important data')
notes = fields.Str(required=True, example='We need this data to save the world')
organization_name = fields.Str(required=False, example='ACME')
data_link = fields.Url(required=False, example='https://duckduckgo.com')
potential_possibilities = fields.Str(required=False)
submission_date = fields.Date(required=True)
decision_date = fields.Date(required=True)
published_at = fields.DateTime(required=True)
feedback_counters = fields.Nested(
FeedbackCounters,
many=False
)
my_feedback = fields.Str(required=False)
class Meta:
object_type = 'submission'
path = 'submissions'
url_template = '{api_url}/submissions/accepted/{ident}'
class ResourceXMLSerializer(schemas.ExtSchema):
id = fields.Integer()
access_url = fields.Url(attribute='frontend_absolute_url')
title = TranslatedStr()
description = TranslatedStr()
openness_score = fields.Integer()
format = fields.Str()
views_count = fields.Int(attribute='computed_views_count')
downloads_count = fields.Int(attribute='computed_downloads_count')
created = fields.DateTime(format='iso8601')
data_date = fields.Date()
type = fields.Function(lambda resource: resource.get_type_display())
file_size = fields.Function(lambda obj: sizeof_fmt(obj.file_size)
if obj.file_size else '')
visualization_types = ListWithoutNoneStrElement(fields.Str())
download_url = fields.Str()
data_special_signs = fields.Nested(SpecialSignSchema,
data_key='special_signs',
many=True)
class UserScheduleItemApiAttrs(ObjectAttrs):
email = fields.Email()
institution = fields.Str(attribute='organization_name')
institution_unit = fields.Str(attribute='organization_unit')
dataset_title = fields.Str()
created = fields.Date()
format = fields.Str()
is_new = fields.Bool()
is_openness_score_increased = fields.Bool()
is_quality_improved = fields.Bool()
description = fields.Str()
state = fields.Str()
recommendation_state = fields.Str()
recommendation_notes = fields.Str()
is_recommendation_issued = fields.Bool()
is_accepted = fields.Bool()
is_completed = fields.Bool()
is_resource_added = fields.Bool()
is_resource_added_notes = fields.Str()
resource_link = fields.Url()
class Meta:
relationships_schema = UserScheduleItemApiRelationships
object_type = 'user_schedule_item'
url_template = '{api_url}/auth/user_schedule_items/{ident}'
ordered = True
model = 'schedules.UserScheduleItem'
@post_dump
def prepare_data(self, data, **kwargs):
request = self.context.get('request')
user = getattr(request, 'user', None) if request else None
if not user or not user.is_superuser:
del data['recommendation_state']
del data['recommendation_notes']
return data
class ExternalResourceSchema(ExtSchema):
title = core_fields.Str(required=False)
url = core_fields.Url(required=False)
class DatasetResourcesCSVSerializer(CSVSerializer):
dataset_url = fields.Url(attribute='frontend_absolute_url',
data_key=_('Dataset URL'))
dataset_title = TranslatedStr(attribute='title', data_key=_('Title'))
dataset_description = TranslatedStr(attribute='notes', data_key=_('Notes'))
dataset_keywords = fields.Function(lambda obj: ', '.join(
(tag.name for tag in getattr(obj, f'tags_{get_language()}'))),
data_key=_('Tag'))
dataset_categories = fields.Function(lambda obj: ', '.join(
(category.title_i18n for category in obj.categories.all())),
data_key=_('Category'))
dataset_update_frequency = fields.Str(attribute='frequency_display',
data_key=_('Update frequency'))
dataset_created = fields.DateTime(attribute='created',
data_key=_('Dataset created'),
format='iso8601')
dataset_verified = fields.DateTime(attribute='verified',
data_key=_('Dataset verified'),
format='iso8601')
views_count = fields.Int(attribute='computed_views_count',
data_key=_("Dataset views count"))
downloads_count = fields.Int(attribute='computed_downloads_count',
data_key=_('Dataset downloads count'))
dataset_resources_count = fields.Int(
attribute='published_resources__count', data_key=_('Number of data'))
dataset_conditions = fields.Method('get_dataset_conditions',
data_key=_('Terms of use'))
dataset_license = fields.Str(attribute='license_name',
data_key=_('License'))
dataset_source = fields.Nested(SourceXMLSchema,
attribute='source',
data_key=_('source'))
organization = fields.Method('get_organization')
resources = fields.Nested(ResourceCSVMetadataSerializer,
many=True,
attribute='published_resources')
@ma.post_dump(pass_many=True)
def unpack_nested_data(self, data, many, **kwargs):
new_result_data = []
for record in data:
resources = record.pop('resources')
organization = record.pop('organization')
record.update(**organization)
for resource in resources:
tmp_record = record.copy()
tmp_record.update(**resource)
new_result_data.append(tmp_record)
return new_result_data
def get_dataset_conditions(self, dataset):
conditions = _(
'This dataset is public information, it can be reused under the following conditions: '
)
terms = [
str(
dataset._meta.get_field(
'license_condition_modification').verbose_name)
if dataset.license_condition_modification else '',
str(
dataset._meta.get_field('license_condition_source').
verbose_name) if dataset.license_condition_source else '',
dataset.license_condition_db_or_copyrighted,
dataset.license_condition_personal_data
]
return conditions + '\n'.join([term for term in terms if term])
@ma.post_dump(pass_many=False)
def prepare_nested_data(self, data, **kwargs):
source = data.get(_('source'))
if source:
source_str =\
'{title_label}: {title}, {url_label}: {url},' \
' {last_import_label}: {last_import}, {frequency_label}: {frequency}'.format(
title=source['title'], title_label=_('name'),
url=source['url'], url_label=_('url'),
last_import=source['last_import_timestamp'], last_import_label=_('last import timestamp'),
frequency=source['update_frequency'], frequency_label=_('Update frequency')
)
data[_('source')] = source_str
data[_('Notes')] = strip_tags(data[_('Notes')])
return data
def get_organization(self, dataset):
context = {
'published_datasets_count':
dataset.organization_published_datasets__count,
'published_resources_count':
dataset.organization_published_resources__count,
}
return InstitutionCSVMetadataSerializer(
many=False, context=context).dump(dataset.organization)
def get_csv_headers(self):
result = []
for field_name, field in self.fields.items():
if field_name == 'organization':
org_headers = [
org_field.data_key for org_field_name, org_field in
InstitutionCSVMetadataSerializer().fields.items()
]
result.extend(org_headers)
elif field_name == 'resources':
res_headers = [
res_field.data_key for res_field_name, res_field in
field.schema.fields.items()
]
result.extend(res_headers)
else:
header = field.data_key or field_name
result.append(header)
return result
class Meta:
ordered = True
