class License(ExtendedModel):
SIGNALS_MAP = {
'updated': (update_related_datasets, ),
'published': (update_related_datasets, ),
'restored': (update_related_datasets, ),
'removed': (null_in_related_datasets, ),
}
name = models.CharField(max_length=200, verbose_name=_('Name'))
title = models.CharField(max_length=250, verbose_name=_('Title'))
url = models.URLField(blank=True, null=True, verbose_name=_('URL'))
def __str__(self):
return self.title
i18n = TranslationField(fields=("name", "title"))
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
tracker = FieldTracker()
slugify_field = 'name'
@classmethod
def accusative_case(cls):
return _("acc: License")
class Meta:
verbose_name = _('License')
verbose_name_plural = _('Licenses')
default_manager_name = "objects"
indexes = [
GinIndex(fields=["i18n"]),
]
class ArticleCategory(ExtendedModel):
SIGNALS_MAP = {
'updated': (update_related_articles, ),
'published': (update_related_articles, ),
'restored': (update_related_articles, ),
'removed': (update_related_articles, ),
}
name = models.CharField(max_length=100,
unique=True,
verbose_name=_("name"))
description = models.CharField(max_length=500,
blank=True,
verbose_name=_("Description"))
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='article_categories_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='article_categories_modified')
def __str__(self):
return self.name_i18n
i18n = TranslationField(fields=("name", "description"))
tracker = FieldTracker()
slugify_field = 'name'
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
class Meta:
verbose_name = _("Article Category")
verbose_name_plural = _("Article Categories")
db_table = "article_category"
default_manager_name = "objects"
indexes = [
GinIndex(fields=['i18n']),
]
class Organization(ExtendedModel):
SIGNALS_MAP = {
'removed':
(remove_related_datasets, search_signals.remove_document_with_related,
core_signals.notify_removed),
}
title = models.CharField(max_length=100, verbose_name=_('Title'))
description = models.TextField(blank=True,
null=True,
verbose_name=_('Description'))
image = models.ImageField(max_length=254,
storage=storages.get_storage('organizations'),
upload_to='%Y%m%d',
blank=True,
null=True,
verbose_name=_('Image URL'))
postal_code = models.CharField(max_length=6,
null=True,
verbose_name=_('Postal code'))
city = models.CharField(max_length=200, null=True, verbose_name=_("City"))
street_type = models.CharField(max_length=50,
null=True,
verbose_name=_("Street type"))
street = models.CharField(max_length=200,
null=True,
verbose_name=_("Street"))
street_number = models.CharField(max_length=200,
null=True,
blank=True,
verbose_name=_("Street number"))
flat_number = models.CharField(max_length=200,
null=True,
blank=True,
verbose_name=_("Flat number"))
email = models.CharField(max_length=300,
null=True,
verbose_name=_("Email"))
epuap = models.CharField(max_length=500,
null=True,
verbose_name=_("EPUAP"))
fax = models.CharField(max_length=50, null=True, verbose_name=_("Fax"))
fax_internal = models.CharField(max_length=20,
null=True,
blank=True,
verbose_name=_('int.'))
institution_type = models.CharField(max_length=50,
choices=INSTITUTION_TYPE_CHOICES,
default=INSTITUTION_TYPE_CHOICES[1][0],
verbose_name=_("Institution type"))
regon = models.CharField(max_length=20, null=True, verbose_name=_("REGON"))
tel = models.CharField(max_length=50, null=True, verbose_name=_("Phone"))
tel_internal = models.CharField(max_length=20,
null=True,
blank=True,
verbose_name=_('int.'))
website = models.CharField(max_length=200,
null=True,
verbose_name=_("Website"))
i18n = TranslationField(fields=('title', 'description', 'slug'))
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='organizations_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='organizations_modified')
def __str__(self):
if self.title:
return self.title
return self.slug
def get_url_path(self):
if self.id:
try:
return reverse("admin:applications_application_change",
kwargs={"object_id": self.id})
except NoReverseMatch:
return ""
return ""
@property
def image_url(self):
try:
return self.image.url
except ValueError:
return ''
@property
def short_description(self):
clean_text = ""
if self.description:
clean_text = ''.join(
BeautifulSoup(self.description,
"html.parser").stripped_strings)
return clean_text
@property
def api_url(self):
return '/institutions/{}'.format(self.id)
@property
def description_html(self):
return format_html(self.description)
@property
def datasets_count(self):
return self.datasets.count()
@classmethod
def accusative_case(cls):
return _("acc: Institution")
@property
def published_datasets(self):
return self.datasets.filter(status='published')
@property
def address_display(self):
city = ' '.join(i.strip() for i in [self.postal_code, self.city] if i)
if not city:
return None
number = '/'.join(i.strip()
for i in [self.street_number, self.flat_number] if i)
addres_line = city
if self.street:
street = ' '.join(i.strip()
for i in [self.street_type, self.street, number]
if i)
addres_line = ', '.join(i for i in [addres_line, street] if i)
return addres_line
@property
def phone_display(self):
if not self.tel:
return None
try:
p = phonenumbers.parse(self.tel, 'PL')
phone = phonenumbers.format_number(
p, phonenumbers.PhoneNumberFormat.INTERNATIONAL)
except phonenumbers.phonenumberutil.NumberParseException:
return None
return _(' int. ').join(i.strip() for i in [phone, self.tel_internal]
if i)
@property
def fax_display(self):
if not self.fax:
return None
try:
p = phonenumbers.parse(self.fax, 'PL')
fax = phonenumbers.format_number(
p, phonenumbers.PhoneNumberFormat.INTERNATIONAL)
except phonenumbers.phonenumberutil.NumberParseException:
return None
return _(' int. ').join(i.strip() for i in [fax, self.fax_internal]
if i)
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
tracker = FieldTracker()
slugify_field = 'title'
short_description.fget.short_description = _("Description")
class Meta:
db_table = "organization"
verbose_name = _("Institution")
verbose_name_plural = _("Institutions")
default_manager_name = "objects"
indexes = [
GinIndex(fields=["i18n"]),
]
class Resource(ExtendedModel):
SIGNALS_MAP = {
'updated': (
revalidate_resource,
search_signals.update_document_with_related,
core_signals.notify_updated
),
'published': (
revalidate_resource,
search_signals.update_document_with_related,
core_signals.notify_published
),
'restored': (
revalidate_resource,
search_signals.update_document_with_related,
core_signals.notify_restored
),
}
file = models.FileField(verbose_name=_("File"), storage=storages.get_storage('resources'),
upload_to='%Y%m%d',
max_length=2000, blank=True, null=True)
packed_file = models.FileField(verbose_name=_("Packed file"), storage=storages.get_storage('resources'),
upload_to='%Y%m%d',
max_length=2000, blank=True, null=True)
file_info = models.TextField(blank=True, null=True, editable=False, verbose_name=_("File info"))
file_encoding = models.CharField(max_length=150, null=True, blank=True, editable=False,
verbose_name=_("File encoding"))
link = models.URLField(verbose_name=_('Resource Link'), max_length=2000, blank=True, null=True)
title = models.CharField(max_length=500, verbose_name=_("title"))
description = models.TextField(blank=True, null=True, verbose_name=_("Description"))
position = models.IntegerField(default=1, verbose_name=_("Position"))
dataset = models.ForeignKey('datasets.Dataset', on_delete=models.CASCADE, related_name='resources',
verbose_name=_('Dataset'))
format = models.CharField(max_length=150, blank=True, null=True, verbose_name=_("Format"),
choices=supported_formats_choices())
type = models.CharField(max_length=10, choices=RESOURCE_TYPE, default='file', editable=False,
verbose_name=_("Type"))
openness_score = models.IntegerField(default=0, verbose_name=_("Openness score"),
validators=[MinValueValidator(0), MaxValueValidator(5)])
created_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='resources_created'
)
modified_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='resources_modified'
)
link_tasks = models.ManyToManyField('TaskResult', verbose_name=_('Download Tasks'),
blank=True,
related_name='link_task_resources',
)
file_tasks = models.ManyToManyField('TaskResult', verbose_name=_('Download Tasks'),
blank=True,
related_name='file_task_resources')
data_tasks = models.ManyToManyField('TaskResult', verbose_name=_('Download Tasks'),
blank=True,
related_name='data_task_resources')
old_file = models.FileField(verbose_name=_("File"), storage=storages.get_storage('resources'), upload_to='',
max_length=2000, blank=True, null=True)
old_resource_type = models.TextField(verbose_name=_("Data type"), null=True)
old_format = models.CharField(max_length=150, blank=True, null=True, verbose_name=_("Format"))
old_customfields = JSONField(blank=True, null=True, verbose_name=_("Customfields"))
old_link = models.URLField(verbose_name=_('Resource Link'), max_length=2000, blank=True, null=True)
downloads_count = models.PositiveIntegerField(default=0)
show_tabular_view = models.BooleanField(verbose_name=_('Tabular view'), default=True)
tabular_data_schema = JSONField(null=True, blank=True)
data_date = models.DateField(null=True, verbose_name=_("Data date"))
verified = models.DateTimeField(blank=True, default=now, verbose_name=_("Update date"))
def __str__(self):
return self.title
@property
def media_type(self):
return self.type or ''
@property
def category(self):
return self.dataset.category if self.dataset else ''
@property
def link_is_valid(self):
task = self.link_tasks.last()
return task.status if task else 'NOT_AVAILABLE'
@property
def file_is_valid(self):
task = self.file_tasks.last()
return task.status if task else 'NOT_AVAILABLE'
@property
def data_is_valid(self):
task = self.data_tasks.last()
return task.status if task else 'NOT_AVAILABLE'
@property
def file_url(self):
if self.file:
_file_url = self.file.url if not self.packed_file else self.packed_file.url
return '%s%s' % (settings.API_URL, _file_url)
return ''
@property
def file_size(self):
return self.file.size if self.file else None
@property
def download_url(self):
if self.file:
return '{}/resources/{}/file'.format(settings.API_URL, self.ident)
return ''
@property
def is_indexable(self):
if self.type == 'file' and not self.file_is_valid:
return False
if self.type in ('api', 'website') and not self.link_is_valid:
return False
return True
@property
def tabular_data(self):
if not self._tabular_data:
if self.format in ('csv', 'tsv', 'xls', 'xlsx', 'ods') and self.file:
self._tabular_data = TabularData(self)
return self._tabular_data
def index_tabular_data(self, force=False):
self.tabular_data.validate()
return self.tabular_data.index(force=force)
def save_file(self, content, filename):
dt = self.created.date() if self.created else now().date()
subdir = dt.isoformat().replace("-", "")
dest_dir = os.path.join(self.file.storage.location, subdir)
os.makedirs(dest_dir, exist_ok=True)
file_path = os.path.join(dest_dir, filename)
with open(file_path, 'wb') as f:
f.write(content.read())
return '%s/%s' % (subdir, filename)
def revalidate(self, **kwargs):
if not self.link or self.link.startswith(settings.BASE_URL):
process_resource_file_task.s(self.id, **kwargs).apply_async(countdown=2)
else:
process_resource_from_url_task.s(self.id, **kwargs).apply_async(countdown=2)
@classmethod
def accusative_case(cls):
return _("acc: Resource")
@property
def _openness_score(self):
if not self.format:
return 0
_, content = content_type_from_file_format(self.format.lower())
return OPENNESS_SCORE.get(content, 0)
@property
def file_size_human_readable(self):
file_size = self.file_size or 0
return sizeof_fmt(file_size)
@property
def title_truncated(self):
title = (self.title[:100] + '..') if len(self.title) > 100 else self.title
return title
_tabular_data = None
i18n = TranslationField(fields=("title", "description"))
tracker = FieldTracker()
slugify_field = 'title'
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
class Meta:
verbose_name = _("Resource")
verbose_name_plural = _("Resources")
db_table = 'resource'
default_manager_name = "objects"
indexes = [GinIndex(fields=["i18n"]), ]
class Application(ExtendedModel):
SIGNALS_MAP = {
'updated': (generate_thumbnail, core_signals.notify_updated),
'published': (generate_thumbnail, core_signals.notify_published),
'restored': (generate_thumbnail, core_signals.notify_restored),
'removed': (search_signals.remove_document, core_signals.notify_removed),
'm2m_added': (search_signals.update_document, core_signals.notify_m2m_added,),
'm2m_removed': (search_signals.update_document, core_signals.notify_m2m_removed,),
'm2m_cleaned': (search_signals.update_document, core_signals.notify_m2m_cleaned,),
}
title = models.CharField(max_length=300, verbose_name=_("Title"))
notes = models.TextField(verbose_name=_("Notes"), null=True)
author = models.CharField(max_length=50, blank=True, null=True, verbose_name=_("Author"))
url = models.URLField(max_length=300, verbose_name=_("App URL"), null=True)
image = models.ImageField(
max_length=200, storage=storages.get_storage('applications'),
upload_to='%Y%m%d', blank=True, null=True, verbose_name=_("Image URL")
)
image_thumb = models.ImageField(
storage=storages.get_storage('applications'),
upload_to='%Y%m%d', blank=True, null=True
)
datasets = models.ManyToManyField('datasets.Dataset',
db_table='application_dataset',
verbose_name=_('Datasets'),
related_name='applications',
related_query_name="application")
tags = models.ManyToManyField('tags.Tag',
blank=True,
db_table='application_tag',
verbose_name=_('Tag'),
related_name='applications',
related_query_name="application")
created_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='applications_created'
)
modified_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='applications_modified'
)
@cached_property
def image_url(self):
try:
return self.image.url
except ValueError:
return ''
@cached_property
def image_thumb_url(self):
try:
return self.image_thumb.url
except ValueError:
return ''
@cached_property
def tags_list(self):
return [tag.name_translated for tag in self.tags.all()]
@cached_property
def users_following_list(self):
return [user.id for user in self.users_following.all()]
def __str__(self):
return self.title
@classmethod
def accusative_case(cls):
return _("acc: Application")
def published_datasets(self):
return self.datasets.filter(status='published')
i18n = TranslationField(fields=("title", "notes"))
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
tracker = FieldTracker()
slugify_field = 'title'
class Meta:
verbose_name = _("Application")
verbose_name_plural = _("Applications")
db_table = "application"
default_manager_name = "objects"
indexes = [GinIndex(fields=["i18n"]), ]
class Category(ExtendedModel):
SIGNALS_MAP = {
'updated': (update_related_datasets, ),
'published': (update_related_datasets, ),
'restored': (update_related_datasets, ),
'removed': (null_in_related_datasets, ),
}
title = models.CharField(max_length=100, verbose_name=_("Title"))
description = models.TextField(null=True, verbose_name=_("Description"))
color = models.CharField(max_length=20,
default="#000000",
null=True,
verbose_name=_("Color"))
image = models.ImageField(max_length=200,
storage=storages.get_storage('common'),
upload_to='',
blank=True,
null=True,
verbose_name=_("Image URL"))
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='categories_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='categories_modified')
@classmethod
def accusative_case(cls):
return _("acc: Category")
def __str__(self):
return self.title_i18n
@property
def image_url(self):
if not self.image or not self.image.url:
return None
return '{}{}'.format(settings.BASE_URL, self.image.url)
i18n = TranslationField(fields=("title", "description"))
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
tracker = FieldTracker()
slugify_field = 'title'
class Meta:
db_table = "category"
verbose_name = _("Category")
verbose_name_plural = _("Categories")
default_manager_name = "objects"
indexes = [
GinIndex(fields=["i18n"]),
]
class Tag(ExtendedModel):
SIGNALS_MAP = {
'updated': (update_related_datasets,
update_related_articles,
update_related_applications
),
'published': (update_related_datasets,
update_related_articles,
update_related_applications
),
'restored': (update_related_datasets,
update_related_articles,
update_related_applications
),
'removed': (update_related_datasets,
update_related_articles,
update_related_applications
),
}
name = models.CharField(unique=True, max_length=100, verbose_name=_("name"))
created_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='tags_created'
)
modified_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='tags_modified'
)
def __str__(self):
return self.name
@classmethod
def accusative_case(cls):
return _("acc: Tag")
i18n = TranslationField(fields=("name",), required_languages=("pl",))
tracker = FieldTracker()
slugify_field = 'name'
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
class Meta:
verbose_name = _("Tag")
verbose_name_plural = _("Tags")
db_table = 'tag'
default_manager_name = "objects"
indexes = [GinIndex(fields=["i18n"]), ]
class Article(ExtendedModel):
SIGNALS_MAP = {
'updated':
(search_signals.update_document, core_signals.notify_updated),
'published':
(search_signals.update_document, core_signals.notify_published),
'restored':
(search_signals.update_document, core_signals.notify_restored),
'removed':
(search_signals.remove_document, core_signals.notify_removed),
}
title = models.CharField(max_length=300, verbose_name=_("Title"))
notes = RichTextUploadingField(verbose_name=_("Notes"), null=True)
license_old_id = models.CharField(max_length=20,
blank=True,
null=True,
verbose_name=_("License ID"))
license = models.ForeignKey('licenses.License',
on_delete=models.DO_NOTHING,
blank=True,
null=True,
verbose_name=_("License ID"))
author = models.CharField(max_length=50,
blank=True,
null=True,
verbose_name=_("Author"))
category = models.ForeignKey(ArticleCategory,
on_delete=models.PROTECT,
verbose_name=_('Category'))
tags = models.ManyToManyField('tags.Tag',
db_table='article_tag',
blank=True,
verbose_name=_("Tags"),
related_name='articles',
related_query_name="article")
datasets = models.ManyToManyField('datasets.Dataset',
db_table='article_dataset',
verbose_name=_('Datasets'),
related_name='articles',
related_query_name="article")
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='articles_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='articles_modified')
@classmethod
def accusative_case(cls):
return _("acc: Article")
def __str__(self):
return self.title
def published_datasets(self):
return self.datasets.filter(status='published')
@property
def tags_list(self):
return [tag.name_translated for tag in self.tags.all()]
i18n = TranslationField(fields=("title", "notes"))
tracker = FieldTracker()
slugify_field = 'title'
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
class Meta:
verbose_name = _("Article")
verbose_name_plural = _("Articles")
db_table = "article"
default_manager_name = "objects"
indexes = [
GinIndex(fields=["i18n"]),
]
class Dataset(ExtendedModel):
SIGNALS_MAP = {
'removed':
(remove_related_resources, search_signals.remove_document_with_related,
core_signals.notify_removed),
}
title = models.CharField(max_length=300,
null=True,
verbose_name=_("Title"))
version = models.CharField(max_length=100,
blank=True,
null=True,
verbose_name=_("Version"))
url = models.CharField(max_length=1000,
blank=True,
null=True,
verbose_name=_("Url"))
notes = models.TextField(verbose_name=_("Notes"), null=True, blank=False)
license_old_id = models.CharField(max_length=20,
blank=True,
null=True,
verbose_name=_("License ID"))
license = models.ForeignKey('licenses.License',
on_delete=models.DO_NOTHING,
blank=True,
null=True,
verbose_name=_("License ID"))
license_condition_db_or_copyrighted = models.CharField(max_length=300,
blank=True,
null=True)
license_condition_modification = models.NullBooleanField(null=True,
blank=True,
default=None)
license_condition_original = models.NullBooleanField(null=True,
blank=True,
default=None)
license_condition_responsibilities = models.TextField(blank=True,
null=True)
license_condition_source = models.NullBooleanField(null=True,
blank=True,
default=None)
license_condition_timestamp = models.NullBooleanField(null=True,
blank=True)
organization = models.ForeignKey('organizations.Organization',
on_delete=models.CASCADE,
related_name='datasets',
verbose_name=_('Institution'))
customfields = JSONField(blank=True,
null=True,
verbose_name=_("Customfields"))
update_frequency = models.CharField(max_length=50,
blank=True,
null=True,
verbose_name=_("Update frequency"))
category = models.ForeignKey('categories.Category',
on_delete=models.SET_NULL,
blank=True,
null=True,
verbose_name=_('Category'))
tags = models.ManyToManyField('tags.Tag',
db_table='dataset_tag',
blank=False,
verbose_name=_("Tag"),
related_name='datasets',
related_query_name="dataset")
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='datasets_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='datasets_modified')
verified = models.DateTimeField(blank=True,
default=now,
verbose_name=_("Update date"))
def __str__(self):
return self.title
@property
def institution(self):
return self.organization
@property
def downloads_count(self):
return sum(res.downloads_count for res in self.resources.all())
@property
def formats(self):
return list(
set(res.format for res in self.resources.all()
if res.format is not None))
@property
def openness_scores(self):
return list(set(res.openness_score for res in self.resources.all()))
@property
def tags_list(self):
return [tag.name_translated for tag in self.tags.all()]
@property
def license_name(self):
return self.license.name if self.license and self.license.name else ''
@property
def license_description(self):
return self.license.title if self.license and self.license.title else ''
@property
def last_modified_resource(self):
return self.resources.all().aggregate(Max('modified'))['modified__max']
last_modified_resource.fget.short_description = _("modified")
@property
def is_license_set(self):
return any([
self.license, self.license_condition_db_or_copyrighted,
self.license_condition_modification,
self.license_condition_original,
self.license_condition_responsibilities
])
@property
def followers_count(self):
return self.users_following.count()
@classmethod
def accusative_case(cls):
return _("acc: Dataset")
i18n = TranslationField(fields=("title", "notes"))
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
tracker = FieldTracker()
slugify_field = 'title'
last_modified_resource.fget.short_description = _("modified")
class Meta:
verbose_name = _("Dataset")
verbose_name_plural = _("Datasets")
db_table = 'dataset'
default_manager_name = "objects"
indexes = [
GinIndex(fields=["i18n"]),
]