class MeetingFile(ExtendedModel):
file = models.FileField(
verbose_name=_('file'), storage=storages.get_storage('meetings'), max_length=2000,
upload_to=meeting_file_path)
meeting = models.ForeignKey(Meeting, on_delete=models.DO_NOTHING, related_name='files')
objects = MeetingFileManager()
trash = MeetingFileTrashManager()
i18n = TranslationField()
tracker = FieldTracker()
@property
def download_url(self):
return self._get_absolute_url(self.file.url, use_lang=False) if self.file else None
@property
def name(self):
return self._get_basename(self.file.name)
def __str__(self):
return self.name
class Meta:
verbose_name = _('meeting file')
verbose_name_plural = _('meeting files')
default_manager_name = "objects"
class LabReport(ExtendedModel):
link = models.URLField(verbose_name=_('Report Link'),
max_length=2000,
blank=True,
null=True)
file = models.FileField(verbose_name=_("File"),
storage=storages.get_storage('lab_reports'),
max_length=2000,
blank=True,
null=True)
lab_event = models.ForeignKey(to=LabEvent,
on_delete=models.DO_NOTHING,
related_name="reports")
objects = SoftDeletableManager()
trash = TrashManager()
i18n = TranslationField(fields=())
tracker = FieldTracker()
@property
def report_type(self):
return 'link' if self.link else 'file'
@property
def download_url(self):
return self._get_absolute_url(self.file.url,
use_lang=False) if self.file else None
def __str__(self):
if self.link:
return self.link
if self.file:
return self.file.name
return super().__str__()
class Meta:
verbose_name = _("Report")
verbose_name_plural = _("Reports")
db_table = 'lab_report'
default_manager_name = "objects"
indexes = [
GinIndex(fields=["i18n"]),
]
def handle(self, *args, **options):
archives_storage = get_storage('datasets_archives')
archive_dirs = [
x[0] for x in os.walk(archives_storage.location)
if x[0] != archives_storage.location
]
for directory in archive_dirs:
non_symlinks = [
os.path.join(directory, f) for f in os.listdir(directory)
if not os.path.islink(os.path.join(directory, f))
]
if len(non_symlinks) > 1:
files_with_dt = sorted(
[(file_path, os.path.getmtime(file_path))
for file_path in non_symlinks],
key=lambda x: x[1])
for f in files_with_dt[1:]:
os.remove(f[0])
def archive_resources_files(dataset_id):
free_space = disk_usage(settings.MEDIA_ROOT).free
if free_space < settings.ALLOWED_MINIMUM_SPACE:
logger.error(
'There is not enough free space on disk, archive creation is canceled.'
)
raise ResourceWarning
logger.debug(
f'Updating dataset resources files archive for dataset {dataset_id}')
resourcefile_model = apps.get_model('resources', 'ResourceFile')
resource_model = apps.get_model('resources', 'Resource')
dataset_model = apps.get_model('datasets', 'Dataset')
ds = dataset_model.raw.get(pk=dataset_id)
def create_full_file_path(_fname):
storage_location = ds.archived_resources_files.storage.location
full_fname = ds.archived_resources_files.field.generate_filename(
ds, _fname)
return os.path.join(storage_location, full_fname)
creation_start = datetime.now()
tmp_filename = f'resources_files_{creation_start.strftime("%Y-%m-%d-%H%M%S%f")}.zip'
symlink_name = 'resources_files.zip'
res_storage = storages.get_storage('resources')
full_symlink_name = ds.archived_resources_files.field.generate_filename(
ds, symlink_name)
full_file_path = create_full_file_path(tmp_filename)
full_symlink_path = create_full_file_path(symlink_name)
full_tmp_symlink_path = create_full_file_path('tmp_resources_files.zip')
abs_path = os.path.dirname(full_file_path)
os.makedirs(abs_path, exist_ok=True)
with zipfile.ZipFile(full_file_path,
'w',
zipfile.ZIP_DEFLATED,
compresslevel=1) as main_zip:
res_location = res_storage.location
if is_enabled('S40_new_file_model.be'):
filtered_files = resourcefile_model.objects.filter(
resource__dataset_id=dataset_id,
resource__status='published',
resource__is_removed=False)
file_details = filtered_files.values_list('file', 'resource_id')
else:
filtered_files = resource_model.objects.filter(
status='published', dataset_id=dataset_id).with_files()
all_resource_files = filtered_files.values('file', 'csv_file',
'jsonld_file', 'pk')
file_details = []
for res in all_resource_files:
res_files = [(res['file'], res['pk'])]
if res['csv_file']:
res_files.append((res['csv_file'], res['pk']))
if res['jsonld_file']:
res_files.append((res['jsonld_file'], res['pk']))
file_details.extend(res_files)
for details in file_details:
split_name = details[0].split('/')
full_path = os.path.join(res_location, details[0])
main_zip.write(
full_path, os.path.join(f'resource_{details[1]}',
split_name[1]))
if not ds.archived_resources_files:
os.symlink(full_file_path, full_symlink_path)
dataset_model.objects.filter(pk=dataset_id).update(
archived_resources_files=full_symlink_name)
else:
old_file_path = os.path.realpath(full_symlink_path)
os.symlink(full_file_path, full_tmp_symlink_path)
os.rename(full_tmp_symlink_path, full_symlink_path)
os.remove(old_file_path)
logger.debug(f'Updated dataset {dataset_id} archive with {tmp_filename}')
class Resource(ExtendedModel):
SIGNALS_MAP = {
'updated': (
revalidate_resource,
search_signals.update_document_with_related,
core_signals.notify_updated
),
'published': (
revalidate_resource,
search_signals.update_document_with_related,
core_signals.notify_published
),
'restored': (
revalidate_resource,
search_signals.update_document_with_related,
core_signals.notify_restored
),
}
file = models.FileField(verbose_name=_("File"), storage=storages.get_storage('resources'),
upload_to='%Y%m%d',
max_length=2000, blank=True, null=True)
packed_file = models.FileField(verbose_name=_("Packed file"), storage=storages.get_storage('resources'),
upload_to='%Y%m%d',
max_length=2000, blank=True, null=True)
file_info = models.TextField(blank=True, null=True, editable=False, verbose_name=_("File info"))
file_encoding = models.CharField(max_length=150, null=True, blank=True, editable=False,
verbose_name=_("File encoding"))
link = models.URLField(verbose_name=_('Resource Link'), max_length=2000, blank=True, null=True)
title = models.CharField(max_length=500, verbose_name=_("title"))
description = models.TextField(blank=True, null=True, verbose_name=_("Description"))
position = models.IntegerField(default=1, verbose_name=_("Position"))
dataset = models.ForeignKey('datasets.Dataset', on_delete=models.CASCADE, related_name='resources',
verbose_name=_('Dataset'))
format = models.CharField(max_length=150, blank=True, null=True, verbose_name=_("Format"),
choices=supported_formats_choices())
type = models.CharField(max_length=10, choices=RESOURCE_TYPE, default='file', editable=False,
verbose_name=_("Type"))
openness_score = models.IntegerField(default=0, verbose_name=_("Openness score"),
validators=[MinValueValidator(0), MaxValueValidator(5)])
created_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='resources_created'
)
modified_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='resources_modified'
)
link_tasks = models.ManyToManyField('TaskResult', verbose_name=_('Download Tasks'),
blank=True,
related_name='link_task_resources',
)
file_tasks = models.ManyToManyField('TaskResult', verbose_name=_('Download Tasks'),
blank=True,
related_name='file_task_resources')
data_tasks = models.ManyToManyField('TaskResult', verbose_name=_('Download Tasks'),
blank=True,
related_name='data_task_resources')
old_file = models.FileField(verbose_name=_("File"), storage=storages.get_storage('resources'), upload_to='',
max_length=2000, blank=True, null=True)
old_resource_type = models.TextField(verbose_name=_("Data type"), null=True)
old_format = models.CharField(max_length=150, blank=True, null=True, verbose_name=_("Format"))
old_customfields = JSONField(blank=True, null=True, verbose_name=_("Customfields"))
old_link = models.URLField(verbose_name=_('Resource Link'), max_length=2000, blank=True, null=True)
downloads_count = models.PositiveIntegerField(default=0)
show_tabular_view = models.BooleanField(verbose_name=_('Tabular view'), default=True)
tabular_data_schema = JSONField(null=True, blank=True)
data_date = models.DateField(null=True, verbose_name=_("Data date"))
verified = models.DateTimeField(blank=True, default=now, verbose_name=_("Update date"))
def __str__(self):
return self.title
@property
def media_type(self):
return self.type or ''
@property
def category(self):
return self.dataset.category if self.dataset else ''
@property
def link_is_valid(self):
task = self.link_tasks.last()
return task.status if task else 'NOT_AVAILABLE'
@property
def file_is_valid(self):
task = self.file_tasks.last()
return task.status if task else 'NOT_AVAILABLE'
@property
def data_is_valid(self):
task = self.data_tasks.last()
return task.status if task else 'NOT_AVAILABLE'
@property
def file_url(self):
if self.file:
_file_url = self.file.url if not self.packed_file else self.packed_file.url
return '%s%s' % (settings.API_URL, _file_url)
return ''
@property
def file_size(self):
return self.file.size if self.file else None
@property
def download_url(self):
if self.file:
return '{}/resources/{}/file'.format(settings.API_URL, self.ident)
return ''
@property
def is_indexable(self):
if self.type == 'file' and not self.file_is_valid:
return False
if self.type in ('api', 'website') and not self.link_is_valid:
return False
return True
@property
def tabular_data(self):
if not self._tabular_data:
if self.format in ('csv', 'tsv', 'xls', 'xlsx', 'ods') and self.file:
self._tabular_data = TabularData(self)
return self._tabular_data
def index_tabular_data(self, force=False):
self.tabular_data.validate()
return self.tabular_data.index(force=force)
def save_file(self, content, filename):
dt = self.created.date() if self.created else now().date()
subdir = dt.isoformat().replace("-", "")
dest_dir = os.path.join(self.file.storage.location, subdir)
os.makedirs(dest_dir, exist_ok=True)
file_path = os.path.join(dest_dir, filename)
with open(file_path, 'wb') as f:
f.write(content.read())
return '%s/%s' % (subdir, filename)
def revalidate(self, **kwargs):
if not self.link or self.link.startswith(settings.BASE_URL):
process_resource_file_task.s(self.id, **kwargs).apply_async(countdown=2)
else:
process_resource_from_url_task.s(self.id, **kwargs).apply_async(countdown=2)
@classmethod
def accusative_case(cls):
return _("acc: Resource")
@property
def _openness_score(self):
if not self.format:
return 0
_, content = content_type_from_file_format(self.format.lower())
return OPENNESS_SCORE.get(content, 0)
@property
def file_size_human_readable(self):
file_size = self.file_size or 0
return sizeof_fmt(file_size)
@property
def title_truncated(self):
title = (self.title[:100] + '..') if len(self.title) > 100 else self.title
return title
_tabular_data = None
i18n = TranslationField(fields=("title", "description"))
tracker = FieldTracker()
slugify_field = 'title'
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
class Meta:
verbose_name = _("Resource")
verbose_name_plural = _("Resources")
db_table = 'resource'
default_manager_name = "objects"
indexes = [GinIndex(fields=["i18n"]), ]
class Organization(ExtendedModel):
INSTITUTION_TYPE_PRIVATE = 'private'
INSTITUTION_TYPE_CHOICES = (
('local', _('Local government')),
('state', _('Public government')),
(INSTITUTION_TYPE_PRIVATE, _('Private entities')),
('other', _('Other')),
)
SIGNALS_MAP = {
'updated': (rdf_signals.update_graph_with_conditional_related,
search_signals.update_document_with_related,
core_signals.notify_updated),
'published':
(rdf_signals.create_graph, search_signals.update_document_with_related,
core_signals.notify_published),
'restored':
(rdf_signals.create_graph, search_signals.update_document_with_related,
core_signals.notify_restored),
'removed': (rdf_signals.delete_graph, remove_related_datasets,
search_signals.remove_document_with_related,
core_signals.notify_removed),
}
title = models.CharField(max_length=200, verbose_name=_('Name'))
description = models.TextField(blank=True,
null=True,
verbose_name=_('Description'))
image = models.ImageField(max_length=254,
storage=storages.get_storage('organizations'),
upload_to='%Y%m%d',
blank=True,
null=True,
verbose_name=_('Image URL'))
postal_code = models.CharField(max_length=6,
null=True,
verbose_name=_('Postal code'))
city = models.CharField(max_length=200, null=True, verbose_name=_("City"))
street_type = models.CharField(max_length=50,
null=True,
verbose_name=_("Street type"))
street = models.CharField(max_length=200,
null=True,
verbose_name=_("Street"))
street_number = models.CharField(max_length=200,
null=True,
blank=True,
verbose_name=_("Street number"))
flat_number = models.CharField(max_length=200,
null=True,
blank=True,
verbose_name=_("Flat number"))
email = models.CharField(max_length=300,
null=True,
verbose_name=_("Email"))
epuap = models.CharField(max_length=500,
null=True,
verbose_name=_("EPUAP"))
fax = models.CharField(max_length=50, null=True, verbose_name=_("Fax"))
fax_internal = models.CharField(max_length=20,
null=True,
blank=True,
verbose_name=_('int.'))
institution_type = models.CharField(max_length=50,
choices=INSTITUTION_TYPE_CHOICES,
default=INSTITUTION_TYPE_CHOICES[1][0],
verbose_name=_("Institution type"))
regon = models.CharField(max_length=20, null=True, verbose_name=_("REGON"))
tel = models.CharField(max_length=50, null=True, verbose_name=_("Phone"))
tel_internal = models.CharField(max_length=20,
null=True,
blank=True,
verbose_name=_('int.'))
website = models.CharField(max_length=200,
null=True,
verbose_name=_("Website"))
abbreviation = models.CharField(max_length=30,
null=True,
verbose_name=_("Abbreviation"))
i18n = TranslationField(fields=('title', 'description', 'slug'))
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='organizations_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='organizations_modified')
def __str__(self):
if self.title:
return self.title
return self.slug
@property
def frontend_url(self):
return f'/institution/{self.ident}'
@property
def frontend_absolute_url(self):
return self._get_absolute_url(self.frontend_url)
@property
def image_url(self):
return self.image.url if self.image else ''
@property
def image_absolute_url(self):
return self._get_absolute_url(self.image_url,
use_lang=False) if self.image_url else ''
@property
def short_description(self):
clean_text = ""
if self.description:
clean_text = ''.join(
BeautifulSoup(self.description,
"html.parser").stripped_strings)
return clean_text
@property
def ckan_datasources(self):
return list(
set([
x.source for x in self.published_datasets
if x.source and x.source.is_ckan
]))
@property
def datasources(self):
return list(
set([x.source for x in self.published_datasets if x.source]))
@property
def sources(self):
_sources = [{
'title': x.title,
'url': x.url,
'source_type': x.source_type
} for x in self.datasources]
return sorted(_sources, key=lambda x: x['title'])
@property
def api_url_base(self):
return 'institutions'
@property
def description_html(self):
return format_html(self.description)
@property
def datasets_count(self):
return self.datasets.count()
@classmethod
def accusative_case(cls):
return _("acc: Institution")
@property
def published_datasets(self):
return self.datasets.filter(status='published')
@property
def published_datasets_count(self):
return self.published_datasets.count()
@property
def published_resources_count(self):
return sum(
[x.published_resources_count for x in self.published_datasets])
@property
def address_display(self):
city = ' '.join(i.strip() for i in [self.postal_code, self.city] if i)
if not city:
return None
number = '/'.join(i.strip()
for i in [self.street_number, self.flat_number] if i)
addres_line = city
if self.street:
street = ' '.join(i.strip()
for i in [self.street_type, self.street, number]
if i)
addres_line = ', '.join(i for i in [addres_line, street] if i)
return addres_line
@property
def phone_display(self):
if not self.tel:
return None
try:
p = phonenumbers.parse(self.tel, 'PL')
phone = phonenumbers.format_number(
p, phonenumbers.PhoneNumberFormat.INTERNATIONAL)
except phonenumbers.phonenumberutil.NumberParseException:
return None
return _(' int. ').join(i.strip() for i in [phone, self.tel_internal]
if i)
@property
def fax_display(self):
if not self.fax:
return None
try:
p = phonenumbers.parse(self.fax, 'PL')
fax = phonenumbers.format_number(
p, phonenumbers.PhoneNumberFormat.INTERNATIONAL)
except phonenumbers.phonenumberutil.NumberParseException:
return None
return _(' int. ').join(i.strip() for i in [fax, self.fax_internal]
if i)
objects = OrganizationManager()
trash = OrganizationTrashManager()
tracker = FieldTracker()
slugify_field = 'title'
short_description.fget.short_description = _("Description")
class Meta:
db_table = "organization"
verbose_name = _("Institution")
verbose_name_plural = _("Institutions")
default_manager_name = "objects"
indexes = [
GinIndex(fields=["i18n"]),
]
class Application(ExtendedModel):
SIGNALS_MAP = {
'updated': (generate_thumbnail, core_signals.notify_updated),
'published': (generate_thumbnail, core_signals.notify_published),
'restored': (generate_thumbnail, core_signals.notify_restored),
'removed': (search_signals.remove_document, core_signals.notify_removed),
'm2m_added': (search_signals.update_document, core_signals.notify_m2m_added,),
'm2m_removed': (search_signals.update_document, core_signals.notify_m2m_removed,),
'm2m_cleaned': (search_signals.update_document, core_signals.notify_m2m_cleaned,),
}
title = models.CharField(max_length=300, verbose_name=_("Title"))
notes = models.TextField(verbose_name=_("Notes"), null=True)
author = models.CharField(max_length=50, blank=True, null=True, verbose_name=_("Author"))
url = models.URLField(max_length=300, verbose_name=_("App URL"), null=True)
image = models.ImageField(
max_length=200, storage=storages.get_storage('applications'),
upload_to='%Y%m%d', blank=True, null=True, verbose_name=_("Image URL")
)
image_thumb = models.ImageField(
storage=storages.get_storage('applications'),
upload_to='%Y%m%d', blank=True, null=True
)
datasets = models.ManyToManyField('datasets.Dataset',
db_table='application_dataset',
verbose_name=_('Datasets'),
related_name='applications',
related_query_name="application")
tags = models.ManyToManyField('tags.Tag',
blank=True,
db_table='application_tag',
verbose_name=_('Tag'),
related_name='applications',
related_query_name="application")
created_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='applications_created'
)
modified_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='applications_modified'
)
@cached_property
def image_url(self):
try:
return self.image.url
except ValueError:
return ''
@cached_property
def image_thumb_url(self):
try:
return self.image_thumb.url
except ValueError:
return ''
@cached_property
def tags_list(self):
return [tag.name_translated for tag in self.tags.all()]
@cached_property
def users_following_list(self):
return [user.id for user in self.users_following.all()]
def __str__(self):
return self.title
@classmethod
def accusative_case(cls):
return _("acc: Application")
def published_datasets(self):
return self.datasets.filter(status='published')
i18n = TranslationField(fields=("title", "notes"))
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
tracker = FieldTracker()
slugify_field = 'title'
class Meta:
verbose_name = _("Application")
verbose_name_plural = _("Applications")
db_table = "application"
default_manager_name = "objects"
indexes = [GinIndex(fields=["i18n"]), ]
class Dataset(ExtendedModel):
LICENSE_CC0 = 1
LICENSE_CC_BY = 2
LICENSE_CC_BY_SA = 3
LICENSE_CC_BY_NC = 4
LICENSE_CC_BY_NC_SA = 5
LICENSE_CC_BY_ND = 6
LICENSE_CC_BY_NC_ND = 7
LICENSES = (
(LICENSE_CC0, "CC0 1.0"),
(LICENSE_CC_BY, "CC BY 4.0"),
(LICENSE_CC_BY_SA, "CC BY-SA 4.0"),
(LICENSE_CC_BY_NC, "CC BY-NC 4.0"),
(LICENSE_CC_BY_NC_SA, "CC BY-NC-SA 4.0"),
(LICENSE_CC_BY_ND, "CC BY-ND 4.0"),
(LICENSE_CC_BY_NC_ND, "CC BY-NC-ND 4.0"),
)
LICENSE_CODE_TO_NAME = dict(LICENSES)
SIGNALS_MAP = {
'updated': (rdf_signals.update_graph_with_conditional_related,
search_signals.update_document_with_related,
core_signals.notify_updated),
'published':
(rdf_signals.create_graph, search_signals.update_document_with_related,
core_signals.notify_published),
'restored':
(rdf_signals.create_graph, search_signals.update_document_with_related,
core_signals.notify_restored),
'removed': (remove_related_resources, rdf_signals.delete_graph,
search_signals.remove_document_with_related,
core_signals.notify_removed),
'post_m2m_added': (
search_signals.update_document_related,
rdf_signals.update_graph,
),
'post_m2m_removed': (
search_signals.update_document_related,
rdf_signals.update_graph,
),
'post_m2m_cleaned': (
search_signals.update_document_related,
rdf_signals.update_graph,
)
}
ext_ident = models.CharField(
max_length=36,
blank=True,
editable=False,
verbose_name=_('external identifier'),
help_text=
_('external identifier of dataset taken during import process (optional)'
))
title = models.CharField(max_length=300,
null=True,
verbose_name=_("Title"))
version = models.CharField(max_length=100,
blank=True,
null=True,
verbose_name=_("Version"))
url = models.CharField(max_length=1000,
blank=True,
null=True,
verbose_name=_("Url"))
notes = models.TextField(verbose_name=_("Notes"), null=True, blank=False)
license_chosen = models.PositiveSmallIntegerField(blank=True,
null=True,
default=None,
verbose_name='',
choices=LICENSES)
license_old_id = models.CharField(max_length=20,
blank=True,
null=True,
verbose_name=_("License ID"))
license = models.ForeignKey('licenses.License',
on_delete=models.DO_NOTHING,
blank=True,
null=True,
verbose_name=_("License ID"))
license_condition_db_or_copyrighted = models.TextField(
blank=True,
null=True,
verbose_name=
_("Conditions for using public information that meets the characteristics of the work or constitute "
"a database (Article 13 paragraph 2 of the Act on the re-use of public sector information)"
))
license_condition_personal_data = models.CharField(
max_length=300,
blank=True,
null=True,
verbose_name=_(
"Conditions for using public sector information containing personal data (Article 14 paragraph 1 "
"point 4 of the Act on the re-use of public Sector Information)"))
license_condition_modification = models.NullBooleanField(
null=True,
blank=True,
default=None,
verbose_name=
_("The recipient should inform about the processing of the information when it modifies it"
))
license_condition_original = models.NullBooleanField(null=True,
blank=True,
default=None)
license_condition_responsibilities = models.TextField(
blank=True,
null=True,
verbose_name=
_("The scope of the provider's responsibility for the information provided"
))
license_condition_source = models.NullBooleanField(
null=True,
blank=True,
default=None,
verbose_name=_(
"The recipient should inform about the date, time of completion and obtaining information from"
" the obliged entity"))
license_condition_timestamp = models.NullBooleanField(null=True,
blank=True)
organization = models.ForeignKey('organizations.Organization',
on_delete=models.CASCADE,
related_name='datasets',
verbose_name=_('Institution'))
customfields = JSONField(blank=True,
null=True,
verbose_name=_("Customfields"))
update_frequency = models.CharField(max_length=50,
blank=True,
null=True,
verbose_name=_("Update frequency"))
is_update_notification_enabled = models.BooleanField(
default=True, verbose_name=_('turn on notification'))
update_notification_frequency = models.PositiveSmallIntegerField(
null=True, blank=True, verbose_name=_('set notifications frequency'))
update_notification_recipient_email = models.EmailField(
blank=True,
verbose_name=_('the person who is the notifications recipient'))
category = models.ForeignKey('categories.Category',
on_delete=models.SET_NULL,
blank=True,
null=True,
verbose_name=_('Category'),
limit_choices_to=Q(code=''))
categories = models.ManyToManyField('categories.Category',
db_table='dataset_category',
verbose_name=_('Categories'),
related_name='datasets',
related_query_name="dataset",
limit_choices_to=~Q(code=''))
tags = models.ManyToManyField('tags.Tag',
db_table='dataset_tag',
blank=False,
verbose_name=_("Tag"),
related_name='datasets',
related_query_name="dataset")
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='datasets_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='datasets_modified')
source = models.ForeignKey(
'harvester.DataSource',
models.CASCADE,
null=True,
blank=True,
verbose_name=_('source'),
related_name='datasource_datasets',
)
verified = models.DateTimeField(blank=True,
default=now,
verbose_name=_("Update date"))
downloads_count = models.PositiveIntegerField(
verbose_name=_('download counter'), default=0)
image = models.ImageField(
max_length=200,
storage=get_storage('datasets'),
upload_to='dataset_logo/%Y%m%d',
blank=True,
null=True,
verbose_name=_('Image URL'),
)
image_alt = models.CharField(max_length=255,
blank=True,
verbose_name=_('Alternative text'))
dcat_vocabularies = JSONField(blank=True,
null=True,
verbose_name=_("Controlled Vocabularies"))
archived_resources_files = models.FileField(
storage=get_storage('datasets_archives'),
blank=True,
null=True,
upload_to=archives_upload_to,
max_length=2000,
verbose_name=_("Archived resources files"))
def __str__(self):
return self.title
@cached_property
def has_high_value_data(self):
return self.resources.filter(has_high_value_data=True).exists()
@cached_property
def has_table(self):
return self.resources.filter(has_table=True).exists()
@cached_property
def has_map(self):
return self.resources.filter(has_map=True).exists()
@cached_property
def has_chart(self):
return self.resources.filter(has_chart=True).exists()
@property
def comment_editors(self):
emails = []
if self.source:
emails.extend(self.source.emails_list)
else:
if self.update_notification_recipient_email:
emails.append(self.update_notification_recipient_email)
elif self.modified_by:
emails.append(self.modified_by.email)
else:
emails.extend(user.email
for user in self.organization.users.all())
return emails
@property
def comment_mail_recipients(self):
return [
config.CONTACT_MAIL,
] + self.comment_editors
@property
def is_imported(self):
return bool(self.source)
@property
def is_imported_from_ckan(self):
return self.is_imported and self.source.is_ckan
@property
def is_imported_from_xml(self):
return self.is_imported and self.source.is_xml
@property
def institution(self):
return self.organization
@property
def source_title(self):
return self.source.name if self.source else None
@property
def source_type(self):
return self.source.source_type if self.source else None
@property
def source_url(self):
return self.source.portal_url if self.source else None
@property
def formats(self):
items = [
x.formats_list for x in self.resources.all() if x.formats_list
]
return sorted(set([item for sublist in items for item in sublist]))
@property
def types(self):
return list(self.resources.values_list('type', flat=True).distinct())
@property
def frontend_url(self):
return f'/dataset/{self.ident}'
@property
def frontend_absolute_url(self):
return self._get_absolute_url(self.frontend_url)
@property
def openness_scores(self):
return list(set(res.openness_score for res in self.resources.all()))
@property
def keywords_list(self):
return [tag.to_dict for tag in self.tags.all()]
@property
def keywords(self):
return self.tags
@property
def tags_list_as_str(self):
return ', '.join(
sorted([str(tag) for tag in self.tags.all()], key=str.lower))
def tags_as_str(self, lang):
return ', '.join(
sorted([tag.name for tag in self.tags.filter(language=lang)],
key=str.lower))
@property
def categories_list_as_html(self):
categories = self.categories.all()
return self.mark_safe('<br>'.join(
category.title for category in categories)) if categories else '-'
@property
def categories_list_str(self):
return ', '.join(self.categories.all().values_list('title', flat=True))
@property
def license_code(self):
license_ = self.LICENSE_CC0
if self.license_condition_source or self.license_condition_modification or self.license_condition_responsibilities:
license_ = self.LICENSE_CC_BY
if self.license_chosen and self.license_chosen > license_:
license_ = self.license_chosen
return license_
@property
def license_name(self):
return self.LICENSE_CODE_TO_NAME.get(self.license_code)
@property
def license_link(self):
url = settings.LICENSES_LINKS.get(self.license_name)
return f'{url}legalcode.{get_language()}'
@property
def license_description(self):
return self.license.title if self.license and self.license.title else ''
@property
def last_modified_resource(self):
return self.resources.all().aggregate(Max('modified'))['modified__max']
last_modified_resource.fget.short_description = _("modified")
@property
def is_license_set(self):
return any([
self.license, self.license_condition_db_or_copyrighted,
self.license_condition_modification,
self.license_condition_original,
self.license_condition_responsibilities
])
@property
def followers_count(self):
return self.users_following.count()
@property
def published_resources_count(self):
return self.resources.count()
@property
def visualization_types(self):
return list(
set(
itertools.chain(
*[r.visualization_types for r in self.resources.all()])))
@property
def model_name(self):
return self._meta.model_name
@classmethod
def accusative_case(cls):
return _("acc: Dataset")
@property
def image_url(self):
return self.image.url if self.image else ''
@property
def image_absolute_url(self):
return self._get_absolute_url(self.image_url,
use_lang=False) if self.image_url else ''
@property
def dataset_logo(self):
if self.image_absolute_url:
return self.mark_safe(
'<a href="%s" target="_blank"><img src="%s" width="%d" alt="%s" /></a>'
% (self.admin_change_url, self.image_absolute_url, 100,
self.image_alt if self.image_alt else ''))
return ''
@property
def computed_downloads_count(self):
return ResourceDownloadCounter.objects.filter(
resource__dataset_id=self.pk).aggregate(count_sum=Sum('count'))['count_sum'] or 0\
if is_enabled('S16_new_date_counters.be') else self.downloads_count
@property
def computed_views_count(self):
return ResourceViewCounter.objects.filter(
resource__dataset_id=self.pk).aggregate(count_sum=Sum('count'))['count_sum'] or 0\
if is_enabled('S16_new_date_counters.be') else self.views_count
def to_rdf_graph(self):
schema = self.get_rdf_serializer_schema()
return schema(many=False).dump(self) if schema else None
def as_sparql_create_query(self):
g = self.to_rdf_graph()
data = ''.join([
f'{s.n3()} {p.n3()} {o.n3()} . '
for s, p, o in g.triples((None, None, None))
])
namespaces_dict = {prefix: ns for prefix, ns in g.namespaces()}
return 'INSERT DATA { %(data)s }' % {'data': data}, namespaces_dict
def clean(self):
_range = UPDATE_FREQUENCY_NOTIFICATION_RANGES.get(
self.update_frequency)
if (_range and self.update_notification_frequency
and self.update_notification_frequency not in range(
_range[0], _range[1] + 1)):
msg = _('The value must be between %(min)s and %(max)s') % {
'min': _range[0],
'max': _range[1]
}
raise ValidationError({'update_notification_frequency': msg})
@property
def frequency_display(self):
return dict(UPDATE_FREQUENCY).get(self.update_frequency)
@property
def dataset_update_notification_recipient(self):
return self.update_notification_recipient_email or self.modified_by.email
@property
def regions(self):
resources_ids = list(self.resources.values_list('pk', flat=True))
return Region.objects.filter(resource__pk__in=resources_ids).distinct()
@cached_property
def showcases_published(self):
return self.showcases.filter(status='published')
def archive_files(self):
archive_resources_files.s(dataset_id=self.pk).apply_async(
countdown=settings.DATASET_ARCHIVE_FILES_TASK_DELAY)
@classmethod
def get_license_data(cls, name, lang=None):
data = None
if name not in [
'CC0', 'CCBY', 'CCBY-SA', 'CCBY-NC', 'CCBY-NC-SA', 'CCBY-ND',
'CCBY-NC-ND'
]:
return data
with open(os.path.join(settings.DATA_DIR, 'datasets',
'licenses.json')) as fp:
json_data = json.load(fp)
try:
lang = lang or get_language()
data = json_data[lang][name]
data = SimpleNamespace(id=uuid4(), **data)
except Exception as exc:
logger.debug(exc)
return data
i18n = TranslationField(fields=("title", "notes", "image_alt"))
objects = DatasetManager()
trash = TrashManager()
tracker = FieldTracker()
slugify_field = 'title'
last_modified_resource.fget.short_description = _("modified")
class Meta:
verbose_name = _("Dataset")
verbose_name_plural = _("Datasets")
db_table = 'dataset'
default_manager_name = "objects"
indexes = [
GinIndex(fields=["i18n"]),
]
class ShowcaseProposal(ShowcaseMixin):
DECISION_CHOICES = (
('accepted', _('Proposal accepted')),
('rejected', _('Proposal rejected')),
)
applicant_email = models.EmailField(verbose_name=_('applicant email'),
blank=True)
image = models.ImageField(max_length=200,
storage=storages.get_storage('showcases'),
upload_to='proposals/image/%Y%m%d',
blank=True,
null=True,
verbose_name=_('image URL'))
illustrative_graphics = models.ImageField(
max_length=200,
storage=storages.get_storage('showcases'),
upload_to='proposals/illustrative_graphics/%Y%m%d',
blank=True,
null=True,
verbose_name=_('illustrative graphics'),
)
file = models.FileField(
verbose_name=_('attachement'),
storage=storages.get_storage('showcases'),
upload_to='proposals/file/%Y%m%d',
max_length=2000,
blank=True,
null=True,
)
datasets = models.ManyToManyField('datasets.Dataset',
blank=True,
verbose_name=_('datasets'),
related_name='showcase_proposals')
keywords = ArrayField(models.CharField(max_length=100),
verbose_name=_('keywords'),
default=list)
report_date = models.DateField(verbose_name=_('report date'))
decision = models.CharField(max_length=8,
verbose_name=_('decision'),
choices=DECISION_CHOICES,
blank=True)
decision_date = models.DateField(verbose_name=_('decision date'),
null=True,
blank=True)
comment = models.TextField(verbose_name=_('comment'), blank=True)
showcase = models.OneToOneField('showcases.Showcase',
on_delete=models.SET_NULL,
null=True,
blank=True,
verbose_name=_('showcase'))
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=True,
null=True,
verbose_name=_('created by'),
related_name='showcase_proposals_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=True,
null=True,
verbose_name=_('modified by'),
related_name='showcase_proposals_modified')
objects = ShowcaseProposalManager()
trash = ShowcaseProposalTrashManager()
i18n = TranslationField()
tracker = FieldTracker()
class Meta(ShowcaseMixin.Meta):
verbose_name = _('Showcase Proposal')
verbose_name_plural = _('Showcase Proposals')
@property
def application_logo(self):
if self.image_absolute_url:
return self.mark_safe(
'<a href="%s" target="_blank"><img src="%s" width="%d" alt="" /></a>'
% (
self.admin_change_url,
self.image_absolute_url,
100,
))
return ''
@property
def applicant_email_link(self):
if self.applicant_email:
return self.mark_safe(
f'<a href="mailto:{self.applicant_email}">{self.applicant_email}</a>'
)
@property
def datasets_links(self):
queryset = self.datasets.order_by('title')
res = '<br>'.join([
f'<a href="{x.frontend_absolute_url}" target="_blank">{x.title}</a>'
for x in queryset
])
return self.mark_safe(res)
@property
def datasets_links_info(self):
res = '\n'.join([
obj.frontend_absolute_url
for obj in self.datasets.order_by('title')
])
return self.mark_safe(res)
@property
def external_datasets_links(self):
res = ''
for x in self.external_datasets:
url = x.get('url')
title = x.get('title')
if url:
res += '<a href="{}" target="_blank">{}</a><br>'.format(
url, title or url)
return self.mark_safe(res)
@property
def external_datasets_info(self):
res = ''
for x in self.external_datasets:
url = x.get('url')
title = x.get('title')
if url:
res += '{}: {}\n'.format(title or url, url)
return self.mark_safe(res)
@cached_property
def image_absolute_url(self):
return self._get_absolute_url(self.image.url,
use_lang=False) if self.image else ''
@property
def is_accepted(self):
return self.decision == 'accepted'
@property
def is_converted_to_showcase(self):
return self.showcase and not self.showcase.is_permanently_removed
@property
def is_rejected(self):
return self.decision == 'rejected'
@property
def keywords_as_str(self):
return ','.join([x for x in self.keywords])
@classmethod
def accusative_case(cls):
return _('acc: showcase proposal')
def convert_to_showcase(self): # noqa
created = False
if self.is_converted_to_showcase:
return created # proposal is after convertion already. Do not convert.
tag_model = apps.get_model('tags.Tag')
data = model_to_dict(
self,
fields=[
'category', 'notes', 'author', 'url', 'title', 'image',
'illustrative_graphics', 'datasets', 'external_datasets',
'keywords', 'created_by', 'modified_by', 'is_mobile_app',
'is_desktop_app', 'mobile_apple_url', 'mobile_google_url',
'desktop_windows_url', 'desktop_linux_url',
'desktop_macos_url', 'license_type', 'file'
])
data['status'] = 'draft'
data['modified_by_id'] = data.pop('modified_by')
data['created_by_id'] = data.pop(
'created_by') or data['modified_by_id']
image = data.pop('image')
illustrative_graphics = data.pop('illustrative_graphics')
file = data.pop('file')
datasets = data.pop('datasets')
keywords = data.pop('keywords')
showcase = Showcase.objects.create(**data)
if image:
showcase.image = showcase.save_file(image,
os.path.basename(image.path))
if illustrative_graphics:
showcase.illustrative_graphics = showcase.save_file(
illustrative_graphics,
os.path.basename(illustrative_graphics.path),
field_name='illustrative_graphics')
if file:
showcase.file = showcase.save_file(file,
os.path.basename(file.path),
field_name='file')
if image or illustrative_graphics or file:
showcase.save()
if datasets:
showcase.datasets.set(datasets)
if keywords:
tag_ids = []
for name in keywords:
tag, created = tag_model.objects.get_or_create(
name=name,
language='pl',
defaults={'created_by_id': data['created_by_id']})
tag_ids.append(tag.id)
if tag_ids:
showcase.tags.set(tag_ids)
self.showcase = showcase
self.save()
created = True
return created
@classmethod
def create(cls, data):
image = data.pop('image', None)
illustrative_graphics = data.pop('illustrative_graphics', None)
datasets_ids = data.pop('datasets', [])
name = cls.slugify(data['title'])
if image:
data['image'] = cls.decode_b64_image(image, name)
if illustrative_graphics:
data['illustrative_graphics'] = cls.decode_b64_image(
illustrative_graphics, name)
obj = cls.objects.create(**data)
if datasets_ids:
obj.datasets.set(datasets_ids)
send_showcase_proposal_mail_task.s(obj.id).apply_async(countdown=1)
return obj
@classmethod
def send_showcase_proposal_mail(cls, obj):
emails = [
config.TESTER_EMAIL
] if settings.DEBUG and config.TESTER_EMAIL else [config.CONTACT_MAIL]
context = {'obj': obj, 'host': settings.BASE_URL}
with translation.override('pl'):
msg_plain = render_to_string('mails/showcaseproposal.txt', context)
msg_html = render_to_string('mails/showcaseproposal.html', context)
mail = EmailMultiAlternatives(
'Powiadomienie - Nowe zgłoszenie PoCoTo',
msg_plain,
from_email=config.NO_REPLY_EMAIL,
to=emails,
connection=get_connection(settings.EMAIL_BACKEND),
)
mail.mixed_subtype = 'related'
mail.attach_alternative(msg_html, 'text/html')
mail.send()
class Application(ApplicationMixin):
SIGNALS_MAP = {
'updated': (generate_thumbnail, core_signals.notify_updated),
'published': (generate_thumbnail, core_signals.notify_published),
'restored': (search_signals.update_document_with_related,
generate_thumbnail, core_signals.notify_restored),
'removed': (search_signals.remove_document_with_related,
core_signals.notify_removed),
'pre_m2m_added': (core_signals.notify_m2m_added, ),
'pre_m2m_removed': (core_signals.notify_m2m_removed, ),
'pre_m2m_cleaned': (core_signals.notify_m2m_cleaned, ),
'post_m2m_added': (
update_application_document,
search_signals.update_document_related,
),
'post_m2m_removed': (
update_application_document,
search_signals.update_document_related,
),
'post_m2m_cleaned': (
update_application_document,
search_signals.update_document_related,
),
}
image = models.ImageField(max_length=200,
storage=storages.get_storage('applications'),
upload_to='%Y%m%d',
blank=True,
null=True,
verbose_name=_("Image URL"))
illustrative_graphics = models.ImageField(
max_length=200,
storage=storages.get_storage('applications'),
upload_to='illustrative_graphics/%Y%m%d',
blank=True,
null=True,
verbose_name=_('illustrative graphics'),
)
illustrative_graphics_alt = models.CharField(
max_length=255,
blank=True,
verbose_name=_('illustrative graphics alternative text'))
image_thumb = models.ImageField(
storage=storages.get_storage('applications'),
upload_to='%Y%m%d',
blank=True,
null=True)
image_alt = models.CharField(max_length=255,
null=True,
blank=True,
verbose_name=_("Alternative text"))
datasets = models.ManyToManyField('datasets.Dataset',
db_table='application_dataset',
verbose_name=_('Datasets'),
related_name='applications',
related_query_name="application")
tags = models.ManyToManyField('tags.Tag',
blank=True,
db_table='application_tag',
verbose_name=_('Tag'),
related_name='applications',
related_query_name="application")
main_page_position = models.PositiveSmallIntegerField(
choices=MAIN_PAGE_ORDERING_CHOICES,
blank=True,
null=True,
unique=True,
verbose_name=_('Positioning on the main page'),
)
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='applications_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='applications_modified')
@property
def frontend_preview_url(self):
return self._get_absolute_url(f'/application/preview/{self.ident}')
@cached_property
def image_url(self):
url = self.image.url if self.image else ''
if url:
return self._get_absolute_url(url, use_lang=False)
return url
@cached_property
def image_absolute_url(self):
return self._get_absolute_url(self.image.url,
use_lang=False) if self.image else ''
@cached_property
def image_thumb_url(self):
url = self.image_thumb.url if self.image_thumb else ''
if url:
return self._get_absolute_url(url, use_lang=False)
return url
@cached_property
def image_thumb_absolute_url(self):
return self._get_absolute_url(
self.image_thumb.url, use_lang=False) if self.image_thumb else ''
@cached_property
def has_image_thumb(self):
return bool(self.image_thumb)
def tags_as_str(self, lang):
return ', '.join(
sorted([tag.name for tag in self.tags.filter(language=lang)],
key=str.lower))
@property
def keywords_list(self):
return [tag.to_dict for tag in self.tags.all()]
@property
def keywords(self):
return self.tags
@property
def preview_link(self):
return self.mark_safe(
f'<a href="{self.frontend_preview_url}" class="btn" target="_blank">{_("Preview")}</a>'
)
@cached_property
def users_following_list(self):
return [user.id for user in self.users_following.all()]
@property
def application_logo(self):
if self.image_thumb_absolute_url or self.image_absolute_url:
return self.mark_safe(
'<a href="%s" target="_blank"><img src="%s" width="%d" alt="%s" /></a>'
% (self.admin_change_url, self.image_thumb_absolute_url
or self.image_absolute_url, 100, self.image_alt
if self.image_alt else f'Logo aplikacji {self.title}'))
return ''
@classmethod
def accusative_case(cls):
return _("acc: Application")
def published_datasets(self):
return self.datasets.filter(status='published')
i18n = TranslationField(fields=("title", "notes", 'image_alt',
'illustrative_graphics_alt'))
objects = SoftDeletableManager()
trash = TrashManager()
tracker = FieldTracker()
slugify_field = 'title'
class Meta(ApplicationMixin.Meta):
verbose_name = _("Application")
verbose_name_plural = _("Applications")
db_table = "application"
indexes = [
GinIndex(fields=["i18n"]),
]
class ApplicationProposal(ApplicationMixin):
DECISION_CHOICES = (
('accepted', _('Proposal accepted')),
('rejected', _('Proposal rejected')),
)
applicant_email = models.EmailField(verbose_name=_('applicant email'),
blank=True)
image = models.ImageField(max_length=200,
storage=storages.get_storage('applications'),
upload_to='proposals/image/%Y%m%d',
blank=True,
null=True,
verbose_name=_('image URL'))
illustrative_graphics = models.ImageField(
max_length=200,
storage=storages.get_storage('applications'),
upload_to='proposals/illustrative_graphics/%Y%m%d',
blank=True,
null=True,
verbose_name=_('illustrative graphics'),
)
datasets = models.ManyToManyField('datasets.Dataset',
blank=True,
verbose_name=_('datasets'),
related_name='application_proposals')
keywords = ArrayField(models.CharField(max_length=100),
verbose_name=_('keywords'),
default=list)
report_date = models.DateField(verbose_name=_('report date'))
decision = models.CharField(max_length=8,
verbose_name=_('decision'),
choices=DECISION_CHOICES,
blank=True)
decision_date = models.DateField(verbose_name=_('decision date'),
null=True,
blank=True)
comment = models.TextField(verbose_name=_('comment'), blank=True)
application = models.OneToOneField('applications.Application',
on_delete=models.CASCADE,
null=True,
blank=True,
verbose_name=_('application'))
created_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=True,
null=True,
verbose_name=_('created by'),
related_name='application_proposals_created')
modified_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=True,
null=True,
verbose_name=_('modified by'),
related_name='application_proposals_modified')
objects = ApplicationProposalManager()
trash = ApplicationProposalTrashManager()
i18n = TranslationField()
tracker = FieldTracker()
class Meta(ApplicationMixin.Meta):
verbose_name = _('Application Proposal')
verbose_name_plural = _('Application Proposals')
@cached_property
def image_absolute_url(self):
return self._get_absolute_url(self.image.url,
use_lang=False) if self.image else ''
@property
def application_logo(self):
if self.image_absolute_url:
return self.mark_safe(
'<a href="%s" target="_blank"><img src="%s" width="%d" alt="" /></a>'
% (
self.admin_change_url,
self.image_absolute_url,
100,
))
return ''
@property
def datasets_admin(self):
objs = self.datasets.order_by('title')
links = [
f'<a href="{x.frontend_absolute_url}" target="_blank">{x.title}</a>'
for x in objs
]
res = '<br>'.join(links)
return self.mark_safe(res)
@property
def external_datasets_admin(self):
res = ''
for x in self.external_datasets:
url = x.get('url')
title = x.get('title')
if url:
res += '<a href="{}" target="_blank">{}</a><br>'.format(
url, title or url)
return self.mark_safe(res)
@property
def is_accepted(self):
return self.decision == 'accepted'
@property
def is_rejected(self):
return self.decision == 'rejected'
@property
def keywords_as_str(self):
return ','.join([x for x in self.keywords])
@classmethod
def convert_to_application(cls, app_proposal_id): # noqa
proposal = cls.objects.filter(id=app_proposal_id,
application__isnull=True).first()
if proposal:
tag_model = apps.get_model('tags.Tag')
data = model_to_dict(proposal,
fields=[
'notes', 'author', 'url', 'title',
'image', 'illustrative_graphics',
'datasets', 'external_datasets',
'keywords', 'created_by', 'modified_by'
])
data['status'] = 'draft'
data['modified_by_id'] = data.pop('modified_by')
data['created_by_id'] = data.pop(
'created_by') or data['modified_by_id']
image = data.pop('image')
illustrative_graphics = data.pop('illustrative_graphics')
datasets = data.pop('datasets')
keywords = data.pop('keywords')
application = Application.objects.create(**data)
if image:
application.image = application.save_file(
image, os.path.basename(image.path))
if illustrative_graphics:
application.illustrative_graphics = application.save_file(
illustrative_graphics,
os.path.basename(illustrative_graphics.path),
field_name='illustrative_graphics')
if image or illustrative_graphics:
application.save()
if datasets:
application.datasets.set(datasets)
if keywords:
tag_ids = []
for name in keywords:
tag, created = tag_model.objects.get_or_create(
name=name,
language='pl',
defaults={'created_by_id': data['created_by_id']})
tag_ids.append(tag.id)
if tag_ids:
application.tags.set(tag_ids)
proposal.application = application
proposal.save()
return application
@classmethod
def create(cls, data):
image = data.pop('image', None)
illustrative_graphics = data.pop('illustrative_graphics', None)
datasets_ids = data.pop('datasets', [])
name = cls.slugify(data['title'])
if image:
data['image'] = cls.decode_b64_image(image, name)
if illustrative_graphics:
data['illustrative_graphics'] = cls.decode_b64_image(
illustrative_graphics, name)
obj = cls.objects.create(**data)
if datasets_ids:
obj.datasets.set(datasets_ids)
return obj
@classmethod
def decode_b64_image(cls, encoded_img, img_name):
data_parts = encoded_img.split(';base64,')
img_data = data_parts[-1].encode('utf-8')
try:
extension = guess_extension(guess_type(encoded_img)[0])
except Exception:
extension = None
name = f'{img_name}{extension}' if extension else img_name
try:
decoded_img = base64.b64decode(img_data)
except Exception:
decoded_img = None
return ContentFile(decoded_img, name=name) if decoded_img else None
@classmethod
def accusative_case(cls):
return _("acc: Application proposal")
@classmethod
def send_application_proposal_mail(cls, data):
dataset_model = apps.get_model('datasets.Dataset')
conn = get_connection(settings.EMAIL_BACKEND)
title = data['title']
img_data = data.get('image')
illustrative_graphics = data.get('illustrative_graphics')
img_name = cls.slugify(
title) if img_data or illustrative_graphics else None
if img_data:
_data = img_data.split(';base64,')[-1].encode('utf-8')
image = MIMEImage(base64.b64decode(_data))
filename = f"{img_name}.{image.get_content_subtype()}"
image.add_header('content-disposition',
'attachment',
filename=filename)
image.add_header('Content-ID', '<app-logo>')
if illustrative_graphics:
_data = illustrative_graphics.split(';base64,')[-1].encode('utf-8')
illustrative_graphics_img = MIMEImage(base64.b64decode(_data))
filename = f'{img_name}_illustrative-graphics.{illustrative_graphics_img.get_content_subtype()}'
illustrative_graphics_img.add_header('content-disposition',
'attachment',
filename=filename)
illustrative_graphics_img.add_header('Content-ID',
'<illustrative-graphics>')
datasets = dataset_model.objects.filter(
id__in=data.get('datasets', []))
data['datasets'] = '\n'.join(ds.frontend_absolute_url
for ds in datasets)
data['dataset_links'] = '<br />'.join(
f"<a href=\"{ds.frontend_absolute_url}\">{ds.title}</a>\n"
for ds in datasets)
external_datasets = data.get('external_datasets', [])
data['external_datasets'] = '\n'.join(
f"{eds.get('title', '(nienazwany)')}: {eds.get('url', '(nie podano url)')}\n"
for eds in external_datasets)
data['external_dataset_links'] = '<br />'.join((
f"{eds.get('title')}: <a href=\"{eds.get('url')}\">{eds.get('url')}</a>\n"
if 'url' in eds else eds.get('title')
) for eds in external_datasets)
data['host'] = settings.BASE_URL
emails = [
config.TESTER_EMAIL
] if settings.DEBUG and config.TESTER_EMAIL else [config.CONTACT_MAIL]
html_template = 'applicationproposal' if is_enabled(
'S39_mail_layout.be') else 'propose-application'
with translation.override('pl'):
msg_plain = render_to_string('mails/propose-application.txt', data)
msg_html = render_to_string(f'mails/{html_template}.html', data)
mail = EmailMultiAlternatives(
'Zgłoszono propozycję aplikacji {}'.format(
title.replace('\n', ' ').replace('\r', '')),
msg_plain,
from_email=config.NO_REPLY_EMAIL,
to=emails,
connection=conn)
mail.mixed_subtype = 'related'
mail.attach_alternative(msg_html, 'text/html')
if img_data:
mail.attach(image)
if illustrative_graphics:
mail.attach(illustrative_graphics_img)
mail.send()
class Newsletter(TimeStampedModel):
NEWSLETTER_LANGUAGES = (
('pl', _('polish')),
('en', _('english')),
)
NEWSLETTER_STATUS_CHOICES = (
('awaits', _('Awaits')),
('sent', _('Sent')),
('error', _('Error')),
)
title = models.CharField(max_length=255, verbose_name=_('title'))
lang = models.CharField(max_length=7,
choices=NEWSLETTER_LANGUAGES,
verbose_name=_('language version'))
planned_sending_date = models.DateField(
verbose_name=_('planned sending date'))
sending_date = models.DateTimeField(verbose_name=_('sending date'),
null=True,
blank=True)
status = models.CharField(max_length=7,
choices=NEWSLETTER_STATUS_CHOICES,
verbose_name=_('status'),
default=NEWSLETTER_STATUS_CHOICES[0][0])
file = models.FileField(verbose_name=_('file'),
storage=storages.get_storage('newsletter'),
upload_to='%Y%m%d',
max_length=2000)
created_by = models.ForeignKey(
settings.AUTH_USER_MODEL,
models.DO_NOTHING,
verbose_name=_('created by'),
related_name='newsletters_created',
)
modified_by = models.ForeignKey(
settings.AUTH_USER_MODEL,
models.DO_NOTHING,
blank=True,
null=True,
verbose_name=_('modified by'),
related_name='newsletters_modified',
)
objects = NewsletterQuerySet.as_manager()
def __str__(self):
return self.title
class Meta:
verbose_name = _('newsletter')
verbose_name_plural = _('newsletters')
db_table = 'newsletter'
@property
def is_sent(self):
return self.status == 'sent'
def clean_fields(self, exclude=None):
super().clean_fields(exclude=exclude)
if not self.lang:
self.lang = get_language()
if not self.is_sent and self.planned_sending_date and self.planned_sending_date <= now(
).date():
if exclude and 'planned_sending_date' in exclude:
raise ValidationError(
_('Planned sending date must be in future!'))
else:
raise ValidationError({
'planned_sending_date':
_('This date must be in future!')
})
@staticmethod
def _get_required_links(html):
soup = BeautifulSoup(html, 'html.parser')
required_links = soup.findAll('a', text='Rezygnacja')
required_links += soup.findAll('a', text='Resignation')
return required_links
def clean(self):
errors = {}
if self.file:
buffer = self.file.read()
file_mime_type = magic.from_buffer(buffer, mime=True)
if file_mime_type != 'text/html':
errors['file'] = _('This is not html file!')
else:
try:
html = buffer.decode('utf8')
except UnicodeDecodeError:
html = buffer.decode('cp1250')
required_links = self._get_required_links(html)
if not required_links:
errors['file'] = _(
'Resignation link in html is required <a href="#">Resign</a>!'
)
if errors:
raise ValidationError(errors)
def send(self):
with open(self.file.path, 'r') as f:
html_template = f.read()
required_links = [
str(x) for x in self._get_required_links(html_template)
]
for obj in Subscription.objects.filter(
# lang=self.lang,
is_active=True):
html_message = str(html_template)
for link in required_links:
updated_link = link.replace(
'href="#"',
'href="{}"'.format(obj.resign_newsletter_absolute_url))
html_message = html_message.replace(link, updated_link)
send_newsletter_mail.s(self.id, obj.id,
html_message).apply_async(countdown=1)
self.sending_date = now()
self.status = 'sent'
self.save()
class Course(ExtendedModel):
COURSE_STATES = {
'planned': 'Planowane',
'current': 'W trakcie',
'finished': 'Zakończone',
}
title = models.CharField(max_length=300, verbose_name=_('title'))
notes = models.TextField(verbose_name=_('description'))
venue = models.CharField(max_length=300, verbose_name=_('venue'))
participants_number = models.PositiveIntegerField(
verbose_name=_('number of participants'))
file = models.FileField(
verbose_name=_('schedule file'),
storage=storages.get_storage('courses'),
upload_to='%Y%m%d',
max_length=2000,
null=True,
blank=True,
)
materials_file = models.FileField(
verbose_name=_('materials file'),
storage=storages.get_storage('courses_materials'),
upload_to='%Y%m%d',
max_length=2000,
null=True,
blank=True,
)
objects = CourseManager()
trash = CourseTrashManager()
i18n = TranslationField()
tracker = FieldTracker()
def __str__(self):
return self.title
class Meta:
default_manager_name = 'objects'
verbose_name = _('course')
verbose_name_plural = _('courses')
def sessions(self):
return self.modules.order_by('start')
@property
def start(self):
return self.modules.earliest('start').start
@property
def end(self):
return self.modules.latest('end').end
@property
def file_type(self):
if self.file:
_name, _ext = os.path.splitext(self.file.name)
return _ext[1:]
@property
def file_url(self):
return self._get_absolute_url(self.file.url,
use_lang=False) if self.file else None
@property
def materials_file_type(self):
if self.materials_file:
_name, _ext = os.path.splitext(self.materials_file.name)
return _ext[1:]
@property
def materials_file_url(self):
return self._get_absolute_url(
self.materials_file.url,
use_lang=False) if self.materials_file else None
class Organization(ExtendedModel):
SIGNALS_MAP = {
'removed':
(remove_related_datasets, search_signals.remove_document_with_related,
core_signals.notify_removed),
}
title = models.CharField(max_length=100, verbose_name=_('Title'))
description = models.TextField(blank=True,
null=True,
verbose_name=_('Description'))
image = models.ImageField(max_length=254,
storage=storages.get_storage('organizations'),
upload_to='%Y%m%d',
blank=True,
null=True,
verbose_name=_('Image URL'))
postal_code = models.CharField(max_length=6,
null=True,
verbose_name=_('Postal code'))
city = models.CharField(max_length=200, null=True, verbose_name=_("City"))
street_type = models.CharField(max_length=50,
null=True,
verbose_name=_("Street type"))
street = models.CharField(max_length=200,
null=True,
verbose_name=_("Street"))
street_number = models.CharField(max_length=200,
null=True,
blank=True,
verbose_name=_("Street number"))
flat_number = models.CharField(max_length=200,
null=True,
blank=True,
verbose_name=_("Flat number"))
email = models.CharField(max_length=300,
null=True,
verbose_name=_("Email"))
epuap = models.CharField(max_length=500,
null=True,
verbose_name=_("EPUAP"))
fax = models.CharField(max_length=50, null=True, verbose_name=_("Fax"))
fax_internal = models.CharField(max_length=20,
null=True,
blank=True,
verbose_name=_('int.'))
institution_type = models.CharField(max_length=50,
choices=INSTITUTION_TYPE_CHOICES,
default=INSTITUTION_TYPE_CHOICES[1][0],
verbose_name=_("Institution type"))
regon = models.CharField(max_length=20, null=True, verbose_name=_("REGON"))
tel = models.CharField(max_length=50, null=True, verbose_name=_("Phone"))
tel_internal = models.CharField(max_length=20,
null=True,
blank=True,
verbose_name=_('int.'))
website = models.CharField(max_length=200,
null=True,
verbose_name=_("Website"))
i18n = TranslationField(fields=('title', 'description', 'slug'))
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='organizations_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='organizations_modified')
def __str__(self):
if self.title:
return self.title
return self.slug
def get_url_path(self):
if self.id:
try:
return reverse("admin:applications_application_change",
kwargs={"object_id": self.id})
except NoReverseMatch:
return ""
return ""
@property
def image_url(self):
try:
return self.image.url
except ValueError:
return ''
@property
def short_description(self):
clean_text = ""
if self.description:
clean_text = ''.join(
BeautifulSoup(self.description,
"html.parser").stripped_strings)
return clean_text
@property
def api_url(self):
return '/institutions/{}'.format(self.id)
@property
def description_html(self):
return format_html(self.description)
@property
def datasets_count(self):
return self.datasets.count()
@classmethod
def accusative_case(cls):
return _("acc: Institution")
@property
def published_datasets(self):
return self.datasets.filter(status='published')
@property
def address_display(self):
city = ' '.join(i.strip() for i in [self.postal_code, self.city] if i)
if not city:
return None
number = '/'.join(i.strip()
for i in [self.street_number, self.flat_number] if i)
addres_line = city
if self.street:
street = ' '.join(i.strip()
for i in [self.street_type, self.street, number]
if i)
addres_line = ', '.join(i for i in [addres_line, street] if i)
return addres_line
@property
def phone_display(self):
if not self.tel:
return None
try:
p = phonenumbers.parse(self.tel, 'PL')
phone = phonenumbers.format_number(
p, phonenumbers.PhoneNumberFormat.INTERNATIONAL)
except phonenumbers.phonenumberutil.NumberParseException:
return None
return _(' int. ').join(i.strip() for i in [phone, self.tel_internal]
if i)
@property
def fax_display(self):
if not self.fax:
return None
try:
p = phonenumbers.parse(self.fax, 'PL')
fax = phonenumbers.format_number(
p, phonenumbers.PhoneNumberFormat.INTERNATIONAL)
except phonenumbers.phonenumberutil.NumberParseException:
return None
return _(' int. ').join(i.strip() for i in [fax, self.fax_internal]
if i)
raw = models.Manager()
objects = SoftDeletableManager()
deleted = DeletedManager()
tracker = FieldTracker()
slugify_field = 'title'
short_description.fget.short_description = _("Description")
class Meta:
db_table = "organization"
verbose_name = _("Institution")
verbose_name_plural = _("Institutions")
default_manager_name = "objects"
indexes = [
GinIndex(fields=["i18n"]),
]
class Showcase(ShowcaseMixin):
MAIN_PAGE_ORDERING_CHOICES = [
(1, _('First')),
(2, _('Second')),
(3, _('Third')),
(4, _('Fourth')),
]
SIGNALS_MAP = {
'updated': (generate_thumbnail, core_signals.notify_updated),
'published': (generate_thumbnail, core_signals.notify_published),
'restored': (search_signals.update_document_with_related,
generate_thumbnail, core_signals.notify_restored),
'removed': (search_signals.remove_document_with_related,
core_signals.notify_removed),
'pre_m2m_added': (core_signals.notify_m2m_added, ),
'pre_m2m_removed': (core_signals.notify_m2m_removed, ),
'pre_m2m_cleaned': (core_signals.notify_m2m_cleaned, ),
'post_m2m_added': (
update_showcase_document,
search_signals.update_document_related,
),
'post_m2m_removed': (
update_showcase_document,
search_signals.update_document_related,
),
'post_m2m_cleaned': (
update_showcase_document,
search_signals.update_document_related,
),
}
image = models.ImageField(max_length=200,
storage=storages.get_storage('showcases'),
upload_to='image/%Y%m%d',
blank=True,
null=True,
verbose_name=_('Image URL'))
illustrative_graphics = models.ImageField(
max_length=200,
storage=storages.get_storage('showcases'),
upload_to='illustrative_graphics/%Y%m%d',
blank=True,
null=True,
verbose_name=_('illustrative graphics'),
)
illustrative_graphics_alt = models.CharField(
max_length=255,
blank=True,
verbose_name=_('illustrative graphics alternative text'))
image_thumb = models.ImageField(storage=storages.get_storage('showcases'),
upload_to='image_thumb/%Y%m%d',
blank=True,
null=True)
image_alt = models.CharField(max_length=255,
null=True,
blank=True,
verbose_name=_('Alternative text'))
file = models.FileField(
verbose_name=_('attachement'),
storage=storages.get_storage('showcases'),
upload_to='file/%Y%m%d',
max_length=2000,
blank=True,
null=True,
)
datasets = models.ManyToManyField('datasets.Dataset',
db_table='showcase_dataset',
verbose_name=_('Datasets'),
related_name='showcases',
related_query_name='showcase')
tags = models.ManyToManyField('tags.Tag',
blank=True,
db_table='showcase_tag',
verbose_name=_('Tag'),
related_name='showcases',
related_query_name='showcase')
main_page_position = models.PositiveSmallIntegerField(
choices=MAIN_PAGE_ORDERING_CHOICES,
blank=True,
null=True,
unique=True,
verbose_name=_('Positioning on the main page'),
)
created_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_('Created by'),
related_name='showcases_created')
modified_by = models.ForeignKey(User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_('Modified by'),
related_name='showcases_modified')
i18n = TranslationField(fields=('title', 'notes', 'image_alt',
'illustrative_graphics_alt'))
objects = ShowcaseManager()
trash = ShowcaseTrashManager()
tracker = FieldTracker()
slugify_field = 'title'
class Meta(ShowcaseMixin.Meta):
verbose_name = _('Showcase')
verbose_name_plural = _('Showcases')
db_table = 'showcase'
indexes = [
GinIndex(fields=['i18n']),
]
@property
def frontend_preview_url(self):
return self._get_absolute_url(f'/showcase/preview/{self.ident}')
@cached_property
def image_url(self):
url = self.image.url if self.image else ''
if url:
return self._get_absolute_url(url, use_lang=False)
return url
@cached_property
def image_absolute_url(self):
return self._get_absolute_url(self.image.url,
use_lang=False) if self.image else ''
@cached_property
def image_thumb_url(self):
url = self.image_thumb.url if self.image_thumb else ''
if url:
return self._get_absolute_url(url, use_lang=False)
return url
@cached_property
def image_thumb_absolute_url(self):
return self._get_absolute_url(
self.image_thumb.url, use_lang=False) if self.image_thumb else ''
@cached_property
def has_image_thumb(self):
return bool(self.image_thumb)
def tags_as_str(self, lang):
return ', '.join(
sorted([tag.name for tag in self.tags.filter(language=lang)],
key=str.lower))
@property
def keywords_list(self):
return [tag.to_dict for tag in self.tags.all()]
@property
def keywords(self):
return self.tags
@property
def preview_link(self):
return self.mark_safe(
f'<a href="{self.frontend_preview_url}" class="btn" target="_blank">{_("Preview")}</a>'
)
@property
def application_logo(self):
if self.image_thumb_absolute_url or self.image_absolute_url:
return self.mark_safe(
'<a href="%s" target="_blank"><img src="%s" width="%d" alt="%s" /></a>'
% (self.admin_change_url, self.image_thumb_absolute_url
or self.image_absolute_url, 100, self.image_alt
if self.image_alt else f'Logo aplikacji {self.title}'))
return ''
@classmethod
def accusative_case(cls):
return _('acc: Showcase')
def published_datasets(self):
return self.datasets.filter(status='published')
def generate_logo_thumbnail(self):
if not self.image:
self.image_thumb = None
else:
image = Image.open(self.image)
if image.mode not in ('L', 'RGB', 'RGBA'):
image = image.convert('RGB')
image.thumbnail(settings.THUMB_SIZE, Image.ANTIALIAS)
temp_handle = BytesIO()
image.save(temp_handle, 'png')
temp_handle.seek(0)
suf = SimpleUploadedFile(os.path.split(self.image.name)[-1],
temp_handle.read(),
content_type='image/png')
thumb_name = '.'.join(suf.name.split('.')[:-1]) + "_thumb.png"
self.image_thumb.save(thumb_name, suf, save=False)
class Category(ExtendedModel):
SIGNALS_MAP = {
'updated': (update_related_datasets,),
'published': (update_related_datasets,),
'restored': (update_related_datasets,),
'removed': (null_in_related_datasets, rdf_signals.update_related_graph),
}
code = models.CharField(max_length=100, verbose_name=_("Code"))
title = models.CharField(max_length=100, verbose_name=_("Title"))
description = models.TextField(null=True, verbose_name=_("Description"))
color = models.CharField(max_length=20, default="#000000", null=True, verbose_name=_("Color"))
image = models.ImageField(
max_length=200, storage=storages.get_storage('common'),
upload_to='', blank=True, null=True, verbose_name=_("Image URL")
)
created_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Created by"),
related_name='categories_created'
)
modified_by = models.ForeignKey(
User,
models.DO_NOTHING,
blank=False,
editable=False,
null=True,
verbose_name=_("Modified by"),
related_name='categories_modified'
)
@classmethod
def accusative_case(cls):
return _("acc: Category")
def __str__(self):
return self.title_i18n
@property
def image_url(self):
if not self.image or not self.image.url:
return None
return '{}{}'.format(settings.BASE_URL, self.image.url)
i18n = TranslationField(fields=("title", "description"))
objects = SoftDeletableManager()
trash = TrashManager()
tracker = FieldTracker()
slugify_field = 'title'
class Meta:
db_table = "category"
verbose_name = _("Category")
verbose_name_plural = _("Categories")
default_manager_name = "objects"
indexes = [GinIndex(fields=["i18n"]), ]
def chart_thumb_path(user, slot=1):
storage = storages.get_storage('chart_thumbs')
delta = timezone.timedelta(days=36500)
token = user._get_or_create_token(2, expiration_delta=delta)
return storage.path(f'{token}-{slot}.png')