def __init__(self, *args, instance=None, **kwargs):
if instance and is_enabled('S41_resource_has_high_value_data.be'):
initial = kwargs.get('initial', {})
initial['has_high_value_data'] = instance.has_high_value_data
kwargs['initial'] = initial
super(DatasetForm, self).__init__(*args, instance=instance, **kwargs)
if 'categories' in self.fields and is_enabled(
'S41_dataset_categories_required.be'):
self.fields['categories'].required = True
try:
self.fields['image'].validators.append(
validate_dataset_image_file_extension)
self.fields['image'].help_text = \
_('Allowed file extensions: jpg, gif, png. For better readability,'
' we recommend .png files with a transparent background')
except KeyError:
pass
if self.fields.get('tags_pl'):
self.fields['tags_pl'].required = True
if 'update_notification_recipient_email' in self.fields:
self.fields['update_notification_recipient_email'].required = True
if instance and instance.modified_by and not instance.update_notification_recipient_email:
self.initial[
'update_notification_recipient_email'] = instance.modified_by.email
def response_change(self, request, obj):
if '_verify_rules' in request.POST:
rules = get_paremeters_from_post(request.POST)
results = obj.verify_rules(rules)
for col, res in results.items():
res, msg_class = process_verification_results(
col, res, obj.tabular_data_schema, rules)
self.message_user(request, res, msg_class)
return HttpResponseRedirect(".")
elif '_change_type' in request.POST:
obj.save()
messages.add_message(request, messages.SUCCESS,
_('Data type changed'))
self.revalidate(request, obj.id)
elif '_map_save' in request.POST:
obj.save()
messages.add_message(request, messages.SUCCESS,
_('Map definition saved'))
self.revalidate(request, obj.id)
elif '_continue' in request.POST or\
(is_enabled('S41_resource_revalidate.be') and '_save' in request.POST):
if is_enabled('S41_resource_revalidate.be'):
obj.save()
self.revalidate(request, obj.id)
return super().response_change(request, obj)
def get_xml_schema_path(version):
if version == '1.3' and not is_enabled(
'S37_validate_resource_link_scheme_harvester.be'):
raise KeyError(version)
if version == '1.4' and not is_enabled(
'S41_xml_harvester_special_signs.be'):
raise KeyError(version)
return settings.HARVESTER_XML_VERSION_TO_SCHEMA_PATH[version]
class ResourceApiAttrs(ObjectAttrs, HighlightObjectMixin):
title = TranslatedStr()
description = TranslatedStr()
category = fields.Str()
format = fields.Str()
media_type = fields.Str(
attribute='type'
) # https://jsonapi.org/format/#document-resource-object-fields
visualization_types = ListWithoutNoneStrElement(fields.Str())
downloads_count =\
fields.Function(
lambda obj: obj.computed_downloads_count if is_enabled('S16_new_date_counters.be') else obj.downloads_count)
openness_score = fields.Integer()
views_count =\
fields.Function(
lambda obj: obj.computed_views_count if is_enabled('S16_new_date_counters.be') else obj.views_count)
modified = fields.DateTime()
created = fields.DateTime()
verified = fields.DateTime()
data_date = fields.Date()
file_url = fields.Str()
file_size = fields.Integer()
csv_file_url = fields.Str()
csv_file_size = fields.Integer()
jsonld_file_url = fields.Str()
jsonld_file_size = fields.Integer()
jsonld_download_url = fields.Str()
download_url = fields.Str()
csv_download_url = fields.Str()
link = fields.Str()
data_special_signs = fields.Nested(SpecialSignSchema,
data_key='special_signs',
many=True)
is_chart_creation_blocked = fields.Bool()
if is_enabled('S35_high_value_data.be'):
has_high_value_data = fields.Boolean()
if is_enabled('S37_resources_admin_region_data.be'):
regions = fields.Method('get_regions')
if is_enabled('S40_new_file_model.be'):
files = fields.Method('get_files')
class Meta:
relationships_schema = ResourceApiRelationships
object_type = 'resource'
api_path = 'resources'
url_template = '{api_url}/resources/{ident}'
model = 'resources.Resource'
def get_regions(self, res):
return RegionSchema(many=True).dump(
getattr(res, 'all_regions', res.regions))
def get_files(self, res):
return ResourceFileSchema(many=True).dump(
getattr(res, 'all_files', res.files))
def save_model(self, request, obj, form, change):
if not obj.id:
obj.created_by = request.user
obj.modified_by = request.user
if is_enabled('S40_new_file_model.be') and obj.file:
obj.file = None
obj.save()
if is_enabled('S40_new_file_model.be') and form.cleaned_data.get(
'file'):
ResourceFile.objects.update_or_create(
resource=obj,
is_main=True,
defaults={'file': form.cleaned_data['file']})
class DatasetApiRelationships(Relationships):
institution = fields.Nested(Relationship,
many=False,
_type='institution',
path='institutions',
url_template='{api_url}/institutions/{ident}')
resources = fields.Nested(Relationship,
many=False,
default=[],
_type='resource',
url_template='{object_url}/resources')
if is_enabled('S42_dataset_showcases.be'):
showcases = fields.Nested(Relationship,
many=False,
default=[],
_type='showcase',
url_template='{object_url}/showcases')
subscription = fields.Nested(
Relationship,
many=False,
_type='subscription',
url_template='{api_url}/auth/subscriptions/{ident}')
def filter_data(self, data, **kwargs):
if not self.context.get('is_listing', False):
if 'resources' in data:
data['resources'] = data['resources'].filter(
status='published')
if 'showcases' in data:
data['showcases'] = data['showcases'].filter(
status='published')
return data
def send_dataset_suggestion_mail_task(obj_id):
model = apps.get_model('suggestions', 'DatasetSubmission')
obj = model.objects.filter(pk=obj_id).first()
result = None
if obj:
conn = get_connection(settings.EMAIL_BACKEND)
emails = [
config.TESTER_EMAIL
] if settings.DEBUG and config.TESTER_EMAIL else [config.CONTACT_MAIL]
context = {'obj': obj, 'host': settings.BASE_URL}
tmpl = 'mails/dataset-submission.html' if is_enabled(
'S39_mail_layout.be') else 'mails/dataset-suggestion.html'
with override('pl'):
msg_plain = render_to_string('mails/dataset-suggestion.txt',
context)
msg_html = render_to_string(tmpl, context)
mail = EmailMultiAlternatives(_('Resource demand reported'),
msg_plain,
from_email=config.NO_REPLY_EMAIL,
to=emails,
connection=conn)
mail.mixed_subtype = 'related'
mail.attach_alternative(msg_html, 'text/html')
result = mail.send()
return {
'sent': bool(result),
'obj_id': obj.id if obj else None,
}
def resource_with_csv_file_converted_to_jsonld(csv2jsonld_csv_file, csv2jsonld_jsonld_file, params_str):
from mcod.resources.models import Resource
params = json.loads(params_str)
obj_id = params.pop('id')
if is_enabled('S40_new_file_model.be'):
res = ResourceFactory(
main_file__file=csv2jsonld_csv_file,
id=obj_id,
type='file',
format='csv',
link=None,
**params,)
ResourceFileFactory.create(
file=csv2jsonld_jsonld_file,
format='jsonld',
resource=res,
is_main=False
)
resource_score, files_score = res.get_openness_score()
Resource.objects.filter(pk=res.pk).update(openness_score=resource_score)
res = Resource.objects.get(pk=res.pk)
res.revalidate()
else:
res = ResourceFactory.create(
id=obj_id,
type='file',
format='csv',
link=None,
file=csv2jsonld_csv_file,
jsonld_file=csv2jsonld_jsonld_file,
**params,
)
res.openness_score = res.get_openness_score()
res.save()
return res
def _update_or_create_resource(self, dataset, data):
if dataset.status == self.dataset_model.STATUS.draft: # TODO: move to SIGNAL_MAP in Dataset?
data['status'] = self.resource_model.STATUS.draft
data['created_by'] = self.import_user
if 'format' in data:
data['format'] = data.get('format') or None
modified = data.pop('modified', None)
modified = modified or data.get('created')
int_ident = data.pop('int_ident', None)
special_signs = data.pop('special_signs', [])
if int_ident:
created = False
obj = self.resource_model.raw.filter(dataset=dataset,
id=int_ident).first()
if obj:
for k, v in data.items():
setattr(obj, k, v)
obj.save()
else:
obj, created = self.resource_model.raw.update_or_create(
dataset=dataset, ext_ident=data['ext_ident'], defaults=data)
if obj and modified: # TODO: find a better way to save modification date with value from data.
self.resource_model.raw.filter(id=obj.id).update(modified=modified)
if is_enabled('S41_xml_harvester_special_signs.be'):
obj.special_signs.set(
obj.special_signs.model.objects.filter(
symbol__in=special_signs))
return obj, created
def resource_with_id_and_filename(filename, dataset, obj_id):
from mcod.resources.models import Resource
full_filename = prepare_file(filename)
with open(full_filename, 'rb') as outfile:
if is_enabled('S40_new_file_model.be'):
res = Resource.objects.create(
id=obj_id,
title='Local file resource',
description='Resource with file',
dataset=dataset,
data_date=datetime.today(),
status='published'
)
ResourceFileFactory.create(
resource_id=res.pk,
file=File(outfile),
)
else:
Resource.objects.create(
id=obj_id,
title='Local file resource',
description='Resource with file',
file=File(outfile),
dataset=dataset,
data_date=datetime.today(),
status='published'
)
def test_change_type_is_run_successfully(self, geo_tabular_data_resource, admin):
client = Client()
client.force_login(admin)
data = {
'title': ['test geo csv'], 'description': ['<p>more than 20 characters</p>'],
'dataset': [geo_tabular_data_resource.dataset_id],
'data_date': [datetime.date(2021, 5, 4)], 'status': ['published'], 'show_tabular_view': ['on'],
'schema_type_0': ['string'],
'schema_type_1': ['string'], 'schema_type_2': ['number'], 'schema_type_3': ['integer'],
'geo_0': [''], 'geo_1': [''], 'geo_2': [''], 'geo_3': [''], 'title_en': [''], 'description_en': [''],
'slug_en': [''], 'Resource_file_tasks-TOTAL_FORMS': ['4'], 'Resource_file_tasks-INITIAL_FORMS': ['1'],
'Resource_file_tasks-MIN_NUM_FORMS': ['0'], 'Resource_file_tasks-MAX_NUM_FORMS': ['1000'],
'Resource_data_tasks-TOTAL_FORMS': ['4'], 'Resource_data_tasks-INITIAL_FORMS': ['1'],
'Resource_data_tasks-MIN_NUM_FORMS': ['0'], 'Resource_data_tasks-MAX_NUM_FORMS': ['1000'],
'Resource_link_tasks-TOTAL_FORMS': ['4'], 'Resource_link_tasks-INITIAL_FORMS': ['1'],
'Resource_link_tasks-MIN_NUM_FORMS': ['0'], 'Resource_link_tasks-MAX_NUM_FORMS': ['1000'],
'_change_type': ''}
if is_enabled('S41_resource_has_high_value_data.be'):
data['has_high_value_data'] = False
resp = client.post(
reverse('admin:resources_resource_change', kwargs={'object_id': geo_tabular_data_resource.id}), data=data,
follow=True
)
content = resp.content.decode()
assert _('Data type changed') in content
def process_for_separate_file_model(resource_id, resource, options,
resource_type, update_file_archive,
forced_file_changed, **kwargs):
Resource = apps.get_model('resources', 'Resource')
ResourceFile = apps.get_model('resources', 'ResourceFile')
openness_score, _ = resource.get_openness_score(options['format'])
qs = Resource.raw.filter(id=resource_id)
if resource_type == 'file':
res_file, created = ResourceFile.objects.get_or_create(
resource_id=resource_id,
is_main=True,
)
if 'filename' in options:
ResourceFile.objects.filter(
pk=res_file.pk).update(file=res_file.save_file(
options['content'], options['filename']))
qs.update(format=options['format'], openness_score=openness_score)
process_resource_res_file_task.s(
res_file.pk,
update_link=False,
update_file_archive=update_file_archive,
**kwargs).apply_async(countdown=2)
else: # API or WWW
ResourceFile.objects.filter(resource_id=resource_id).delete()
if is_enabled('S41_resource_bulk_download.be') and forced_file_changed:
resource.dataset.archive_files()
qs.update(type=resource_type,
format=options['format'],
openness_score=openness_score)
def test_map_save_is_run_successfully(self, geo_tabular_data_resource, admin):
geo_tabular_data_resource.revalidate()
geo_tabular_data_resource.refresh_from_db()
client = Client()
client.force_login(admin)
data = {
'title': ['test geo csv'], 'description': ['<p>more than 20 characters</p>'],
'dataset': [geo_tabular_data_resource.dataset_id],
'data_date': [datetime.date(2021, 5, 4)], 'status': ['published'], 'show_tabular_view': ['on'],
'schema_type_0': ['string'],
'schema_type_1': ['string'], 'schema_type_2': ['integer'], 'schema_type_3': ['integer'],
'geo_0': [''], 'geo_1': ['label'], 'geo_2': ['l'], 'geo_3': ['b'], 'title_en': [''], 'description_en': [''],
'slug_en': [''], 'Resource_file_tasks-TOTAL_FORMS': ['4'], 'Resource_file_tasks-INITIAL_FORMS': ['1'],
'Resource_file_tasks-MIN_NUM_FORMS': ['0'], 'Resource_file_tasks-MAX_NUM_FORMS': ['1000'],
'Resource_data_tasks-TOTAL_FORMS': ['4'], 'Resource_data_tasks-INITIAL_FORMS': ['1'],
'Resource_data_tasks-MIN_NUM_FORMS': ['0'], 'Resource_data_tasks-MAX_NUM_FORMS': ['1000'],
'Resource_link_tasks-TOTAL_FORMS': ['4'], 'Resource_link_tasks-INITIAL_FORMS': ['1'],
'Resource_link_tasks-MIN_NUM_FORMS': ['0'], 'Resource_link_tasks-MAX_NUM_FORMS': ['1000'],
'_map_save': [''],
}
if is_enabled('S41_resource_has_high_value_data.be'):
data['has_high_value_data'] = False
resp = client.post(geo_tabular_data_resource.admin_change_url, data=data, follow=True)
content = resp.content.decode()
assert _('Map definition saved') in content
def get_queryset(self):
if is_enabled('S40_new_file_model.be'):
qs = super(ResourceManager,
self).get_queryset().with_prefetched_files()
else:
qs = super(ResourceManager, self).get_queryset()
return qs
def resource_with_xls_file_converted_to_csv(res_id, example_xls_file, buzzfeed_dataset, buzzfeed_editor):
if is_enabled('S40_new_file_model.be'):
from mcod.resources.models import Resource
params = {
'id': res_id,
'type': 'file',
'format': 'xls',
'link': None,
'filename': 'example_xls_file.xls',
'openness_score': 1
}
res = create_res(buzzfeed_dataset, buzzfeed_editor, **params)
resource_score, files_score = res.get_openness_score()
Resource.objects.filter(pk=res.pk).update(openness_score=resource_score)
res.revalidate()
else:
res = ResourceFactory.create(
id=res_id,
type='file',
format='xls',
link=None,
file=example_xls_file,
)
res.revalidate()
res.increase_openness_score()
return res
class DatasetApiAttrs(ObjectAttrs, HighlightObjectMixin):
title = TranslatedStr()
slug = TranslatedStr()
notes = TranslatedStr()
categories = fields.Nested(DatasetCategoryAttr, many=True)
category = fields.Nested(DatasetCategoryAttr, many=False)
formats = fields.List(fields.String())
types = fields.List(fields.String())
keywords = KeywordsList(TranslatedStr())
openness_scores = fields.List(fields.Int())
license_chosen = fields.Integer()
license_condition_db_or_copyrighted = fields.String()
license_condition_personal_data = fields.String()
license_condition_modification = fields.Boolean()
license_condition_original = fields.Boolean()
license_condition_responsibilities = fields.String()
license_condition_source = fields.Boolean()
license_condition_timestamp = fields.Boolean()
license_name = fields.String()
license_description = fields.String()
update_frequency = TransUpdateFreqField()
views_count =\
fields.Function(
lambda obj: obj.computed_views_count if is_enabled('S16_new_date_counters.be') else obj.views_count)
downloads_count =\
fields.Function(
lambda obj: obj.computed_downloads_count if is_enabled('S16_new_date_counters.be') else obj.downloads_count)
url = fields.String()
followed = fields.Boolean()
modified = fields.DateTime()
resource_modified = fields.DateTime()
created = fields.DateTime()
verified = fields.DateTime()
visualization_types = ListWithoutNoneStrElement(fields.Str())
source = fields.Nested(SourceSchema)
image_url = fields.Str()
image_alt = TranslatedStr()
if is_enabled('S35_high_value_data.be'):
has_high_value_data = fields.Boolean()
if is_enabled('S37_resources_admin_region_data.be'):
regions = fields.Nested(RegionSchema, many=True)
class Meta:
relationships_schema = DatasetApiRelationships
object_type = 'dataset'
url_template = '{api_url}/datasets/{ident}'
model = 'datasets.Dataset'
class ResourceForm(forms.ModelForm):
title = forms.CharField(widget=forms.Textarea(attrs={
'style': 'width: 99%',
'rows': 2
}),
label=_("Title"))
title_en = forms.CharField(widget=forms.Textarea(attrs={
'style': 'width: 99%',
'rows': 2
}),
label=_("Title") + " (EN)",
required=False)
description = forms.CharField(
widget=CKEditorWidget,
label=_("Description"),
min_length=settings.DESCRIPTION_FIELD_MIN_LENGTH,
max_length=settings.DESCRIPTION_FIELD_MAX_LENGTH,
validators=[ContainsLetterValidator()],
)
description_en = forms.CharField(
widget=CKEditorWidget,
label=_("Description") + " (EN)",
required=False,
min_length=settings.DESCRIPTION_FIELD_MIN_LENGTH,
max_length=settings.DESCRIPTION_FIELD_MAX_LENGTH,
validators=[ContainsLetterValidator()],
)
special_signs = SpecialSignMultipleChoiceField(
queryset=SpecialSign.objects.published(),
required=False,
label=_('Special Signs'),
widget=FilteredSelectMultiple(_('special signs'), False),
)
if is_enabled('S37_resources_admin_region_data.be'):
regions = RegionsMultipleChoiceField(required=False,
label=_('Regions'))
if is_enabled('S41_resource_has_high_value_data.be'):
has_high_value_data = forms.ChoiceField(
label=_('has high value data').capitalize(),
help_text=
('Zaznaczając dane dodane w zbiorze danych zostaną określone jako wysokiej wartości. Są to dane, '
'których ponowne wykorzystanie wiąże się z istotnymi korzyściami dla społeczeństwa, środowiska '
'i gospodarki'),
choices=[(True, _('Yes')), (False, _('No'))],
widget=forms.RadioSelect(attrs={'class': 'inline'}),
required=True)
def get_readonly_fields(self, request, obj=None):
read_only_fields = super(DatasetAdmin,
self).get_readonly_fields(request, obj)
archived_resources = [
'archived_resources_files'
] if is_enabled('S41_resource_bulk_download.be') else []
read_only_fields = archived_resources + read_only_fields
return read_only_fields
def validate_link(self, data, **kwargs):
value = data.get('link')
if value and "://" in value and is_enabled(
'S37_validate_resource_link_scheme_harvester.be'):
scheme = value.split("://")[0].lower()
if scheme != 'https':
raise ValidationError(_('Required scheme is https://'),
field_name='link')
def get_data(resource):
resource.revalidate()
assert resource.tabular_data_schema
data = resource.__dict__
data['dataset'] = resource.dataset_id
if is_enabled('S41_resource_has_high_value_data.be'):
data['has_high_value_data'] = False
return data
def queryset(self, request, queryset):
val = self.value()
if not val:
return queryset
empty_val = '' if is_enabled(
'S41_resource_AP_optimization.be') else None
val = empty_val if val == 'N/A' else val
return queryset.filter(**{self.qs_param: val})
def resource_created_with_file():
from mcod.resources.models import Resource
res = Resource.objects.all().latest('id')
if is_enabled('S40_new_file_model.be'):
assert res.file.name == ''
assert res.main_file.name != ''
else:
assert res.file.name != ''
def resource_views_count_is(resource_id, counter_type, val):
model = apps.get_model('resources', 'resource')
obj = model.objects.get(pk=resource_id)
if is_enabled('S16_new_date_counters.be'):
current_count = getattr(obj, f'computed_{counter_type}')
else:
current_count = getattr(obj, counter_type)
assert current_count == val
def test_xls_resource_display_csv_file_data(self, admin, resource_with_xls_file):
client = Client()
client.force_login(admin)
resp = client.get(resource_with_xls_file.admin_change_url)
content = resp.content.decode()
if is_enabled('S40_new_file_model.be'):
assert 'csv_converted_file' in content
else:
assert 'csv_file' in content
class AddResourceForm(ResourceForm, LinkOrFileUploadForm):
data_date = forms.DateField(initial=today,
widget=AdminDateWidget,
label=_("Data date"))
from_resource = forms.ModelChoiceField(queryset=Resource.objects.all(),
widget=forms.HiddenInput(),
required=False)
if is_enabled('S37_validate_resource_link_scheme.be'):
link = forms.URLField(widget=ResourceLinkWidget(
attrs={
'style': 'width: 99%',
'placeholder': 'https://'
}))
def clean_link(self):
link = super().clean_link()
if link and not link.startswith('https:') and is_enabled(
'S37_validate_resource_link_scheme.be'):
self.add_error('link', _('Required scheme is https://'))
return link
def clean_status(self):
dataset = self.cleaned_data.get('dataset')
if dataset and dataset.status == 'draft':
if self.cleaned_data['status'] == 'published':
error_message = _(
"You can't set status of this resource to published, because it's dataset is still a draft. "
"Set status of this resource to draft or set stauts to published for that dataset: "
)
error_message += "<a href='{}'>{}</a>".format(
dataset.admin_change_url, dataset.title)
raise forms.ValidationError(mark_safe(error_message))
return self.cleaned_data['status']
def clean_switcher(self):
selected_field = super().clean_switcher()
if selected_field == 'link':
self.fields['data_date'].required = False
def clean_data_date(self):
data_date = self.cleaned_data.get('data_date')
if not data_date:
self.cleaned_data['data_date'] = today()
return self.cleaned_data['data_date']
def clean_file(self):
file = self.cleaned_data.get('file')
if file:
_name, ext = os.path.splitext(file.name)
if ext.lower() not in SUPPORTED_FILE_EXTENSIONS:
self.add_error(
'file',
_('Invalid file extension: %(ext)s.') %
{'ext': ext or '-'})
return file
def no_data_resource(dataset):
resource = Resource()
resource.title = "No data resource"
resource.type = "file"
resource.format = 'JPG'
if not is_enabled('S40_new_file_model.be'):
resource.file = File(open(prepare_file('buzzfeed-logo.jpg'), 'rb'))
resource.file.open('rb')
resource.dataset = dataset
resource.save()
if is_enabled('S40_new_file_model.be'):
ResourceFile.objects.create(file=File(
open(prepare_file('buzzfeed-logo.jpg'), 'rb')),
is_main=True,
resource=resource,
format='JPG')
resource = Resource.objects.get(pk=resource.pk)
return resource
def get_context(self, resource):
if is_enabled('S40_new_file_model.be'):
context = self.get_link_or_file_context(resource)
else:
if resource.link and not resource.file:
context = self.get_link_context_data(resource.link)
else:
context = self.get_file_context_data(resource.file)
return context
def create_res(ds, editor, **kwargs):
from mcod.resources.models import Resource, ResourceFile
_fname = kwargs.pop('filename')
_kwargs = {
'title': 'Analysis of fake news sites and viral posts',
'description': 'Over the past four years, BuzzFeed News has maintained lists of sites that '
'publish completely fabricated stories. As we encounter new ones and debunk '
'their content, we add them to the list.',
'file': _fname,
'link': f'https://falconframework.org/media/resources/{_fname}',
'format': 'csv',
'openness_score': 3,
'views_count': 10,
'downloads_count': 20,
'dataset': ds,
'created_by': editor,
'modified_by': editor,
'data_date': datetime.today(),
}
if is_enabled('S41_resource_has_high_value_data'):
_kwargs['has_high_value_data'] = False
_kwargs.update(**kwargs)
if is_enabled('S40_new_file_model.be'):
with open(os.path.join(settings.RESOURCES_MEDIA_ROOT, _fname), 'rb') as f:
from mcod.resources.link_validation import session
adapter = requests_mock.Adapter()
adapter.register_uri('GET', _kwargs['link'], content=f.read(), headers={'Content-Type': 'application/csv'})
session.mount('https://falconframework.org', adapter)
_kwargs.pop('file')
res = Resource.objects.create(**_kwargs)
ResourceFile.objects.create(
is_main=True,
resource=res,
file=os.path.join(settings.RESOURCES_MEDIA_ROOT, _fname),
format='csv'
)
else:
res = Resource(**_kwargs)
res.tracker.saved_data['link'] = res.link
res.save()
res = Resource.objects.get(pk=res.pk)
return res
class MonitoringReportsAdmin(ReportsAdmin):
app_models = [
'applications.ApplicationProposal',
'suggestions.DatasetSubmission',
'suggestions.DatasetComment',
'suggestions.ResourceComment',
]
if is_enabled('S39_showcases.be'):
app_models.append('showcases.ShowcaseProposal')
class Media:
pass
def process_resource_file_task(resource_id,
update_link=True,
update_file_archive=False,
**kwargs):
Resource = apps.get_model('resources', 'Resource')
resource = Resource.raw.get(id=resource_id)
format, file_info, file_encoding, p, file_mimetype, analyze_exc = resource.analyze_file(
)
if not resource.file_extension and format:
Resource.raw.filter(pk=resource_id).update(file=resource.save_file(
resource.file, f'{resource.file_basename}.{format}'))
resource = Resource.raw.get(id=resource_id)
Resource.raw.filter(pk=resource_id).update(
format=format,
file_mimetype=file_mimetype,
file_info=file_info,
file_encoding=file_encoding,
type='file',
link=resource.file_url if update_link else resource.link,
openness_score=resource.get_openness_score(format))
resource = Resource.raw.get(id=resource_id)
resource.check_support()
if resource.format == 'csv' and resource.file_encoding is None:
raise FileEncodingValidationError([{
'code':
'unknown-encoding',
'message':
'Nie udało się wykryć kodowania pliku.'
}])
if analyze_exc:
raise analyze_exc
process_resource_file_data_task.s(resource_id,
**kwargs).apply_async(countdown=2)
if update_link:
process_resource_from_url_task.s(resource_id,
update_file=False,
**kwargs).apply_async(countdown=2)
if is_enabled('S41_resource_bulk_download.be') and update_file_archive:
resource.dataset.archive_files()
return json.dumps({
'uuid': str(resource.uuid),
'link': resource.link,
'format': resource.format,
'type': resource.type,
'path': resource.file.path,
'url': resource.file_url
})
