def convert_package_extra_to_kind(dataset_schema=None, package_extra_list=[PackageExtra]):
if not dataset_schema:
dataset_schema = DatasetSchemaDcatApOp("")
if not package_extra_list:
return
kind = KindSchemaDcatApOp(uri_util.create_uri_for_schema(KindSchemaDcatApOp),
graph_name=dataset_schema.graph_name)
for package_extra in package_extra_list:
if package_extra.value:
if package_extra.key == CONTACT_NAME:
length = str(len(kind.organisationDASHname_vcard))
kind.organisationDASHname_vcard[length] = ResourceValue(package_extra.value)
elif package_extra.key == CONTACT_EMAIL:
length = str(len(kind.hasEmail_vcard))
kind.hasEmail_vcard[length] = SchemaGeneric(package_extra.value, graph_name=dataset_schema.graph_name)
elif package_extra.key == CONTACT_TELEPHONE:
telephone_number = re.sub('[^+\/0-9]', '', package_extra.value)
if telephone_number != package_extra.value:
logging.info(u"Migration: {2} telephone number migrated from {0} to {1}".format(package_extra.value,
telephone_number, dataset_schema.uri))
if telephone_number:
#remove space
telephone_number = telephone_number.replace(' ','')
telephone = TelephoneSchemaDcatApOp(
"telephone-" + package_extra_list[0].id + str(create_blank_node_replacement_uri()),
graph_name=dataset_schema.graph_name)
length_value = str(len(telephone.hasValue_vcard))
telephone.hasValue_vcard[length_value] = SchemaGeneric(
telephone_number,
graph_name=dataset_schema.graph_name)
telephone.type_rdf['0'] = NAMESPACE_DCATAPOP.vcard + VOICE
telephone.type_rdf['1'] = NAMESPACE_DCATAPOP.vcard + WORK
length_telephone = str(len(kind.hasTelephone_vcard))
kind.hasTelephone_vcard[length_telephone] = telephone
elif package_extra.key == CONTACT_ADDRESS:
# TODO: The address could be parses but the data are not formatted correctly.
address = AddressSchemaDcatApOp(uri_util.create_uri_for_schema(AddressSchemaDcatApOp),
graph_name=dataset_schema.graph_name)
length = str(len(address.streetDASHaddress_vcard))
address.streetDASHaddress_vcard[length] = ResourceValue(package_extra.value,
datatype=NAMESPACE_DCATAPOP.vcard + ADDRESS)
address.postalDASHcode_vcard = ""
address.locality_vcard = ""
address.countryDASHname_vcard = ""
length_address = str(len(kind.hasAddress_vcard))
kind.hasAddress_vcard[length_address] = address
elif package_extra.key == CONTACT_WEBPAGE:
document = DocumentSchemaDcatApOp(new_documentation_uri(), graph_name=dataset_schema.graph_name)
document.url_schema['0'] = ResourceValue(package_extra.value,
datatype=NAMESPACE_DCATAPOP.foaf + DOCUMENT)
document.topic_foaf['0'] = SchemaGeneric(dataset_schema.uri)
document.title_dcterms['0'] = ResourceValue("title_" + document.uri, lang='en')
document.type_dcterms['0'] = SchemaGeneric("default_type_dcterms")
length = str(len(kind.homePage_foaf))
kind.homePage_foaf[length] = document
return kind
def create_dataset_schema_for_package_dict(data_dict):
name = data_dict.get('name')
uri = uri_util.new_dataset_uri_from_name(name)
dataset = DatasetDcatApOp(uri)
# Catalog Record
catalogRecord = CatalogRecordSchemaDcatApOp(
uri_util.new_catalog_record_uri())
date = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
catalogRecord.issued_dcterms['0'] = ResourceValue(date,
datatype=XSD.datetime)
catalogRecord.modified_dcterms['0'] = ResourceValue(date,
datatype=XSD.datetime)
catalogRecord.primaryTopic_foaf['0'] = SchemaGeneric(dataset.schema.uri)
dataset.schema_catalog_record = catalogRecord
generated_dataset = __dataset_old_model_transformation(
dataset, data_dict, dataset.schema)
# Generate DOI if requested
if _DOI_GENERATION_KEY in data_dict:
doi = generate_doi_for_dataset(dataset, data_dict[_DOI_GENERATION_KEY])
generated_dataset.set_doi(doi)
return generated_dataset
def set_home_page(self, data_dict):
try:
home_pages = data_dict.get('home_page')
if home_pages:
for home_page in home_pages.split(" "):
if home_page:
home_page_length = str(len(self.schema.homepage_foaf))
document = DocumentSchemaDcatApOp(
uri_util.create_uri_for_schema(
DocumentSchemaDcatApOp))
document.url_schema[str(len(
document.url_schema))] = ResourceValue(home_page)
# TODO add correct default values for the three properties
document.topic_foaf['0'] = SchemaGeneric(
"default_topic_foaf")
document.title_dcterms['0'] = ResourceValue("title_" +
home_page)
document.type_dcterms['0'] = SchemaGeneric(
"default_type_dcterms")
self.schema.homepage_foaf[home_page_length] = document
except BaseException as e:
log.error("Failed to set home page to catalog {0}".format(
self.schema.uri))
def set_landing_page(dataset_schema, package):
document_schema = DocumentSchemaDcatApOp(uri_util.create_uri_for_schema(DocumentSchemaDcatApOp),
graph_name=dataset_schema.graph_name)
document_schema.url_schema['0'] = ResourceValue(package.url)
document_schema.topic_foaf['0'] = SchemaGeneric(dataset_schema.uri)
document_schema.title_dcterms['0'] = ResourceValue("title_" + package.url, lang='en')
document_schema.type_dcterms['0'] = SchemaGeneric("default_type_dcterms")
dataset_schema.landingPage_dcat[str(len(dataset_schema.landingPage_dcat))] = document_schema
def test_at_most_one_by_language(self):
ds = DatasetDcatApOp(
"http://data.europa.eu/88u/dataset/dgt-translation-memory-V1-2")
if ds.get_description_from_ts():
ds.schema.title_dcterms['1'] = ResourceValue("new title", "fr")
ds.schema.title_dcterms['2'] = ResourceValue("new title2", "it")
validator = ValidationSchema(ds.schema,
ds.schema.get_schema_type())
report = validator.validate()
# validation_result = True
for result in report:
if result.get("property") == "title_dcterms" and result.get(
"constraint") == "card_1..n_en":
validation_result = False if result.get(
"result") == ValidationTypeResult.error else True
break
self.assertTrue(validation_result,
" Test validation of test_at_least_one_en failed")
# test the case of empty value
ds.schema.title_dcterms['0'].value_or_uri = ''
validator = ValidationSchema(ds.schema,
ds.schema.get_schema_type())
report = validator.validate()
for result in report:
if result.get("property") == "title_dcterms" and result.get(
"constraint") == "card_1..n_en":
validation_result = False if result.get(
"result") == ValidationTypeResult.error else True
self.assertTrue(not validation_result,
" Test validation of test_at_least_one_en failed")
ds.schema.title_dcterms['0'].value_or_uri = None
validator = ValidationSchema(ds.schema,
ds.schema.get_schema_type())
report = validator.validate()
for result in report:
if result.get("property") == "title_dcterms" and result.get(
"constraint") == "card_1..n_en":
validation_result = False if result.get(
"result") == ValidationTypeResult.error else True
self.assertTrue(not validation_result,
" Test validation of test_at_least_one_en failed")
# No english title
ds.schema.title_dcterms['0'] = ResourceValue("new title", "de")
validator = ValidationSchema(ds.schema,
ds.schema.get_schema_type())
report = validator.validate()
for result in report:
if result.get("property") == "title_dcterms" and result.get(
"constraint") == "card_1..n_en":
validation_result = False if result.get(
"result") == ValidationTypeResult.error else True
self.assertTrue(not validation_result,
" Test validation of test_at_least_one_en failed")
def set_distribution_titles(configuration_file=CONFIGURATION_FILE_PATH, distribution=None,
resource=Resource()):
if not distribution:
distribution = DistributionSchemaDcatApOp('')
if resource.name:
distribution.title_dcterms['0'] = ResourceValue(resource.name or '', LanguagesConstants.LANGUAGE_CODE_EN)
condition = TermTranslation.term == resource.name
titles = find_any_in_database(configuration_file, condition, TermTranslation) # type: list[TermTranslation]
for title in titles:
length = str(len(distribution.title_dcterms))
distribution.title_dcterms[length] = ResourceValue(title.term_translation, title.lang_code)
def set_keyword(dataset_schema=None, tag=Tag(), configuration_file=CONFIGURATION_FILE_PATH):
if not dataset_schema:
dataset_schema = DatasetSchemaDcatApOp("")
length = str(len(dataset_schema.keyword_dcat))
condition = TermTranslation.term == tag.name
keywords_translation = find_any_in_database(configuration_file, condition, TermTranslation) # type: list[TermTranslation]
length = str(len(dataset_schema.keyword_dcat))
dataset_schema.keyword_dcat[length] = ResourceValue(value_or_uri=tag.name, lang='en')
for keyword in keywords_translation:
length = str(len(dataset_schema.keyword_dcat))
dataset_schema.keyword_dcat[length] = ResourceValue(value_or_uri=keyword.term_translation, lang=keyword.lang_code)
def set_package_titles(configuration_file=CONFIGURATION_FILE_PATH, dataset_schema=None,
package=Package()):
if not dataset_schema:
dataset_schema = DatasetSchemaDcatApOp("")
if package.title:
dataset_schema.title_dcterms['0'] = ResourceValue(package.title, lang=LanguagesConstants.LANGUAGE_CODE_EN)
condition = TermTranslation.term == package.title
titles = find_any_in_database(configuration_file, condition, TermTranslation) # type: list[TermTranslation]
for title in titles:
if title.term_translation:
length = str(len(dataset_schema.title_dcterms))
dataset_schema.title_dcterms[length] = ResourceValue(title.term_translation, lang=title.lang_code)
def set_package_descriptions(configuration_file=CONFIGURATION_FILE_PATH, dataset_schema=None,
package=Package()):
if not dataset_schema:
dataset_schema = DatasetSchemaDcatApOp("")
description = package.notes or package.description
dataset_schema.description_dcterms['0'] = ResourceValue(description, lang=LanguagesConstants.LANGUAGE_CODE_EN)
condition = TermTranslation.term == u'{0}'.format(description)
descriptions = find_any_in_database(configuration_file, condition, TermTranslation) # type: list[TermTranslation]
for description in descriptions:
length = str(len(dataset_schema.description_dcterms))
dataset_schema.description_dcterms[length] = ResourceValue(description.term_translation,
lang=description.lang_code)
def edit_save_to_ts():
ds1 = DatasetDcatApOp("http://data.europa.eu/88u/dataset/dgt-translation-memory-V1-2")
if ds1.get_description_from_ts():
ds1.privacy_state = "public"
ds1.schema.ckanName_dcatapop['0'].value_or_uri = "NEW CKAN NAME"
ds1.schema.keyword_dcat['fr'] = ResourceValue(u'la réussite', lang="fr")
ds1.schema.keyword_dcat['grg'] = ResourceValue(u'επιτυχία', lang="gr")
ds1.schema.contactPoint_dcat['0'].hasTelephone_vcard['0'].hasValue_vcard['0'].uri = "TEL:213232323"
if ds1.save_to_ts():
print " Save done"
ds1after = DatasetDcatApOp("http://data.europa.eu/88u/dataset/dgt-translation-memory-V1-2")
ds1after.get_description_from_ts()
pass
def set_alternative_titles(configuration_file=CONFIGURATION_FILE_PATH, dataset_schema=None,
package_extra=PackageExtra()):
if not dataset_schema:
dataset_schema = DatasetSchemaDcatApOp("")
dataset_schema.alternative_dcterms['0'] = ResourceValue(value_or_uri=package_extra.value,
lang=LanguagesConstants.LANGUAGE_CODE_EN)
condition = TermTranslation.term == package_extra.value
alternative_titles = \
find_any_in_database(configuration_file, condition, TermTranslation) # type: list[TermTranslation]
for title in alternative_titles:
length = str(len(dataset_schema.alternative_dcterms))
dataset_schema.alternative_dcterms[length] = ResourceValue(value_or_uri=title.term_translation,
lang=title.lang_code)
def test_create_ds(self):
ds1 = DatasetDcatApOp("t1")
ds2 = DatasetDcatApOp("t2")
ds1.schema.description_dcterms['5'] = ResourceValue("rien ds 111")
ds2.schema.description_dcterms['5'] = ResourceValue("rien ds 222")
ds1.schema.ckanName_dcatapop['6'] = ResourceValue("ckan name ds1")
ds1.schema.description_dcterms['4'] = ResourceValue("rien 111")
ds2.schema.ckanName_dcatapop['6'] = ResourceValue("ckan name ds2")
self.assertNotEqual(ds1.schema.description_dcterms['5'].value_or_uri,
ds2.schema.description_dcterms['5'].value_or_uri,
"ddd")
self.assertNotEqual(len(ds1.schema.description_dcterms),
len(ds2.schema.description_dcterms))
def set_variables_for_languages(cls, dict, parameter, result_dict, type):
from ckanext.ecportal.model.schemas.generic_schema import ResourceValue
for idx, lang in enumerate(LanguagesConstants.get_languages_as_list()):
param = parameter + '-' + lang
if dict.get(param):
rvl = ResourceValue(dict.get(param), lang, type=type)
result_dict[str(len(result_dict))] = rvl
def resource_show(context, data_dict):
'''Return the metadata of a resource.
:param id: the id of the resource
:type id: string
:rtype: dictionary
'''
model = context['model']
id = _get_or_bust(data_dict, 'id')
uri_prefix = config.get('ckan.ecodp.uri_prefix')
uri = '{0}/{1}/{2}'.format(config.get('ckan.ecodp.uri_prefix'),
'distribution', id)
resource = DistributionSchemaDcatApOp(uri)
if not resource.get_description_from_ts():
uri = '{0}/{1}/{2}'.format(config.get('ckan.ecodp.uri_prefix'),
'document', id)
resource = DocumentSchemaDcatApOp(uri)
if not resource.get_description_from_ts():
raise NotFound('Resource {0} not found.'.format(id))
context['resource'] = resource
# _check_access('resource_show', context, data_dict)
resource_dict = resource.schema_dictaze()
resource_dict['format'] = ''
if isinstance(resource, DistributionSchemaDcatApOp):
resource_dict['url'] = resource.downloadURL_dcat.get(
'0', SchemaGeneric('')).uri or resource.accessURL_dcat.get(
'0', SchemaGeneric('')).uri
resource_dict['format'] = resource.format_dcterms.get(
'0', SchemaGeneric('')).uri.split('/')[-1]
elif isinstance(resource, DocumentSchemaDcatApOp):
resource_dict['url'] = resource.url_schema.get(
'0', ResourceValue('')).value_or_uri
resource_dict['format'] = resource.format_dcterms.get(
'0', SchemaGeneric('')).uri.split('/')[-1]
if isinstance(resource, DistributionSchemaDcatApOp):
resource_dict['id'] = resource.uri.split('/')[-1]
for item in plugins.PluginImplementations(plugins.IResourceController):
resource_dict = item.before_show(resource_dict)
try:
prev_url = resource_dict['url']
prev_format = resource_dict.get('format', '')
prev_res = {'url': prev_url, 'format': prev_format, 'id': id}
except Exception as e:
import traceback
log.error('{0}'.format(e))
log.error(traceback.print_exc())
resource_dict['can_be_previewed'] = _resource_preview(
{'resource': prev_res})
return resource_dict
def convert_temporal_coverage(self, date_from, date_to):
from ckanext.ecportal.model.schemas.dcatapop_period_of_time_schema import PeriodOfTimeSchemaDcatApOp
from ckanext.ecportal.lib.dataset_util import ResourceValue
return_dict = {}
period = PeriodOfTimeSchemaDcatApOp(
uri=uri_util.create_uri_for_schema(PeriodOfTimeSchemaDcatApOp))
if date_from:
period.startDate_schema['0'] = ResourceValue(date_from,
datatype=XSD.date)
if date_to:
period.endDate_schema['0'] = ResourceValue(date_to,
datatype=XSD.date)
if period.startDate_schema or period.endDate_schema:
return_dict['0'] = period
return return_dict
def test_edit_save_to_ts(self):
self.test_get_description_from_ts()
dataset = DatasetDcatApOp(TRANSLATION_MEMORY_V_1_2)
if dataset.get_description_from_ts():
dataset.privacy_state = PRIVACY_STATE_PUBLIC
dataset.schema.ckanName_dcatapop[
'0'].value_or_uri = "NEW CKAN NAME"
dataset.schema.keyword_dcat[LanguagesConstants.LANGUAGE_CODE_FR] = \
ResourceValue(u'la réussite', lang=LanguagesConstants.LANGUAGE_CODE_FR)
dataset.schema.keyword_dcat[LanguagesConstants.LANGUAGE_CODE_EL] = \
ResourceValue(u'επιτυχία', lang=LanguagesConstants.LANGUAGE_CODE_EL)
dataset.schema.contactPoint_dcat['0'].hasTelephone_vcard[
'0'].hasValue_vcard['0'].uri = "TEL:213232323"
if dataset.save_to_ts():
print " Save done"
ds1after = DatasetDcatApOp(TRANSLATION_MEMORY_V_1_2)
ds1after.get_description_from_ts()
pass
def set_splitted_labels(cls, dict, parameter):
from ckanext.ecportal.model.schemas.generic_schema import ResourceValue
result_dict = {}
labels = dict.get(parameter)
if labels:
for label in labels.split(" "):
if label:
label_length = str(len(result_dict))
result_dict[label_length] = ResourceValue(label)
return result_dict
def set_document(configuration_file=CONFIGURATION_FILE_PATH, dataset_schema=None,
resource=Resource, file_types=dict, documentation_types=dict):
if not dataset_schema:
dataset_schema = DatasetSchemaDcatApOp("")
uri_prefix = 'http://data.europa.eu/88u'
uri = '{0}/document/{1}'.format(uri_prefix, resource.id)
document = DocumentSchemaDcatApOp(uri)
type = resource.resource_type or resource.extras
type_uri = ''
if MAIN_DOCUMENTATION in type:
type_uri = 'http://publications.europa.eu/resource/authority/documentation-type/DOCUMENTATION_MAIN'
elif RELATED_DOCUMENTATION in type:
type_uri = 'http://publications.europa.eu/resource/authority/documentation-type/DOCUMENTATION_RELATED'
elif WEB_RELATED_DOCUMENTATION in type:
type_uri = 'http://publications.europa.eu/resource/authority/documentation-type/WEBPAGE_RELATED'
else:
type_uri = type
log.warn('nor mapping for type {0}'.format(type_uri))
if type_uri:
document.type_dcterms['0'] = SchemaGeneric(type_uri, default_type={'0': SchemaGeneric(DOCUMENTATIONTYPE_DCATAPOP)})
else:
log.warn('Could not map type {2} for documentation {1} of ds {0}'.format(document.uri, dataset_schema.uri, type_uri))
file_type = ''
if resource.format:
tmp_format = resource.format
if '/' in tmp_format:
tmp_format = tmp_format.split('/')[-1]
file_type = next((key for key, value in file_types.iteritems() if tmp_format == key.split('/')[-1].lower()), None)
if not file_type:
file_type = mapping_file_formats_manual.get(resource.format,resource.format)
if file_type == resource.format:
log.warn('No mapping for format {0} of documentation {1} in ds {2}'.format(file_type, document.uri, dataset_schema.uri))
if file_type:
document.format_dcterms['0'] = MediaTypeOrExtentSchemaDcatApOp(file_type,
graph_name=dataset_schema.graph_name)
else:
log.warn('No format for document {0} of ds {1}'.format(document.uri,dataset_schema.uri))
if resource.url:
document.url_schema['0'] = ResourceValue(resource.url)
# document = DocumentSchemaDcatApOp(uri_util.create_uri_for_schema(DocumentSchemaDcatApOp))
# document.url_schema[str(len(document.url_schema))] = ResourceValue(landing_page)
document.topic_foaf['0'] = SchemaGeneric(dataset_schema.uri)
# document.title_dcterms['0'] = ResourceValue("title_" + landing_page, lang='en')
# document.type_dcterms['0'] = SchemaGeneric("default_type_dcterms")
set_document_descriptions(configuration_file, document, resource)
length = str(len(dataset_schema.page_foaf))
dataset_schema.page_foaf[length] = document
def set_temporal_to(dataset_schema=None, package_extra=PackageExtra()):
if not dataset_schema:
dataset_schema = DatasetSchemaDcatApOp("")
length = str(len(dataset_schema.temporal_dcterms))
period = dataset_schema.temporal_dcterms.get('0', None)
if not period:
period = PeriodOfTimeSchemaDcatApOp("period_of_time-" + str(create_blank_node_replacement_uri()),
graph_name=dataset_schema.graph_name)
period.endDate_schema['0'] = ResourceValue(package_extra.value,
datatype=NAMESPACE_DCATAPOP.xsd + DATE_TIME)
dataset_schema.temporal_dcterms['0'] = period
def set_rights(self, data_dict):
try:
rights = data_dict.get('rights')
if rights:
rs = RightsStatementSchemaDcatApOp(
uri_util.new_rightstatement_uri())
rs.label_rdfs['0'] = ResourceValue(rights)
self.schema.rights_dcterms['0'] = rs
except BaseException as e:
log.error("Failed to set rights to catalog {0}".format(
self.schema.uri))
def test_change_DOI_structure(self):
result = False
for id in IDENTIFIER_LIST:
schema = IdentifierSchemaDcatApOp(id)
schema.get_description_from_ts()
if schema.notation_skos.get(
'0', ResourceValue('')
).value_or_uri == "http://publications.europa.eu/resource/authority/notation-type/DOI":
doi = schema.uri
schema.notation_skos['0'] = ResourceValue(
doi,
datatype=
"http://publications.europa.eu/resource/authority/notation-type/DOI"
)
result = schema.save_to_ts()
print "change {0} result {1}".format(id, result)
self.assertTrue(result)
def set_document_descriptions(configuration_file=CONFIGURATION_FILE_PATH, document=None,
resource=Resource()):
if not document:
document = DocumentSchemaDcatApOp('')
if resource.name:
document.title_dcterms['0'] = ResourceValue(resource.name, LanguagesConstants.LANGUAGE_CODE_EN)
name_condition = TermTranslation.term == resource.name
titles = find_any_in_database(configuration_file, name_condition,
TermTranslation) # type: list[TermTranslation]
for title in titles:
length = str(len(document.title_dcterms))
document.title_dcterms[length] = ResourceValue(title.term_translation, title.lang_code)
if resource.description:
document.description_dcterms['0'] = ResourceValue(resource.description, LanguagesConstants.LANGUAGE_CODE_EN)
condition = TermTranslation.term == resource.description
descriptions = find_any_in_database(configuration_file, condition,
TermTranslation) # type: list[TermTranslation]
for description in descriptions:
length = str(len(document.description_dcterms))
document.description_dcterms[length] = ResourceValue(description.term_translation, description.lang_code)
def set_splitted_variables_for_languages(cls, dict, parameter, result_dict,
type):
from ckanext.ecportal.model.schemas.generic_schema import ResourceValue
for idx, lang in enumerate(LanguagesConstants.get_languages_as_list()):
param = parameter + '-' + lang
variable = dict.get(param)
if variable:
variables = variable.split(" ")
for variable in variables:
rvl = ResourceValue(variable, lang, type=type)
length = str(len(result_dict))
result_dict[length] = rvl
def convert_keywords(self, keyword_string):
from ckanext.ecportal.lib.dataset_util import ResourceValue
tags = keyword_string if isinstance(keyword_string, list) else [
tag.strip() for tag in keyword_string.split(',') if tag.strip()
]
return_dict = {}
if tags:
i = 0
for keyword in tags:
return_dict[str(i)] = ResourceValue(keyword, DEFAULT_LANGUAGE)
i += 1
return return_dict
def update_dataset_for_package_dict(dataset, data_dict):
"""
:param DatasetDcatApOp dataset:
:param data_dict:
:return:
"""
date = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
dataset.schema_catalog_record.modified_dcterms = {}
dataset.schema_catalog_record.modified_dcterms['0'] = ResourceValue(
date, datatype=XSD.datetime)
dataset.schema_catalog_record.numberOfViews_dcatapop = {
'0':
max(dataset.schema_catalog_record.numberOfViews_dcatapop.values()
or [ResourceValue("0")],
key=lambda x: int(x.value_or_uri))
}
old_dataset = dataset.schema #needed for keeping the resource ids
dataset.schema = DatasetSchemaDcatApOp(dataset.dataset_uri)
return __dataset_old_model_transformation(dataset, data_dict, old_dataset)
def test_save_to_ts(self):
ds1 = DatasetDcatApOp(
"http://data.europa.eu/88u/dataset/dgt-translation-memory-V1-2")
if ds1.get_description_from_ts():
ds1.privacy_state = "public"
ds1.schema.ckanName_dcatapop['0'].value_or_uri = "NEW CKAN NAME"
ds1.schema.ckanName_dcatapop['1'] = ResourceValue(
"Second CKAN NAME")
# ckan_name_new = ds1.schema.ckanName_dcatapop['1'] = ResourceValue("another ckan Name")
ds1.schema.contactPoint_dcat['0'].hasTelephone_vcard[
'0'].hasValue_vcard['0'].uri = "TEL:213232323"
ds1.save_to_ts()
ds1after = DatasetDcatApOp(
"http://data.europa.eu/88u/dataset/dgt-translation-memory-V1-2"
)
ds1after.get_description_from_ts()
ckan_name_new = ds1after.schema.ckanName_dcatapop['0'].value_or_uri
lenc = len(ds1after.schema.ckanName_dcatapop)
msg = "Expected name {0}, New value {1}. Expected length {2}, Get {3}"
self.assertTrue(
ckan_name_new == "NEW CKAN NAME" and lenc == 2,
msg.format("NEW CKAN NAME", ckan_name_new, 2, lenc))
# check if the generation of uris from memeber name workds in the case of DASH and DOT
self.assertTrue("organisation-name" in ds1after.ttl_as_in_ts,
"generation of uri from member failed")
ds_new = DatasetDcatApOp("http://newdcatap.com")
ds_new.schema.ckanName_dcatapop['0'] = ResourceValue("ckan Name new")
ds_new.save_to_ts()
ds_new_from_ts = DatasetDcatApOp("http://newdcatap.com")
ds_new_from_ts.get_description_from_ts()
self.assertTrue(
ds_new_from_ts.schema.ckanName_dcatapop['0'].value_or_uri ==
ds_new.schema.ckanName_dcatapop['0'].value_or_uri,
"New dataset is not saved")
def set_doi(self, doi):
"""
Set a DOI for this catalogue.
:param str doi: The DOI to set.
"""
if doi:
update_doi = True
catalog = CatalogDcatApOp(self.schema.uri)
catalog.get_description_from_ts()
for dois in catalog.schema.identifier_adms.values():
if hasattr(dois, 'notation_skos'):
if dois.notation_skos.get(
"0"
).value_or_uri == controlled_vocabulary_util.DOI_URI:
update_doi = False
self.schema.identifier_adms['0'] = dois
break
if update_doi:
org = ckan_helper.get_organization({"name": "publ"})
identifier = IdentifierSchemaDcatApOp(
uri_util.create_uri_for_schema(IdentifierSchemaDcatApOp))
identifier.notation_skos = {
"0":
ResourceValue(doi,
datatype=controlled_vocabulary_util.DOI_URI)
}
identifier.issued_dcterms = {
"0":
ResourceValue(
datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
datatype=XSD.date)
}
identifier.schemeAgency_adms = {"0": ResourceValue(org[1])}
identifier_adms_length = len(self.schema.identifier_adms)
self.schema.identifier_adms[str(
identifier_adms_length)] = identifier
def set_catalog_record(package, package_extra_list, dataset_schema):
uri = uri_util.new_catalog_record_uri()
schema_catalog_record = CatalogRecordSchemaDcatApOp(uri)
schema_catalog_record.primaryTopic_foaf['0'] = dataset_schema
schema_catalog_record.modified_dcterms['0'] = ResourceValue(str(package.metadata_modified),
datatype=NAMESPACE_DCATAPOP.xsd + DATE_TIME) # 0...1
ts = get_metadata_created_timestamp(package.id)
schema_catalog_record.issued_dcterms['0'] = ResourceValue(str(ts),
datatype=NAMESPACE_DCATAPOP.xsd + DATE_TIME) # 0...1
count = 0
for value in [row.value for row in package_extra_list if row.key == 'metadata_language']:
if value:
schema_catalog_record.language_dcterms['{0}'.format(count)] = LinguisticSystemSchemaDcatApOp(value)
tracking = model.TrackingSummary.get_for_package(dataset_schema.uri)
if tracking:
schema_catalog_record.numberOfViews_dcatapop['0'] = ResourceValue(tracking['total'], datatype=XSD.integer)
return schema_catalog_record
def establish_resource_schema_from_distribution(distribution):
'''
:param DistributionSchemaDcatApOp resource:
:return:
'''
schema = {
"datastore_active":
False,
"id":
distribution.uri,
"size":
distribution.byteSize_dcat.get('0', ResourceValue('')).value_or_uri,
"state":
distribution.status_adms.get('0', SchemaGeneric('')).uri,
"hash":
distribution.checksum_spdx.get(
'0', ChecksumSchemaDcatApOp('')).checksumValue_spdx.get(
'0', ResourceValue('')).value_or_uri,
"description":
util._get_translated_term_from_dcat_object(distribution,
'description_dcterms',
'en'),
"format":
distribution.format_dcterms.get('0', SchemaGeneric('')).uri,
"tracking_summary":
None,
"last_modified":
distribution.modified_dcterms.get('0', ResourceValue('')).value_or_uri,
"download_total_resource":
distribution.numberOfDownloads_dcatapop.get(
'0', ResourceValue('')).value_or_uri,
"url_type":
None,
"mimetype":
None,
"name":
util._get_translated_term_from_dcat_object(distribution,
'title_dcterms', 'en'),
"created":
distribution.issued_dcterms.get('0', ResourceValue('')).value_or_uri,
"url":
next((value.uri for value in distribution.accessURL_dcat.values()),
''),
"iframe_code":
distribution.iframe_dcatapop.get('0', ResourceValue('')).value_or_uri,
"mimetype_inner":
"",
"position":
None,
"resource_type":
distribution.type_dcterms.get('0', SchemaGeneric('')).uri
}
return schema
def convert_translations_of_splitted_parameters(self,
dict,
parameter,
type='literal'):
from ckanext.ecportal.model.schemas.generic_schema import ResourceValue
result_dict = {}
if dict and parameter:
variable = dict.get(parameter)
if variable:
variables = variable.split(" ")
for variable in variables:
rv = ResourceValue(variable, DEFAULT_LANGUAGE, type=type)
length = str(len(result_dict))
result_dict[length] = rv
self.set_splitted_variables_for_languages(
dict, parameter, result_dict, type)
return result_dict