def resource_show(context, data_dict):
'''Return the metadata of a resource.
:param id: the id of the resource
:type id: string
:rtype: dictionary
'''
model = context['model']
id = _get_or_bust(data_dict, 'id')
uri_prefix = config.get('ckan.ecodp.uri_prefix')
uri = '{0}/{1}/{2}'.format(config.get('ckan.ecodp.uri_prefix'),
'distribution', id)
resource = DistributionSchemaDcatApOp(uri)
if not resource.get_description_from_ts():
uri = '{0}/{1}/{2}'.format(config.get('ckan.ecodp.uri_prefix'),
'document', id)
resource = DocumentSchemaDcatApOp(uri)
if not resource.get_description_from_ts():
raise NotFound('Resource {0} not found.'.format(id))
context['resource'] = resource
# _check_access('resource_show', context, data_dict)
resource_dict = resource.schema_dictaze()
resource_dict['format'] = ''
if isinstance(resource, DistributionSchemaDcatApOp):
resource_dict['url'] = resource.downloadURL_dcat.get(
'0', SchemaGeneric('')).uri or resource.accessURL_dcat.get(
'0', SchemaGeneric('')).uri
resource_dict['format'] = resource.format_dcterms.get(
'0', SchemaGeneric('')).uri.split('/')[-1]
elif isinstance(resource, DocumentSchemaDcatApOp):
resource_dict['url'] = resource.url_schema.get(
'0', ResourceValue('')).value_or_uri
resource_dict['format'] = resource.format_dcterms.get(
'0', SchemaGeneric('')).uri.split('/')[-1]
if isinstance(resource, DistributionSchemaDcatApOp):
resource_dict['id'] = resource.uri.split('/')[-1]
for item in plugins.PluginImplementations(plugins.IResourceController):
resource_dict = item.before_show(resource_dict)
try:
prev_url = resource_dict['url']
prev_format = resource_dict.get('format', '')
prev_res = {'url': prev_url, 'format': prev_format, 'id': id}
except Exception as e:
import traceback
log.error('{0}'.format(e))
log.error(traceback.print_exc())
resource_dict['can_be_previewed'] = _resource_preview(
{'resource': prev_res})
return resource_dict
def convert_package_extra_to_kind(dataset_schema=None, package_extra_list=[PackageExtra]):
if not dataset_schema:
dataset_schema = DatasetSchemaDcatApOp("")
if not package_extra_list:
return
kind = KindSchemaDcatApOp(uri_util.create_uri_for_schema(KindSchemaDcatApOp),
graph_name=dataset_schema.graph_name)
for package_extra in package_extra_list:
if package_extra.value:
if package_extra.key == CONTACT_NAME:
length = str(len(kind.organisationDASHname_vcard))
kind.organisationDASHname_vcard[length] = ResourceValue(package_extra.value)
elif package_extra.key == CONTACT_EMAIL:
length = str(len(kind.hasEmail_vcard))
kind.hasEmail_vcard[length] = SchemaGeneric(package_extra.value, graph_name=dataset_schema.graph_name)
elif package_extra.key == CONTACT_TELEPHONE:
telephone_number = re.sub('[^+\/0-9]', '', package_extra.value)
if telephone_number != package_extra.value:
logging.info(u"Migration: {2} telephone number migrated from {0} to {1}".format(package_extra.value,
telephone_number, dataset_schema.uri))
if telephone_number:
#remove space
telephone_number = telephone_number.replace(' ','')
telephone = TelephoneSchemaDcatApOp(
"telephone-" + package_extra_list[0].id + str(create_blank_node_replacement_uri()),
graph_name=dataset_schema.graph_name)
length_value = str(len(telephone.hasValue_vcard))
telephone.hasValue_vcard[length_value] = SchemaGeneric(
telephone_number,
graph_name=dataset_schema.graph_name)
telephone.type_rdf['0'] = NAMESPACE_DCATAPOP.vcard + VOICE
telephone.type_rdf['1'] = NAMESPACE_DCATAPOP.vcard + WORK
length_telephone = str(len(kind.hasTelephone_vcard))
kind.hasTelephone_vcard[length_telephone] = telephone
elif package_extra.key == CONTACT_ADDRESS:
# TODO: The address could be parses but the data are not formatted correctly.
address = AddressSchemaDcatApOp(uri_util.create_uri_for_schema(AddressSchemaDcatApOp),
graph_name=dataset_schema.graph_name)
length = str(len(address.streetDASHaddress_vcard))
address.streetDASHaddress_vcard[length] = ResourceValue(package_extra.value,
datatype=NAMESPACE_DCATAPOP.vcard + ADDRESS)
address.postalDASHcode_vcard = ""
address.locality_vcard = ""
address.countryDASHname_vcard = ""
length_address = str(len(kind.hasAddress_vcard))
kind.hasAddress_vcard[length_address] = address
elif package_extra.key == CONTACT_WEBPAGE:
document = DocumentSchemaDcatApOp(new_documentation_uri(), graph_name=dataset_schema.graph_name)
document.url_schema['0'] = ResourceValue(package_extra.value,
datatype=NAMESPACE_DCATAPOP.foaf + DOCUMENT)
document.topic_foaf['0'] = SchemaGeneric(dataset_schema.uri)
document.title_dcterms['0'] = ResourceValue("title_" + document.uri, lang='en')
document.type_dcterms['0'] = SchemaGeneric("default_type_dcterms")
length = str(len(kind.homePage_foaf))
kind.homePage_foaf[length] = document
return kind
def set_home_page(self, data_dict):
try:
home_pages = data_dict.get('home_page')
if home_pages:
for home_page in home_pages.split(" "):
if home_page:
home_page_length = str(len(self.schema.homepage_foaf))
document = DocumentSchemaDcatApOp(
uri_util.create_uri_for_schema(
DocumentSchemaDcatApOp))
document.url_schema[str(len(
document.url_schema))] = ResourceValue(home_page)
# TODO add correct default values for the three properties
document.topic_foaf['0'] = SchemaGeneric(
"default_topic_foaf")
document.title_dcterms['0'] = ResourceValue("title_" +
home_page)
document.type_dcterms['0'] = SchemaGeneric(
"default_type_dcterms")
self.schema.homepage_foaf[home_page_length] = document
except BaseException as e:
log.error("Failed to set home page to catalog {0}".format(
self.schema.uri))
def set_landing_page(dataset_schema, package):
document_schema = DocumentSchemaDcatApOp(uri_util.create_uri_for_schema(DocumentSchemaDcatApOp),
graph_name=dataset_schema.graph_name)
document_schema.url_schema['0'] = ResourceValue(package.url)
document_schema.topic_foaf['0'] = SchemaGeneric(dataset_schema.uri)
document_schema.title_dcterms['0'] = ResourceValue("title_" + package.url, lang='en')
document_schema.type_dcterms['0'] = SchemaGeneric("default_type_dcterms")
dataset_schema.landingPage_dcat[str(len(dataset_schema.landingPage_dcat))] = document_schema
def set_document(configuration_file=CONFIGURATION_FILE_PATH, dataset_schema=None,
resource=Resource, file_types=dict, documentation_types=dict):
if not dataset_schema:
dataset_schema = DatasetSchemaDcatApOp("")
uri_prefix = 'http://data.europa.eu/88u'
uri = '{0}/document/{1}'.format(uri_prefix, resource.id)
document = DocumentSchemaDcatApOp(uri)
type = resource.resource_type or resource.extras
type_uri = ''
if MAIN_DOCUMENTATION in type:
type_uri = 'http://publications.europa.eu/resource/authority/documentation-type/DOCUMENTATION_MAIN'
elif RELATED_DOCUMENTATION in type:
type_uri = 'http://publications.europa.eu/resource/authority/documentation-type/DOCUMENTATION_RELATED'
elif WEB_RELATED_DOCUMENTATION in type:
type_uri = 'http://publications.europa.eu/resource/authority/documentation-type/WEBPAGE_RELATED'
else:
type_uri = type
log.warn('nor mapping for type {0}'.format(type_uri))
if type_uri:
document.type_dcterms['0'] = SchemaGeneric(type_uri, default_type={'0': SchemaGeneric(DOCUMENTATIONTYPE_DCATAPOP)})
else:
log.warn('Could not map type {2} for documentation {1} of ds {0}'.format(document.uri, dataset_schema.uri, type_uri))
file_type = ''
if resource.format:
tmp_format = resource.format
if '/' in tmp_format:
tmp_format = tmp_format.split('/')[-1]
file_type = next((key for key, value in file_types.iteritems() if tmp_format == key.split('/')[-1].lower()), None)
if not file_type:
file_type = mapping_file_formats_manual.get(resource.format,resource.format)
if file_type == resource.format:
log.warn('No mapping for format {0} of documentation {1} in ds {2}'.format(file_type, document.uri, dataset_schema.uri))
if file_type:
document.format_dcterms['0'] = MediaTypeOrExtentSchemaDcatApOp(file_type,
graph_name=dataset_schema.graph_name)
else:
log.warn('No format for document {0} of ds {1}'.format(document.uri,dataset_schema.uri))
if resource.url:
document.url_schema['0'] = ResourceValue(resource.url)
# document = DocumentSchemaDcatApOp(uri_util.create_uri_for_schema(DocumentSchemaDcatApOp))
# document.url_schema[str(len(document.url_schema))] = ResourceValue(landing_page)
document.topic_foaf['0'] = SchemaGeneric(dataset_schema.uri)
# document.title_dcterms['0'] = ResourceValue("title_" + landing_page, lang='en')
# document.type_dcterms['0'] = SchemaGeneric("default_type_dcterms")
set_document_descriptions(configuration_file, document, resource)
length = str(len(dataset_schema.page_foaf))
dataset_schema.page_foaf[length] = document
def establish_resource_schema_from_distribution(distribution):
'''
:param DistributionSchemaDcatApOp resource:
:return:
'''
schema = {
"datastore_active":
False,
"id":
distribution.uri,
"size":
distribution.byteSize_dcat.get('0', ResourceValue('')).value_or_uri,
"state":
distribution.status_adms.get('0', SchemaGeneric('')).uri,
"hash":
distribution.checksum_spdx.get(
'0', ChecksumSchemaDcatApOp('')).checksumValue_spdx.get(
'0', ResourceValue('')).value_or_uri,
"description":
util._get_translated_term_from_dcat_object(distribution,
'description_dcterms',
'en'),
"format":
distribution.format_dcterms.get('0', SchemaGeneric('')).uri,
"tracking_summary":
None,
"last_modified":
distribution.modified_dcterms.get('0', ResourceValue('')).value_or_uri,
"download_total_resource":
distribution.numberOfDownloads_dcatapop.get(
'0', ResourceValue('')).value_or_uri,
"url_type":
None,
"mimetype":
None,
"name":
util._get_translated_term_from_dcat_object(distribution,
'title_dcterms', 'en'),
"created":
distribution.issued_dcterms.get('0', ResourceValue('')).value_or_uri,
"url":
next((value.uri for value in distribution.accessURL_dcat.values()),
''),
"iframe_code":
distribution.iframe_dcatapop.get('0', ResourceValue('')).value_or_uri,
"mimetype_inner":
"",
"position":
None,
"resource_type":
distribution.type_dcterms.get('0', SchemaGeneric('')).uri
}
return schema
def establish_resource_schema_from_documentation(doc):
'''
:param DocumentSchemaDcatApOp resource:
:return:
'''
schema = {
"datastore_active":
False,
"id":
doc.uri,
"size":
None,
"state":
None,
"hash":
"",
"description":
util._get_translated_term_from_dcat_object(doc, 'description_dcterms',
'en'),
"format":
doc.format_dcterms.get('0', SchemaGeneric('')).uri,
"tracking_summary":
None,
"last_modified":
None,
"download_total_resource":
None,
"url_type":
None,
"mimetype":
None,
"name":
util._get_translated_term_from_dcat_object(doc, 'title_dcterms', 'en'),
"created":
None,
"url":
next((value.value_or_uri for value in doc.url_schema.values()), ''),
"iframe_code":
None,
"mimetype_inner":
None,
"position":
None,
"resource_type":
doc.type_dcterms.get('0', SchemaGeneric('')).uri
}
return schema
def test_register_doi(self):
'''
integration test for doi registration
:return:
'''
import pickle
from doi.configuration.doi_configuration import DOIConfiguration
file_path = "/applications/ecodp/users/ecodp/ckan/ecportal/src/ckanext-ecportal/ckanext/ecportal/test/data/datasets/doi-test1.pickle"
# Set up test configuration
_TEST_CONFIG = DOIConfiguration()
_TEST_CONFIG.doi_prefix = '10.2899'
_TEST_CONFIG.doi_db_connection_string = 'postgresql://*****:*****@127.0.0.1/ecodp'
_TEST_CONFIG.email_host = 'ms1.cube-lux.lan'
_TEST_CONFIG.email_port = 25
_TEST_CONFIG.email_is_authenticated = False
_TEST_CONFIG.email_username = ''
_TEST_CONFIG.email_password = ''
_TEST_CONFIG.report_log_directory = '/tmp'
_TEST_CONFIG.report_sender_email = '*****@*****.**'
_TEST_CONFIG.report_receiver_email = '*****@*****.**'
_TEST_CONFIG.submission_doi_ra_url = 'https://ra-publications-dev.medra.org/servlet/ws/doidata'
_TEST_CONFIG.submission_doi_ra_user = '******'
_TEST_CONFIG.submission_doi_ra_password = '******'
_TEST_CONFIG.submission_doi_sender_email = '*****@*****.**'
_TEST_CONFIG.submission_doi_from_company = 'Publications Office'
_TEST_CONFIG.submission_doi_to_company = 'OP'
# Get dataset from pickle file
with open(file_path, "rb") as ds_file:
ds = pickle.load(ds_file) # type: DatasetDcatApOp
# Get dataset by URI from triplestore
# ds = DatasetDcatApOp("http://data.europa.eu/88u/dataset/the-community-fishing-fleet-register")
# ds.get_description_from_ts()
from doi.facade.doi_facade import DOIFacade
facade_doi_test = DOIFacade(_TEST_CONFIG)
# Update the dataset with the new genearted DOI
doi_str = facade_doi_test.generate_doi("PUBL", uri=ds.dataset_uri)
from ckanext.ecportal.model.schemas.generic_schema import SchemaGeneric
identifier = SchemaGeneric(doi_str)
identifier.type_rdf = {"0": SchemaGeneric("http:testdoi")}
ds.schema.identifier_adms = {'0': identifier}
doi_dict = ds.build_DOI_dict()
facade_doi_test.register_doi(doi_str, doi_dict)
pass
def create_dataset_schema_for_package_dict(data_dict):
name = data_dict.get('name')
uri = uri_util.new_dataset_uri_from_name(name)
dataset = DatasetDcatApOp(uri)
# Catalog Record
catalogRecord = CatalogRecordSchemaDcatApOp(
uri_util.new_catalog_record_uri())
date = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
catalogRecord.issued_dcterms['0'] = ResourceValue(date,
datatype=XSD.datetime)
catalogRecord.modified_dcterms['0'] = ResourceValue(date,
datatype=XSD.datetime)
catalogRecord.primaryTopic_foaf['0'] = SchemaGeneric(dataset.schema.uri)
dataset.schema_catalog_record = catalogRecord
generated_dataset = __dataset_old_model_transformation(
dataset, data_dict, dataset.schema)
# Generate DOI if requested
if _DOI_GENERATION_KEY in data_dict:
doi = generate_doi_for_dataset(dataset, data_dict[_DOI_GENERATION_KEY])
generated_dataset.set_doi(doi)
return generated_dataset
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
super(PeriodOfTimeSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(PeriodOfTimeSchemaDcatApOp.rdf_type,
graph_name)
self.startDate_schema = {} # type: dict[str,ResourceValue] #Datetime
self.endDate_schema = {} # type: dict[str,ResourceValue] #Datetime
def factory(self, type, uri=''):
from ckanext.ecportal.model.schemas.dcatapop_dataset_schema import DatasetSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_agent_schema import AgentSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_category_schema import CorporateSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_category_schema import DataThemeSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_category_schema import DatasetTypeSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_data_extension_schema import DataExtensionSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_distribution_schema import DistributionSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_document_schema import DocumentSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_empty_classes_schema import FrequencySchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_empty_classes_schema import LinguisticSystemSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_empty_classes_schema import LocationSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_empty_classes_schema import ProvenanceStatementSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_empty_classes_schema import StandardSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_group_schema import DatasetGroupSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_identifier_schema import IdentifierSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_kind_schema import KindSchemaDcatApOp
from ckanext.ecportal.model.schemas.dcatapop_period_of_time_schema import PeriodOfTimeSchemaDcatApOp
from ckanext.ecportal.model.schemas.generic_schema import ResourceValue
from ckanext.ecportal.model.schemas.generic_schema import SchemaGeneric
if schema.DATASET_DCATAPOP == type:
return DatasetSchemaDcatApOp(uri)
elif schema.DISTRIBUTION_DCAT == type:
return DistributionSchemaDcatApOp(uri)
elif schema.DOCUMENT_FOAF == type:
return DocumentSchemaDcatApOp(uri)
else:
return SchemaGeneric(uri)
def set_splitted_uris(cls, dict, parameter):
from ckanext.ecportal.model.schemas.generic_schema import SchemaGeneric
import ckanext.ecportal.lib.uri_util as uri_util
result_dict = {}
uris = dict.get(parameter)
if uris:
for uri in uris.split(" "):
if uri:
if uri_util.is_uri_valid(uri):
url_length = str(len(result_dict))
result_dict[url_length] = SchemaGeneric(uri)
else:
url_length = str(len(result_dict))
result_dict[url_length] = SchemaGeneric(uri)
log.warn("Url [{0}] is not valid".format(uri))
return result_dict
def __init__(self, uri, graph_name="dcatapop-public"):
super(IdentifierSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(IdentifierSchemaDcatApOp.rdf_type,
graph_name)
self.schemeAgency_adms = {} # type: dict[str,ResourceValue]
self.issued_dcterms = {} # type: dict[str,ResourceValue]
self.notation_skos = {} # type: dict[str,ResourceValue]
self.creator_dcterms = {} # type:dict[str, AgentSchemaDcatApOp]
def __init__(self, uri, graph_name):
super(ConceptSchemeSchemaSkos, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(self.type_rdf, graph_name)
self.label_rdfs = {} # type: dict[str,ResourceValue] #Literal
self.versionInfo_owl = {} # type: dict[str,ResourceValue] #Literal
self.prefLabel_at = {} # type: dict[str,ResourceValue] #Literal
self.tableDOTid_at = {} # type: dict[str,ResourceValue] #Literal
self.tableDOTversionDOTnumber_at = {
} # type: dict[str,ResourceValue] #Literal
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
super(KindSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(KindSchemaDcatApOp.rdf_type,
graph_name)
self.organisationDASHname_vcard = {
} # type:dict[str,ResourceValue] Literal
self.hasEmail_vcard = {} # type: dict[str,SchemaGeneric]
self.hasTelephone_vcard = {} # type:dict[str,TelephoneSchemaDcatApOp]
self.hasAddress_vcard = {} # type: dict[str,AddressSchemaDcatApOp]
self.homePage_foaf = {} # type: dict[str, DocumentSchemaDcatApOp]
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
super(AddressSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(AddressSchemaDcatApOp.rdf_type,
graph_name)
self.streetDASHaddress_vcard = {
} # type: dict[str,ResourceValue] Literal
self.postalDASHcode_vcard = {} # type: dict[str,ResourceValue] Literal
self.locality_vcard = {} # type: dict[str,ResourceValue] Literal
self.countryDASHname_vcard = {
} # type: dict[str,ResourceValue] Literal
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
if not '{0}/{1}'.format(uri_util.PREFIX,'document') in uri and uri != DocumentSchemaDcatApOp.rdf_type:
uri = uri_util.new_documentation_uri()
super(DocumentSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(DocumentSchemaDcatApOp.rdf_type, graph_name)
self.format_dcterms = {} # type: dict[str,MediaTypeOrExtentSchemaDcatApOp]
self.title_dcterms = {} # type: dict[str,ResourceValue] #Literal
self.type_dcterms = {} # type: dict[str,ResourceValue] #Literal #DocumentationType
self.topic_foaf = {} # type: dict[str,SchemaGeneric] TODO: To be verified
self.url_schema = {} # type: dict[str,ResourceValue] #anyURI
self.description_dcterms = {} # type: dict[str,ResourceValue]
self.language_dcterms = {} # type: dict[str,LinguisticSystemSchemaDcatApOp]
def test_at_most_one(self):
ds = DatasetDcatApOp(
"http://data.europa.eu/88u/dataset/dgt-translation-memory-V1-2")
if ds.get_description_from_ts():
validation_error = False
# ds.schema.theme_dcat['1'] = ResourceValue("New CKAN")
validator = ValidationSchema(ds.schema,
ds.schema.get_schema_type())
report = validator.validate()
for result in report:
if result.get(
"property") == "accessRights_dcterms" and result.get(
"constraint") == "card_0..1":
validation_result = False if result.get(
"result") == ValidationTypeResult.error else True
break
self.assertTrue(validation_result,
" Test validation of must_have_one failed")
# add another member
ds.schema.accessRights_dcterms['1'] = SchemaGeneric("newthem")
validator = ValidationSchema(ds.schema,
ds.schema.get_schema_type())
report = validator.validate()
for result in report:
if result.get(
"property") == "accessRights_dcterms" and result.get(
"constraint") == "card_0..1":
validation_result = True if result.get(
"result") == ValidationTypeResult.error else False
break
self.assertTrue(
validation_result,
" Test validation of must_have_one (more than 1) failed")
pass
ds.schema.accessRights_dcterms = None
validator = ValidationSchema(ds.schema,
ds.schema.get_schema_type())
report = validator.validate()
for result in report:
if result.get(
"property") == "accessRights_dcterms" and result.get(
"constraint") == "card_0..1":
validation_result = False if result.get(
"result") == ValidationTypeResult.error else True
break
self.assertTrue(
validation_result,
" Test validation of must_have_one (cardinality 0) failed")
pass
def __init__(self, uri, graph_name):
super(ConceptSchemaSkos, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(self.type_rdf, graph_name)
self.identifier_dc = {} # type: dict[str,ResourceValue] #Literal
self.prefLabel_skos = {} # type: dict[str,ResourceValue] #Literal
self.altLabel_skos = {} # type: dict[str,ResourceValue] #Literal
self.inScheme_skos = {} # type: dict[str,ResourceValue] #Literal
self.authorityDASHcode_at = {
} # type: dict[str,ResourceValue] #Literal
self.opDASHcode_at = {} # type: dict[str,ResourceValue] #Literal
self.opDASHcode_atold = {} # type: dict[str,ResourceValue] #Literal
self.startDOTuse_at = {} # type: dict[str,ResourceValue] #Literal
self.deprecated_at = {} # type: dict[str,ResourceValue] #Literal
def convert_theme_dcat(self, data_dict, param, theme_dcat_dict):
from ckanext.ecportal.lib.dataset_util import SchemaGeneric
if param == 'theme':
groups = data_dict.get(param, [])
if groups:
theme_dcat_dict = {}
if not isinstance(groups, list):
groups = [groups]
for group in groups:
if group:
length = str(len(theme_dcat_dict))
theme_dcat_dict[length] = SchemaGeneric(group)
elif param == 'concepts_eurovoc':
themes = data_dict.get(param, [])
if not isinstance(themes, list):
themes = [themes]
for theme in themes:
if theme:
length = str(len(theme_dcat_dict))
theme_dcat_dict[length] = SchemaGeneric(theme)
else: # theme not yet in screens
log.warning('Unknown theme for {0}'.format(param))
return theme_dcat_dict
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
if not '{0}/{1}'.format(
uri_util.PREFIX, 'record'
) in uri and uri != CatalogRecordSchemaDcatApOp.rdf_type:
uri = uri_util.new_catalog_record_uri()
super(CatalogRecordSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(
CatalogRecordSchemaDcatApOp.rdf_type, graph_name)
self.conformsTo_dcterms = {} # type: dict[str,ResourceValue]
self.primaryTopic_foaf = {} # type: dict[str,DatasetSchemaDcatApOp]
self.modified_dcterms = {} # type: dict[str,ResourceValue] #datetime
self.description_dcterms = {} # type: dict[str,ResourceValue] #Literal
self.issued_dcterms = {} # type: dict[str,ResourceValue] #datetime
self.title_dcterms = {} # type: dict[str,ResourceValue] #Literal
self.language_dcterms = {
} # type: dict[str, LinguisticSystemSchemaDcatApOp]
self.numberOfViews_dcatapop = {} # type: dict[str,ResourceValue]
self.source_dcterms = {} # type: dict[str,CatalogRecord]
self.status_adms = {} # type: dict[str,ResourceValue] #DatasetStatus
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
super(CatalogSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(CatalogSchemaDcatApOp.rdf_type, graph_name)
self.hasPart_dcterms = {} # type: dict[str,CatalogSchemaDcatApOp]
self.isPartOf_dcterms = {} # type: dict[str,CatalogSchemaDcatApOp]
self.modified_dcterms = {} # type: dict[str,ResourceValue] #datetime
self.description_dcterms = {} # type: dict[str,ResourceValue] #Literal
self.issued_dcterms = {} # type: dict[str,ResourceValue] #Literal
self.language_dcterms = {} # type: dict[str, LinguisticSystemSchemaDcatApOp]
self.license_dcterms = {} # type: dict[str, LicenseDocumentDcatApOp]
self.publisher_dcterms = {} # type: dict[str, AgentSchemaDcatApOp]
self.rights_dcterms = {} # type: dict[str, RightsStatementSchemaDcatApOp]
self.spatial_dcterms = {} # type: dict[str, LocationSchemaDcatApOp]
self.record_dcat = {} # type: dict[str,CatalogRecord]
self.dataset_dcat = {} # type: dict[str,DatasetSchemaDcatApOp]
self.themeTaxonomy_dcat = {} # type: dict[str,ResourceValue] #ConceptScheme
self.title_dcterms = {} # type: dict[str,ResourceValue] #ConceptScheme #Literal
self.homepage_foaf = {} # type: dict[str,DocumentSchemaDcatApOp]
self.identifier_adms = {} # type: dict[str, IdentifierSchemaDcatApOp]
def test_register_doi(self):
"""
integration test for doi registration
:return:
"""
file_path = "/applications/ecodp/users/ecodp/ckan/ecportal/src/ckanext-ecportal/ckanext/ecportal/test/data/catalogs/my-first-catalogue.pickle"
# Set up test configuration
_TEST_CONFIG = DOIConfiguration()
_TEST_CONFIG.doi_prefix = '10.2899'
_TEST_CONFIG.doi_db_connection_string = 'postgresql://*****:*****@127.0.0.1/ecodp'
_TEST_CONFIG.email_host = 'ms1.cube-lux.lan'
_TEST_CONFIG.email_port = 25
_TEST_CONFIG.email_is_authenticated = False
_TEST_CONFIG.email_username = ''
_TEST_CONFIG.email_password = ''
_TEST_CONFIG.report_log_directory = '/tmp'
_TEST_CONFIG.report_sender_email = '*****@*****.**'
_TEST_CONFIG.report_receiver_email = '*****@*****.**'
_TEST_CONFIG.submission_doi_ra_url = 'https://ra-publications-dev.medra.org/servlet/ws/doidata'
_TEST_CONFIG.submission_doi_ra_user = '******'
_TEST_CONFIG.submission_doi_ra_password = '******'
_TEST_CONFIG.submission_doi_sender_email = '*****@*****.**'
_TEST_CONFIG.submission_doi_from_company = 'Publications Office'
_TEST_CONFIG.submission_doi_to_company = 'OP'
# Get catalogs from pickle file
with open(file_path, "rb") as catalog_file:
catalog = pickle.load(catalog_file) # type: CatalogDcatApOp
facade_doi_test = DOIFacade(_TEST_CONFIG)
# Update the catalog with the new generated DOI
doi_str = facade_doi_test.generate_doi("ODP", catalog.catalog_uri)
catalog.schema.identifier_adms = {'0': SchemaGeneric(doi_str)}
doi_dict = catalog.build_DOI_dict()
facade_doi_test.register_doi(doi_str, doi_dict)
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
if not '{0}/{1}'.format(
uri_util.PREFIX, 'distribution'
) in uri and uri != DistributionSchemaDcatApOp.rdf_type:
uri = uri_util.new_distribution_uri()
super(DistributionSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(DistributionSchemaDcatApOp.rdf_type,
graph_name)
self.mediaType_dcat = {
} # type: dict[str,MediaTypeOrExtentSchemaDcatApOp]
self.numberOfDownloads_dcatapop = {} # type: dict[str,ResourceValue]
self.modified_dcterms = {} # type: dict[str,ResourceValue] #dateTime
self.checksum_spdx = {} # type: dict[str,ChecksumSchemaDcatApOp]
self.description_dcterms = {} # type: dict[str,ResourceValue] #Literal
self.format_dcterms = {
} # type: dict[str,MediaTypeOrExtentSchemaDcatApOp]
self.issued_dcterms = {} # type: dict[str,ResourceValue] #dateTime
self.title_dcterms = {} # type: dict[str,ResourceValue] #Literal
self.type_dcterms = {
} # type: dict[str,SchemaGeneric] #DistributionType
self.extensionLiteral_dcatapop = {
} # type: dict[str,DataExtensionSchemaDcatApOp]
self.extensionValue_dcatapop = {
} # type: dict[str,DataExtensionSchemaDcatApOp]
self.iframe_dcatapop = {} # type: dict[str,ResourceValue]
self.language_dcterms = {
} # type: dict[str,LinguisticSystemSchemaDcatApOp]
self.license_dcterms = {} # type: dict[str,LicenseDocumentDcatApOp]
self.page_foaf = {} # type: dict[str,DocumentSchemaDcatApOp]
self.rights_dcterms = {
} # type: dict[str,RightsStatementSchemaDcatApOp]
self.status_adms = {} # type: dict[str,SchemaGeneric] #DatasetStatus
self.accessURL_dcat = {} # type: dict[str,SchemaGeneric]
self.downloadURL_dcat = {} # type: dict[str,SchemaGeneric]
self.byteSize_dcat = {} # type: dict[str,ResourceValue]
self.conformsTo_dcterms = {} # type: dict[str,StandardSchemaDcatApOp]
def test_controlled_vocabularies_values(self):
ds = DatasetDcatApOp(
"http://data.europa.eu/88u/dataset/dgt-translation-memory-V1-2")
if ds.get_description_from_ts():
validation_result = True
validator = ValidationSchema(ds.schema,
ds.schema.get_schema_type())
report = validator.validate()
for result in report:
if result.get("property") == "theme_dcat" and result.get(
"constraint") == "controlled_vocabulary":
validation_result = False if result.get(
"result") == ValidationTypeResult.error else True
if not validation_result:
break
self.assertTrue(validation_result,
" Test validation of validation failed")
ds.schema.accrualPeriodicity_dcterms['1'] = SchemaGeneric(
"http://doNoBelongToControledVocabulary")
validation_result = True
validator = ValidationSchema(ds.schema,
ds.schema.get_schema_type())
report = validator.validate()
for result in report:
if result.get("property"
) == "accrualPeriodicity_dcterms" and result.get(
"constraint") == "controlled_vocabulary":
validation_result = False if result.get(
"result") == ValidationTypeResult.error else True
if not validation_result:
break
self.assertTrue(not validation_result,
" Test validation of validation failed")
pass
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
super(ProvenanceStatementSchemaDcatApOp,
self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(
NAMESPACE_DCATAPOP.spdx + "ProvenanceStatement", graph_name)
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
super(FrequencySchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(
NAMESPACE_DCATAPOP.dcterms + "Frequency", graph_name)
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
super(MediaTypeOrExtentSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(
NAMESPACE_DCATAPOP.dcterms + "MediaTypeOrExtent", graph_name)
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
super(LiteralSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(NAMESPACE_DCATAPOP.rdfs + "Literal",
graph_name)
def __init__(self, uri=None, graph_name=DCATAPOP_PUBLIC_GRAPH_NAME):
super(RightsStatementSchemaDcatApOp, self).__init__(uri, graph_name)
self.type_rdf['0'] = SchemaGeneric(
NAMESPACE_DCATAPOP.dcterms + "RightsStatement", graph_name)
self.label_rdfs = {}