def _dge_harvest_list_dataset_field_labels(name_field=None, value_field=None): ''' Returns the available values that the given dataset name_field may have to the given value_field ''' result = {} if name_field is not None: dataset = sh.scheming_get_schema('dataset', 'dataset') values = sh.scheming_field_by_name(dataset.get('dataset_fields'), name_field) or [] if values and values['choices']: for option in values['choices']: if option and option['value']: if value_field: if option['value'] == value_field: return { option.get('value'): { 'label': option.get('label'), 'description': option.get('description'), 'dcat_ap': option.get('dcat_ap'), 'notation': option.get('notation') } } else: result[option.get('value')] = { 'label': option.get('label'), 'description': option.get('description'), 'dcat_ap': option.get('dcat_ap'), 'notation': option.get('notation') } return result
def fluent_form_languages(field=None, entity_type=None, object_type=None, schema=None): """ Return a list of language codes for this form (or form field) 1. return field['form_languages'] if it is defined 2. return schema['form_languages'] if it is defined 3. get schema from entity_type + object_type then return schema['form_languages'] if they are defined 4. return languages from site configuration """ if field and 'form_languages' in field: return field['form_languages'] if schema and 'form_languages' in schema: return schema['form_languages'] if entity_type and object_type: # late import for compatibility with older ckanext-scheming from ckanext.scheming.helpers import scheming_get_schema schema = scheming_get_schema(entity_type, object_type) if schema and 'form_languages' in schema: return schema['form_languages'] langs = [] for l in get_available_locales(): if l.language not in langs: langs.append(l.language) return langs
def fluent_form_languages(field=None, entity_type=None, object_type=None, schema=None): """ Return a list of language codes for this form (or form field) 1. return field['form_languages'] if it is defined 2. return schema['form_languages'] if it is defined 3. get schema from entity_type + object_type then return schema['form_languages'] if they are defined 4. return languages from site configuration """ if field and 'form_languages' in field: return field['form_languages'] if schema and 'form_languages' in schema: return schema['form_languages'] if entity_type and object_type: # late import for compatibility with older ckanext-scheming from ckanext.scheming.helpers import scheming_get_schema schema = scheming_get_schema(entity_type, object_type) if schema and 'form_languages' in schema: return schema['form_languages'] langs = [] for l in get_available_locales(): if l.language not in langs: langs.append(l.language) return langs
def before_index(self, data_dict): dataset = sh.scheming_get_schema('dataset', 'dataset') if ('res_format' in data_dict): #Get format field formats = sh.scheming_field_by_name(dataset.get('resource_fields'), 'format') #Create SOLR field data_dict['res_format_label'] = [] for res_format in data_dict['res_format']: #Get format label res_format_label = sh.scheming_choices_label( formats['choices'], res_format) if res_format_label: #Add label to new SOLR field data_dict['res_format_label'].append(res_format_label) if ('frequency' in data_dict): #Get frequency field frequency = data_dict['frequency'] if frequency: freq = json.loads(frequency) ftype = freq['type'] fvalue = freq['value'] data_dict['frequency_id'] = '{value}-{type}'.format( type=ftype, value=fvalue) data_dict[ 'frequency_label'] = helpers.csc_dataset_display_frequency( fvalue, ftype) #log.info('Frecuency = {f1}, frequency_id={f2}, frequency_label={f3}'.format(f1=frequency, f2=data_dict['frequency_id'], f3=data_dict['frequency_label'])) if ('theme' in data_dict): #Get theme field categoria = sh.scheming_field_by_name( dataset.get('dataset_fields'), 'theme') #Get theme value valor_categoria = data_dict['theme'] #Empty theme values data_dict['theme'] = [] data_dict['theme_id'] = [] data_dict['theme_es'] = [] data_dict['theme_gl'] = [] #Get key values valores = valor_categoria.replace('[', '').replace(']', '') categorias = valores.split('", "') #Get translated label for each key for term_categoria in list(categorias): clean_term = term_categoria.replace('"', '') data_dict['theme'].append(clean_term) data_dict['theme_id'].append(helpers.csc_theme_id(clean_term)) #Look for label in the scheme for option in categoria.get('choices'): if option['value'] == clean_term: #Add label for each language data_dict['theme_es'].append(option['label']['es']) data_dict['theme_gl'].append(option['label']['gl']) return data_dict
def get_required_fields_name(dataset_type): dataset_scheme = h.scheming_get_schema('dataset', dataset_type) fields = dataset_scheme['dataset_fields'] required_fields_name = [] for f in fields: if f.get('required'): required_fields_name.append(f.get('field_name')) return required_fields_name
def get_required_fields_name(dataset_type): dataset_scheme = h.scheming_get_schema('dataset', dataset_type) fields = dataset_scheme['dataset_fields'] required_fields_name = [] for f in fields: if f.get('required'): required_fields_name.append(f.get('field_name')) return required_fields_name
def has_published_date_field_in_schema(dataset_type): if not dataset_type: return False dataset_scheme = h.scheming_get_schema('dataset', dataset_type) fields = dataset_scheme['dataset_fields'] pd = h.scheming_field_by_name(fields, "published_date") if not pd: return False return True
def resource_required(dataset_type): if not dataset_type: return False dataset_scheme = h.scheming_get_schema('dataset', dataset_type) fields = dataset_scheme['dataset_fields'] for f in fields: if f.get('validators') and re.search("ab_ps_resource_required", f.get('validators')): return True return False
def get_required_fields_name_label_dict(dataset_type): if not dataset_type: return {} dataset_scheme = h.scheming_get_schema('dataset', dataset_type) fields = dataset_scheme['dataset_fields'] required_dict = {} for f in fields: if f.get('required'): required_dict[f.get('field_name')] = f.get('label') return required_dict
def _get_classfication_field(dataset_type): scheme = h.scheming_get_schema('dataset', dataset_type) if not scheme: return None fields = scheme.get('resource_fields') if not fields: return None f = h.scheming_field_by_name(fields, "classification") if f: return f else: return None
def _csc_dcat_list_resource_field_values(name_field=None): ''' Returns the available values that the given resource name_field may have ''' result = [] if name_field is not None: dataset = sh.scheming_get_schema('dataset', 'dataset') values = sh.scheming_field_by_name(dataset.get('resource_fields'), name_field) or [] if values and values['choices']: for option in values['choices']: if option and option['value']: result.append(option['value']) return result
def dge_list_themes(themes=None): ''' Given an theme list values, get theirs translated labels :param themes: value theme list :type string list :rtype (string, string) list ''' dataset = sh.scheming_get_schema('dataset', 'dataset') formats = sh.scheming_field_by_name(dataset.get('dataset_fields'), 'theme') label_list = [] for theme in themes: label = sh.scheming_choices_label(formats['choices'], theme) if label: label_list.append((dge_theme_id(theme), label)) return label_list
def dge_resource_format_label(res_format=None): ''' Given an format, get its label :param res_format: format :type string :rtype string ''' if format: dataset = sh.scheming_get_schema('dataset', 'dataset') formats = sh.scheming_field_by_name(dataset.get('resource_fields'), 'format') res_format_label = sh.scheming_choices_label(formats['choices'], res_format) if res_format_label: return res_format_label return res_format
def after_search(self, search_results, search_params): if not is_frontend(): return search_results # Translate the unselected search facets. facets = search_results.get('search_facets') if not facets: return search_results desired_lang_code = pylons.request.environ['CKAN_LANG'] fallback_lang_code = pylons.config.get('ckan.locale_default', 'es') # Look up translations for all of the facets in one db query. dataset = sh.scheming_get_schema('dataset', 'dataset') categoria = sh.scheming_field_by_name(dataset.get('dataset_fields'), 'theme') dict_categoria = {} for option in categoria.get('choices'): label_option = (option.get('label')).get(desired_lang_code, None) if not label_option: label_option = (option.get('label')).get( fallback_lang_code, None) dict_categoria[helpers.dge_theme_id( option.get('value'))] = label_option facet = facets.get('theme_id', None) if facet: for item in facet.get('items', None): item['display_name'] = dict_categoria.get( item.get('name'), item.get('display_name')) item['class'] = item.get('name') facet = facets.get('administration_level', None) if facet: for item in facet.get('items', None): item[ 'display_name'] = helpers.dge_get_translated_administration_level( item.get('name')) return search_results
def after_search(self, search_results, search_params): # Translate the unselected search facets. facets = search_results.get('search_facets') if not facets: return search_results desired_lang_code = request.environ.get('CKAN_LANG') fallback_lang_code = config.get('ckan.locale_default', 'es') # Look up translations for all of the facets in one db query. dataset = sh.scheming_get_schema('dataset', 'dataset') categoria = sh.scheming_field_by_name(dataset.get('dataset_fields'), 'theme') dict_categoria = {} for option in categoria.get('choices'): label_option = (option.get('label')).get(desired_lang_code, None) if not label_option: label_option = (option.get('label')).get( fallback_lang_code, None) dict_categoria[helpers.csc_theme_id( option.get('value'))] = label_option facet = facets.get('theme_id', None) if facet: for item in facet.get('items', None): item['display_name'] = dict_categoria.get( item.get('name'), item.get('display_name')) item['class'] = item.get('name') facet = facets.get('frequency_id', None) if facet: for item in facet.get('items', None): #log.info("facet {facet}".format(facet=facet)) value = item.get('name', '').split('-') item['display_name'] = helpers.csc_dataset_display_frequency( value[0], value[1]) return search_results
def dge_list_reduce_resource_format_label(resources=None, field_name='format'): ''' Given an resource list, get label of resource_format :param resources: resource dict :type dict list :param field_name: field_name of resource :type string :rtype string list ''' format_list = h.dict_list_reduce(resources, field_name) dataset = sh.scheming_get_schema('dataset', 'dataset') formats = sh.scheming_field_by_name(dataset.get('resource_fields'), 'format') label_list = [] for res_format in format_list: res_format_label = sh.scheming_choices_label(formats['choices'], res_format) if res_format_label: label_list.append(res_format_label) return label_list
def dge_harvest_catalog_show(context, data_dict): method_log_prefix = '[%s][dge_harvest_catalog_show]' % __name__ output = None try: log.debug('%s Init method. Inputs context=%s, data_dict=%s' % (method_log_prefix, context, data_dict)) ini = datetime.datetime.now() toolkit.check_access('dge_harvest_catalog_show', context, data_dict) page = 1 data_dict['page'] = page limit = data_dict.get('limit', -1) _format = data_dict.get('format') if _format == RDF_FORMAT: filepath = config.get('ckanext.dge_harvest.rdf.filepath', '/tmp/catalog.rdf') elif _format == CSV_FORMAT: filepath = config.get('ckanext.dge_harvest.csv.filepath', '/tmp/catalog.csv') columnsfilepath = config.get( 'ckanext.dge_harvest.csv.columns.filepath', '/usr/lib/ckan/default/src/ckanext-dge-harvest/ckanext/dge_harvest/commands/columns.json' ) else: filepath = '/tmp/catalog.' + _format query = _dge_harvest_search_ckan_datasets(context, data_dict) dataset_dicts = query['results'] total_datasets = query['count'] log.debug('%s Total_datasets obtenidos en la query: %s' % (method_log_prefix, total_datasets)) if limit > -1 and limit < total_datasets: total_datasets = limit num = len(dataset_dicts) log.debug('%s Total_datasets a exportar: %s' % (method_log_prefix, total_datasets)) while (total_datasets > num): page = page + 1 data_dict['page'] = page query = _dge_harvest_search_ckan_datasets(context, data_dict) dataset_dicts.extend(query['results']) total_datasets = query['count'] num = len(dataset_dicts) log.debug('%s Total_datasets obtenidos en la query: %s' % (method_log_prefix, total_datasets)) log.debug('%s Total_datasets a exportar: %s' % (method_log_prefix, num)) if _format == RDF_FORMAT: serializer = DGERDFSerializer() #log.debug("%s DATASET_DICTS = %s" % (method_log_prefix,dataset_dicts)) output = serializer.serialize_catalog( {}, dataset_dicts, _format=data_dict.get('format'), pagination_info=None) elif _format == CSV_FORMAT and columnsfilepath: #log.info('%s Dataset_dicts de partida =%s' % (method_log_prefix, dataset_dicts)) organizations = {} themes = dhh.dge_harvest_dict_theme_option_label() spatial_coverages = dhh.dge_harvest_dict_spatial_coverage_option_label( ) _dataset = sh.scheming_get_schema('dataset', 'dataset') res_format = sh.scheming_field_by_name( _dataset.get('resource_fields'), 'format') format_values = res_format['choices'] formats = {} datasets = [] num = 0 for dataset in dataset_dicts: ds = {} #Id #ds['id'] = _encode_value(dataset.get('id', None)) #ulr ds['url'] = dataset_uri(dataset) #Description descriptions = _from_dict_to_string( dataset.get(dhc.DS_DESCRIPTION, None)) ds['description'] = _encode_value(descriptions, True) #Title titles = _from_dict_to_string( dataset.get(dhc.DS_TITLE_TRANSLATED, None)) ds['title'] = _encode_value(titles, True) #Theme theme_values = dataset.get(dhc.DS_THEME, None) theme_labels = [] if theme_values: for value in theme_values: theme = themes.get(value) if theme and theme.get('label'): theme_labels.append(theme.get('label').get('es')) theme_value = _from_list_to_string(theme_labels) ds['theme'] = _encode_value(theme_value, True) #Keywords tags = dataset.get(dhc.DS_TAGS) value = None if tags and len(tags) > 0: for tag in tags: stag = tag.get('name', None) if stag: if value: value = '%s%s%s' % (value, MAIN_SEPARATOR, stag) else: value = stag ds['tags'] = _encode_value(value, True) #Identifier ds['identifier'] = _encode_value( dataset.get('identifier', None), True) #Created ds['issued_date'] = _encode_value( _from_iso8601_date_to_string( dataset.get(dhc.DS_ISSUED_DATE, None))) #Modified ds['modified_date'] = _encode_value( _from_iso8601_date_to_string( dataset.get(dhc.DS_MODIFIED_DATE, None))) #Accrual Periodicity frequency = dataset.get(dhc.DS_FREQUENCY) if (frequency): stype = frequency.get('type', '') if stype and len(stype) > 0: stype = 'http://www.w3.org/2006/time#' + stype svalue = frequency.get('value', '') sfrequency = '[TYPE]%s[VALUE]%s' % (stype, svalue) ds['frequency'] = _encode_value(sfrequency, True) #Language languages = _from_list_to_string(dataset.get(dhc.DS_LANGUAGE)) ds['language'] = _encode_value(languages, True) #Publisher publisher = dataset.get(dhc.DS_PUBLISHER, None) if publisher: if publisher in organizations: ds['publisher'] = _encode_value( organizations.get(publisher, None), True) else: organization = h.get_organization(publisher, False) if organization: organizations[publisher] = organization.get( 'title', organization.get('display_name', None)) ds['publisher'] = _encode_value( organizations.get(publisher), True) #License ds['license_id'] = _encode_value(dataset.get(dhc.DS_LICENSE), True) #Spatial spatial_values = dataset.get(dhc.DS_SPATIAL, None) spatial_labels = [] if spatial_values: for value in spatial_values: spatial = spatial_coverages.get(value) if spatial and spatial.get('label') and spatial.get( 'label').get('es'): spatial_labels.append( spatial.get('label').get('es')) spatials = _from_list_to_string(spatial_labels) ds['spatial'] = _encode_value(spatials, True) #Temporal temporal_coverage = dataset.get(dhc.DS_TEMPORAL_COVERAGE) if temporal_coverage: value = None for tc in temporal_coverage.itervalues(): if tc: tc_from = _from_iso8601_date_to_string( tc.get('from', None)) tc_to = _from_iso8601_date_to_string( tc.get('to', None)) if tc_from or tc_to: if value: value = '%s%s%s-%s' % (value, MAIN_SEPARATOR, (tc_from or ''), (tc_to or '')) else: value = '%s-%s' % ((tc_from or ''), (tc_to or '')) ds['coverage_new'] = _encode_value(value, True) #Valid ds['valid'] = _encode_value( _from_iso8601_date_to_string( dataset.get(dhc.DS_VALID, None)), True) #References references = _from_list_to_string( dataset.get(dhc.DS_REFERENCE, None)) ds['references'] = _encode_value(references, True) #Normative conforms_to = _from_list_to_string( dataset.get(dhc.DS_NORMATIVE, None)) ds['conforms_to'] = _encode_value(conforms_to, True) #Resources resources = dataset.get(dhc.DS_RESOURCES) sresources = [] if resources: for resource in resources: sresource = None if resource: name = _from_dict_to_string( resource.get(dhc.DS_RESOURCE_NAME_TRANSLATED, None), 'TITLE_') if not name: name = '' url = resource.get(dhc.DS_RESOURCE_ACCESS_URL, '') if url: url = '[ACCESS_URL]%s' % (url) format_value = resource.get( dhc.DS_RESOURCE_FORMAT, None) format = None if format_value: if format_value in formats: format = formats.get(format_value, None) else: formats[ format_value] = sh.scheming_choices_label( format_values, format_value) format = formats.get(format_value, None) if format: format = '[MEDIA_TYPE]%s' % (format) size = resource.get(dhc.DS_RESOURCE_BYTE_SIZE, '') if size: size = '[BYTE_SIZE]%s' % (size) relation = _from_list_to_string( resource.get(dhc.DS_RESOURCE_RELATION, None), SECONDARY_SEPARATOR) relations = '' if relation: relations = '[RELATION]%s' % (relation) sresource = '%s%s%s%s%s' % (name, url, format, size, relations) if sresource and len(sresource) > 0: sresources.append(sresource) if len(sresources) > 0: value = None for item in sresources: if value: value = '%s%s%s' % (value, MAIN_SEPARATOR, item) else: value = item ds['resources'] = _encode_value(value, True) num = num + 1 datasets.append(ds) #log.debug('%s Datasets con datos a exportar=%s' % (method_log_prefix, datasets)) log.debug('%s Numero de datasets con datos a exportar...%s' % (method_log_prefix, num)) output = losser.losser.table(datasets, columnsfilepath, csv=True, pretty=False) if filepath: file = None try: file = open(filepath, "w") file.write(output) file.close() except: if file and not file.closed: file.close() end = datetime.datetime.now() log.debug( "%s Time in serialize %s catalog [%s] with %s datasets ... %s milliseconds" % (method_log_prefix, _format, filepath, total_datasets, int((end - ini).total_seconds() * 1000))) except Exception, e: log.error("%s Exception %s: %s" % (method_log_prefix, type(e).__name__, e)) output = None
def graph_from_dataset(self, dataset_dict, dataset_ref): ''' Given a CKAN dataset dict, creates an RDF graph The class RDFLib graph (accessible via `self.g`) should be updated on this method `dataset_dict` is a dict with the dataset metadata like the one returned by `package_show`. `dataset_ref` is an rdflib URIRef object that must be used to reference the dataset when working with the graph. ''' method_log_prefix = '[%s][graph_from_dataset]' % type( self).__name__ #log.debug('%s Init method. Inputs dataset_dict=%r, dataset_ref=%r' % (method_log_prefix, dataset_dict, dataset_ref)) #log.debug('%s Init method. Inputs, dataset_ref=%r' % (method_log_prefix, dataset_ref)) try: g = self.g for prefix, namespace in namespaces.iteritems(): g.bind(prefix, namespace) g.add((dataset_ref, RDF.type, DCAT.Dataset)) # Title self._add_translated_triple_field_from_dict( dataset_dict, dataset_ref, DCT.title, DS_TITLE_TRANSLATED, None) # Description self._add_translated_triple_field_from_dict( dataset_dict, dataset_ref, DCT.description, DS_DESCRIPTION, None) # Theme value = self._get_dict_value(dataset_dict, DS_THEME) if value: themes = dataset_dict.get(EXPORT_AVAILABLE_THEMES, {}) for theme in value: #self._add_resource_list_triple(dataset_ref, DCAT.theme, value) theme_values = themes.get(theme, {}) labels = theme_values.get('label') descriptions = theme_values.get('description') dcat_ap = theme_values.get('dcat_ap') notation = theme_values.get('notation') self._add_resource_list_triple( dataset_ref, DCAT.theme, theme, labels, descriptions, dcat_ap, notation) # Tags for tag in dataset_dict.get('tags', []): self.g.add( (dataset_ref, DCAT.keyword, Literal(tag['name']))) # Identifier self._add_triple_from_dict( dataset_dict, dataset_ref, DCT.identifier, DS_IDENTIFIER, None, False, False) # Issued, Modified dates self._add_date_triple(dataset_ref, DCT.issued, self._get_value_from_dict( dataset_dict, DS_ISSUED_DATE, ['metadata_created'])) self._add_date_triple(dataset_ref, DCT.modified, self._get_value_from_dict( dataset_dict, DS_MODIFIED_DATE, ['metadata_modified'])) self._add_date_triple(dataset_ref, DCT.valid, self._get_value_from_dict( dataset_dict, DS_VALID, None)) # Accrual periodicity frequency = dataset_dict.get(DS_FREQUENCY) if frequency: ftypes = {'seconds': TIME.seconds, 'minutes': TIME.minutes, 'hours': TIME.hours, 'days': TIME.days, 'weeks': TIME.weeks, 'months': TIME.months, 'years': TIME.years} ftype = frequency.get('type') fvalue = frequency.get('value') if ftype and ftype in ftypes.keys() and fvalue: duration = BNode() frequency = BNode() g.add((frequency, RDF.type, DCT.Frequency)) g.add((duration, RDF.type, TIME.DurationDescription)) g.add((dataset_ref, DCT.accrualPeriodicity, frequency)) g.add((frequency, RDF.value, duration)) g.add((duration, ftypes.get(ftype), Literal( fvalue, datatype=XSD.decimal))) # Languages self._add_triple_from_dict( dataset_dict, dataset_ref, DCT.language, DS_LANGUAGE, None, True, False) # Publisher pub_dir3 = False publishers = dataset_dict.get( EXPORT_AVAILABLE_PUBLISHERS, {}) organization_id = dataset_dict.get('owner_org') if organization_id in publishers: publisher = publishers.get(organization_id) else: org = h.get_organization(organization_id, False) publisher = [None, None, None] if org: publisher = [org.get('title'), None, None] if org['extras']: for extra in org.get('extras'): if extra and 'key' in extra and extra['key'] == ORG_PROP_ID_UD_ORGANICA: notation = extra.get('value') if notation and notation != '': pub_dir3 = True publisher[1] = PUBLISHER_PREFIX + notation publisher[2] = notation if pub_dir3: publishers[organization_id] = publisher dataset_dict[EXPORT_AVAILABLE_PUBLISHERS] = publishers else: #publisher organizations = cdh.csc_dcat_organizations_available() publisher_ref = config.get('ckanext.csc_dcat.catalog.publisher', None) if publisher_ref and len(publisher_ref.strip()) > 0: publisher_ref = publisher_ref.strip() publisher = [publisher_ref, None, None] s_publisher = publisher_ref.upper().split('/') if s_publisher and len(s_publisher) > 0: organization_minhap = s_publisher[-1] org = organizations.get(organization_minhap, None) if org: publisher = [org[1], PUBLISHER_PREFIX + organization_minhap, organization_minhap] if publisher[1]: self._add_resource_list_triple( dataset_ref, DCT.publisher, publisher[1], publisher[0], None, None, publisher[2]) else: g.add((dataset_ref, DCT.publisher, URIRef(publisher[0]))) # Spatial Coverage value = self._get_dict_value(dataset_dict, DS_SPATIAL) if value: self._add_resource_list_triple( dataset_ref, DCT.spatial, value) # Temporal temporal_coverage = self._get_dataset_value( dataset_dict, DS_TEMPORAL_COVERAGE) i = 1 if temporal_coverage: for key, value in temporal_coverage.items(): if (value): start = end = None if 'from' in value: start = value.get('from') if 'to' in value: end = value.get('to') if start or end: temporal_extent = URIRef( "%s/%s-%s" % (dataset_ref, 'PeriodOfTime', i)) g.add( (temporal_extent, RDF.type, DCT.PeriodOfTime)) if start: self._add_date_triple( temporal_extent, SCHEMA.startDate, start) if end: self._add_date_triple( temporal_extent, SCHEMA.endDate, end) g.add((dataset_ref, DCT.temporal, temporal_extent)) i = i+1 # References value = self._get_dict_value(dataset_dict, DS_REFERENCE) if value: self._add_resource_list_triple( dataset_ref, DCT.references, value) # Conforms To value = self._get_dict_value(dataset_dict, DS_NORMATIVE) if value: self._add_resource_list_triple( dataset_ref, DCT.conformsTo, value) # License (dataset license) if dataset_dict.get(DS_LICENSE): g.add((dataset_ref, DCT.license, URIRef( dataset_dict.get(DS_LICENSE)))) # Distributions/Resources for resource_dict in dataset_dict.get('resources', []): uri_resource = '%s/resource/%s' % ( dataset_ref, resource_dict['id']) distribution = URIRef(uri_resource) g.add((dataset_ref, DCAT.distribution, distribution)) g.add((distribution, RDF.type, DCAT.Distribution)) # Identifier self._add_triple_from_dict( resource_dict, distribution, DCT.identifier, DS_RESOURCE_IDENTIFIER, None, False, False) # Title self._add_translated_triple_field_from_dict( resource_dict, distribution, DCT.title, DS_RESOURCE_NAME_TRANSLATED, None) # License (dataset license) if dataset_dict.get(DS_LICENSE): g.add((distribution, DCT.license, URIRef( dataset_dict.get(DS_LICENSE)))) # Access URL if resource_dict.get(DS_RESOURCE_ACCESS_URL): g.add((distribution, DCAT.accessURL, Literal( resource_dict.get(DS_RESOURCE_ACCESS_URL), datatype=XSD.anyURI))) # Format if resource_dict.get(DS_RESOURCE_FORMAT, None): imt = URIRef("%s/format" % uri_resource) g.add((imt, RDF.type, DCT.IMT)) g.add((distribution, DCT['format'], imt)) format = resource_dict.get( DS_RESOURCE_FORMAT, None) formats = dataset_dict.get( EXPORT_AVAILABLE_RESOURCE_FORMATS, {}) label = None if format and format in formats: label = formats.get(format, None) else: _dataset = sh.scheming_get_schema( 'dataset', 'dataset') res_format = sh.scheming_field_by_name(_dataset.get('resource_fields'), 'format') formats[format] = sh.scheming_choices_label( res_format['choices'], format) label = formats.get(format, None) dataset_dict[EXPORT_AVAILABLE_RESOURCE_FORMATS] = formats if label: g.add((imt, RDFS.label, Literal(label))) g.add((imt, RDF.value, Literal( resource_dict[DS_RESOURCE_FORMAT]))) # Size if resource_dict.get(DS_RESOURCE_BYTE_SIZE): try: g.add((distribution, DCAT.byteSize, Literal(float(resource_dict[DS_RESOURCE_BYTE_SIZE]), datatype=XSD.decimal))) except (ValueError, TypeError): g.add((distribution, DCAT.byteSize, Literal(resource_dict[DS_RESOURCE_BYTE_SIZE]))) # Relation value = self._get_dict_value( dataset_dict, DS_NORMATIVE) if value: self._add_resource_list_triple( distribution, DCT.relation, value) except Exception, e: log.error("%s [dataset_ref: %s]. Unexpected Error %s: %s" % ( method_log_prefix, dataset_ref, type(e).__name__, e))
def _get_process_state_field(dataset_type): dataset_scheme = h.scheming_get_schema('dataset', dataset_type) if not dataset_scheme: return None fields = dataset_scheme['dataset_fields'] return h.scheming_field_by_name(fields, "process_state")
def before_index(self, data_dict): dataset = sh.scheming_get_schema('dataset', 'dataset') if ('res_format' in data_dict): #Get format field formats = sh.scheming_field_by_name(dataset.get('resource_fields'), 'format') #Create SOLR field data_dict['res_format_label'] = [] for res_format in data_dict['res_format']: #Get format label res_format_label = sh.scheming_choices_label( formats['choices'], res_format) if res_format_label: #Add label to new SOLR field data_dict['res_format_label'].append(res_format_label) if ('publisher' in data_dict): organismo = data_dict['publisher'] if is_frontend(): publisher = toolkit.get_action('dge_organization_publisher')( { 'model': model }, { 'id': organismo }) else: publisher = h.get_organization(organismo) data_dict['publisher'] = publisher.get('id') data_dict['publisher_display_name'] = publisher.get('display_name') administration_level_code = helpers.dge_get_organization_administration_level_code( publisher) if not administration_level_code or administration_level_code not in TRANSLATED_UNITS: administration_level_code = DEFAULT_UNIT data_dict['administration_level'] = administration_level_code data_dict['administration_level_es'] = TRANSLATED_UNITS[ administration_level_code]['es'] or '' data_dict['administration_level_en'] = TRANSLATED_UNITS[ administration_level_code]['en'] or '' data_dict['administration_level_ca'] = TRANSLATED_UNITS[ administration_level_code]['ca'] or '' data_dict['administration_level_eu'] = TRANSLATED_UNITS[ administration_level_code]['eu'] or '' data_dict['administration_level_gl'] = TRANSLATED_UNITS[ administration_level_code]['gl'] or '' if ('theme' in data_dict): #Get theme field categoria = sh.scheming_field_by_name( dataset.get('dataset_fields'), 'theme') #Get theme value valor_categoria = data_dict['theme'] #Empty theme values data_dict['theme'] = [] data_dict['theme_id'] = [] data_dict['theme_es'] = [] data_dict['theme_en'] = [] data_dict['theme_ca'] = [] data_dict['theme_eu'] = [] data_dict['theme_gl'] = [] #Get key values valores = valor_categoria.replace('[', '').replace(']', '') categorias = valores.split('", "') #Get translated label for each key for term_categoria in list(categorias): clean_term = term_categoria.replace('"', '') data_dict['theme'].append(clean_term) data_dict['theme_id'].append(helpers.dge_theme_id(clean_term)) #Look for label in the scheme for option in categoria.get('choices'): if option['value'] == clean_term: #Add label for each language data_dict['theme_es'].append(option['label']['es']) data_dict['theme_en'].append(option['label']['en']) data_dict['theme_ca'].append(option['label']['ca']) data_dict['theme_eu'].append(option['label']['eu']) data_dict['theme_gl'].append(option['label']['gl']) return data_dict