def after_search(self, results, params): _org_cache = {} is_popular_first = toolkit.asbool( params.get('extras', {}).get('ext_popular_first', False)) for item in results['results']: item['five_star_rating'] = spc_utils._get_stars_from_solr( item['id']) item['ga_view_count'] = spc_utils.ga_view_count(item['name']) item['short_notes'] = h.whtext.truncate(item.get('notes', '')) org_name = item['organization']['name'] try: organization = _org_cache[org_name] except KeyError: organization = h.get_organization(org_name) _org_cache[org_name] = organization item['organization_image_url'] = organization.get( 'image_display_url') or h.url_for_static( '/base/images/placeholder-organization.png', qualified=True) if is_popular_first: results['results'].sort(key=lambda i: i.get('ga_view_count', 0), reverse=True) spc_utils.store_search_query(params) return results
def after_search(self, results, params): _org_cache = {} try: for item in results['search_facets']['type']['items']: item['display_name'] = toolkit._(item['display_name']) except KeyError: pass try: for item in results['search_facets']['member_countries']['items']: item[ 'display_name'] = toolkit.h.spc_member_countries_facet_label( item) except KeyError: pass is_popular_first = toolkit.asbool( params.get('extras', {}).get('ext_popular_first', False)) for item in results['results']: if len(item) == 1: # it's shortened search, probably initiated by bulk download continue if item.get('id'): item['tracking_summary'] = ( model.TrackingSummary.get_for_package(item['id'])) item['five_star_rating'] = spc_utils._get_stars_from_solr( item['id']) if _package_is_native(item['id']): item['isPartOf'] = 'pdh.pacificdatahub' else: src_type = _get_isPartOf(item['id']) if src_type: item['isPartOf'] = src_type if item.get('name'): item['ga_view_count'] = spc_utils.ga_view_count(item['name']) item['short_notes'] = h.truncate(item.get('notes', '')) if item.get('organization'): org_name = item['organization']['name'] try: organization = _org_cache[org_name] except KeyError: organization = h.get_organization(org_name) _org_cache[org_name] = organization item['organization_image_url'] = organization.get( 'image_display_url') or h.url_for_static( '/base/images/placeholder-organization.png', qualified=True) if is_popular_first: results['results'].sort(key=lambda i: i.get('ga_view_count', 0), reverse=True) spc_utils.store_search_query(params) return results
def _get_organizations_objs(organizations_branch, depth=0): organizations = [] for tree_obj in organizations_branch: organization = ckan_helpers.get_organization(org=tree_obj['name']) organization['depth'] = depth if 'children' in tree_obj and len(tree_obj['children']) > 0: organization['children'] = _get_organizations_objs(tree_obj['children'], depth=depth+1) organizations.append(organization) return organizations
def _get_organizations_objs(organizations_branch, depth=0): organizations = [] for tree_obj in organizations_branch: organization = ckan_helpers.get_organization(org=tree_obj['name']) organization['depth'] = depth if 'children' in tree_obj and len(tree_obj['children']) > 0: organization['children'] = _get_organizations_objs(tree_obj['children'], depth=depth + 1) organizations.append(organization) return organizations
def linked_organization(org): organization = helpers.get_organization(org) if organization: return tags.literal(u'{icon} {link}'.format( icon=helpers.icon_html( organization['image_display_url'], alt='', inline=False), link=tags.link_to(organization['title'], url_for( controller='organization', action='read', id=organization['name'])))) return 'Not Existed'
def after_insert(self, mapper, connection, instance): if db.yammer_user_table is not None: p = package.Package().get(instance.id) else: p = None if p is not None: edits = self.get_edit_type(p) org = h.get_organization(org=p.owner_org) for action in edits: if action == 'create' and p.state != 'deleted' and p.private is not True: print('posting to yammer') self.yammer_post('created', p, org['display_name']) else: pass
def after_update(self, mapper, connection, instance): #get the package details from the mapper if db.yammer_user_table is not None: p = package.Package().get(instance.id) else: p = None if p != None: edits = self.get_edit_type(p) org = h.get_organization(org=p.owner_org) for action in edits: if action == 'update' and p.state != 'deleted' and p.state != 'draft' and p.private is not True: self.yammer_post('updated', p, org['display_name']) elif action == 'delete' and p.state == 'deleted': self.yammer_post('deleted', p, org['display_name']) else: pass
def before_index(self, data_dict): dataset = sh.scheming_get_schema('dataset', 'dataset') if ('res_format' in data_dict): #Get format field formats = sh.scheming_field_by_name(dataset.get('resource_fields'), 'format') #Create SOLR field data_dict['res_format_label'] = [] for res_format in data_dict['res_format']: #Get format label res_format_label = sh.scheming_choices_label( formats['choices'], res_format) if res_format_label: #Add label to new SOLR field data_dict['res_format_label'].append(res_format_label) if ('publisher' in data_dict): organismo = data_dict['publisher'] if is_frontend(): publisher = toolkit.get_action('dge_organization_publisher')( { 'model': model }, { 'id': organismo }) else: publisher = h.get_organization(organismo) data_dict['publisher'] = publisher.get('id') data_dict['publisher_display_name'] = publisher.get('display_name') administration_level_code = helpers.dge_get_organization_administration_level_code( publisher) if not administration_level_code or administration_level_code not in TRANSLATED_UNITS: administration_level_code = DEFAULT_UNIT data_dict['administration_level'] = administration_level_code data_dict['administration_level_es'] = TRANSLATED_UNITS[ administration_level_code]['es'] or '' data_dict['administration_level_en'] = TRANSLATED_UNITS[ administration_level_code]['en'] or '' data_dict['administration_level_ca'] = TRANSLATED_UNITS[ administration_level_code]['ca'] or '' data_dict['administration_level_eu'] = TRANSLATED_UNITS[ administration_level_code]['eu'] or '' data_dict['administration_level_gl'] = TRANSLATED_UNITS[ administration_level_code]['gl'] or '' if ('theme' in data_dict): #Get theme field categoria = sh.scheming_field_by_name( dataset.get('dataset_fields'), 'theme') #Get theme value valor_categoria = data_dict['theme'] #Empty theme values data_dict['theme'] = [] data_dict['theme_id'] = [] data_dict['theme_es'] = [] data_dict['theme_en'] = [] data_dict['theme_ca'] = [] data_dict['theme_eu'] = [] data_dict['theme_gl'] = [] #Get key values valores = valor_categoria.replace('[', '').replace(']', '') categorias = valores.split('", "') #Get translated label for each key for term_categoria in list(categorias): clean_term = term_categoria.replace('"', '') data_dict['theme'].append(clean_term) data_dict['theme_id'].append(helpers.dge_theme_id(clean_term)) #Look for label in the scheme for option in categoria.get('choices'): if option['value'] == clean_term: #Add label for each language data_dict['theme_es'].append(option['label']['es']) data_dict['theme_en'].append(option['label']['en']) data_dict['theme_ca'].append(option['label']['ca']) data_dict['theme_eu'].append(option['label']['eu']) data_dict['theme_gl'].append(option['label']['gl']) return data_dict
def org_uri_from_dataset_dict(dataset_dict, field_name): """Get the human-readable URI of a catalogue organization in the CKAN dataset_dict based on the field_name.""" uri = '{0}/organization/{1}'.format(catalog_uri().rstrip('/'), helpers.get_organization(dataset_dict[field_name])['name']) return uri
def dge_harvest_catalog_show(context, data_dict): method_log_prefix = '[%s][dge_harvest_catalog_show]' % __name__ output = None try: log.debug('%s Init method. Inputs context=%s, data_dict=%s' % (method_log_prefix, context, data_dict)) ini = datetime.datetime.now() toolkit.check_access('dge_harvest_catalog_show', context, data_dict) page = 1 data_dict['page'] = page limit = data_dict.get('limit', -1) _format = data_dict.get('format') if _format == RDF_FORMAT: filepath = config.get('ckanext.dge_harvest.rdf.filepath', '/tmp/catalog.rdf') elif _format == CSV_FORMAT: filepath = config.get('ckanext.dge_harvest.csv.filepath', '/tmp/catalog.csv') columnsfilepath = config.get( 'ckanext.dge_harvest.csv.columns.filepath', '/usr/lib/ckan/default/src/ckanext-dge-harvest/ckanext/dge_harvest/commands/columns.json' ) else: filepath = '/tmp/catalog.' + _format query = _dge_harvest_search_ckan_datasets(context, data_dict) dataset_dicts = query['results'] total_datasets = query['count'] log.debug('%s Total_datasets obtenidos en la query: %s' % (method_log_prefix, total_datasets)) if limit > -1 and limit < total_datasets: total_datasets = limit num = len(dataset_dicts) log.debug('%s Total_datasets a exportar: %s' % (method_log_prefix, total_datasets)) while (total_datasets > num): page = page + 1 data_dict['page'] = page query = _dge_harvest_search_ckan_datasets(context, data_dict) dataset_dicts.extend(query['results']) total_datasets = query['count'] num = len(dataset_dicts) log.debug('%s Total_datasets obtenidos en la query: %s' % (method_log_prefix, total_datasets)) log.debug('%s Total_datasets a exportar: %s' % (method_log_prefix, num)) if _format == RDF_FORMAT: serializer = DGERDFSerializer() #log.debug("%s DATASET_DICTS = %s" % (method_log_prefix,dataset_dicts)) output = serializer.serialize_catalog( {}, dataset_dicts, _format=data_dict.get('format'), pagination_info=None) elif _format == CSV_FORMAT and columnsfilepath: #log.info('%s Dataset_dicts de partida =%s' % (method_log_prefix, dataset_dicts)) organizations = {} themes = dhh.dge_harvest_dict_theme_option_label() spatial_coverages = dhh.dge_harvest_dict_spatial_coverage_option_label( ) _dataset = sh.scheming_get_schema('dataset', 'dataset') res_format = sh.scheming_field_by_name( _dataset.get('resource_fields'), 'format') format_values = res_format['choices'] formats = {} datasets = [] num = 0 for dataset in dataset_dicts: ds = {} #Id #ds['id'] = _encode_value(dataset.get('id', None)) #ulr ds['url'] = dataset_uri(dataset) #Description descriptions = _from_dict_to_string( dataset.get(dhc.DS_DESCRIPTION, None)) ds['description'] = _encode_value(descriptions, True) #Title titles = _from_dict_to_string( dataset.get(dhc.DS_TITLE_TRANSLATED, None)) ds['title'] = _encode_value(titles, True) #Theme theme_values = dataset.get(dhc.DS_THEME, None) theme_labels = [] if theme_values: for value in theme_values: theme = themes.get(value) if theme and theme.get('label'): theme_labels.append(theme.get('label').get('es')) theme_value = _from_list_to_string(theme_labels) ds['theme'] = _encode_value(theme_value, True) #Keywords tags = dataset.get(dhc.DS_TAGS) value = None if tags and len(tags) > 0: for tag in tags: stag = tag.get('name', None) if stag: if value: value = '%s%s%s' % (value, MAIN_SEPARATOR, stag) else: value = stag ds['tags'] = _encode_value(value, True) #Identifier ds['identifier'] = _encode_value( dataset.get('identifier', None), True) #Created ds['issued_date'] = _encode_value( _from_iso8601_date_to_string( dataset.get(dhc.DS_ISSUED_DATE, None))) #Modified ds['modified_date'] = _encode_value( _from_iso8601_date_to_string( dataset.get(dhc.DS_MODIFIED_DATE, None))) #Accrual Periodicity frequency = dataset.get(dhc.DS_FREQUENCY) if (frequency): stype = frequency.get('type', '') if stype and len(stype) > 0: stype = 'http://www.w3.org/2006/time#' + stype svalue = frequency.get('value', '') sfrequency = '[TYPE]%s[VALUE]%s' % (stype, svalue) ds['frequency'] = _encode_value(sfrequency, True) #Language languages = _from_list_to_string(dataset.get(dhc.DS_LANGUAGE)) ds['language'] = _encode_value(languages, True) #Publisher publisher = dataset.get(dhc.DS_PUBLISHER, None) if publisher: if publisher in organizations: ds['publisher'] = _encode_value( organizations.get(publisher, None), True) else: organization = h.get_organization(publisher, False) if organization: organizations[publisher] = organization.get( 'title', organization.get('display_name', None)) ds['publisher'] = _encode_value( organizations.get(publisher), True) #License ds['license_id'] = _encode_value(dataset.get(dhc.DS_LICENSE), True) #Spatial spatial_values = dataset.get(dhc.DS_SPATIAL, None) spatial_labels = [] if spatial_values: for value in spatial_values: spatial = spatial_coverages.get(value) if spatial and spatial.get('label') and spatial.get( 'label').get('es'): spatial_labels.append( spatial.get('label').get('es')) spatials = _from_list_to_string(spatial_labels) ds['spatial'] = _encode_value(spatials, True) #Temporal temporal_coverage = dataset.get(dhc.DS_TEMPORAL_COVERAGE) if temporal_coverage: value = None for tc in temporal_coverage.itervalues(): if tc: tc_from = _from_iso8601_date_to_string( tc.get('from', None)) tc_to = _from_iso8601_date_to_string( tc.get('to', None)) if tc_from or tc_to: if value: value = '%s%s%s-%s' % (value, MAIN_SEPARATOR, (tc_from or ''), (tc_to or '')) else: value = '%s-%s' % ((tc_from or ''), (tc_to or '')) ds['coverage_new'] = _encode_value(value, True) #Valid ds['valid'] = _encode_value( _from_iso8601_date_to_string( dataset.get(dhc.DS_VALID, None)), True) #References references = _from_list_to_string( dataset.get(dhc.DS_REFERENCE, None)) ds['references'] = _encode_value(references, True) #Normative conforms_to = _from_list_to_string( dataset.get(dhc.DS_NORMATIVE, None)) ds['conforms_to'] = _encode_value(conforms_to, True) #Resources resources = dataset.get(dhc.DS_RESOURCES) sresources = [] if resources: for resource in resources: sresource = None if resource: name = _from_dict_to_string( resource.get(dhc.DS_RESOURCE_NAME_TRANSLATED, None), 'TITLE_') if not name: name = '' url = resource.get(dhc.DS_RESOURCE_ACCESS_URL, '') if url: url = '[ACCESS_URL]%s' % (url) format_value = resource.get( dhc.DS_RESOURCE_FORMAT, None) format = None if format_value: if format_value in formats: format = formats.get(format_value, None) else: formats[ format_value] = sh.scheming_choices_label( format_values, format_value) format = formats.get(format_value, None) if format: format = '[MEDIA_TYPE]%s' % (format) size = resource.get(dhc.DS_RESOURCE_BYTE_SIZE, '') if size: size = '[BYTE_SIZE]%s' % (size) relation = _from_list_to_string( resource.get(dhc.DS_RESOURCE_RELATION, None), SECONDARY_SEPARATOR) relations = '' if relation: relations = '[RELATION]%s' % (relation) sresource = '%s%s%s%s%s' % (name, url, format, size, relations) if sresource and len(sresource) > 0: sresources.append(sresource) if len(sresources) > 0: value = None for item in sresources: if value: value = '%s%s%s' % (value, MAIN_SEPARATOR, item) else: value = item ds['resources'] = _encode_value(value, True) num = num + 1 datasets.append(ds) #log.debug('%s Datasets con datos a exportar=%s' % (method_log_prefix, datasets)) log.debug('%s Numero de datasets con datos a exportar...%s' % (method_log_prefix, num)) output = losser.losser.table(datasets, columnsfilepath, csv=True, pretty=False) if filepath: file = None try: file = open(filepath, "w") file.write(output) file.close() except: if file and not file.closed: file.close() end = datetime.datetime.now() log.debug( "%s Time in serialize %s catalog [%s] with %s datasets ... %s milliseconds" % (method_log_prefix, _format, filepath, total_datasets, int((end - ini).total_seconds() * 1000))) except Exception, e: log.error("%s Exception %s: %s" % (method_log_prefix, type(e).__name__, e)) output = None
def validator(key, data, errors, context): if errors[key]: return value = data[key] if value is not missing: if value: return output = {} prefix = field['autogeneration_field'] if not prefix: prefix = DEFAULT_TITLE_FIELD log.debug('[multilanguage_url] Creating field using the field %s', prefix) prefix = prefix + '-' extras = data.get(key[:-1] + ('__extras',), {}) locale_default = config.get('ckan.locale_default', 'es') if locale_default: title_lang = prefix + locale_default if title_lang in extras and extras[title_lang]: dataset_title = extras[title_lang] # Generate title prefix field_prefix = field['organization_field'] organization_prefix = field['organization_prefix'] publisher_id = data.get((field_prefix,)) if not publisher_id and field_prefix in extras: publisher_id = extras[field_prefix] if publisher_id and organization_prefix: organization = h.get_organization(publisher_id) if not organization: organization = toolkit.get_action('dge_organization_publisher')({'model': model}, {'id': publisher_id}) if organization and organization['extras']: for extra in organization['extras']: if extra['key'] == organization_prefix and extra['state'] == 'active': dataset_title = extra['value'] + '-' + dataset_title break data[key] = munge.munge_title_to_name(dataset_title) log.debug('[multilanguage_url] Created name "%s" for package from language %s', data[key], locale_default) return locale_order = config.get('ckan.locale_order', '').split() for l in locale_order: title_lang = prefix + l if title_lang in extras and extras[title_lang]: dataset_title = extras[title_lang] # Generate title prefix field_prefix = field['organization_field'] organization_prefix = field['organization_prefix'] publisher_id = data.get((field_prefix,)) if not publisher_id and field_prefix in extras: publisher_id = extras[field_prefix] if publisher_id and organization_prefix: organization = h.get_organization(publisher_id) if not organization: organization = toolkit.get_action('dge_organization_publisher')({'model': model}, {'id': publisher_id}) if organization and organization['extras']: for extra in organization['extras']: if extra['key'] == organization_prefix and extra['state'] == 'active': dataset_title = extra['value'] + '-' + dataset_title break data[key] = munge.munge_title_to_name(dataset_title) log.debug('[multilanguage_url] Created name "%s" for package from language %s', data[key], l) break