def after_search(self, results, params):
_org_cache = {}
is_popular_first = toolkit.asbool(
params.get('extras', {}).get('ext_popular_first', False))
for item in results['results']:
item['five_star_rating'] = spc_utils._get_stars_from_solr(
item['id'])
item['ga_view_count'] = spc_utils.ga_view_count(item['name'])
item['short_notes'] = h.whtext.truncate(item.get('notes', ''))
org_name = item['organization']['name']
try:
organization = _org_cache[org_name]
except KeyError:
organization = h.get_organization(org_name)
_org_cache[org_name] = organization
item['organization_image_url'] = organization.get(
'image_display_url') or h.url_for_static(
'/base/images/placeholder-organization.png',
qualified=True)
if is_popular_first:
results['results'].sort(key=lambda i: i.get('ga_view_count', 0),
reverse=True)
spc_utils.store_search_query(params)
return results
def after_search(self, results, params):
_org_cache = {}
try:
for item in results['search_facets']['type']['items']:
item['display_name'] = toolkit._(item['display_name'])
except KeyError:
pass
try:
for item in results['search_facets']['member_countries']['items']:
item[
'display_name'] = toolkit.h.spc_member_countries_facet_label(
item)
except KeyError:
pass
is_popular_first = toolkit.asbool(
params.get('extras', {}).get('ext_popular_first', False))
for item in results['results']:
if len(item) == 1:
# it's shortened search, probably initiated by bulk download
continue
if item.get('id'):
item['tracking_summary'] = (
model.TrackingSummary.get_for_package(item['id']))
item['five_star_rating'] = spc_utils._get_stars_from_solr(
item['id'])
if _package_is_native(item['id']):
item['isPartOf'] = 'pdh.pacificdatahub'
else:
src_type = _get_isPartOf(item['id'])
if src_type:
item['isPartOf'] = src_type
if item.get('name'):
item['ga_view_count'] = spc_utils.ga_view_count(item['name'])
item['short_notes'] = h.truncate(item.get('notes', ''))
if item.get('organization'):
org_name = item['organization']['name']
try:
organization = _org_cache[org_name]
except KeyError:
organization = h.get_organization(org_name)
_org_cache[org_name] = organization
item['organization_image_url'] = organization.get(
'image_display_url') or h.url_for_static(
'/base/images/placeholder-organization.png',
qualified=True)
if is_popular_first:
results['results'].sort(key=lambda i: i.get('ga_view_count', 0),
reverse=True)
spc_utils.store_search_query(params)
return results
def _get_organizations_objs(organizations_branch, depth=0):
organizations = []
for tree_obj in organizations_branch:
organization = ckan_helpers.get_organization(org=tree_obj['name'])
organization['depth'] = depth
if 'children' in tree_obj and len(tree_obj['children']) > 0:
organization['children'] = _get_organizations_objs(tree_obj['children'], depth=depth+1)
organizations.append(organization)
return organizations
def _get_organizations_objs(organizations_branch, depth=0):
organizations = []
for tree_obj in organizations_branch:
organization = ckan_helpers.get_organization(org=tree_obj['name'])
organization['depth'] = depth
if 'children' in tree_obj and len(tree_obj['children']) > 0:
organization['children'] = _get_organizations_objs(tree_obj['children'], depth=depth + 1)
organizations.append(organization)
return organizations
def linked_organization(org):
organization = helpers.get_organization(org)
if organization:
return tags.literal(u'{icon} {link}'.format(
icon=helpers.icon_html(
organization['image_display_url'], alt='', inline=False),
link=tags.link_to(organization['title'],
url_for(
controller='organization',
action='read',
id=organization['name']))))
return 'Not Existed'
def after_insert(self, mapper, connection, instance):
if db.yammer_user_table is not None:
p = package.Package().get(instance.id)
else:
p = None
if p is not None:
edits = self.get_edit_type(p)
org = h.get_organization(org=p.owner_org)
for action in edits:
if action == 'create' and p.state != 'deleted' and p.private is not True:
print('posting to yammer')
self.yammer_post('created', p, org['display_name'])
else:
pass
def after_update(self, mapper, connection, instance):
#get the package details from the mapper
if db.yammer_user_table is not None:
p = package.Package().get(instance.id)
else:
p = None
if p != None:
edits = self.get_edit_type(p)
org = h.get_organization(org=p.owner_org)
for action in edits:
if action == 'update' and p.state != 'deleted' and p.state != 'draft' and p.private is not True:
self.yammer_post('updated', p, org['display_name'])
elif action == 'delete' and p.state == 'deleted':
self.yammer_post('deleted', p, org['display_name'])
else:
pass
def before_index(self, data_dict):
dataset = sh.scheming_get_schema('dataset', 'dataset')
if ('res_format' in data_dict):
#Get format field
formats = sh.scheming_field_by_name(dataset.get('resource_fields'),
'format')
#Create SOLR field
data_dict['res_format_label'] = []
for res_format in data_dict['res_format']:
#Get format label
res_format_label = sh.scheming_choices_label(
formats['choices'], res_format)
if res_format_label:
#Add label to new SOLR field
data_dict['res_format_label'].append(res_format_label)
if ('publisher' in data_dict):
organismo = data_dict['publisher']
if is_frontend():
publisher = toolkit.get_action('dge_organization_publisher')(
{
'model': model
}, {
'id': organismo
})
else:
publisher = h.get_organization(organismo)
data_dict['publisher'] = publisher.get('id')
data_dict['publisher_display_name'] = publisher.get('display_name')
administration_level_code = helpers.dge_get_organization_administration_level_code(
publisher)
if not administration_level_code or administration_level_code not in TRANSLATED_UNITS:
administration_level_code = DEFAULT_UNIT
data_dict['administration_level'] = administration_level_code
data_dict['administration_level_es'] = TRANSLATED_UNITS[
administration_level_code]['es'] or ''
data_dict['administration_level_en'] = TRANSLATED_UNITS[
administration_level_code]['en'] or ''
data_dict['administration_level_ca'] = TRANSLATED_UNITS[
administration_level_code]['ca'] or ''
data_dict['administration_level_eu'] = TRANSLATED_UNITS[
administration_level_code]['eu'] or ''
data_dict['administration_level_gl'] = TRANSLATED_UNITS[
administration_level_code]['gl'] or ''
if ('theme' in data_dict):
#Get theme field
categoria = sh.scheming_field_by_name(
dataset.get('dataset_fields'), 'theme')
#Get theme value
valor_categoria = data_dict['theme']
#Empty theme values
data_dict['theme'] = []
data_dict['theme_id'] = []
data_dict['theme_es'] = []
data_dict['theme_en'] = []
data_dict['theme_ca'] = []
data_dict['theme_eu'] = []
data_dict['theme_gl'] = []
#Get key values
valores = valor_categoria.replace('[', '').replace(']', '')
categorias = valores.split('", "')
#Get translated label for each key
for term_categoria in list(categorias):
clean_term = term_categoria.replace('"', '')
data_dict['theme'].append(clean_term)
data_dict['theme_id'].append(helpers.dge_theme_id(clean_term))
#Look for label in the scheme
for option in categoria.get('choices'):
if option['value'] == clean_term:
#Add label for each language
data_dict['theme_es'].append(option['label']['es'])
data_dict['theme_en'].append(option['label']['en'])
data_dict['theme_ca'].append(option['label']['ca'])
data_dict['theme_eu'].append(option['label']['eu'])
data_dict['theme_gl'].append(option['label']['gl'])
return data_dict
def org_uri_from_dataset_dict(dataset_dict, field_name):
"""Get the human-readable URI of a catalogue organization in the CKAN dataset_dict based on the field_name."""
uri = '{0}/organization/{1}'.format(catalog_uri().rstrip('/'),
helpers.get_organization(dataset_dict[field_name])['name'])
return uri
def dge_harvest_catalog_show(context, data_dict):
method_log_prefix = '[%s][dge_harvest_catalog_show]' % __name__
output = None
try:
log.debug('%s Init method. Inputs context=%s, data_dict=%s' %
(method_log_prefix, context, data_dict))
ini = datetime.datetime.now()
toolkit.check_access('dge_harvest_catalog_show', context, data_dict)
page = 1
data_dict['page'] = page
limit = data_dict.get('limit', -1)
_format = data_dict.get('format')
if _format == RDF_FORMAT:
filepath = config.get('ckanext.dge_harvest.rdf.filepath',
'/tmp/catalog.rdf')
elif _format == CSV_FORMAT:
filepath = config.get('ckanext.dge_harvest.csv.filepath',
'/tmp/catalog.csv')
columnsfilepath = config.get(
'ckanext.dge_harvest.csv.columns.filepath',
'/usr/lib/ckan/default/src/ckanext-dge-harvest/ckanext/dge_harvest/commands/columns.json'
)
else:
filepath = '/tmp/catalog.' + _format
query = _dge_harvest_search_ckan_datasets(context, data_dict)
dataset_dicts = query['results']
total_datasets = query['count']
log.debug('%s Total_datasets obtenidos en la query: %s' %
(method_log_prefix, total_datasets))
if limit > -1 and limit < total_datasets:
total_datasets = limit
num = len(dataset_dicts)
log.debug('%s Total_datasets a exportar: %s' %
(method_log_prefix, total_datasets))
while (total_datasets > num):
page = page + 1
data_dict['page'] = page
query = _dge_harvest_search_ckan_datasets(context, data_dict)
dataset_dicts.extend(query['results'])
total_datasets = query['count']
num = len(dataset_dicts)
log.debug('%s Total_datasets obtenidos en la query: %s' %
(method_log_prefix, total_datasets))
log.debug('%s Total_datasets a exportar: %s' %
(method_log_prefix, num))
if _format == RDF_FORMAT:
serializer = DGERDFSerializer()
#log.debug("%s DATASET_DICTS = %s" % (method_log_prefix,dataset_dicts))
output = serializer.serialize_catalog(
{},
dataset_dicts,
_format=data_dict.get('format'),
pagination_info=None)
elif _format == CSV_FORMAT and columnsfilepath:
#log.info('%s Dataset_dicts de partida =%s' % (method_log_prefix, dataset_dicts))
organizations = {}
themes = dhh.dge_harvest_dict_theme_option_label()
spatial_coverages = dhh.dge_harvest_dict_spatial_coverage_option_label(
)
_dataset = sh.scheming_get_schema('dataset', 'dataset')
res_format = sh.scheming_field_by_name(
_dataset.get('resource_fields'), 'format')
format_values = res_format['choices']
formats = {}
datasets = []
num = 0
for dataset in dataset_dicts:
ds = {}
#Id
#ds['id'] = _encode_value(dataset.get('id', None))
#ulr
ds['url'] = dataset_uri(dataset)
#Description
descriptions = _from_dict_to_string(
dataset.get(dhc.DS_DESCRIPTION, None))
ds['description'] = _encode_value(descriptions, True)
#Title
titles = _from_dict_to_string(
dataset.get(dhc.DS_TITLE_TRANSLATED, None))
ds['title'] = _encode_value(titles, True)
#Theme
theme_values = dataset.get(dhc.DS_THEME, None)
theme_labels = []
if theme_values:
for value in theme_values:
theme = themes.get(value)
if theme and theme.get('label'):
theme_labels.append(theme.get('label').get('es'))
theme_value = _from_list_to_string(theme_labels)
ds['theme'] = _encode_value(theme_value, True)
#Keywords
tags = dataset.get(dhc.DS_TAGS)
value = None
if tags and len(tags) > 0:
for tag in tags:
stag = tag.get('name', None)
if stag:
if value:
value = '%s%s%s' % (value, MAIN_SEPARATOR,
stag)
else:
value = stag
ds['tags'] = _encode_value(value, True)
#Identifier
ds['identifier'] = _encode_value(
dataset.get('identifier', None), True)
#Created
ds['issued_date'] = _encode_value(
_from_iso8601_date_to_string(
dataset.get(dhc.DS_ISSUED_DATE, None)))
#Modified
ds['modified_date'] = _encode_value(
_from_iso8601_date_to_string(
dataset.get(dhc.DS_MODIFIED_DATE, None)))
#Accrual Periodicity
frequency = dataset.get(dhc.DS_FREQUENCY)
if (frequency):
stype = frequency.get('type', '')
if stype and len(stype) > 0:
stype = 'http://www.w3.org/2006/time#' + stype
svalue = frequency.get('value', '')
sfrequency = '[TYPE]%s[VALUE]%s' % (stype, svalue)
ds['frequency'] = _encode_value(sfrequency, True)
#Language
languages = _from_list_to_string(dataset.get(dhc.DS_LANGUAGE))
ds['language'] = _encode_value(languages, True)
#Publisher
publisher = dataset.get(dhc.DS_PUBLISHER, None)
if publisher:
if publisher in organizations:
ds['publisher'] = _encode_value(
organizations.get(publisher, None), True)
else:
organization = h.get_organization(publisher, False)
if organization:
organizations[publisher] = organization.get(
'title',
organization.get('display_name', None))
ds['publisher'] = _encode_value(
organizations.get(publisher), True)
#License
ds['license_id'] = _encode_value(dataset.get(dhc.DS_LICENSE),
True)
#Spatial
spatial_values = dataset.get(dhc.DS_SPATIAL, None)
spatial_labels = []
if spatial_values:
for value in spatial_values:
spatial = spatial_coverages.get(value)
if spatial and spatial.get('label') and spatial.get(
'label').get('es'):
spatial_labels.append(
spatial.get('label').get('es'))
spatials = _from_list_to_string(spatial_labels)
ds['spatial'] = _encode_value(spatials, True)
#Temporal
temporal_coverage = dataset.get(dhc.DS_TEMPORAL_COVERAGE)
if temporal_coverage:
value = None
for tc in temporal_coverage.itervalues():
if tc:
tc_from = _from_iso8601_date_to_string(
tc.get('from', None))
tc_to = _from_iso8601_date_to_string(
tc.get('to', None))
if tc_from or tc_to:
if value:
value = '%s%s%s-%s' % (value,
MAIN_SEPARATOR,
(tc_from or ''),
(tc_to or ''))
else:
value = '%s-%s' % ((tc_from or ''),
(tc_to or ''))
ds['coverage_new'] = _encode_value(value, True)
#Valid
ds['valid'] = _encode_value(
_from_iso8601_date_to_string(
dataset.get(dhc.DS_VALID, None)), True)
#References
references = _from_list_to_string(
dataset.get(dhc.DS_REFERENCE, None))
ds['references'] = _encode_value(references, True)
#Normative
conforms_to = _from_list_to_string(
dataset.get(dhc.DS_NORMATIVE, None))
ds['conforms_to'] = _encode_value(conforms_to, True)
#Resources
resources = dataset.get(dhc.DS_RESOURCES)
sresources = []
if resources:
for resource in resources:
sresource = None
if resource:
name = _from_dict_to_string(
resource.get(dhc.DS_RESOURCE_NAME_TRANSLATED,
None), 'TITLE_')
if not name:
name = ''
url = resource.get(dhc.DS_RESOURCE_ACCESS_URL, '')
if url:
url = '[ACCESS_URL]%s' % (url)
format_value = resource.get(
dhc.DS_RESOURCE_FORMAT, None)
format = None
if format_value:
if format_value in formats:
format = formats.get(format_value, None)
else:
formats[
format_value] = sh.scheming_choices_label(
format_values, format_value)
format = formats.get(format_value, None)
if format:
format = '[MEDIA_TYPE]%s' % (format)
size = resource.get(dhc.DS_RESOURCE_BYTE_SIZE, '')
if size:
size = '[BYTE_SIZE]%s' % (size)
relation = _from_list_to_string(
resource.get(dhc.DS_RESOURCE_RELATION, None),
SECONDARY_SEPARATOR)
relations = ''
if relation:
relations = '[RELATION]%s' % (relation)
sresource = '%s%s%s%s%s' % (name, url, format,
size, relations)
if sresource and len(sresource) > 0:
sresources.append(sresource)
if len(sresources) > 0:
value = None
for item in sresources:
if value:
value = '%s%s%s' % (value, MAIN_SEPARATOR, item)
else:
value = item
ds['resources'] = _encode_value(value, True)
num = num + 1
datasets.append(ds)
#log.debug('%s Datasets con datos a exportar=%s' % (method_log_prefix, datasets))
log.debug('%s Numero de datasets con datos a exportar...%s' %
(method_log_prefix, num))
output = losser.losser.table(datasets,
columnsfilepath,
csv=True,
pretty=False)
if filepath:
file = None
try:
file = open(filepath, "w")
file.write(output)
file.close()
except:
if file and not file.closed:
file.close()
end = datetime.datetime.now()
log.debug(
"%s Time in serialize %s catalog [%s] with %s datasets ... %s milliseconds"
% (method_log_prefix, _format, filepath, total_datasets,
int((end - ini).total_seconds() * 1000)))
except Exception, e:
log.error("%s Exception %s: %s" %
(method_log_prefix, type(e).__name__, e))
output = None
def validator(key, data, errors, context):
if errors[key]:
return
value = data[key]
if value is not missing:
if value:
return
output = {}
prefix = field['autogeneration_field']
if not prefix:
prefix = DEFAULT_TITLE_FIELD
log.debug('[multilanguage_url] Creating field using the field %s', prefix)
prefix = prefix + '-'
extras = data.get(key[:-1] + ('__extras',), {})
locale_default = config.get('ckan.locale_default', 'es')
if locale_default:
title_lang = prefix + locale_default
if title_lang in extras and extras[title_lang]:
dataset_title = extras[title_lang]
# Generate title prefix
field_prefix = field['organization_field']
organization_prefix = field['organization_prefix']
publisher_id = data.get((field_prefix,))
if not publisher_id and field_prefix in extras:
publisher_id = extras[field_prefix]
if publisher_id and organization_prefix:
organization = h.get_organization(publisher_id)
if not organization:
organization = toolkit.get_action('dge_organization_publisher')({'model': model}, {'id': publisher_id})
if organization and organization['extras']:
for extra in organization['extras']:
if extra['key'] == organization_prefix and extra['state'] == 'active':
dataset_title = extra['value'] + '-' + dataset_title
break
data[key] = munge.munge_title_to_name(dataset_title)
log.debug('[multilanguage_url] Created name "%s" for package from language %s',
data[key], locale_default)
return
locale_order = config.get('ckan.locale_order', '').split()
for l in locale_order:
title_lang = prefix + l
if title_lang in extras and extras[title_lang]:
dataset_title = extras[title_lang]
# Generate title prefix
field_prefix = field['organization_field']
organization_prefix = field['organization_prefix']
publisher_id = data.get((field_prefix,))
if not publisher_id and field_prefix in extras:
publisher_id = extras[field_prefix]
if publisher_id and organization_prefix:
organization = h.get_organization(publisher_id)
if not organization:
organization = toolkit.get_action('dge_organization_publisher')({'model': model}, {'id': publisher_id})
if organization and organization['extras']:
for extra in organization['extras']:
if extra['key'] == organization_prefix and extra['state'] == 'active':
dataset_title = extra['value'] + '-' + dataset_title
break
data[key] = munge.munge_title_to_name(dataset_title)
log.debug('[multilanguage_url] Created name "%s" for package from language %s',
data[key], l)
break