def showcase_create(context, data_dict):
'''Upload the image and continue with package creation.'''
# force type to 'showcase'
data_dict['type'] = 'showcase'
# If get_uploader is available (introduced for IUploader in CKAN 2.5), use
# it, otherwise use the default uploader.
# https://github.com/ckan/ckan/pull/2510
try:
upload = uploader.get_uploader('showcase')
except AttributeError:
upload = uploader.Upload('showcase')
# schema images
imgs = ['icon', 'featured_image', 'image_1', 'image_2', 'image_3']
for image in imgs:
if data_dict[image]:
upload.update_data_dict(data_dict, image,
image+'_upload', 'clear_'+ image + '_upload')
upload.upload(uploader.get_max_image_size())
pkg = toolkit.get_action('package_create')(context, data_dict)
return pkg
def pages_upload(context, data_dict):
try:
p.toolkit.check_access('ckanext_pages_upload', context, data_dict)
except p.toolkit.NotAuthorized:
p.toolkit.abort(401, p.toolkit._('Not authorized to see this page'))
if p.toolkit.check_ckan_version(min_version='2.5'):
upload = uploader.get_uploader('page_images')
else:
upload = uploader.Upload('page_images')
upload.update_data_dict(data_dict, 'image_url', 'upload', 'clear_upload')
upload.upload(uploader.get_max_image_size())
image_url = data_dict.get('image_url')
if not image_mime_type_validator(upload.filepath):
raise ValidationError(tk._('You can upload image file only'))
if image_url:
image_url = h.url_for_static('uploads/page_images/%s' % image_url,
qualified=True)
return image_url
def showcase_create(context, data_dict):
'''Upload the image and continue with package creation.'''
# force type to 'showcase'
data_dict['type'] = 'showcase'
# If get_uploader is available (introduced for IUploader in CKAN 2.5), use
# it, otherwise use the default uploader.
# https://github.com/ckan/ckan/pull/2510
try:
upload = uploader.get_uploader('showcase')
except AttributeError:
upload = uploader.Upload('showcase')
if 'image_upload' in data_dict:
# mimetype is needed before uploading to AWS S3
upload.mimetype = getattr(data_dict['image_upload'], 'type',
'application/octet-stream')
upload.update_data_dict(data_dict, 'image_url', 'image_upload',
'clear_upload')
upload.upload(uploader.get_max_image_size())
pkg = toolkit.get_action('package_create')(context, data_dict)
return pkg
def pages_upload(context, data_dict):
try:
p.toolkit.check_access('ckanext_pages_upload', context, data_dict)
except p.toolkit.NotAuthorized:
p.toolkit.abort(401, p.toolkit._('Not authorized to see this page'))
upload = uploader.Upload('page_images')
upload.update_data_dict(data_dict, 'image_url', 'upload', 'clear_upload')
upload.upload()
image_url = data_dict.get('image_url')
if image_url:
image_url = h.url_for_static('uploads/page_images/%s' % image_url,
qualified=True)
return {'url': image_url}
def experience_update(context, data_dict):
# If get_uploader is available (introduced for IUploader in CKAN 2.5), use
# it, otherwise use the default uploader.
# https://github.com/ckan/ckan/pull/2510
try:
upload = uploader.get_uploader('experience', data_dict['image_url'])
except AttributeError:
upload = uploader.Upload('experience', data_dict['image_url'])
upload.update_data_dict(data_dict, 'image_url',
'image_upload', 'clear_upload')
upload.upload(uploader.get_max_image_size())
pkg = toolkit.get_action('package_update')(context, data_dict)
return pkg
def showcase_upload(context, data_dict):
''' Uploads images to be used in showcase content.
'''
toolkit.check_access('ckanext_showcase_upload', context, data_dict)
try:
upload = uploader.get_uploader('showcase_image')
except AttributeError:
upload = uploader.Upload('showcase_image')
upload.update_data_dict(data_dict, 'image_url', 'upload', 'clear_upload')
upload.upload(uploader.get_max_image_size())
image_url = data_dict.get('image_url')
if image_url:
image_url = h.url_for_static(
'uploads/showcase_image/{}'.format(image_url), qualified=True)
return {'url': image_url}
def pages_upload(context, data_dict):
try:
p.toolkit.check_access('ckanext_pages_upload', context, data_dict)
except p.toolkit.NotAuthorized:
p.toolkit.abort(401, p.toolkit._('Not authorized to see this page'))
if p.toolkit.check_ckan_version(min_version='2.5'):
upload = uploader.get_uploader('page_images')
else:
upload = uploader.Upload('page_images')
upload.update_data_dict(data_dict, 'image_url',
'upload', 'clear_upload')
upload.upload(uploader.get_max_image_size())
image_url = data_dict.get('image_url')
if image_url and image_url[0:6] not in {'http:/', 'https:'}:
image_url = h.url_for_static(
'uploads/page_images/%s' % image_url,
qualified=True
)
return {'url': image_url, 'fileName': upload.filename, 'uploaded': 1}
def ogdch_xml_upload(context, data_dict):
data = data_dict.get('data')
org_id = data_dict.get('organization')
# Don't use uploader.get_uploader(), as this will return the S3Uploader.
# We want to process the file locally and then delete it.
upload = uploader.Upload('dataset_xml')
upload.update_data_dict(data, 'dataset_xml',
'file_upload', 'clear_upload')
upload.upload()
dataset_filename = data.get('dataset_xml')
if not dataset_filename:
h.flash_error('Error uploading file.')
return
full_file_path = os.path.join(upload.storage_path, dataset_filename)
data_rdfgraph = rdflib.ConjunctiveGraph()
profile = SwissDCATAPProfile(data_rdfgraph)
try:
data_rdfgraph.parse(full_file_path, "xml")
except Exception as e:
h.flash_error(
'Error parsing the RDF file during dataset import: {0}'
.format(e))
os.remove(full_file_path)
return
for dataset_ref in data_rdfgraph.subjects(RDF.type, DCAT.Dataset):
dataset_dict = {}
profile.parse_dataset(dataset_dict, dataset_ref)
dataset_dict['owner_org'] = org_id
_create_or_update_dataset(dataset_dict)
# Clean up the file as we have no further use for it.
os.remove(full_file_path)
def hdx_group_or_org_create(context, data_dict, is_org=False):
# Overriding default so that orgs can have multiple images
model = context['model']
user = context['user']
session = context['session']
data_dict['is_organization'] = is_org
if is_org:
check_access('organization_create', context, data_dict)
else:
check_access('group_create', context, data_dict)
# get the schema
group_type = data_dict.get('type')
group_plugin = lib_plugins.lookup_group_plugin(group_type)
try:
schema = group_plugin.form_to_db_schema_options({
'type': 'create', 'api': 'api_version' in context,
'context': context})
except AttributeError:
schema = group_plugin.form_to_db_schema()
# try:
# customization = json.loads(group.extras['customization'])
# except:
customization = {'image_sq': '', 'image_rect': ''}
try:
data_dict['customization'] = json.loads(data_dict['customization'])
except:
data_dict['customization'] = {}
if 'image_sq_upload' in data_dict and data_dict['image_sq_upload'] != '' and data_dict['image_sq_upload'] != None:
# If old image was uploaded remove it
if customization['image_sq']:
remove_image(customization['image_sq'])
upload1 = uploader.Upload('group', customization['image_sq'])
upload1.update_data_dict(data_dict, 'image_sq',
'image_sq_upload', 'clear_upload')
if 'image_rect_upload' in data_dict and data_dict['image_rect_upload'] != '' and data_dict[
'image_rect_upload'] != None:
if customization['image_rect']:
remove_image(customization['image_rect'])
upload2 = uploader.Upload('group', customization['image_rect'])
upload2.update_data_dict(data_dict, 'image_rect',
'image_rect_upload', 'clear_upload')
storage_path = uploader.get_storage_path()
##Rearrange things the way we need them
try:
if data_dict['image_sq'] != '' and data_dict['image_sq'] != None:
data_dict['customization']['image_sq'] = data_dict['image_sq']
else:
data_dict['customization']['image_sq'] = customization['image_sq']
except KeyError:
data_dict['customization']['image_sq'] = ''
try:
if data_dict['image_rect'] != '' and data_dict['image_rect'] != None:
data_dict['customization']['image_rect'] = data_dict['image_rect']
else:
data_dict['customization']['image_rect'] = customization['image_rect']
except KeyError:
data_dict['customization']['image_rect'] = ''
data_dict['customization'] = json.dumps(data_dict['customization'])
if 'api_version' not in context:
# old plugins do not support passing the schema so we need
# to ensure they still work
try:
group_plugin.check_data_dict(data_dict, schema)
except TypeError:
group_plugin.check_data_dict(data_dict)
data, errors = lib_plugins.plugin_validate(
group_plugin, context, data_dict, schema,
'organization_create' if is_org else 'group_create')
log.debug('group_create validate_errs=%r user=%s group=%s data_dict=%r',
errors, context.get('user'), data_dict.get('name'), data_dict)
if errors:
session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Create object %s') % data.get("name")
group = model_save.group_dict_save(data, context)
# Needed to let extensions know the group id
session.flush()
if is_org:
plugin_type = plugins.IOrganizationController
else:
plugin_type = plugins.IGroupController
for item in plugins.PluginImplementations(plugin_type):
item.create(group)
if is_org:
activity_type = 'new organization'
else:
activity_type = 'new group'
user_id = model.User.by_name(user.decode('utf8')).id
activity_dict = {
'user_id': user_id,
'object_id': group.id,
'activity_type': activity_type,
}
activity_dict['data'] = {
'group': ckan.lib.dictization.table_dictize(group, context)
}
activity_create_context = {
'model': model,
'user': user,
'defer_commit': True,
'ignore_auth': True,
'session': session
}
logic.get_action('activity_create')(activity_create_context, activity_dict)
try:
upload1.upload(uploader.get_max_image_size())
except:
pass
try:
upload2.upload(uploader.get_max_image_size())
except:
pass
if not context.get('defer_commit'):
model.repo.commit()
context["group"] = group
context["id"] = group.id
# creator of group/org becomes an admin
# this needs to be after the repo.commit or else revisions break
member_dict = {
'id': group.id,
'object': user_id,
'object_type': 'user',
'capacity': 'admin',
}
member_create_context = {
'model': model,
'user': user,
'ignore_auth': True, # we are not a member of the group at this point
'session': session
}
logic.get_action('member_create')(member_create_context, member_dict)
log.debug('Created object %s' % group.name)
return model_dictize.group_dictize(group, context)
def get_uploader(self, upload_to, old_filename=None):
return uploader.Upload(upload_to, old_filename)
def package_update(context, data_dict):
'''Update a dataset (package).
You must be authorized to edit the dataset and the groups that it belongs
to.
Plugins may change the parameters of this function depending on the value
of the dataset's ``type`` attribute, see the ``IDatasetForm`` plugin
interface.
For further parameters see ``package_create()``.
:param id: the name or id of the dataset to update
:type id: string
:returns: the updated dataset (if 'return_package_dict' is True in the
context, which is the default. Otherwise returns just the
dataset id)
:rtype: dictionary
'''
model = context['model']
user = context['user']
name_or_id = data_dict.get("id") or data_dict['name']
pkg = model.Package.get(name_or_id)
if pkg is None:
raise NotFound(_('Package was not found.'))
context["package"] = pkg
data_dict["id"] = pkg.id
# FIXME: first modifications to package_updade begin here:
# tag strings are reconstructed because validators are stripping
# tags passed and only taking taks as tag_string values
# image upload support has also been added here
old_data = get_action('package_show')(context, {'id': pkg.id})
'''
Constructing the tag_string from the given tags.
There must be at least one tag, otherwise the tag_string will be empty and a validation error
will be raised.
'''
if not data_dict.get('tag_string'):
data_dict['tag_string'] = ', '.join(
h.dict_list_reduce(data_dict.get('tags', {}), 'name'))
for key, value in old_data.iteritems():
if key not in data_dict:
data_dict[key] = value
# data_dict['resources'] = data_dict.get('resources', old_data.get('resources'))
# iso_topic_cat = data_dict.get('iso_topic_string', [])
# if isinstance(iso_topic_cat, basestring):
# iso_topic_cat = [iso_topic_cat]
#
# data_dict['iso_topic_string'] = ','.join(iso_topic_cat)
# Set the package last modified date
data_dict['record_last_modified'] = str(datetime.date.today())
# If the Created Date has not yet been set, then set it
if data_dict['edc_state'] == 'DRAFT' and not data_dict.get(
'record_create_date'):
data_dict['record_create_date'] = str(datetime.date.today())
# If the Publish Date has not yet been set, then set it
if data_dict['edc_state'] == 'PUBLISHED' and not data_dict.get(
'record_publish_date'):
data_dict['record_publish_date'] = str(datetime.date.today())
# If the Archive Date has not yet been set, then set it
if data_dict['edc_state'] == 'ARCHIVED' and not data_dict.get(
'record_archive_date'):
data_dict['record_archive_date'] = str(datetime.date.today())
_check_access('package_update', context, data_dict)
# get the schema
package_plugin = lib_plugins.lookup_package_plugin(pkg.type)
if 'schema' in context:
schema = context['schema']
else:
schema = package_plugin.update_package_schema()
image_url = old_data.get('image_url', None)
upload = uploader.Upload('edc', image_url)
upload.update_data_dict(data_dict, 'image_url', 'image_upload',
'clear_upload')
# Adding image display url for the uploaded image
image_url = data_dict.get('image_url')
data_dict['image_display_url'] = image_url
if image_url and not image_url.startswith('http'):
image_url = munge.munge_filename(image_url)
data_dict['image_display_url'] = h.url_for_static(
'uploads/edc/%s' % data_dict.get('image_url'), qualified=True)
if 'api_version' not in context:
# check_data_dict() is deprecated. If the package_plugin has a
# check_data_dict() we'll call it, if it doesn't have the method we'll
# do nothing.
check_data_dict = getattr(package_plugin, 'check_data_dict', None)
if check_data_dict:
try:
package_plugin.check_data_dict(data_dict, schema)
except TypeError:
# Old plugins do not support passing the schema so we need
# to ensure they still work.
package_plugin.check_data_dict(data_dict)
# FIXME: modifications to package_update end here^
data, errors = lib_plugins.plugin_validate(package_plugin, context,
data_dict, schema,
'package_update')
log.debug('package_update validate_errs=%r user=%s package=%s data=%r',
errors, context.get('user'),
context.get('package').name if context.get('package') else '',
data)
if errors:
model.Session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
# avoid revisioning by updating directly
model.Session.query(model.Package).filter_by(id=pkg.id).update(
{"metadata_modified": datetime.datetime.utcnow()})
model.Session.refresh(pkg)
pkg = model_save.package_dict_save(data, context)
context_org_update = context.copy()
context_org_update['ignore_auth'] = True
context_org_update['defer_commit'] = True
_get_action('package_owner_org_update')(context_org_update, {
'id': pkg.id,
'organization_id': pkg.owner_org
})
for item in plugins.PluginImplementations(plugins.IPackageController):
item.edit(pkg)
item.after_update(context, data)
upload.upload(uploader.get_max_image_size())
# TODO the next two blocks are copied from ckan/ckan/logic/action/update.py
# This codebase is currently hard to maintain because large chunks of the
# CKAN action API and the CKAN controllers are simply overriden. This is
# probably worse than just forking CKAN would have been, because in that
# case at least we could track changes. - @deniszgonjanin
# Needed to let extensions know the new resources ids
model.Session.flush()
if data.get('resources'):
for index, resource in enumerate(data['resources']):
resource['id'] = pkg.resources[index].id
# Create default views for resources if necessary
if data.get('resources'):
logic.get_action('package_create_default_resource_views')(
{
'model': context['model'],
'user': context['user'],
'ignore_auth': True
}, {
'package': data
})
if not context.get('defer_commit'):
model.repo.commit()
log.debug('Updated object %s' % pkg.name)
return_id_only = context.get('return_id_only', False)
# Make sure that a user provided schema is not used on package_show
context.pop('schema', None)
# we could update the dataset so we should still be able to read it.
context['ignore_auth'] = True
output = data_dict['id'] if return_id_only \
else _get_action('package_show')(context, {'id': data_dict['id']})
'''
Send state change notifications if required; Added by Khalegh Mamakani
Using a thread to run the job in the background so that package_update will not wait for notifications sending.
'''
old_state = old_data.get('edc_state')
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
dataset_url = config.get('ckan.site_url') + h.url_for(
controller='package', action="read", id=data_dict['name'])
import threading
notify_thread = threading.Thread(target=check_record_state,
args=(context, old_state, data_dict,
g.site_title, g.site_url,
dataset_url))
notify_thread.start()
return output
def _group_or_org_update(context, data_dict, is_org=False):
model = context['model']
user = context['user']
session = context['session']
id = _get_or_bust(data_dict, 'id')
group = model.Group.get(id)
context["group"] = group
if group is None:
raise NotFound('Group was not found.')
# get the schema
group_plugin = lib_plugins.lookup_group_plugin(group.type)
try:
schema = group_plugin.form_to_db_schema_options({'type':'update',
'api':'api_version' in context,
'context': context})
except AttributeError:
schema = group_plugin.form_to_db_schema()
upload = uploader.Upload('group', group.image_url)
upload.update_data_dict(data_dict, 'image_url',
'image_upload', 'clear_upload')
if is_org:
_check_access('organization_update', context, data_dict)
else:
_check_access('group_update', context, data_dict)
if 'api_version' not in context:
# old plugins do not support passing the schema so we need
# to ensure they still work
try:
group_plugin.check_data_dict(data_dict, schema)
except TypeError:
group_plugin.check_data_dict(data_dict)
data, errors = _validate(data_dict, schema, context)
log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r',
errors, context.get('user'),
context.get('group').name if context.get('group') else '',
data_dict)
if errors:
session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
# when editing an org we do not want to update the packages if using the
# new templates.
if ((not is_org)
and not converters.asbool(
config.get('ckan.legacy_templates', False))
and 'api_version' not in context):
context['prevent_packages_update'] = True
group = model_save.group_dict_save(data, context)
if is_org:
plugin_type = plugins.IOrganizationController
else:
plugin_type = plugins.IGroupController
for item in plugins.PluginImplementations(plugin_type):
item.edit(group)
if is_org:
activity_type = 'changed organization'
else:
activity_type = 'changed group'
activity_dict = {
'user_id': model.User.by_name(user.decode('utf8')).id,
'object_id': group.id,
'activity_type': activity_type,
}
# Handle 'deleted' groups.
# When the user marks a group as deleted this comes through here as
# a 'changed' group activity. We detect this and change it to a 'deleted'
# activity.
if group.state == u'deleted':
if session.query(ckan.model.Activity).filter_by(
object_id=group.id, activity_type='deleted').all():
# A 'deleted group' activity for this group has already been
# emitted.
# FIXME: What if the group was deleted and then activated again?
activity_dict = None
else:
# We will emit a 'deleted group' activity.
activity_dict['activity_type'] = 'deleted group'
if activity_dict is not None:
activity_dict['data'] = {
'group': dictization.table_dictize(group, context)
}
activity_create_context = {
'model': model,
'user': user,
'defer_commit': True,
'ignore_auth': True,
'session': session
}
_get_action('activity_create')(activity_create_context, activity_dict)
# TODO: Also create an activity detail recording what exactly changed
# in the group.
upload.upload(uploader.get_max_image_size())
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.group_dictize(group, context)
def _group_or_org_create(context, data_dict, is_org=False):
model = context['model']
user = context['user']
session = context['session']
data_dict['is_organization'] = is_org
upload = uploader.Upload('group')
upload.update_data_dict(data_dict, 'image_url', 'image_upload',
'clear_upload')
# get the schema
group_plugin = lib_plugins.lookup_group_plugin(
group_type=data_dict.get('type'))
try:
schema = group_plugin.form_to_db_schema_options({
'type':
'create',
'api':
'api_version' in context,
'context':
context
})
except AttributeError:
schema = group_plugin.form_to_db_schema()
if 'api_version' not in context:
# old plugins do not support passing the schema so we need
# to ensure they still work
try:
group_plugin.check_data_dict(data_dict, schema)
except TypeError:
group_plugin.check_data_dict(data_dict)
data, errors = _validate(data_dict, schema, context)
log.debug('group_create validate_errs=%r user=%s group=%s data_dict=%r',
errors, context.get('user'), data_dict.get('name'), data_dict)
if errors:
session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Create object %s') % data.get("name")
group = model_save.group_dict_save(data, context)
if user:
admins = [model.User.by_name(user.decode('utf8'))]
else:
admins = []
model.setup_default_user_roles(group, admins)
# Needed to let extensions know the group id
session.flush()
if is_org:
plugin_type = plugins.IOrganizationController
else:
plugin_type = plugins.IGroupController
for item in plugins.PluginImplementations(plugin_type):
item.create(group)
if is_org:
activity_type = 'new organization'
else:
activity_type = 'new group'
user_id = model.User.by_name(user.decode('utf8')).id
activity_dict = {
'user_id': user_id,
'object_id': group.id,
'activity_type': activity_type,
}
activity_dict['data'] = {
'group': ckan.lib.dictization.table_dictize(group, context)
}
activity_create_context = {
'model': model,
'user': user,
'defer_commit': True,
'ignore_auth': True,
'session': session
}
logic.get_action('activity_create')(activity_create_context, activity_dict)
upload.upload(uploader.get_max_image_size())
if not context.get('defer_commit'):
model.repo.commit()
context["group"] = group
context["id"] = group.id
# creator of group/org becomes an admin
# this needs to be after the repo.commit or else revisions break
member_dict = {
'id': group.id,
'object': user_id,
'object_type': 'user',
'capacity': 'admin',
}
member_create_context = {
'model': model,
'user': user,
'ignore_auth': True, # we are not a member of the group at this point
'session': session
}
logic.get_action('member_create')(member_create_context, member_dict)
log.debug('Created object %s' % group.name)
return model_dictize.group_dictize(group, context)
def action_user_update(context, data_dict):
''' Modified from CKAN: user_update
Update a user account.
Normal users can only update their own user accounts. Sysadmins can update
any user account.
For further parameters see ``user_create()``.
:param id: the name or id of the user to update
:type id: string
:returns: the updated user account
:rtype: dictionary
'''
model = context['model']
user = context['user']
session = context['session']
schema = context.get('schema') or logic.schema.default_update_user_schema()
# Modify the schema by adding translation related keys
add_translation_modify_schema(schema)
upload = uploader.Upload('user')
upload.update_data_dict(data_dict, 'image_url', 'image_upload',
'clear_upload')
ignore_missing = toolkit.get_validator('ignore_missing')
convert_to_extras = toolkit.get_converter('convert_to_extras')
schema['job_title'] = [ignore_missing, unicode, convert_to_extras]
schema['telephone_number'] = [ignore_missing, unicode, convert_to_extras]
schema['main_organization'] = [ignore_missing, unicode, convert_to_extras]
schema['image_url'] = [ignore_missing, unicode, convert_to_extras]
schema['linkedin'] = [ignore_missing, unicode, convert_to_extras]
schema['facebook'] = [ignore_missing, unicode, convert_to_extras]
schema['twitter'] = [ignore_missing, unicode, convert_to_extras]
schema['blog'] = [ignore_missing, to_list_json, convert_to_extras]
schema['www_page'] = [ignore_missing, to_list_json, convert_to_extras]
# Add the localized keys for the localized fields to the schema
schema = add_languages_modify(schema, _localized_fields)
not_empty = toolkit.get_validator('not_empty')
schema['fullname'] = [not_empty, unicode]
id = logic.get_or_bust(data_dict, 'id')
user_obj = model.User.get(id)
context['user_obj'] = user_obj
if user_obj is None:
raise NotFound('User was not found.')
# If the translations are not in the data_dict, the user has not added any translations or the user has deleted all translations.
# Therefore, the translations are not sent with the POST so we need to empty and update the translations here.
if 'translations' not in data_dict:
data_dict['translations'] = []
toolkit.check_access('user_update', context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
session.rollback()
raise ValidationError(errors)
for extra in data['extras'] if 'extras' in data else []:
user_obj.extras[extra['key']] = extra['value']
user = model_save.user_dict_save(data, context)
activity_dict = {
'user_id': user.id,
'object_id': user.id,
'activity_type': 'changed user'
}
activity_create_context = {
'model': model,
'user': user,
'defer_commit': True,
'ignore_auth': True,
'session': session
}
toolkit.get_action('activity_create')(activity_create_context,
activity_dict)
# Attempt to update drupal user
_update_drupal_user(context, data_dict)
# TODO: Also create an activity detail recording what exactly changed in
# the user.
upload.upload(uploader.get_max_image_size())
if not context.get('defer_commit'):
model.repo.commit()
user_data = user_dictize(user, context)
for key, value in user.extras.iteritems():
if key in user_data:
log.warning(
"Trying to override user data with extra variable '%s'", key)
continue
user_data[key] = value
return user_data
def logic_action_update_package_update(context, data_dict):
'''Update a dataset (package).
You must be authorized to edit the dataset and the groups that it belongs
to.
It is recommended to call
:py:func:`ckan.logic.action.get.package_show`, make the desired changes to
the result, and then call ``package_update()`` with it.
Plugins may change the parameters of this function depending on the value
of the dataset's ``type`` attribute, see the
:py:class:`~ckan.plugins.interfaces.IDatasetForm` plugin interface.
For further parameters see
:py:func:`~ckan.logic.action.create.package_create`.
:param id: the name or id of the dataset to update
:type id: string
:returns: the updated dataset (if ``'return_package_dict'`` is ``True`` in
the context, which is the default. Otherwise returns just the
dataset id)
:rtype: dictionary
'''
import ckan.lib.plugins as lib_plugins
import datetime
import ckan.lib.dictization.model_save as model_save
import ckan.plugins as plugins
import ckan.lib.uploader as uploader
_check_access = ckan.logic.check_access
model = context['model']
user = context['user']
name_or_id = data_dict.get("id") or data_dict['name']
pkg = model.Package.get(name_or_id)
if pkg is None:
raise NotFound(_('Package was not found.'))
context["package"] = pkg
data_dict["id"] = pkg.id
data_dict['type'] = pkg.type
upload = uploader.Upload('package', '')
upload.update_data_dict(data_dict, 'image_url', 'image_upload',
'clear_upload')
_check_access('package_update', context, data_dict)
# get the schema
package_plugin = lib_plugins.lookup_package_plugin(pkg.type)
if 'schema' in context:
schema = context.pop('schema')
else:
schema = package_plugin.update_package_schema()
if 'api_version' not in context:
# check_data_dict() is deprecated. If the package_plugin has a
# check_data_dict() we'll call it, if it doesn't have the method we'll
# do nothing.
check_data_dict = getattr(package_plugin, 'check_data_dict', None)
if check_data_dict:
try:
package_plugin.check_data_dict(data_dict, schema)
except TypeError:
# Old plugins do not support passing the schema so we need
# to ensure they still work.
package_plugin.check_data_dict(data_dict)
data, errors = lib_plugins.plugin_validate(package_plugin, context,
data_dict, schema,
'package_update')
log.debug('package_update validate_errs=%r user=%s package=%s data=%r',
errors, context.get('user'),
context.get('package').name if context.get('package') else '',
data)
if errors:
model.Session.rollback()
raise ckan.logic.ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
#avoid revisioning by updating directly
model.Session.query(model.Package).filter_by(id=pkg.id).update(
{"metadata_modified": datetime.datetime.utcnow()})
model.Session.refresh(pkg)
upload.upload(uploader.get_max_image_size())
generateThumbs(upload.filepath)
pkg = model_save.package_dict_save(data, context)
context_org_update = context.copy()
context_org_update['ignore_auth'] = True
context_org_update['defer_commit'] = True
ckan.logic.get_action('package_owner_org_update')(
context_org_update, {
'id': pkg.id,
'organization_id': pkg.owner_org
})
# Needed to let extensions know the new resources ids
model.Session.flush()
if data.get('resources'):
for index, resource in enumerate(data['resources']):
resource['id'] = pkg.resources[index].id
for item in plugins.PluginImplementations(plugins.IPackageController):
item.edit(pkg)
item.after_update(context, data)
# Create default views for resources if necessary
if data.get('resources'):
ckan.logic.get_action('package_create_default_resource_views')(
{
'model': context['model'],
'user': context['user'],
'ignore_auth': True
}, {
'package': data
})
if not context.get('defer_commit'):
model.repo.commit()
log.debug('Updated object %s' % pkg.name)
return_id_only = context.get('return_id_only', False)
# Make sure that a user provided schema is not used on package_show
context.pop('schema', None)
# we could update the dataset so we should still be able to read it.
context['ignore_auth'] = True
output = data_dict['id'] if return_id_only \
else ckan.logic.get_action('package_show')(context, {'id': data_dict['id']})
return output
def logic_action_create_package_create(context, data_dict):
'''Create a new dataset (package).
You must be authorized to create new datasets. If you specify any groups
for the new dataset, you must also be authorized to edit these groups.
Plugins may change the parameters of this function depending on the value
of the ``type`` parameter, see the
:py:class:`~ckan.plugins.interfaces.IDatasetForm` plugin interface.
:param name: the name of the new dataset, must be between 2 and 100
characters long and contain only lowercase alphanumeric characters,
``-`` and ``_``, e.g. ``'warandpeace'``
:type name: string
:param title: the title of the dataset (optional, default: same as
``name``)
:type title: string
:param author: the name of the dataset's author (optional)
:type author: string
:param author_email: the email address of the dataset's author (optional)
:type author_email: string
:param maintainer: the name of the dataset's maintainer (optional)
:type maintainer: string
:param maintainer_email: the email address of the dataset's maintainer
(optional)
:type maintainer_email: string
:param license_id: the id of the dataset's license, see
:py:func:`~ckan.logic.action.get.license_list` for available values
(optional)
:type license_id: license id string
:param notes: a description of the dataset (optional)
:type notes: string
:param url: a URL for the dataset's source (optional)
:type url: string
:param version: (optional)
:type version: string, no longer than 100 characters
:param state: the current state of the dataset, e.g. ``'active'`` or
``'deleted'``, only active datasets show up in search results and
other lists of datasets, this parameter will be ignored if you are not
authorized to change the state of the dataset (optional, default:
``'active'``)
:type state: string
:param type: the type of the dataset (optional),
:py:class:`~ckan.plugins.interfaces.IDatasetForm` plugins
associate themselves with different dataset types and provide custom
dataset handling behaviour for these types
:type type: string
:param resources: the dataset's resources, see
:py:func:`resource_create` for the format of resource dictionaries
(optional)
:type resources: list of resource dictionaries
:param tags: the dataset's tags, see :py:func:`tag_create` for the format
of tag dictionaries (optional)
:type tags: list of tag dictionaries
:param extras: the dataset's extras (optional), extras are arbitrary
(key: value) metadata items that can be added to datasets, each extra
dictionary should have keys ``'key'`` (a string), ``'value'`` (a
string)
:type extras: list of dataset extra dictionaries
:param relationships_as_object: see :py:func:`package_relationship_create`
for the format of relationship dictionaries (optional)
:type relationships_as_object: list of relationship dictionaries
:param relationships_as_subject: see :py:func:`package_relationship_create`
for the format of relationship dictionaries (optional)
:type relationships_as_subject: list of relationship dictionaries
:param groups: the groups to which the dataset belongs (optional), each
group dictionary should have one or more of the following keys which
identify an existing group:
``'id'`` (the id of the group, string), or ``'name'`` (the name of the
group, string), to see which groups exist
call :py:func:`~ckan.logic.action.get.group_list`
:type groups: list of dictionaries
:param owner_org: the id of the dataset's owning organization, see
:py:func:`~ckan.logic.action.get.organization_list` or
:py:func:`~ckan.logic.action.get.organization_list_for_user` for
available values (optional)
:type owner_org: string
:returns: the newly created dataset (unless 'return_id_only' is set to True
in the context, in which case just the dataset id will
be returned)
:rtype: dictionary
'''
import ckan.lib.plugins as lib_plugins
import datetime
import ckan.lib.dictization.model_save as model_save
import ckan.plugins as plugins
import ckan.lib.uploader as uploader
_check_access = ckan.logic.check_access
model = context['model']
user = context['user']
if 'type' not in data_dict:
package_plugin = lib_plugins.lookup_package_plugin()
try:
# use first type as default if user didn't provide type
package_type = package_plugin.package_types()[0]
except (AttributeError, IndexError):
package_type = 'dataset'
# in case a 'dataset' plugin was registered w/o fallback
package_plugin = lib_plugins.lookup_package_plugin(package_type)
data_dict['type'] = package_type
else:
package_plugin = lib_plugins.lookup_package_plugin(data_dict['type'])
if 'schema' in context:
schema = context['schema']
else:
schema = package_plugin.create_package_schema()
upload = uploader.Upload('package', '')
upload.update_data_dict(data_dict, 'image_url', 'image_upload',
'clear_upload')
_check_access('package_create', context, data_dict)
if 'api_version' not in context:
# check_data_dict() is deprecated. If the package_plugin has a
# check_data_dict() we'll call it, if it doesn't have the method we'll
# do nothing.
check_data_dict = getattr(package_plugin, 'check_data_dict', None)
if check_data_dict:
try:
check_data_dict(data_dict, schema)
except TypeError:
# Old plugins do not support passing the schema so we need
# to ensure they still work
package_plugin.check_data_dict(data_dict)
data, errors = lib_plugins.plugin_validate(package_plugin, context,
data_dict, schema,
'package_create')
log.debug('package_create validate_errs=%r user=%s package=%s data=%r',
errors, context.get('user'), data.get('name'), data_dict)
if errors:
model.Session.rollback()
raise ckan.logic.ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Create object %s') % data.get("name")
admins = []
if user:
user_obj = model.User.by_name(user.decode('utf8'))
if user_obj:
admins = [user_obj]
data['creator_user_id'] = user_obj.id
upload.upload(uploader.get_max_image_size())
generateThumbs(upload.filepath)
if 'raw_image_url' in data_dict:
data['extras'].append({
'key': 'image_url',
'value': data_dict['raw_image_url']
})
generateThumbs(
config.get('ckan.storage_path') + '/storage/uploads/package/' +
data_dict['raw_image_url'])
pkg = model_save.package_dict_save(data, context)
model.setup_default_user_roles(pkg, admins)
# Needed to let extensions know the package and resources ids
model.Session.flush()
data['id'] = pkg.id
if data.get('resources'):
for index, resource in enumerate(data['resources']):
resource['id'] = pkg.resources[index].id
context_org_update = context.copy()
context_org_update['ignore_auth'] = True
context_org_update['defer_commit'] = True
ckan.logic.get_action('package_owner_org_update')(
context_org_update, {
'id': pkg.id,
'organization_id': pkg.owner_org
})
for item in plugins.PluginImplementations(plugins.IPackageController):
item.create(pkg)
item.after_create(context, data)
# Make sure that a user provided schema is not used in create_views
# and on package_show
context.pop('schema', None)
# Create default views for resources if necessary
if data.get('resources'):
ckan.logic.get_action('package_create_default_resource_views')(
{
'model': context['model'],
'user': context['user'],
'ignore_auth': True
}, {
'package': data
})
if not context.get('defer_commit'):
model.repo.commit()
## need to let rest api create
context["package"] = pkg
## this is added so that the rest controller can make a new location
context["id"] = pkg.id
log.debug('Created object %s' % pkg.name)
return_id_only = context.get('return_id_only', False)
output = context['id'] if return_id_only \
else ckan.logic.get_action('package_show')(context, {'id': context['id']})
return output
def hdx_group_or_org_update(context, data_dict, is_org=False):
# Overriding default so that orgs can have multiple images
model = context['model']
user = context['user']
session = context['session']
id = _get_or_bust(data_dict, 'id')
group = model.Group.get(id)
context["group"] = group
if group is None:
raise NotFound('Group was not found.')
if is_org:
check_access('organization_update', context, data_dict)
else:
check_access('group_update', context, data_dict)
# get the schema
group_plugin = lib_plugins.lookup_group_plugin(group.type)
try:
schema = group_plugin.form_to_db_schema_options({'type': 'update',
'api': 'api_version' in context,
'context': context})
except AttributeError:
schema = group_plugin.form_to_db_schema()
try:
customization = json.loads(group.extras['customization'])
except:
customization = {'image_sq': '', 'image_rect': ''}
try:
data_dict['customization'] = json.loads(data_dict['customization'])
except:
data_dict['customization'] = {}
# If we're removing the image
if 'clear_image_sq' in data_dict and data_dict['clear_image_sq']:
remove_image(customization['image_sq'])
data_dict['customization']['image_sq'] = ''
customization['image_rect'] = ''
if 'clear_image_rect' in data_dict and data_dict['clear_image_rect']:
remove_image(customization['image_rect'])
data_dict['customization']['image_rect'] = ''
customization['image_rect'] = ''
if 'image_sq_upload' in data_dict and data_dict['image_sq_upload'] != '' and data_dict['image_sq_upload'] != None:
# Although weird, the FieldStorage instance has a boolean value of False so we need to compare to None
# If old image was uploaded remove it
if customization['image_sq']:
remove_image(customization['image_sq'])
upload1 = uploader.Upload('group', customization['image_sq'])
upload1.update_data_dict(data_dict, 'image_sq',
'image_sq_upload', 'clear_upload')
if 'image_rect_upload' in data_dict and data_dict['image_rect_upload'] != '' \
and data_dict['image_rect_upload'] != None:
# Although weird, the FieldStorage instance has a boolean value of False so we need to compare to None
if customization['image_rect']:
remove_image(customization['image_rect'])
upload2 = uploader.Upload('group', customization['image_rect'])
upload2.update_data_dict(data_dict, 'image_rect',
'image_rect_upload', 'clear_upload')
storage_path = uploader.get_storage_path()
##Rearrange things the way we need them
try:
if data_dict['image_sq'] != '' and data_dict['image_sq'] != None:
data_dict['customization']['image_sq'] = data_dict['image_sq']
else:
data_dict['customization']['image_sq'] = customization['image_sq']
except KeyError:
data_dict['customization']['image_sq'] = ''
try:
if data_dict['image_rect'] != '' and data_dict['image_rect'] != None:
data_dict['customization']['image_rect'] = data_dict['image_rect']
else:
data_dict['customization']['image_rect'] = customization['image_rect']
except KeyError:
data_dict['customization']['image_rect'] = ''
data_dict['customization'] = json.dumps(data_dict['customization'])
if 'api_version' not in context:
# old plugins do not support passing the schema so we need
# to ensure they still work
try:
group_plugin.check_data_dict(data_dict, schema)
except TypeError:
group_plugin.check_data_dict(data_dict)
data, errors = _validate(data_dict, schema, context)
log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r',
errors, context.get('user'),
context.get('group').name if context.get('group') else '',
data_dict)
if errors:
session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
# when editing an org we do not want to update the packages if using the
# new templates.
if ((not is_org)
and not converters.asbool(
config.get('ckan.legacy_templates', False))
and 'api_version' not in context):
context['prevent_packages_update'] = True
group = model_save.group_dict_save(data, context)
if is_org:
plugin_type = plugins.IOrganizationController
else:
plugin_type = plugins.IGroupController
for item in plugins.PluginImplementations(plugin_type):
item.edit(group)
if is_org:
activity_type = 'changed organization'
else:
activity_type = 'changed group'
activity_dict = {
'user_id': model.User.by_name(user.decode('utf8')).id,
'object_id': group.id,
'activity_type': activity_type,
}
# Handle 'deleted' groups.
# When the user marks a group as deleted this comes through here as
# a 'changed' group activity. We detect this and change it to a 'deleted'
# activity.
if group.state == u'deleted':
if session.query(ckan.model.Activity).filter_by(
object_id=group.id, activity_type='deleted').all():
# A 'deleted group' activity for this group has already been
# emitted.
# FIXME: What if the group was deleted and then activated again?
activity_dict = None
else:
# We will emit a 'deleted group' activity.
activity_dict['activity_type'] = 'deleted group'
if activity_dict is not None:
activity_dict['data'] = {
'group': dictization.table_dictize(group, context)
}
activity_create_context = {
'model': model,
'user': user,
'defer_commit': True,
'ignore_auth': True,
'session': session
}
get_action('activity_create')(activity_create_context, activity_dict)
# TODO: Also create an activity detail recording what exactly changed
# in the group.
try:
upload1.upload(uploader.get_max_image_size())
except:
pass
try:
upload2.upload(uploader.get_max_image_size())
except:
pass
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.group_dictize(group, context)