def after_update(self, context, pkg_dict):
'''after_update - uploads package zipfile to s3'''
# Obtain logger
logger = logging.getLogger(__name__)
# Check context object
# If originating from resource create or update, skip package zipfile
# upload for now
# For more information, read documentation in 'before_create_or_update'
if 'resource_create_or_update' not in context:
logger.info(
"Package after_update without originating from resource create/update"
)
# Check if required config options exist
if not upload.config_exists():
# Log an error
logger.error(
"Required S3 config options missing. Please check if required config options exist."
)
raise Exception('Required S3 config options missing')
else:
upload.upload_package_zipfile_to_s3(context, pkg_dict)
else:
# Skip package_zipfile upload
logger.info(
"Package after_update originating from resource create/update... Skipping package zipfile upload"
)
def after_create_or_update(self, context, resource):
'''Uploads resource zip file to S3
Done after create/update instead of before to ensure metadata is generated correctly'''
if 'upload' not in resource: return
upload.upload_resource_zipfile_to_s3(context, resource)
# Remove 'resource_create_or_update' in context. See documentation in 'before_create_or_update'
# for more details
if 'resource_create_or_update' in context and upload.config_exists() and resource['upload'] != '':
context.pop('resource_create_or_update')
pkg = plugins.toolkit.get_action('package_show')(data_dict={'id': resource['package_id']})
upload.upload_package_zipfile_to_s3(context, pkg)
def before_create_or_update(self, context, resource):
'''before_create_or_update - our own function. NOT a CKAN hook.
Contains shared code performed regardless of whether we are
creating or updating.
'''
# Check if required config options exist
if not upload.config_exists():
# Log an error
logger = logging.getLogger(__name__)
logger.error(
"Required S3 config options missing. Please check if required config options exist."
)
raise Exception('Required S3 config options missing')
else:
# If resource is an API, don't do anything special
if resource.get('format') == 'API':
return
# Only upload to S3 if not blacklisted
elif not upload.is_blacklisted(resource):
upload.upload_resource_to_s3(context, resource)
else:
# If blacklisted, the resource file is uploaded to CKAN.
#
# However, in the CKAN source resource_create/resource_update, package_update is
# called before the file is uploaded.
#
# This causes a problem as our package after_update attempts to upload
# the package zipfile and it cannot locate the resource file.
#
# To solve this, we add the field 'resource_create_or_update' into the context object,
# and look for it in the package after_update.
#
# We remove this field from the context object in resource after_create and after_update.
#
# We don't actually use the value context['resource_create_or_update'], we just check
# the existence of 'resource_create_or_update' in context.
context['resource_create_or_update'] = True
logger = logging.getLogger(__name__)
logger.info(
"Resource %s from package %s is blacklisted and not uploaded to S3."
% (resource['name'], resource['package_id']))