def call_action(self, action, data_dict=None, context=None, apikey=None,
files=None, requests_kwargs=None):
"""
:param action: the action name, e.g. 'package_create'
:param data_dict: the dict to pass to the action as JSON,
defaults to {}
:param context: always set to None for RemoteCKAN
:param apikey: API key for authentication
:param files: None or {field-name: file-to-be-sent, ...}
This function parses the response from the server as JSON and
returns the decoded value. When an error is returned this
function will convert it back to an exception that matches the
one the action function itself raised.
"""
if context:
raise CKANAPIError("RemoteCKAN.call_action does not support "
"use of context parameter, use apikey instead")
if files and self.get_only:
raise CKANAPIError("RemoteCKAN: files may not be sent when "
"get_only is True")
url, data, headers = prepare_action(
action, data_dict, apikey or self.apikey, files)
headers['User-Agent'] = self.user_agent
url = self.address.rstrip('/') + '/' + url
requests_kwargs = requests_kwargs or {}
if self.get_only:
status, response = self._request_fn_get(url, data_dict, headers, requests_kwargs)
else:
status, response = self._request_fn(url, data, headers, files, requests_kwargs)
return reverse_apicontroller_action(url, status, response)
def call_action(self,
action,
data_dict=None,
context=None,
apikey=None,
files=None,
requests_kwargs=None,
progress=None):
"""
:param action: the action name, e.g. 'package_create'
:param data_dict: the dict to pass to the action as JSON,
defaults to {}
:param context: always set to None for RemoteCKAN
:param apikey: API key for authentication
:param files: None or {field-name: file-to-be-sent, ...}
:param progress: A callable that takes an instance of
:class:`requests_toolbelt.MultipartEncoder` as parameter and returns
a callback funtion. The callback function will be called every time
data is read from the file-to-be-sent and it will be passed the
instance of :class:`requests_toolbelt.MultipartEncoderMonitor`. This
monitor has the attribute `bytes_read` that can be used to display
a progress bar. An example is implemented in ckanapi.cli.
This function parses the response from the server as JSON and
returns the decoded value. When an error is returned this
function will convert it back to an exception that matches the
one the action function itself raised.
"""
if context:
raise CKANAPIError("RemoteCKAN.call_action does not support "
"use of context parameter, use apikey instead")
if files and self.get_only:
raise CKANAPIError("RemoteCKAN: files may not be sent when "
"get_only is True")
url, data, headers = prepare_action(action, data_dict, apikey
or self.apikey, files)
headers['User-Agent'] = self.user_agent
url = self.address.rstrip('/') + '/' + url
requests_kwargs = requests_kwargs or {}
if not self.session:
self.session = requests.Session()
if self.get_only:
status, response = self._request_fn_get(url, data_dict, headers,
requests_kwargs)
else:
status, response = self._request_fn(url, data, headers, files,
requests_kwargs, progress)
return reverse_apicontroller_action(url, status, response)
def test_restore_catalog_failing_destination_portal(
self, mock_action, mock_push_thm, mock_push_dst):
identifiers = [ds['identifier'] for ds in self.catalog.datasets]
mock_action.return_value.organization_list.return_value = \
['org_1', 'org_2']
mock_action.return_value.organization_show.side_effect = [
{
'packages': [{
'id': identifiers[0]
}]
},
{
'packages': [{
'id': identifiers[1]
}]
},
]
mock_push_dst.side_effect = CKANAPIError('Broken destination portal')
pushed = restore_catalog_to_ckan(self.catalog, 'origin', 'destination',
'apikey')
mock_push_dst.assert_any_call(self.catalog, 'org_1', identifiers[0],
'destination', 'apikey', None, False,
False, None, None)
mock_push_dst.assert_any_call(self.catalog, 'org_2', identifiers[1],
'destination', 'apikey', None, False,
False, None, None)
expected = {'org_1': [], 'org_2': []}
self.assertDictEqual(expected, pushed)
def _request_fn(self, url, data, headers, files, requests_kwargs,
progress):
if files: # use streaming
newfiles = dict([(k, (getattr(files[k], 'name',
'upload_filename'), files[k]))
for k in files])
intersect = set(data.keys()) & set(newfiles.keys())
if intersect:
raise CKANAPIError('field-name for files ("{}")'.format(
', '.join(list(intersect))) +
' cannot also be field name in data_dict.')
data.update(newfiles)
m = MultipartEncoder(data)
if progress:
m = MultipartEncoderMonitor(m, progress(m))
headers.update({'Content-Type': m.content_type})
r = self.session.post(url,
data=m,
headers=headers,
allow_redirects=False,
**requests_kwargs)
else:
r = self.session.post(url,
data=data,
headers=headers,
files=files,
allow_redirects=False,
**requests_kwargs)
# allow_redirects=False because: if a post is redirected (e.g. 301 due
# to a http to https redirect), then the second request is made to the
# new URL, but *without* the data. This gives a confusing "No request
# body data" error. It is better to just return the 301 to the user, so
# we disallow redirects.
return r.status_code, r.text
def test_restore_failing_organization_to_ckan(self, mock_push_thm,
mock_push_dst):
# Continua subiendo el segundo dataset a pesar que el primero falla
effects = [
CKANAPIError('broken dataset'),
self.catalog.datasets[1]['identifier']
]
mock_push_dst.side_effect = effects
identifiers = [ds['identifier'] for ds in self.catalog.datasets]
pushed = restore_organization_to_ckan(self.catalog, 'owner_org',
'portal', 'apikey', identifiers)
self.assertEqual([identifiers[1]], pushed)
mock_push_thm.assert_called_with(self.catalog, 'portal', 'apikey')
mock_push_dst.assert_called_with(self.catalog,
'owner_org',
identifiers[1],
'portal',
'apikey',
catalog_id=None,
demote_superThemes=False,
demote_themes=False,
download_strategy=None,
generate_new_access_url=None,
origin_tz=DEFAULT_TIMEZONE,
dst_tz=DEFAULT_TIMEZONE)
def reverse_apicontroller_action(url, status, response):
"""
Make an API call look like a direct action call by reversing the
exception -> HTTP response translation that ApiController.action does
"""
try:
parsed = json.loads(response)
if parsed.get('success'):
return parsed['result']
if hasattr(parsed, 'get'):
err = parsed.get('error', {})
else:
err = {}
except (AttributeError, ValueError):
err = {}
etype = err.get('__type')
emessage = err.get('message', '').split(': ', 1)[-1]
if etype == 'Search Query Error':
# I refuse to eval(emessage), even if it would be more correct
raise SearchQueryError(emessage)
elif etype == 'Search Error':
# I refuse to eval(emessage), even if it would be more correct
raise SearchError(emessage)
elif etype == 'Search Index Error':
raise SearchIndexError(emessage)
elif etype == 'Validation Error':
raise ValidationError(err)
elif etype == 'Not Found Error':
raise NotFound(emessage)
elif etype == 'Authorization Error':
raise NotAuthorized(err)
# don't recognize the error
raise CKANAPIError(repr([url, status, response]))
def test_resource_upload_error(self, mock_portal):
mock_portal.return_value.action.resource_patch = MagicMock(
side_effect=CKANAPIError('broken resource'))
resources = {self.distribution_id: 'tests/samples/resource_sample.csv'}
res = resources_update('portal', 'key', self.catalog.distributions,
resources)
mock_portal.return_value.action.resource_patch.assert_called_with(
id=self.distribution_id, resource_type='file.upload', upload=ANY)
self.assertEqual([], res)
def test_restore_catalog_failing_origin_portal(self, mock_action,
mock_push_thm,
mock_push_dst):
mock_action.return_value.organization_list.side_effect = \
CKANAPIError('Broken origin portal')
pushed = restore_catalog_to_ckan(self.catalog, 'origin', 'destination',
'apikey')
self.assertDictEqual({}, pushed)
mock_push_thm.assert_not_called()
mock_push_dst.assert_not_called()
def test_resource_upload_error(self, mock_portal):
mock_portal.return_value.action.resource_patch = MagicMock(
side_effect=CKANAPIError('broken resource'))
resources = {self.distribution_id: 'tests/samples/resource_sample.csv'}
res = resources_update('portal', 'key', self.catalog.distributions,
resources)
_, _, kwargs = \
mock_portal.return_value.action.resource_patch.mock_calls[0]
self.assertEqual(self.distribution_id, kwargs['id'])
self.assertEqual('Convocatorias abiertas durante el año 2015',
kwargs['name'])
self.assertEqual('file.upload', kwargs['resource_type'])
self.assertEqual([], res)
def call_action(self, action, data_dict=None, context=None, apikey=None,
files=None):
"""
:param action: the action name, e.g. 'package_create'
:param data_dict: the dict to pass to the action, defaults to {}
:param context: an override for the context to use for this action,
remember to include a 'user' when necessary
:param apikey: not supported
:param files: not supported
"""
if not data_dict:
data_dict = []
if context is None:
context = self.context
if apikey:
# FIXME: allow use of apikey to set a user in context?
raise CKANAPIError("LocalCKAN.call_action does not support "
"use of apikey parameter, use context['user'] instead")
if files:
raise CKANAPIError("TestAppCKAN.call_action does not support "
"file uploads, consider contributing it if you need it")
# copy dicts because actions may modify the dicts they are passed
return self._get_action(action)(dict(context), dict(data_dict))
def call_action(self,
action,
data_dict=None,
context=None,
apikey=None,
files=None,
requests_kwargs=None):
"""
:param action: the action name, e.g. 'package_create'
:param data_dict: the dict to pass to the action, defaults to {}
:param context: an override for the context to use for this action,
remember to include a 'user' when necessary
:param apikey: not supported
:param files: None or {field-name: file-to-be-sent, ...}
:param requests_kwargs: ignored for LocalCKAN (requests not used)
"""
# copy dicts because actions may modify the dicts they are passed
# (CKAN...you so crazy)
data_dict = dict(data_dict or [])
context = dict(self.context if context is None else context)
if apikey:
# FIXME: allow use of apikey to set a user in context?
raise CKANAPIError(
"LocalCKAN.call_action does not support "
"use of apikey parameter, use context['user'] instead")
to_close = []
try:
for fieldname in files or []:
f = files[fieldname]
if isinstance(f, tuple):
# requests accepts (filename, file...) tuples
filename, f = f[:2]
else:
filename = f.name
try:
f.seek(0)
except (AttributeError, IOError):
f = _write_temp_file(f)
to_close.append(f)
field_storage = FieldStorage()
field_storage.file = f
field_storage.filename = filename
data_dict[fieldname] = field_storage
return self._get_action(action)(context, data_dict)
finally:
for f in to_close:
f.close()
def test_restore_catalog_failing_destination_portal(
self, mock_action, mock_push_thm, mock_push_dst):
identifiers = [ds['identifier'] for ds in self.catalog.datasets]
mock_action.return_value.organization_list.return_value = \
['org_1', 'org_2']
mock_action.return_value.organization_show.side_effect = [
{
'packages': [{
'id': identifiers[0]
}]
},
{
'packages': [{
'id': identifiers[1]
}]
},
]
mock_push_dst.side_effect = CKANAPIError('Broken destination portal')
pushed = restore_catalog_to_ckan(self.catalog, 'origin', 'destination',
'apikey')
mock_push_dst.assert_any_call(self.catalog,
'org_1',
identifiers[0],
'destination',
'apikey',
catalog_id=None,
demote_superThemes=False,
demote_themes=False,
download_strategy=None,
generate_new_access_url=None,
origin_tz=DEFAULT_TIMEZONE,
dst_tz=DEFAULT_TIMEZONE)
mock_push_dst.assert_any_call(self.catalog,
'org_2',
identifiers[1],
'destination',
'apikey',
catalog_id=None,
demote_superThemes=False,
demote_themes=False,
download_strategy=None,
generate_new_access_url=None,
origin_tz=DEFAULT_TIMEZONE,
dst_tz=DEFAULT_TIMEZONE)
expected = {'org_1': [], 'org_2': []}
self.assertDictEqual(expected, pushed)
def test_restore_failing_organization_to_ckan(self, mock_push_thm,
mock_push_dst):
# Continua subiendo el segundo dataset a pesar que el primero falla
effects = [
CKANAPIError('broken dataset'),
self.catalog.datasets[1]['identifier']
]
mock_push_dst.side_effect = effects
identifiers = [ds['identifier'] for ds in self.catalog.datasets]
pushed = restore_organization_to_ckan(self.catalog, 'owner_org',
'portal', 'apikey', identifiers)
self.assertEqual([identifiers[1]], pushed)
mock_push_thm.assert_called_with(self.catalog, 'portal', 'apikey')
mock_push_dst.assert_called_with(self.catalog, 'owner_org',
identifiers[1], 'portal', 'apikey',
None, False, False, None, None)
def call_action(self,
action,
data_dict=None,
context=None,
apikey=None,
files=None):
"""
:param action: the action name, e.g. 'package_create'
:param data_dict: the dict to pass to the action as JSON,
defaults to {}
:param context: not supported
:param files: None or {field-name: file-to-be-sent, ...}
This function parses the response from the server as JSON and
returns the decoded value. When an error is returned this
function will convert it back to an exception that matches the
one the action function itself raised.
"""
if context:
raise CKANAPIError("TestAppCKAN.call_action does not support "
"use of context parameter, use apikey instead")
url, data, headers = prepare_action(action, data_dict, apikey
or self.apikey, files)
kwargs = {}
if files:
# Convert the list of (fieldname, file_object) tuples into the
# (fieldname, filename, file_contents) tuples that webtests needs.
upload_files = []
for fieldname, file_ in files.items():
if hasattr(file_, 'name'):
filename = os.path.split(file_.name)[1]
else:
filename = fieldname
upload_files.append((fieldname, filename, file_.read()))
kwargs['upload_files'] = upload_files
r = self.test_app.post('/' + url,
params=data,
headers=headers,
expect_errors=True,
**kwargs)
return reverse_apicontroller_action(url, r.status, r.body)