def _test_package(self, resources):
context = {
'model': model,
'ignore_auth': True,
'session': model.Session,
'user': '******'
}
title = " ".join(random.sample(words, 3))
name = title.lower().replace(' ', '-') + '-' + str(uuid.uuid4())
notes = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus a velit lectus. Maecenas magna turpis, consequat et nibh a, porta tincidunt arcu. Pellentesque et venenatis ligula. Nam luctus luctus odio rutrum pulvinar. Cras vestibulum dolor eget elit cursus facilisis. Fusce eu nulla a justo euismod feugiat dictum a nisi. Donec porta, est nec euismod elementum, ligula odio fermentum orci, nec aliquam leo neque sed felis. Ut sed neque et neque gravida efficitur. Praesent efficitur tortor nunc, sed placerat ligula elementum nec. Nunc a augue et leo feugiat ornare in in urna. Nam ut sapien rutrum erat tincidunt suscipit. Pellentesque pulvinar diam eget nisl hendrerit, interdum posuere odio lobortis."""
pkg = {
'title': title,
'name': name,
'resources': resources,
'notes': notes,
'license_id': 'uk-ogl'
}
pkg = get_action('package_create')(context, pkg)
for res in pkg['resources']:
update_resource(
self.config,
res['id']) # This causes test to depend on internet
return pkg
def test_ipipe_notified(self, url):
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
res_id = self._test_resource(url)['id']
update_resource(self.config, res_id, 'queue1')
assert len(testipipe.calls) == 1
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') is None
assert params.get('resource_id') == res_id
def test_ipipe_notified(self, url):
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
res_id = self._test_resource(url)['id']
update_resource(self.config, res_id, 'queue1')
assert len(testipipe.calls) == 1
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') is None
assert params.get('resource_id') == res_id
def test_url_with_30x_follows_and_records_redirect(self, url):
redirect_url = url + u'?status=200&content=test&content-type=text/csv'
url += u'?status=301&location=%s' % quote_plus(redirect_url)
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result
assert_equal(result['url_redirected_to'], redirect_url)
def test_update_with_zero_length(self, client):
url = client + '/?status=200&content-type=csv'
# i.e. no content
res_id = self._test_resource(url)['id']
result = update_resource(res_id)
assert not result, result
self.assert_archival_error('Content-length after streaming was 0', res_id)
def test_file_too_large_2(self, client):
url = client + '/?status=200&content_long=test_contents_greater_than_the_max_length&no-content-length&content-type=csv'
# no size info in headers - it stops only after downloading the content
res_id = self._test_resource(url)['id']
result = update_resource(res_id)
assert not result, result
self.assert_archival_error('Content-length 1000001 exceeds maximum allowed value 1000000', res_id)
def test_file_too_large_1(self, client):
url = client + '/?status=200&content=short&length=1000001&content-type=csv'
# will stop after receiving the header
res_id = self._test_resource(url)['id']
result = update_resource(res_id)
assert not result, result
self.assert_archival_error('Content-length 1000001 exceeds maximum allowed value 1000000', res_id)
def test_ipipe_notified(self, client):
url = client + '/?status=200&content=test&content-type=csv'
testipipe = plugins.get_plugin('testipipe')
testipipe.reset()
res_id = self._test_resource(url)['id']
update_resource(res_id, 'queue1')
assert len(testipipe.calls) == 1
operation, queue, params = testipipe.calls[0]
assert operation == 'archived'
assert queue == 'queue1'
assert params.get('package_id') is None
assert params.get('resource_id') == res_id
def test_content_length_repeated(self, client):
url = client + '/?status=200&content=content&repeat-length&content-type=csv'
# listing the Content-Length header twice causes requests to
# store the value as a comma-separated list
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(res_id))
assert result, result
def test_resource_hash_and_content_length(self, url):
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result['size'] == len('test')
from hashlib import sha1
assert result['hash'] == sha1('test').hexdigest(), result
_remove_archived_file(result.get('cache_filepath'))
def test_resource_hash_and_content_length(self, url):
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result['size'] == len('test')
from hashlib import sha1
assert result['hash'] == sha1('test').hexdigest(), result
_remove_archived_file(result.get('cache_filepath'))
def test_update_with_zero_length(self, url):
# i.e. no content
res_id = self._test_resource(url)['id']
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error('Content-length after streaming was 0',
res_id)
def test_url_with_30x_follows_and_records_redirect(self, url):
redirect_url = url + u'?status=200&content=test&content-type=text/csv'
url += u'?status=301&location=%s' % quote_plus(redirect_url)
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result
assert_equal(result['url_redirected_to'], redirect_url)
def _test_package(self, resources):
context = {'model': model, 'ignore_auth': True, 'session': model.Session, 'user': '******'}
title = " ".join(random.sample(words, 3))
name = title.lower().replace(' ', '-') + '-' + str(uuid.uuid4())
notes = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus a velit lectus. Maecenas magna turpis, consequat et nibh a, porta tincidunt arcu. Pellentesque et venenatis ligula. Nam luctus luctus odio rutrum pulvinar. Cras vestibulum dolor eget elit cursus facilisis. Fusce eu nulla a justo euismod feugiat dictum a nisi. Donec porta, est nec euismod elementum, ligula odio fermentum orci, nec aliquam leo neque sed felis. Ut sed neque et neque gravida efficitur. Praesent efficitur tortor nunc, sed placerat ligula elementum nec. Nunc a augue et leo feugiat ornare in in urna. Nam ut sapien rutrum erat tincidunt suscipit. Pellentesque pulvinar diam eget nisl hendrerit, interdum posuere odio lobortis."""
pkg = {'title': title,
'name': name,
'resources': resources,
'notes': notes,
'license_id': 'uk-ogl'}
pkg = get_action('package_create')(context, pkg)
for res in pkg['resources']:
update_resource(self.config, res['id']) # This causes test to depend on internet
return pkg
def test_resource_hash_and_content_length(self, client):
url = client + '/?status=200&content=test&content-type=csv'
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(res_id))
assert result['size'] == len('test')
from hashlib import sha1
assert result['hash'] == sha1('test'.encode('utf-8')).hexdigest(), result
_remove_archived_file(result.get('cache_filepath'))
def test_file_too_large_1(self, url):
# will stop after receiving the header
res_id = self._test_resource(url)['id']
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error(
'Content-length 1000001 exceeds maximum allowed value 1000000',
res_id)
def test_file_too_large_2(self, url):
# no size info in headers - it stops only after downloading the content
res_id = self._test_resource(url)['id']
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error(
'Content-length 1000001 exceeds maximum allowed value 1000000',
res_id)
def test_resource_hash_and_content_length(self, url):
res_id = self._test_resource(url)["id"]
result = json.loads(update_resource(self.config, res_id))
assert result["size"] == len("test")
from hashlib import sha1
assert result["hash"] == sha1("test").hexdigest(), result
_remove_archived_file(result.get("cache_filepath"))
def test_wms_1_3(self):
with MockWmsServer(wms_version='1.3').serve() as url:
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result, result
assert result['request_type'] == 'WMS 1.3'
with open(result['cache_filepath']) as f:
content = f.read()
assert '<WMT_MS_Capabilities' in content, content[:1000]
_remove_archived_file(result.get('cache_filepath'))
def test_wms_1_3(self):
with MockWmsServer(wms_version='1.3').serve() as url:
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result, result
assert result['request_type'] == 'WMS 1.3'
with open(result['cache_filepath']) as f:
content = f.read()
assert '<WMT_MS_Capabilities' in content, content[:1000]
_remove_archived_file(result.get('cache_filepath'))
def test_wms_1_3(self):
with MockWmsServer(wms_version="1.3").serve() as url:
res_id = self._test_resource(url)["id"]
result = json.loads(update_resource(self.config, res_id))
assert result, result
assert result["request_type"] == "WMS 1.3"
with open(result["cache_filepath"]) as f:
content = f.read()
assert "<WMT_MS_Capabilities" in content, content[:1000]
_remove_archived_file(result.get("cache_filepath"))
def test_wms_1_3(self, client):
url = client + '/WMS_1_3/'
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(res_id))
assert result, result
assert result['request_type'] == 'WMS 1.3'
with open(result['cache_filepath']) as f:
content = f.read()
assert '<WMT_MS_Capabilities' in content, content[:1000]
_remove_archived_file(result.get('cache_filepath'))
def test_archived_file(self, url):
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result['cache_filepath']
assert os.path.exists(result['cache_filepath'])
with open(result['cache_filepath']) as f:
content = f.readlines()
assert len(content) == 1
assert content[0] == "test"
_remove_archived_file(result.get('cache_filepath'))
def test_archived_file(self, url):
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result['cache_filepath']
assert os.path.exists(result['cache_filepath'])
with open(result['cache_filepath']) as f:
content = f.readlines()
assert len(content) == 1
assert content[0] == "test"
_remove_archived_file(result.get('cache_filepath'))
def test_archived_file(self, client):
url = client + '/?status=200&content=test&content-type=csv'
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(res_id))
assert result['cache_filepath']
assert os.path.exists(result['cache_filepath'])
with open(result['cache_filepath']) as f:
content = f.readlines()
assert len(content) == 1
assert content[0] == "test"
_remove_archived_file(result.get('cache_filepath'))
def test_server_error(self, url):
res_id = self._test_resource(url)['id']
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error('Server reported status error: 500 Internal Server Error', res_id)
def test_content_length_repeated(self, url):
# listing the Content-Length header twice causes requests to
# store the value as a comma-separated list
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result, result
def test_update_url_with_unknown_content_type(self, url):
res_id = self._test_resource(
url, format='foo')['id'] # format has no effect
result = json.loads(update_resource(self.config, res_id))
assert result, result
assert result['mimetype'] == 'application/foo' # stored from the header
def test_bad_url(self):
res_id = self._test_resource('http:host.com')['id'] # no slashes
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error('Failed to parse', res_id)
def test_update_url_with_unknown_content_type(self, url):
res_id = self._test_resource(url, format='foo')['id'] # format has no effect
result = json.loads(update_resource(self.config, res_id))
assert result, result
assert result['mimetype'] == 'application/foo' # stored from the header
def test_content_length_repeated(self, url):
# listing the Content-Length header twice causes requests to
# store the value as a comma-separated list
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result, result
def test_update_url_with_unknown_content_type(self, url):
res_id = self._test_resource(url, format="foo")["id"] # format has no effect
result = json.loads(update_resource(self.config, res_id))
assert result, result
assert result["mimetype"] == "application/foo" # stored from the header
def test_server_error(self, url):
res_id = self._test_resource(url)['id']
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error(
'Server reported status error: 500 Internal Server Error', res_id)
def _get_update_resource_json(self, id):
result = update_resource(resource_id=id)
assert result, "update_resource returned: {}".format(result)
return json.loads(result)
def test_file_too_large_1(self, url):
# will stop after receiving the header
res_id = self._test_resource(url)['id']
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error('Content-length 1000001 exceeds maximum allowed value 1000000', res_id)
def test_bad_url(self):
res_id = self._test_resource('http:host.com')['id'] # no slashes
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error('Failed to parse', res_id)
def test_server_error(self, client):
url = client + '/?status=500&content=test&content-type=csv'
res_id = self._test_resource(url)['id']
result = update_resource(res_id)
assert not result, result
self.assert_archival_error('Server reported status error: 500 INTERNAL SERVER ERROR', res_id)
def test_file_not_found(self, client):
url = client + '/?status=404&content=test&content-type=csv'
res_id = self._test_resource(url)['id']
result = update_resource(res_id)
assert not result, result
self.assert_archival_error('Server reported status error: 404 NOT FOUND', res_id)
def test_file_too_large_2(self, url):
# no size info in headers - it stops only after downloading the content
res_id = self._test_resource(url)['id']
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error('Content-length 1000001 exceeds maximum allowed value 1000000', res_id)
def test_content_length_not_integer(self, url):
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result, result
def test_bad_url(self):
res_id = self._test_resource("http:host.com")["id"] # no slashes
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error("Failed to parse", res_id)
def test_file_not_found(self, url):
res_id = self._test_resource(url)['id']
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error(
'Server reported status error: 404 Not Found', res_id)
def test_content_length_not_integer(self, client):
url = client + '/?status=200&content=content&length=abc&content-type=csv'
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(res_id))
assert result, result
def test_file_url(self):
res_id = self._test_resource('file:///home/root/test.txt')['id'] # scheme not allowed
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error('Invalid url scheme', res_id)
def test_file_not_found(self, url):
res_id = self._test_resource(url)['id']
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error('Server reported status error: 404 Not Found', res_id)
def test_content_length_not_integer(self, url):
res_id = self._test_resource(url)['id']
result = json.loads(update_resource(self.config, res_id))
assert result, result
def test_update_with_zero_length(self, url):
# i.e. no content
res_id = self._test_resource(url)['id']
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error('Content-length after streaming was 0', res_id)
def test_file_url(self):
res_id = self._test_resource('file:///home/root/test.txt')[
'id'] # scheme not allowed
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error('Invalid url scheme', res_id)
def test_bad_url(self):
res_id = self._test_resource('http:host.com')['id'] # no slashes
result = update_resource(res_id)
assert not result, result
self.assert_archival_error('URL parsing failure', res_id)
def test_update_url_with_unknown_content_type(self, client):
url = client + '/?content-type=application/foo&content=test'
res_id = self._test_resource(url, format='foo')['id'] # format has no effect
result = json.loads(update_resource(res_id))
assert result, result
assert result['mimetype'] == 'application/foo' # stored from the header
def test_file_url(self):
res_id = self._test_resource("file:///home/root/test.txt")["id"] # scheme not allowed
result = update_resource(self.config, res_id)
assert not result, result
self.assert_archival_error("Invalid url scheme", res_id)