def test_resource_hash_and_content_length(self, url):
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result['size'] == len('test')
from hashlib import sha1
assert result['hash'] == sha1('test').hexdigest(), result
_remove_archived_file(result.get('cache_filepath'))
def test_url_with_30x_follows_and_records_redirect(self, url):
redirect_url = url + u'?status=200&content=test&content-type=text/csv'
url += u'?status=301&location=%s' % quote_plus(redirect_url)
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result
assert_equal(result['url_redirected_to'], redirect_url)
def test_update_with_zero_length(self, url):
# i.e. no content
res_id = self._test_resource(url)
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error('Content-length after streaming was 0',
res_id)
def test_resource_hash_and_content_length(self, url):
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result['size'] == len('test')
from hashlib import sha1
assert result['hash'] == sha1('test').hexdigest(), result
_remove_archived_file(result.get('cache_filepath'))
def test_url_with_30x_follows_and_records_redirect(self, url):
redirect_url = url + u'?status=200&content=test&content-type=text/csv'
url += u'?status=301&location=%s' % quote_plus(redirect_url)
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result
assert_equal(result['url_redirected_to'], redirect_url)
def test_update_url_with_unknown_content_type(self, url):
context = json.dumps(self.fake_context)
resource = self.fake_resource
resource['format'] = 'arfle-barfle-gloop'
resource['url'] = url
data = json.dumps(resource)
result = update(context, data)
assert not result, result
def test_file_too_large_1(self, url):
# will stop after receiving the header
res_id = self._test_resource(url)
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error(
'Content-length 1000001 exceeds maximum allowed value 1000000',
res_id)
def test_file_too_large_2(self, url):
# no size info in headers - it stops only after downloading the content
res_id = self._test_resource(url)
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error(
'Content-length 1000001 exceeds maximum allowed value 1000000',
res_id)
def test_update_url_with_unknown_content_type(self, url):
context = json.dumps(self.fake_context)
resource = self.fake_resource
resource['format'] = 'arfle-barfle-gloop'
resource['url'] = url
data = json.dumps(resource)
result = update(context, data)
assert not result, result
def test_update_with_zero_length(self, url):
# i.e. no content
context = json.dumps(self.fake_context)
resource = self.fake_resource
resource['format'] = 'arfle-barfle-gloop'
resource['url'] = url
data = json.dumps(resource)
result = update(context, data)
assert not result, result
def test_update_with_zero_length(self, url):
# i.e. no content
context = json.dumps(self.fake_context)
resource = self.fake_resource
resource['format'] = 'arfle-barfle-gloop'
resource['url'] = url
data = json.dumps(resource)
result = update(context, data)
assert not result, result
def test_wms_1_3(self):
with MockWmsServer(wms_version='1.3').serve() as url:
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result, result
assert result['request_type'] == 'WMS 1.3'
with open(result['cache_filepath']) as f:
content = f.read()
assert '<WMT_MS_Capabilities' in content, content[:1000]
_remove_archived_file(result.get('cache_filepath'))
def test_wms_1_3(self):
with MockWmsServer(wms_version='1.3').serve() as url:
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result, result
assert result['request_type'] == 'WMS 1.3'
with open(result['cache_filepath']) as f:
content = f.read()
assert '<WMT_MS_Capabilities' in content, content[:1000]
_remove_archived_file(result.get('cache_filepath'))
def test_resource_hash_and_content_length(self, url):
context = json.dumps(self.fake_context)
resource = self.fake_resource
resource['url'] = url
data = json.dumps(resource)
result = json.loads(update(context, data))
assert result['resource']['size'] == unicode(len('test'))
from hashlib import sha1
assert result['resource']['hash'] == sha1('test').hexdigest(), result
self._remove_archived_file(result.get('file_path'))
def test_resource_hash_and_content_length(self, url):
context = json.dumps(self.fake_context)
resource = self.fake_resource
resource['url'] = url
data = json.dumps(resource)
result = json.loads(update(context, data))
assert result['resource']['size'] == unicode(len('test'))
from hashlib import sha1
assert result['resource']['hash'] == sha1('test').hexdigest(), result
self._remove_archived_file(result.get('file_path'))
def test_update_all_content_types(self, url):
context = json.dumps(self.fake_context)
resource = self.fake_resource
resource['format'] = 'arfle-barfle-gloop'
resource['url'] = url
data = json.dumps(resource)
from ckanext.archiver import default_settings
tmp = default_settings.DATA_FORMATS
default_settings.DATA_FORMATS = 'all'
try:
result = update(context, data)
finally:
default_settings.DATA_FORMATS = tmp
def test_archived_file(self, url):
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result['cache_filepath']
assert os.path.exists(result['cache_filepath'])
with open(result['cache_filepath']) as f:
content = f.readlines()
assert len(content) == 1
assert content[0] == "test"
_remove_archived_file(result.get('cache_filepath'))
def test_update_all_content_types(self, url):
context = json.dumps(self.fake_context)
resource = self.fake_resource
resource['format'] = 'arfle-barfle-gloop'
resource['url'] = url
data = json.dumps(resource)
from ckanext.archiver import default_settings
tmp = default_settings.DATA_FORMATS
default_settings.DATA_FORMATS = 'all'
try:
result = update(context, data)
finally:
default_settings.DATA_FORMATS = tmp
def test_archived_file(self, url):
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result['cache_filepath']
assert os.path.exists(result['cache_filepath'])
with open(result['cache_filepath']) as f:
content = f.readlines()
assert len(content) == 1
assert content[0] == "test"
_remove_archived_file(result.get('cache_filepath'))
def test_archived_file(self, url):
context = json.dumps(self.fake_context)
resource = self.fake_resource
resource['url'] = url
data = json.dumps(resource)
result = json.loads(update(context, data))
assert result['file_path']
assert os.path.exists(result['file_path'])
with open(result['file_path']) as f:
content = f.readlines()
assert len(content) == 1
assert content[0] == "test"
self._remove_archived_file(result.get('file_path'))
def test_archived_file(self, url):
context = json.dumps(self.fake_context)
resource = self.fake_resource
resource['url'] = url
data = json.dumps(resource)
result = json.loads(update(context, data))
assert result['file_path']
assert os.path.exists(result['file_path'])
with open(result['file_path']) as f:
content = f.readlines()
assert len(content) == 1
assert content[0] == "test"
self._remove_archived_file(result.get('file_path'))
def test_file_too_large_1(self, url):
# will stop after receiving the header
res_id = self._test_resource(url)
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error('Content-length 1000001 exceeds maximum allowed value 1000000', res_id)
def test_bad_url(self):
res_id = self._test_resource('http:host.com') # no slashes
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error('Failed to parse', res_id)
def test_content_length_repeated(self, url):
# listing the Content-Length header twice causes requests to
# store the value as a comma-separated list
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result, result
def test_content_length_not_integer(self, url):
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result, result
def test_update_url_with_unknown_content_type(self, url):
res_id = self._test_resource(url, format='foo') # format has no effect
result = json.loads(update(self.config, res_id))
assert result, result
assert result['mimetype'] == 'application/foo' # stored from the header
def test_content_length_repeated(self, url):
# listing the Content-Length header twice causes requests to
# store the value as a comma-separated list
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result, result
def test_server_error(self, url):
res_id = self._test_resource(url)
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error(
'Server reported status error: 500 Internal Server Error', res_id)
def test_file_not_found(self, url):
res_id = self._test_resource(url)
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error(
'Server reported status error: 404 Not Found', res_id)
def test_update_with_zero_length(self, url):
# i.e. no content
res_id = self._test_resource(url)
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error('Content-length after streaming was 0', res_id)
def test_server_error(self, url):
res_id = self._test_resource(url)
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error('Server reported status error: 500 Internal Server Error', res_id)
def test_content_length_not_integer(self, url):
res_id = self._test_resource(url)
result = json.loads(update(self.config, res_id))
assert result, result
def test_file_not_found(self, url):
res_id = self._test_resource(url)
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error('Server reported status error: 404 Not Found', res_id)
def test_bad_url(self):
res_id = self._test_resource('http:host.com') # no slashes
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error('Failed to parse', res_id)
def test_update_url_with_unknown_content_type(self, url):
res_id = self._test_resource(url, format='foo') # format has no effect
result = json.loads(update(self.config, res_id))
assert result, result
assert result['mimetype'] == 'application/foo' # stored from the header
def test_file_too_large_2(self, url):
# no size info in headers - it stops only after downloading the content
res_id = self._test_resource(url)
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error('Content-length 1000001 exceeds maximum allowed value 1000000', res_id)
def test_file_url(self):
res_id = self._test_resource(
'file:///home/root/test.txt') # scheme not allowed
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error('Invalid url scheme', res_id)
def test_file_url(self):
res_id = self._test_resource('file:///home/root/test.txt') # scheme not allowed
result = update(self.config, res_id)
assert not result, result
self.assert_archival_error('Invalid url scheme', res_id)