def test_validate_status_code(self, mock_get):
self.engine = CkanDatasetEngine(endpoint="http://localhost:5000/api/3/action",
apikey=TEST_CKAN_DATASET_SERVICE['APIKEY'])
result_data = {'resources': self.test_resource_name, 'version': '1.0'}
mock_get.return_value = MockJsonResponse(201, result=result_data)
self.assertRaises(AssertionError, self.engine.validate)
def __init__(self, engine_url, api_key, model_name,
dataset_notes="CKAN Dataset",
resource_description="CKAN Resource",
date_format_string="%Y%m%d"):
if engine_url.endswith('/'):
engine_url = engine_url[:-1]
if not engine_url.endswith('api/action') and not engine_url.endswith('api/3/action'):
engine_url += '/api/action'
self.dataset_engine = CkanDatasetEngine(endpoint=engine_url, apikey=api_key)
self.model_name = model_name
self.dataset_notes = dataset_notes
self.resource_description = resource_description
self.date_format_string = date_format_string
def get_engine(self, request=None):
"""
Retrieves dataset service engine
"""
# Get Token for HydroShare interactions
if self.engine == self.HYDROSHARE:
# Constants
HYDROSHARE_OAUTH_PROVIDER_NAME = 'hydroshare'
user = request.user
try:
# social = user.social_auth.get(provider='google-oauth2')
social = user.social_auth.get(
provider=HYDROSHARE_OAUTH_PROVIDER_NAME)
apikey = social.extra_data['access_token'] # noqa: F841
except ObjectDoesNotExist:
# User is not associated with that provider
# Need to prompt for association
raise AuthException(
"HydroShare authentication required. To automate the authentication prompt "
"decorate your controller function with the @ensure_oauth('hydroshare') decorator."
)
return HydroShareDatasetEngine(endpoint=self.endpoint,
username=self.username,
password=self.password,
apikey=self.apikey)
return CkanDatasetEngine(endpoint=self.endpoint,
username=self.username,
password=self.password,
apikey=self.apikey)
def setUp(self):
# Create Test Engine
self.engine = CkanDatasetEngine(endpoint=TEST_CKAN_DATASET_SERVICE['ENDPOINT'],
apikey=TEST_CKAN_DATASET_SERVICE['APIKEY'])
# Test Dataset Name
self.test_dataset_name = random_string_generator(10)
# Test Resource Variables
self.test_resource_name = random_string_generator(10)
self.test_resource_url = 'http://home.byu.edu'
# File paths
self.tests_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
self.files_path = os.path.join(self.tests_path, 'files')
self.support_path = os.path.join(self.tests_path, 'support')
def setUp(self):
# Create Test Engine
self.engine = CkanDatasetEngine(
endpoint=TEST_CKAN_DATASET_SERVICE['ENDPOINT'],
apikey=TEST_CKAN_DATASET_SERVICE['APIKEY'])
# Create Test Dataset
self.test_dataset_name = random_string_generator(10)
dataset_result = self.engine.create_dataset(
name=self.test_dataset_name, version='1.0')
self.test_dataset = dataset_result['result']
# Create Test Resource
self.test_resource_name = random_string_generator(10)
self.test_resource_url = 'http://home.byu.edu'
resource_result = self.engine.create_resource(
self.test_dataset_name, url=self.test_resource_url, format='zip')
self.test_resource = resource_result['result']
def setUp(self):
# Create Test Engine
self.engine = CkanDatasetEngine(endpoint=TEST_CKAN_DATASET_SERVICE['ENDPOINT'],
apikey=TEST_CKAN_DATASET_SERVICE['APIKEY'])
# Create Test Dataset
self.test_dataset_name = random_string_generator(10)
dataset_result = self.engine.create_dataset(name=self.test_dataset_name, version='1.0')
self.test_dataset = dataset_result['result']
# Create Test Resource
self.test_resource_name = random_string_generator(10)
self.test_resource_url = 'http://home.byu.edu'
resource_result = self.engine.create_resource(self.test_dataset_name, url=self.test_resource_url, format='zip')
self.test_resource = resource_result['result']
class TestCkanDatasetEngine(unittest.TestCase):
def setUp(self):
# Create Test Engine
self.engine = CkanDatasetEngine(
endpoint=TEST_CKAN_DATASET_SERVICE['ENDPOINT'],
apikey=TEST_CKAN_DATASET_SERVICE['APIKEY'])
# Create Test Dataset
self.test_dataset_name = random_string_generator(10)
dataset_result = self.engine.create_dataset(
name=self.test_dataset_name, version='1.0')
self.test_dataset = dataset_result['result']
# Create Test Resource
self.test_resource_name = random_string_generator(10)
self.test_resource_url = 'http://home.byu.edu'
resource_result = self.engine.create_resource(
self.test_dataset_name, url=self.test_resource_url, format='zip')
self.test_resource = resource_result['result']
def tearDown(self):
# Delete test resource and dataset
self.engine.delete_resource(resource_id=self.test_resource['id'])
self.engine.delete_dataset(dataset_id=self.test_dataset_name)
def test_list_datasets_defaults(self):
# Execute
result = self.engine.list_datasets()
# Verify Success
self.assertTrue(result['success'])
def test_list_datasets_with_resources(self):
# Execute
result = self.engine.list_datasets(with_resources=True)
# Verify Success
self.assertTrue(result['success'])
def test_list_datasets_with_params(self):
# Setup
limit = 500
number_all = len(self.engine.list_datasets()['result'])
# Execute twice with offsets different
result_page_1 = self.engine.list_datasets(limit=limit,
offset=1,
console=False)
result_page_2 = self.engine.list_datasets(limit=limit, offset=2)
# Verify success
self.assertTrue(result_page_1['success'])
self.assertTrue(result_page_2['success'])
# Count the results
page_1_count = len(result_page_1['result'])
page_2_count = len(result_page_2['result'])
# Verify count (should be less than or equal to limit)
self.assertLessEqual(page_1_count, limit)
self.assertLessEqual(page_2_count, limit)
# If there are more than 5 datasets, the results should be different
if number_all > 5:
self.assertNotEqual(result_page_1, result_page_2)
def test_search_resources(self):
# Execute
result = self.engine.search_resources(query={'format': 'zip'})
# Verify Success
self.assertTrue(result['success'])
# Check search results if they exist
search_results = result['result']['results']
if len(search_results) > 1:
for result in search_results:
self.assertIn('zip', result['format'].lower())
def test_search_datasets(self):
# Execute
version = '1.0'
result = self.engine.search_datasets(query={'version': version},
console=False)
# Verify Success
self.assertTrue(result['success'])
# Check search results if they exist
search_results = result['result']['results']
if len(search_results) > 1:
for result in search_results:
self.assertIn('version', result)
self.assertEqual(result['version'], version)
def test_create_dataset(self):
# Setup
new_dataset_name = random_string_generator(10)
# Execute
result = self.engine.create_dataset(name=new_dataset_name)
# Verify Success
self.assertTrue(result['success'])
# Should return the new one
self.assertEqual(new_dataset_name, result['result']['name'])
# Delete
self.engine.delete_dataset(dataset_id=new_dataset_name)
def test_create_resource_url(self):
# Setup
new_resource_name = random_string_generator(5)
new_resource_url = 'http://home.byu.edu'
# Execute
result = self.engine.create_resource(dataset_id=self.test_dataset_name,
url=new_resource_url,
name=new_resource_name)
# Verify Success
self.assertTrue(result['success'])
# Verify name and url
self.assertEqual(new_resource_name, result['result']['name'])
self.assertEqual(new_resource_url, result['result']['url'])
# Delete
self.engine.delete_resource(resource_id=result['result']['id'])
def test_create_resource_file_upload(self):
# Prepare
file_to_upload = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'support',
'upload_test.txt')
# Execute
result = self.engine.create_resource(dataset_id=self.test_dataset_name,
file=file_to_upload,
console=False)
print result
# Verify Success
self.assertTrue(result['success'], result)
# Verify name and url_type (which should be upload if file upload)
self.assertEqual(result['result']['name'], 'upload_test.txt')
self.assertEqual(result['result']['url_type'], 'upload')
# Delete
self.engine.delete_resource(resource_id=result['result']['id'])
def test_get_dataset(self):
# Execute
result = self.engine.get_dataset(dataset_id=self.test_dataset_name,
console=True)
# Verify Success
self.assertTrue(result['success'])
# Verify Name
self.assertEqual(result['result']['name'], self.test_dataset_name)
def test_get_resource(self):
# Execute
result = self.engine.get_resource(resource_id=self.test_resource['id'],
console=True)
# Verify Success
self.assertTrue(result['success'])
# Verify Properties
self.assertEqual(result['result']['url'], self.test_resource_url)
def test_update_dataset(self):
# Setup
test_version = '2.0'
# Execute
result = self.engine.update_dataset(dataset_id=self.test_dataset_name,
version=test_version)
# Verify Success
self.assertTrue(result['success'])
# Verify new version property
self.assertEqual(result['result']['version'], test_version)
def test_update_resource_property_change(self):
# Setup
new_format = 'web'
# Execute
result = self.engine.update_resource(
resource_id=self.test_resource['id'], format=new_format)
# Verify Success
self.assertTrue(result['success'])
# Verify new format
self.assertEqual(result['result']['format'], new_format)
def test_update_resource_url_change(self):
# Setup
new_url = 'http://www.utah.edu'
# Execute
result = self.engine.update_resource(
resource_id=self.test_resource['id'], url=new_url)
# Verify Success
self.assertTrue(result['success'])
# Verify New URL Property
self.assertEqual(result['result']['url'], new_url)
self.assertNotEqual(result['result']['url'], self.test_resource['url'])
def test_update_resource_file_upload(self):
# Setup
file_to_upload = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'support',
'upload_test.txt')
# Execute
result = self.engine.update_resource(
resource_id=self.test_resource['id'],
file=file_to_upload,
console=False)
print result
# Verify Success
self.assertTrue(result['success'])
# Verify Name (should be the same as the file uploaded by default)
self.assertEqual(result['result']['name'], 'upload_test.txt')
# URL should be different than original when file upload executes
# self.assertNotEqual(result['result']['url'], self.test_resource['url'])
def test_delete_resource(self):
# Execute
result = self.engine.delete_resource(
resource_id=self.test_resource['id'])
# Verify Success
self.assertTrue(result['success'])
# Delete requests should return nothing
self.assertEqual(result['result'], None)
def test_delete_dataset(self):
# Execute
result = self.engine.delete_dataset(dataset_id=self.test_dataset_name,
console=False)
# Confirm Success
self.assertTrue(result['success'])
# Delete requests should return nothing
self.assertEqual(result['result'], None)
def test_download_resource(self):
pass
# self.engine.download_resouce(resource_id, location='../tests/files/', local_file_name='test_resource.test')
def test_download_dataset(self):
pass
class CKANDatasetManager(object):
"""
This class is used to find, zip and upload files to a CKAN data server
#note: this does not delete the original files
"""
def __init__(self, engine_url, api_key, model_name,
dataset_notes="CKAN Dataset",
resource_description="CKAN Resource",
date_format_string="%Y%m%d"):
if engine_url.endswith('/'):
engine_url = engine_url[:-1]
if not engine_url.endswith('api/action') and not engine_url.endswith('api/3/action'):
engine_url += '/api/action'
self.dataset_engine = CkanDatasetEngine(endpoint=engine_url, apikey=api_key)
self.model_name = model_name
self.dataset_notes = dataset_notes
self.resource_description = resource_description
self.date_format_string = date_format_string
def initialize_run(self, watershed, subbasin, date_string):
"""
Initialize run for watershed upload/download
"""
self.watershed = watershed.lower()
self.subbasin = subbasin.lower()
self.date_string = date_string
self.date = datetime.datetime.strptime(self.date_string, self.date_format_string)
self.dataset_name = '%s-%s-%s-%s' % (self.model_name,
self.watershed,
self.subbasin,
self.date.strftime("%Y%m%d"))
self.resource_name = '%s-%s-%s-%s' % (self.model_name,
self.watershed,
self.subbasin,
self.date_string)
def make_tarfile(self, file_path):
"""
This function packages the dataset into a tar.gz file and
returns the path
"""
base_path = os.path.dirname(file_path)
output_tar_file = os.path.join(base_path, "%s.tar.gz" % self.resource_name)
if not os.path.exists(output_tar_file):
with tarfile.open(output_tar_file, "w:gz") as tar:
tar.add(file_path, arcname=os.path.basename(file_path))
return output_tar_file
def make_directory_tarfile(self, directory_path, search_string="*"):
"""
This function packages all of the datasets into a tar.gz file and
returns the path
"""
base_path = os.path.dirname(directory_path)
output_tar_file = os.path.join(base_path, "%s.tar.gz" % self.resource_name)
if not os.path.exists(output_tar_file):
directory_files = glob(os.path.join(directory_path,search_string))
with tarfile.open(output_tar_file, "w:gz") as tar:
for directory_file in directory_files:
tar.add(directory_file, arcname=os.path.basename(directory_file))
return output_tar_file
def get_dataset_id(self):
"""
This function gets the id of a dataset
"""
# Use the json module to load CKAN's response into a dictionary.
response_dict = self.dataset_engine.search_datasets({ 'name': self.dataset_name })
if response_dict['success']:
if int(response_dict['result']['count']) > 0:
return response_dict['result']['results'][0]['id']
return None
else:
return None
def create_dataset(self):
"""
This function creates a dataset if it does not exist
"""
dataset_id = self.get_dataset_id()
#check if dataset exists
if not dataset_id:
#if it does not exist, create the dataset
result = self.dataset_engine.create_dataset(name=self.dataset_name,
notes=self.dataset_notes,
version='1.0',
tethys_app='erfp_tool',
waterhsed=self.watershed,
subbasin=self.subbasin,
month=self.date.month,
year=self.date.year)
dataset_id = result['result']['id']
return dataset_id
def upload_resource(self, file_path, overwrite=False, file_format='tar.gz'):
"""
This function uploads a resource to a dataset if it does not exist
"""
#create dataset for each watershed-subbasin combo if needed
dataset_id = self.create_dataset()
if dataset_id:
#check if dataset already exists
resource_results = self.dataset_engine.search_resources({'name':self.resource_name},
datset_id=dataset_id)
try:
#determine if results are exact or similar
same_ckan_resource_id = ""
if resource_results['result']['count'] > 0:
for resource in resource_results['result']['results']:
if resource['name'] == self.resource_name:
same_ckan_resource_id = resource['id']
break
if overwrite and same_ckan_resource_id:
#delete resource
"""
CKAN API CURRENTLY DOES NOT WORK FOR UPDATE - bug = needs file or url,
but requres both and to have only one ...
#update existing resource
print resource_results['result']['results'][0]
update_results = self.dataset_engine.update_resource(resource_results['result']['results'][0]['id'],
file=file_to_upload,
url="",
date_uploaded=datetime.datetime.utcnow().strftime("%Y%m%d%H%M"))
"""
self.dataset_engine.delete_resource(same_ckan_resource_id)
if not same_ckan_resource_id or overwrite:
#upload resources to the dataset
return self.dataset_engine.create_resource(dataset_id,
name=self.resource_name,
file=file_path,
format=file_format,
tethys_app="erfp_tool",
watershed=self.watershed,
subbasin=self.subbasin,
forecast_date=self.date_string,
description=self.resource_description)
else:
print "Resource", self.resource_name ,"exists. Skipping ..."
except Exception,e:
print e
pass
class TestCkanDatasetEngine(unittest.TestCase):
def setUp(self):
# Create Test Engine
self.engine = CkanDatasetEngine(endpoint=TEST_CKAN_DATASET_SERVICE['ENDPOINT'],
apikey=TEST_CKAN_DATASET_SERVICE['APIKEY'])
# Create Test Dataset
self.test_dataset_name = random_string_generator(10)
dataset_result = self.engine.create_dataset(name=self.test_dataset_name, version='1.0')
self.test_dataset = dataset_result['result']
# Create Test Resource
self.test_resource_name = random_string_generator(10)
self.test_resource_url = 'http://home.byu.edu'
resource_result = self.engine.create_resource(self.test_dataset_name, url=self.test_resource_url, format='zip')
self.test_resource = resource_result['result']
def tearDown(self):
# Delete test resource and dataset
self.engine.delete_resource(resource_id=self.test_resource['id'])
self.engine.delete_dataset(dataset_id=self.test_dataset_name)
def test_list_datasets_defaults(self):
# Execute
result = self.engine.list_datasets()
# Verify Success
self.assertTrue(result['success'])
def test_list_datasets_with_resources(self):
# Execute
result = self.engine.list_datasets(with_resources=True)
# Verify Success
self.assertTrue(result['success'])
def test_list_datasets_with_params(self):
# Setup
limit = 500
number_all = len(self.engine.list_datasets()['result'])
# Execute twice with offsets different
result_page_1 = self.engine.list_datasets(limit=limit, offset=1, console=False)
result_page_2 = self.engine.list_datasets(limit=limit, offset=2)
# Verify success
self.assertTrue(result_page_1['success'])
self.assertTrue(result_page_2['success'])
# Count the results
page_1_count = len(result_page_1['result'])
page_2_count = len(result_page_2['result'])
# Verify count (should be less than or equal to limit)
self.assertLessEqual(page_1_count, limit)
self.assertLessEqual(page_2_count, limit)
# If there are more than 5 datasets, the results should be different
if number_all > 5:
self.assertNotEqual(result_page_1, result_page_2)
def test_search_resources(self):
# Execute
result = self.engine.search_resources(query={'format': 'zip'})
# Verify Success
self.assertTrue(result['success'])
# Check search results if they exist
search_results = result['result']['results']
if len(search_results) > 1:
for result in search_results:
self.assertIn('zip', result['format'].lower())
def test_search_datasets(self):
# Execute
version = '1.0'
result = self.engine.search_datasets(query={'version': version}, console=False)
# Verify Success
self.assertTrue(result['success'])
# Check search results if they exist
search_results = result['result']['results']
if len(search_results) > 1:
for result in search_results:
self.assertIn('version', result)
self.assertEqual(result['version'], version)
def test_create_dataset(self):
# Setup
new_dataset_name = random_string_generator(10)
# Execute
result = self.engine.create_dataset(name=new_dataset_name)
# Verify Success
self.assertTrue(result['success'])
# Should return the new one
self.assertEqual(new_dataset_name, result['result']['name'])
# Delete
self.engine.delete_dataset(dataset_id=new_dataset_name)
def test_create_resource_url(self):
# Setup
new_resource_name = random_string_generator(5)
new_resource_url = 'http://home.byu.edu'
# Execute
result = self.engine.create_resource(dataset_id=self.test_dataset_name,
url=new_resource_url,
name=new_resource_name)
# Verify Success
self.assertTrue(result['success'])
# Verify name and url
self.assertEqual(new_resource_name, result['result']['name'])
self.assertEqual(new_resource_url, result['result']['url'])
# Delete
self.engine.delete_resource(resource_id=result['result']['id'])
def test_create_resource_file_upload(self):
# Prepare
file_to_upload = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'support', 'upload_test.txt')
# Execute
result = self.engine.create_resource(dataset_id=self.test_dataset_name, file=file_to_upload, console=False)
print result
# Verify Success
self.assertTrue(result['success'], result)
# Verify name and url_type (which should be upload if file upload)
self.assertEqual(result['result']['name'], 'upload_test.txt')
self.assertEqual(result['result']['url_type'], 'upload')
# Delete
self.engine.delete_resource(resource_id=result['result']['id'])
def test_get_dataset(self):
# Execute
result = self.engine.get_dataset(dataset_id=self.test_dataset_name, console=True)
# Verify Success
self.assertTrue(result['success'])
# Verify Name
self.assertEqual(result['result']['name'], self.test_dataset_name)
def test_get_resource(self):
# Execute
result = self.engine.get_resource(resource_id=self.test_resource['id'], console=True)
# Verify Success
self.assertTrue(result['success'])
# Verify Properties
self.assertEqual(result['result']['url'], self.test_resource_url)
def test_update_dataset(self):
# Setup
test_version = '2.0'
# Execute
result = self.engine.update_dataset(dataset_id=self.test_dataset_name, version=test_version)
# Verify Success
self.assertTrue(result['success'])
# Verify new version property
self.assertEqual(result['result']['version'], test_version)
def test_update_resource_property_change(self):
# Setup
new_format = 'web'
# Execute
result = self.engine.update_resource(resource_id=self.test_resource['id'], format=new_format)
# Verify Success
self.assertTrue(result['success'])
# Verify new format
self.assertEqual(result['result']['format'], new_format)
def test_update_resource_url_change(self):
# Setup
new_url = 'http://www.utah.edu'
# Execute
result = self.engine.update_resource(resource_id=self.test_resource['id'], url=new_url)
# Verify Success
self.assertTrue(result['success'])
# Verify New URL Property
self.assertEqual(result['result']['url'], new_url)
self.assertNotEqual(result['result']['url'], self.test_resource['url'])
def test_update_resource_file_upload(self):
# Setup
file_to_upload = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'support', 'upload_test.txt')
# Execute
result = self.engine.update_resource(resource_id=self.test_resource['id'], file=file_to_upload, console=False)
print result
# Verify Success
self.assertTrue(result['success'])
# Verify Name (should be the same as the file uploaded by default)
self.assertEqual(result['result']['name'], 'upload_test.txt')
# URL should be different than original when file upload executes
# self.assertNotEqual(result['result']['url'], self.test_resource['url'])
def test_delete_resource(self):
# Execute
result = self.engine.delete_resource(resource_id=self.test_resource['id'])
# Verify Success
self.assertTrue(result['success'])
# Delete requests should return nothing
self.assertEqual(result['result'], None)
def test_delete_dataset(self):
# Execute
result = self.engine.delete_dataset(dataset_id=self.test_dataset_name, console=False)
# Confirm Success
self.assertTrue(result['success'])
# Delete requests should return nothing
self.assertEqual(result['result'], None)
def test_download_resource(self):
pass
# self.engine.download_resouce(resource_id, location='../tests/files/', local_file_name='test_resource.test')
def test_download_dataset(self):
pass
# self.engine.download_dataset(dataset_id, console=True)
class TestCkanDatasetEngine(unittest.TestCase):
def setUp(self):
# Create Test Engine
self.engine = CkanDatasetEngine(endpoint=TEST_CKAN_DATASET_SERVICE['ENDPOINT'],
apikey=TEST_CKAN_DATASET_SERVICE['APIKEY'])
# Test Dataset Name
self.test_dataset_name = random_string_generator(10)
# Test Resource Variables
self.test_resource_name = random_string_generator(10)
self.test_resource_url = 'http://home.byu.edu'
# File paths
self.tests_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
self.files_path = os.path.join(self.tests_path, 'files')
self.support_path = os.path.join(self.tests_path, 'support')
def tearDown(self):
pass
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_list_datasets_defaults(self, mock_post):
mock_post.return_value = MockJsonResponse(200, result='Datasetname')
# Execute
result = self.engine.list_datasets()
# Verify Success
self.assertTrue(result['success'])
self.assertIn('Datasetname', result['result'])
@mock.patch('tethys_dataset_services.engines.ckan_engine.log')
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_list_datasets_defaults_no_json(self, mock_post, mock_log):
mock_post.return_value = MockJsonResponse(201, result='Datasetname', json_format=False)
# Execute
self.engine.list_datasets()
mock_log.exception.assert_called()
call_args = mock_log.exception.call_args_list
self.assertIn('Status Code 201', call_args[0][0][0])
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_list_datasets_with_resources(self, mock_post):
mock_post.return_value = MockJsonResponse(200, result='Datasetname')
# Execute
result = self.engine.list_datasets(with_resources=True)
# Verify Success
self.assertTrue(result['success'])
self.assertIn('Datasetname', result['result'])
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_list_datasets_with_params(self, mock_post):
data_list = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10']
mock_post.return_value = MockJsonResponse(200, result=data_list)
# Setup
limit = 10
number_all = len(self.engine.list_datasets()['result'])
# Execute twice with offsets different
data_list = ['2', '3', '4', '5', '6', '7', '8', '9', '10']
mock_post.return_value = MockJsonResponse(200, result=data_list)
result_page_1 = self.engine.list_datasets(limit=limit, offset=1, console=False)
data_list = ['3', '4', '5', '6', '7', '8', '9', '10']
mock_post.return_value = MockJsonResponse(200, result=data_list)
result_page_2 = self.engine.list_datasets(limit=limit, offset=2)
# Verify success
self.assertTrue(result_page_1['success'])
self.assertTrue(result_page_2['success'])
# Count the results
page_1_count = len(result_page_1['result'])
page_2_count = len(result_page_2['result'])
# Verify count (should be less than or equal to limit)
self.assertLessEqual(page_1_count, limit)
self.assertLessEqual(page_2_count, limit)
# If there are more than 5 datasets, the results should be different
if number_all > 5:
self.assertNotEqual(result_page_1, result_page_2)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_search_resources(self, mock_post):
result_data = {'results': [{'format': 'ZIP'}, {'format': 'ZIP'}]}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.search_resources(query={'format': 'zip', 'contents': 'html'})
# Verify Success
self.assertTrue(result['success'])
# Check search results if they exist
search_results = result['result']['results']
if len(search_results) > 1:
for result in search_results:
self.assertIn('zip', result['format'].lower())
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_search_datasets(self, mock_post):
version = '1.0'
result_data = {'results': [{'version': version}, {'version': version}]}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.search_datasets(query={'version': version}, console=False)
# Verify Success
self.assertTrue(result['success'])
# Check search results if they exist
search_results = result['result']['results']
if len(search_results) > 1:
for result in search_results:
self.assertIn('version', result)
self.assertEqual(result['version'], version)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_search_datasets_filtered(self, mock_post):
version = '1.0'
result_data = {'results': [{'version': version}, {'version': version}]}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.search_datasets(filtered_query={'version': version}, console=False)
# Verify Success
self.assertTrue(result['success'])
# Check search results if they exist
search_results = result['result']['results']
if len(search_results) > 1:
for result in search_results:
self.assertIn('version', result)
self.assertEqual(result['version'], version)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_search_datasets_no_queries(self, mock_post):
version = '1.0'
result_data = {'results': [{'version': version}, {'version': version}]}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
self.assertRaises(Exception, self.engine.search_datasets, console=False)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_create_dataset(self, mock_post):
# Setup
new_dataset_name = random_string_generator(10)
result_data = {'name': new_dataset_name}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.create_dataset(name=new_dataset_name)
# Verify Success
self.assertTrue(result['success'])
# Should return the new one
self.assertEqual(new_dataset_name, result['result']['name'])
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_create_resource_url(self, mock_post):
# Setup
new_resource_name = random_string_generator(5)
new_resource_url = 'http://home.byu.edu'
result_data = {'name': new_resource_name, 'url': new_resource_url,
'id': self.test_dataset_name}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.create_resource(dataset_id=self.test_dataset_name,
url=new_resource_url,
name=new_resource_name)
# Verify Success
self.assertTrue(result['success'])
# Verify name and url
self.assertEqual(new_resource_name, result['result']['name'])
self.assertEqual(new_resource_url, result['result']['url'])
def test_create_resource_url_file(self):
file_name = 'upload_test.txt'
file_to_upload = os.path.join(self.support_path, file_name)
# Setup
new_resource_url = 'http://home.byu.edu'
# Execute file=file_to_upload
self.assertRaises(IOError, self.engine.create_resource, dataset_id=self.test_dataset_name,
url=new_resource_url,
file=file_to_upload)
def test_create_resource_no_url_file(self):
# Execute
self.assertRaises(IOError, self.engine.create_resource, dataset_id=self.test_dataset_name)
def test_create_resource_file_not_exist(self):
file_name = 'upload_test1.txt'
file_to_upload = os.path.join(self.support_path, file_name)
# Execute file=file_to_upload
self.assertRaises(IOError, self.engine.create_resource, dataset_id=self.test_dataset_name,
file=file_to_upload)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_create_resource_file_upload(self, mock_post):
# Prepare
file_name = 'upload_test.txt'
file_to_upload = os.path.join(self.support_path, file_name)
result_data = {'name': file_name, 'url_type': 'upload',
'id': self.test_dataset_name}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.create_resource(dataset_id=self.test_dataset_name, file=file_to_upload, console=False)
# Verify Success
self.assertTrue(result['success'], result)
# Verify name and url_type (which should be upload if file upload)
self.assertEqual(result['result']['name'], 'upload_test.txt')
self.assertEqual(result['result']['url_type'], 'upload')
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_create_resource_file_upload_no_ext(self, mock_post):
# Prepare
file_name = 'upload_test.txt'
upload_file_name = 'testfile'
file_to_upload = os.path.join(self.support_path, file_name)
result_data = {'name': upload_file_name, 'url_type': 'upload', 'id': self.test_dataset_name}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.create_resource(dataset_id=self.test_dataset_name,
file=file_to_upload,
name=upload_file_name,
console=False)
# Verify Success
self.assertTrue(result['success'], result)
# Verify name and url_type (which should be upload if file upload)
self.assertEqual(upload_file_name, result['result']['name'])
self.assertEqual(result['result']['url_type'], 'upload')
@mock.patch('tethys_dataset_services.engines.ckan_engine.pprint')
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_get_dataset(self, mock_post, _):
result_data = {'name': self.test_dataset_name, 'id': self.test_dataset_name}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.get_dataset(dataset_id=self.test_dataset_name, console=True)
# Verify Success
self.assertTrue(result['success'])
# Verify Name
self.assertEqual(result['result']['name'], self.test_dataset_name)
@mock.patch('tethys_dataset_services.engines.ckan_engine.pprint')
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_get_resource(self, mock_post, mock_pprint):
result_data = {'name': self.test_dataset_name, 'url': self.test_resource_url}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.get_resource(resource_id=self.test_resource_name, console=True)
# Verify Success
self.assertTrue(result['success'])
# Verify Properties
self.assertEqual(result['result']['url'], self.test_resource_url)
mock_pprint.pprint.assert_called()
@mock.patch('tethys_dataset_services.engines.ckan_engine.log')
@mock.patch('tethys_dataset_services.engines.ckan_engine.pprint')
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_get_resource_console_error(self, mock_post, mock_pprint, mock_log):
mock_pprint.pprint.side_effect = Exception('Fake Exception')
result_data = {'name': self.test_dataset_name, 'url': self.test_resource_url}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.get_resource(resource_id=self.test_resource_name, console=True)
# Verify Success
self.assertTrue(result['success'])
# Verify Properties
self.assertEqual(result['result']['url'], self.test_resource_url)
mock_pprint.pprint.assert_called()
mock_log.exception.assert_called()
@mock.patch('tethys_dataset_services.engines.ckan_engine.log')
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_get_resource_get_error(self, mock_post, mock_log):
result_data = {'name': self.test_dataset_name, 'url': self.test_resource_url}
mock_post.return_value = MockJsonResponse(200, result=result_data, success=False)
# Execute
result = self.engine.get_resource(resource_id=self.test_resource_name, console=True)
# Verify Success
self.assertFalse(result['success'])
# Verify Properties
self.assertIn('failed message', result['error']['message'])
self.assertEqual(result['result']['url'], self.test_resource_url)
mock_log.error.assert_called()
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_update_dataset(self, mock_post):
# Setup
test_version = '2.0'
result_data = {'version': test_version, 'resources': self.test_resource_name,
'tags': 'tag_test'}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.update_dataset(dataset_id=self.test_dataset_name, version=test_version)
# Verify Success
self.assertTrue(result['success'])
# Verify new version property
self.assertEqual(result['result']['version'], test_version)
self.assertEqual(result['result']['resources'], self.test_resource_name,)
self.assertEqual(result['result']['tags'], 'tag_test')
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_update_resource_property_change(self, mock_post):
# Setup
new_format = 'web'
result_data = {'format': new_format, 'url': self.test_resource_url}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.update_resource(resource_id=self.test_resource_name, format=new_format)
# Verify Success
self.assertTrue(result['success'])
# Verify new format
self.assertEqual(result['result']['format'], new_format)
self.assertEqual(result['result']['url'], self.test_resource_url)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_update_resource_url_change(self, mock_post):
# Setup
new_url = 'http://www.utah.edu'
result_data = {'url': new_url}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.update_resource(resource_id=self.test_resource_name, url=new_url)
# Verify Success
self.assertTrue(result['success'])
# Verify New URL Property
self.assertEqual(result['result']['url'], new_url)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_update_resource_file_upload(self, mock_post):
# Setup
file_name = 'upload_test.txt'
file_to_upload = os.path.join(self.support_path, file_name)
result_data = {'name': file_name, 'id': self.test_dataset_name,
'url': self.test_resource_url}
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.update_resource(resource_id=self.test_resource_name, file=file_to_upload, console=False)
# Verify Success
self.assertTrue(result['success'])
# Verify Name (should be the same as the file uploaded by default)
self.assertEqual(result['result']['name'], file_name)
self.assertEqual(result['result']['url'], self.test_resource_url)
# URL should be different than original when file upload executes
# self.assertNotEqual(result['result']['url'], self.test_resource['url'])
def test_update_resource_url_file(self):
file_name = 'upload_test.txt'
file_to_upload = os.path.join(self.support_path, file_name)
# Setup
new_resource_url = 'http://home.byu.edu'
# Execute file=file_to_upload
self.assertRaises(IOError, self.engine.update_resource, resource_id=self.test_resource_name,
url=new_resource_url,
file=file_to_upload)
def test_update_resource_file_not_exist(self):
file_name = 'upload_test1.txt'
file_to_upload = os.path.join(self.support_path, file_name)
# Execute file=file_to_upload
self.assertRaises(IOError, self.engine.update_resource, resource_id=self.test_resource_name,
file=file_to_upload)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_delete_resource(self, mock_post):
result_data = None
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.delete_resource(resource_id=self.test_resource_name)
# Verify Success
self.assertTrue(result['success'])
# Delete requests should return nothing
self.assertEqual(result['result'], None)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_delete_dataset(self, mock_post):
result_data = None
mock_post.return_value = MockJsonResponse(200, result=result_data)
# Execute
result = self.engine.delete_dataset(dataset_id=self.test_dataset_name, console=False)
# Confirm Success
self.assertTrue(result['success'])
# Delete requests should return nothing
self.assertEqual(result['result'], None)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_download_resource(self, mock_post):
location = self.files_path
local_file_name = 'test_resource.test'
location_final = os.path.join(self.files_path, local_file_name)
result_data = {'url': self.test_resource_url}
mock_post.return_value = MockJsonResponse(200, result=result_data)
result = self.engine.download_resource(self.test_resource_name, location=location,
local_file_name=local_file_name)
# Result will return the local file path. Check here
self.assertEqual(location_final, result)
# Delete the file
if os.path.isfile(location_final):
os.remove(location_final)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_download_resource_no_location(self, mock_post):
local_file_name = 'test_resource.test'
location_check = os.path.join('./', local_file_name)
result_data = {'url': self.test_resource_url}
mock_post.return_value = MockJsonResponse(200, result=result_data)
result = self.engine.download_resource(self.test_resource_name,
local_file_name=local_file_name)
# Result will return the local file path. Check here
self.assertEqual(location_check, result)
# Delete the file
if os.path.isfile(location_check):
os.remove(location_check)
@mock.patch('tethys_dataset_services.engines.ckan_engine.CkanDatasetEngine.get_resource')
def test_download_resource_not_exist(self, mock_ckan):
mock_ckan.return_value = {'success': False}
self.assertRaises(Exception, self.engine.download_resource, self.test_dataset_name)
mock_ckan.assert_called_with(self.test_dataset_name, console=False)
@mock.patch('sys.stdout', new_callable=StringIO)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.get')
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_download_resource_request_get_exception(self, mock_post, mock_get, mock_print):
mock_get.side_effect = Exception('Requests.get Exception')
location = self.files_path
local_file_name = 'test_resource.test'
result_data = {'url': self.test_resource_url}
mock_post.return_value = MockJsonResponse(200, result=result_data)
self.engine.download_resource(self.test_resource_name, location=location,
local_file_name=local_file_name)
output = mock_print.getvalue()
# check results
self.assertIn('Requests.get Exception', output)
@mock.patch('tethys_dataset_services.engines.ckan_engine.warnings')
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_download_resouce(self, mock_post, mock_warnings):
location = self.files_path
local_file_name = 'test_resource.test'
location_final = os.path.join(self.files_path, local_file_name)
result_data = {'url': self.test_resource_url}
mock_post.return_value = MockJsonResponse(200, result=result_data)
result = self.engine.download_resouce(self.test_resource_name, location=location,
local_file_name=local_file_name)
# Result will return None instead of the local path file.
self.assertEqual(None, result)
# Check if file is created and delete it
if os.path.isfile(location_final):
os.remove(location_final)
else:
raise AssertionError('Resource has not been downloaded')
# Delete the file
if os.path.isfile(location_final):
os.remove(location_final)
mock_warnings.warn.assert_called()
@mock.patch('tethys_dataset_services.engines.ckan_engine.pprint')
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.post')
def test_download_dataset(self, mock_post, _):
location = self.files_path
location_final = os.path.join(self.files_path, 'resource1.txt')
result_check = [location_final]
result_data = {'resources': [{'name': 'resource1', 'id': 'resource2',
'format': 'txt', 'url': self.test_resource_url}]}
mock_post.return_value = MockJsonResponse(200, result=result_data)
result = self.engine.download_dataset(self.test_dataset_name, location=location,
console=True)
# Result will return list of the local file path. Check here
self.assertEqual(result_check, result)
# Delete the file
if os.path.isfile(location_final):
os.remove(location_final)
@mock.patch('tethys_dataset_services.engines.ckan_engine.CkanDatasetEngine.get_dataset')
def test_download_dataset_not_exist(self, mock_ckan):
mock_ckan.return_value = {'success': False}
self.assertRaises(Exception, self.engine.download_dataset, self.test_dataset_name)
mock_ckan.assert_called_with(self.test_dataset_name, console=False)
def test_type(self):
response = self.engine.type
expected_response = 'CKAN'
# Check Response
self.assertEqual(response, expected_response)
def test_prepare_request(self):
method = 'resource_show'
result = self.engine._prepare_request(method, apikey=TEST_CKAN_DATASET_SERVICE['APIKEY'])
# Check Result, result[0] is url, result[1] is data_dict, result[2] is headers
self.assertIn(TEST_CKAN_DATASET_SERVICE['APIKEY'], result[2]['X-CKAN-API-Key'])
self.assertIn(method, result[0])
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.get')
def test_validate(self, mock_get):
mock_get.side_effect = requests.exceptions.MissingSchema
self.assertRaises(AssertionError, self.engine.validate)
@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.get')
def test_validate_status_code(self, mock_get):
self.engine = CkanDatasetEngine(endpoint="http://*****:*****@mock.patch('tethys_dataset_services.engines.ckan_engine.requests.get')
def test_validate_no_version(self, mock_get):
mock_get.return_value = MockResponse(200, json='')
self.assertRaises(AssertionError, self.engine.validate)