def setUp(self):
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
Secret.set('testsecret')
self.cluster = FunctionalTestCluster(verbose=self._get_test_verbosity())
def setUp(self):
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
self._reset_config()
Secret.set('testsecret')
SlaveRegistry.reset_singleton()
self.cluster = FunctionalTestCluster(verbose=self._get_test_verbosity())
self._network = Network()
def setUp(self):
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
Secret.set('testsecret')
self.test_app_base_dir = tempfile.TemporaryDirectory()
self.test_conf_file_path = self._create_test_config_file({
'secret':
Secret.get(),
'base_directory':
self.test_app_base_dir.name,
# Set the max log file size to a low value so that we cause at least one rollover during the test.
'max_log_file_size':
1024 * 5,
})
self.cluster = FunctionalTestCluster(
conf_file_path=self.test_conf_file_path,
verbose=self._get_test_verbosity(),
)
def setUp(self):
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
Secret.set('testsecret')
self.test_app_base_dir = tempfile.TemporaryDirectory()
self.test_conf_file_path = self._create_test_config_file({
'secret': Secret.get(),
'base_directory': self.test_app_base_dir.name,
# Set the max log file size to a low value so that we cause at least one rollover during the test.
'max_log_file_size': 1024 * 5,
})
self.cluster = FunctionalTestCluster(
conf_file_path=self.test_conf_file_path,
verbose=self._get_test_verbosity(),
)
class BaseFunctionalTestCase(TestCase):
"""
This is the base class for all functional tests. This class has two main purposes:
- Make available a `FunctionalTestCluster` object for use in functional tests (self.cluster)
- Implement any helper assertion methods that might be useful for making our tests easier to read and write
"""
def setUp(self):
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
Secret.set('testsecret')
self.test_app_base_dir = tempfile.TemporaryDirectory()
self.test_conf_file_path = self._create_test_config_file({
'secret': Secret.get(),
'base_directory': self.test_app_base_dir.name,
# Set the max log file size to a low value so that we cause at least one rollover during the test.
'max_log_file_size': 1024 * 5,
})
self.cluster = FunctionalTestCluster(
conf_file_path=self.test_conf_file_path,
verbose=self._get_test_verbosity(),
)
def _create_test_config_file(self, conf_values_to_set=None):
"""
Create a temporary conf file just for this test.
:return: The path to the conf file
:rtype: str
"""
# Copy default conf file to tmp location
repo_dir = path.dirname(path.dirname(path.dirname(path.dirname(path.realpath(__file__)))))
self._conf_template_path = path.join(repo_dir, 'conf', 'default_clusterrunner.conf')
test_conf_file_path = tempfile.NamedTemporaryFile().name
shutil.copy(self._conf_template_path, test_conf_file_path)
os.chmod(test_conf_file_path, ConfigFile.CONFIG_FILE_MODE)
conf_file = ConfigFile(test_conf_file_path)
# Set custom conf file values for this test
conf_values_to_set = conf_values_to_set or {}
for conf_key, conf_value in conf_values_to_set.items():
conf_file.write_value(conf_key, conf_value, BASE_CONFIG_FILE_SECTION)
return test_conf_file_path
def tearDown(self):
# Clean up files created during this test
with suppress(FileNotFoundError):
os.remove(self.test_conf_file_path)
# Give the cluster a bit of extra time to finish working (before forcefully killing it and failing the test)
with suppress(TestClusterTimeoutError):
self.cluster.block_until_build_queue_empty(timeout=5)
# Kill processes and make sure all processes exited with 0 exit code
services = self.cluster.kill()
# only check the exit code if not on Windows as Popen.terminate kills the process on Windows and the exit
# code is not zero.
# TODO: remove the is_windows() check after we can handle exit on Windows gracefully.
if not is_windows():
for service in services:
self.assertEqual(
service.return_code,
0,
'Service running on url: {} should exit with code 0, but exited with code {}.'.format(
service.url,
service.return_code,
),
)
# Remove the temp dir. This will delete the log files, so should be run after cluster shuts down.
self.test_app_base_dir.cleanup()
def _get_test_verbosity(self):
"""
Get test verbosity from an env variable. We need to use an env var since Nose does not support specifying
command-line test configuration natively. (But if we need more of these configuration paramaters, we should
instead look at the 'nose-testconfig' plugin instead of adding tons of environment variables.)
:return: Whether or not tests should be run verbosely
:rtype: bool
"""
is_verbose = os.getenv('CR_VERBOSE') not in ('0', '', None) # default value of is_verbose is False
return is_verbose
def assert_build_status_contains_expected_data(self, build_id, expected_data):
"""
Assert that the build status endpoint contains the expected fields and values. This assertion does an API
request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
:param expected_data: A dict of expected keys and values in the build status response
:type expected_data: dict
"""
build_status = self.cluster.master_api_client.get_build_status(build_id).get('build')
self.assertIsInstance(build_status, dict, 'Build status API request should return a dict.')
self.assertDictContainsSubset(expected_data, build_status,
'Build status API response should contain the expected status data.')
def assert_build_has_successful_status(self, build_id):
"""
Assert that the build status endpoint contains fields signifying the build was successful (had no failures).
This assertion does an API request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
"""
expected_successful_build_params = {
'result': 'NO_FAILURES',
'status': 'FINISHED',
}
self.assert_build_status_contains_expected_data(build_id, expected_successful_build_params)
def assert_build_has_failure_status(self, build_id):
"""
Assert that the build status endpoint contains fields signifying the build was failed. This assertion does an
API request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
"""
expected_failure_build_params = {
'result': 'FAILURE',
'status': 'FINISHED',
}
self.assert_build_status_contains_expected_data(build_id, expected_failure_build_params)
def assert_build_has_canceled_status(self, build_id):
"""
Assert that the build status endpoint contains fields signifying the build was failed. This assertion does an
API request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
"""
expected_failure_build_params = {
'result': 'FAILURE',
'status': 'CANCELED',
}
self.assert_build_status_contains_expected_data(build_id, expected_failure_build_params)
def assert_build_artifact_contents_match_expected(self, build_id, expected_build_artifact_contents):
"""
Assert that artifact files for this build have the expected contents.
:param build_id: The id of the build whose artifacts to check
:type build_id: int
:param expected_build_artifact_contents: A list of FSItems corresponding to the expected artifact dir contents
:type expected_build_artifact_contents: list[FSItem]
"""
build_artifacts_dir_path = os.path.join(self.test_app_base_dir.name, 'results', 'master', str(build_id))
self.assert_directory_contents_match_expected(build_artifacts_dir_path, expected_build_artifact_contents)
def assert_directory_contents_match_expected(self, dir_path, expected_dir_contents):
"""
Assert that the specified directory has the expected contents.
:param dir_path: The path of the directory whose artifacts to check
:type dir_path: string
:param expected_dir_contents: A list of FSItems corresponding to the expected directory contents
:type expected_dir_contents: list[FSItem]
"""
if expected_dir_contents is not None:
dir_path = os.path.abspath(dir_path) # converts path to absolute, removes trailing slash if present
expected_dir_name = os.path.basename(dir_path)
expected_build_artifacts = Directory(expected_dir_name, expected_dir_contents)
expected_build_artifacts.assert_matches_path(dir_path, allow_extra_items=False)
class BaseFunctionalTestCase(TestCase):
"""
This is the base class for all functional tests. This class has two main purposes:
- Make available a `FunctionalTestCluster` object for use in functional tests (self.cluster)
- Implement any helper assertion methods that might be useful for making our tests easier to read and write
"""
def setUp(self):
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
Secret.set('testsecret')
self.cluster = FunctionalTestCluster(
verbose=self._get_test_verbosity())
self._network = Network()
def _create_test_config_file(self, conf_values_to_set=None):
"""
Create a temporary conf file just for this test.
:return: The path to the conf file
:rtype: str
"""
# Copy default conf file to tmp location
repo_dir = path.dirname(
path.dirname(path.dirname(path.dirname(path.realpath(__file__)))))
self._conf_template_path = path.join(repo_dir, 'conf',
'default_clusterrunner.conf')
test_conf_file_path = tempfile.NamedTemporaryFile().name
shutil.copy(self._conf_template_path, test_conf_file_path)
os.chmod(test_conf_file_path, ConfigFile.CONFIG_FILE_MODE)
conf_file = ConfigFile(test_conf_file_path)
# Set custom conf file values for this test
conf_values_to_set = conf_values_to_set or {}
for conf_key, conf_value in conf_values_to_set.items():
conf_file.write_value(conf_key, conf_value,
BASE_CONFIG_FILE_SECTION)
return test_conf_file_path
def tearDown(self):
# Give the cluster a bit of extra time to finish working (before forcefully killing it and failing the test)
with suppress(TestClusterTimeoutError):
self.cluster.block_until_build_queue_empty(timeout=5)
# Kill processes and make sure all processes exited with 0 exit code
services = self.cluster.kill()
# only check the exit code if not on Windows as Popen.terminate kills the process on Windows and the exit
# code is not zero.
# TODO: remove the is_windows() check after we can handle exit on Windows gracefully.
if not is_windows():
for service in services:
self.assertEqual(
service.return_code,
0,
'Service running on url: {} should exit with code 0, but exited with code {}.'
.format(
service.url,
service.return_code,
),
)
# Remove the temp dir. This will delete the log files, so should be run after cluster shuts down.
self.cluster.master_app_base_dir.cleanup()
[
slave_app_base_dir.cleanup()
for slave_app_base_dir in self.cluster.slaves_app_base_dirs
]
def _get_test_verbosity(self):
"""
Get test verbosity from an env variable. We need to use an env var since Nose does not support specifying
command-line test configuration natively. (But if we need more of these configuration paramaters, we should
instead look at the 'nose-testconfig' plugin instead of adding tons of environment variables.)
:return: Whether or not tests should be run verbosely
:rtype: bool
"""
is_verbose = os.getenv('CR_VERBOSE') not in (
'0', '', None) # default value of is_verbose is False
return is_verbose
def assert_build_status_contains_expected_data(self, build_id,
expected_data):
"""
Assert that the build status endpoint contains the expected fields and values. This assertion does an API
request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
:param expected_data: A dict of expected keys and values in the build status response
:type expected_data: dict
"""
build_status = self.cluster.master_api_client.get_build_status(
build_id).get('build')
self.assertIsInstance(
build_status, dict,
'Build status API request should return a dict.')
self.assertDictContainsSubset(
expected_data, build_status,
'Build status API response should contain the expected status data.'
)
def assert_build_has_successful_status(self, build_id):
"""
Assert that the build status endpoint contains fields signifying the build was successful (had no failures).
This assertion does an API request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
"""
expected_successful_build_params = {
'result': 'NO_FAILURES',
'status': 'FINISHED',
}
self.assert_build_status_contains_expected_data(
build_id, expected_successful_build_params)
def assert_build_has_failure_status(self, build_id):
"""
Assert that the build status endpoint contains fields signifying the build was failed. This assertion does an
API request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
"""
expected_failure_build_params = {
'result': 'FAILURE',
'status': 'FINISHED',
}
self.assert_build_status_contains_expected_data(
build_id, expected_failure_build_params)
def assert_build_has_canceled_status(self, build_id):
"""
Assert that the build status endpoint contains fields signifying the build was failed. This assertion does an
API request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
"""
expected_failure_build_params = {
'result': 'FAILURE',
'status': 'CANCELED',
}
self.assert_build_status_contains_expected_data(
build_id, expected_failure_build_params)
def assert_build_artifact_contents_match_expected(
self, master_api, build_id, expected_build_artifact_contents):
"""
Assert that artifact files for this build have the expected contents.
:type master_api: app.util.url_builder.UrlBuilder
:param build_id: The id of the build whose artifacts to check
:type build_id: int
:param expected_build_artifact_contents: A list of FSItems corresponding to the expected artifact dir contents
:type expected_build_artifact_contents: list[FSItem]
"""
with tempfile.TemporaryDirectory() as build_artifacts_dir_path:
self._download_and_extract_results(master_api, build_id,
build_artifacts_dir_path)
self.assert_directory_contents_match_expected(
build_artifacts_dir_path, expected_build_artifact_contents)
def assert_directory_contents_match_expected(self, dir_path,
expected_dir_contents):
"""
Assert that the specified directory has the expected contents.
:param dir_path: The path of the directory whose artifacts to check
:type dir_path: string
:param expected_dir_contents: A list of FSItems corresponding to the expected directory contents
:type expected_dir_contents: list[FSItem]
"""
if expected_dir_contents is not None:
dir_path = os.path.abspath(
dir_path
) # converts path to absolute, removes trailing slash if present
expected_dir_name = os.path.basename(dir_path)
expected_build_artifacts = Directory(expected_dir_name,
expected_dir_contents)
expected_build_artifacts.assert_matches_path(
dir_path, allow_extra_items=False)
def _download_and_extract_results(self, master_api, build_id,
download_dir):
"""
:type master_api: app.util.url_builder.UrlBuilder
:type build_id: int
:type download_dir: str
"""
download_artifacts_url = master_api.url('build', build_id, 'result')
download_filepath = os.path.join(download_dir,
BuildArtifact.ARTIFACT_FILE_NAME)
response = self._network.get(download_artifacts_url)
if response.status_code == http.client.OK:
# save tar file to disk, decompress, and delete
with open(download_filepath, 'wb') as file:
chunk_size = 500 * 1024
for chunk in response.iter_content(chunk_size):
file.write(chunk)
extract_tar(download_filepath, delete=True)
class BaseFunctionalTestCase(TestCase):
"""
This is the base class for all functional tests. This class has two main purposes:
- Make available a `FunctionalTestCluster` object for use in functional tests (self.cluster)
- Implement any helper assertion methods that might be useful for making our tests easier to read and write
"""
def setUp(self):
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
self._reset_config()
Secret.set('testsecret')
SlaveRegistry.reset_singleton()
self.cluster = FunctionalTestCluster(verbose=self._get_test_verbosity())
self._network = Network()
def _reset_config(self):
Configuration.reset_singleton()
config = Configuration.singleton()
conf_loader = BaseConfigLoader()
conf_loader.configure_defaults(config)
conf_loader.configure_postload(config)
def tearDown(self):
# Give the cluster a bit of extra time to finish working (before forcefully killing it and failing the test)
with suppress(TestClusterTimeoutError):
self.cluster.block_until_build_queue_empty(timeout=5)
# Kill processes and make sure all processes exited with 0 exit code
services = self.cluster.kill()
# only check the exit code if not on Windows as Popen.terminate kills the process on Windows and the exit
# code is not zero.
# TODO: remove the is_windows() check after we can handle exit on Windows gracefully.
if not is_windows():
for service in services:
self.assertEqual(
service.return_code,
0,
'Service running on url: {} should exit with code 0, but exited with code {}.'.format(
service.url,
service.return_code,
),
)
# Remove the temp dir. This will delete the log files, so should be run after cluster shuts down.
self.cluster.master_app_base_dir.cleanup()
[slave_app_base_dir.cleanup() for slave_app_base_dir in self.cluster.slaves_app_base_dirs]
def _get_test_verbosity(self):
"""
Get test verbosity from an env variable. We need to use an env var since Nose does not support specifying
command-line test configuration natively. (But if we need more of these configuration paramaters, we should
instead look at the 'nose-testconfig' plugin instead of adding tons of environment variables.)
:return: Whether or not tests should be run verbosely
:rtype: bool
"""
is_verbose = os.getenv('CR_VERBOSE') not in ('0', '', None) # default value of is_verbose is False
return is_verbose
def assert_build_status_contains_expected_data(self, build_id, expected_data):
"""
Assert that the build status endpoint contains the expected fields and values. This assertion does an API
request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
:param expected_data: A dict of expected keys and values in the build status response
:type expected_data: dict
"""
build_status = self.cluster.master_api_client.get_build_status(build_id).get('build')
self.assertIsInstance(build_status, dict, 'Build status API request should return a dict.')
self.assertDictContainsSubset(expected_data, build_status,
'Build status API response should contain the expected status data.')
def assert_build_has_successful_status(self, build_id):
"""
Assert that the build status endpoint contains fields signifying the build was successful (had no failures).
This assertion does an API request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
"""
expected_successful_build_params = {
'result': 'NO_FAILURES',
'status': 'FINISHED',
}
self.assert_build_status_contains_expected_data(build_id, expected_successful_build_params)
def assert_build_has_failure_status(self, build_id):
"""
Assert that the build status endpoint contains fields signifying the build was failed. This assertion does an
API request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
"""
expected_failure_build_params = {
'result': 'FAILURE',
'status': 'FINISHED',
}
self.assert_build_status_contains_expected_data(build_id, expected_failure_build_params)
def assert_build_has_canceled_status(self, build_id):
"""
Assert that the build status endpoint contains fields signifying the build was failed. This assertion does an
API request to the master service of self.cluster.
:param build_id: The id of the build whose status to check
:type build_id: int
"""
expected_failure_build_params = {
'result': 'FAILURE',
'status': 'CANCELED',
}
self.assert_build_status_contains_expected_data(build_id, expected_failure_build_params)
def assert_build_artifact_contents_match_expected(self, master_api, build_id, expected_build_artifact_contents):
"""
Assert that artifact files for this build have the expected contents.
:type master_api: app.util.url_builder.UrlBuilder
:param build_id: The id of the build whose artifacts to check
:type build_id: int
:param expected_build_artifact_contents: A list of FSItems corresponding to the expected artifact dir contents
:type expected_build_artifact_contents: list[FSItem]
"""
with tempfile.TemporaryDirectory() as build_artifacts_dir_path:
self._download_and_extract_zip_results(master_api, build_id, build_artifacts_dir_path)
self.assert_directory_contents_match_expected(build_artifacts_dir_path, expected_build_artifact_contents)
# Also check the tar archive even though it is deprecated.
with tempfile.TemporaryDirectory() as build_artifacts_dir_path:
self._download_and_extract_tar_results(master_api, build_id, build_artifacts_dir_path)
self.assert_directory_contents_match_expected(build_artifacts_dir_path, expected_build_artifact_contents)
def assert_directory_contents_match_expected(self, dir_path, expected_dir_contents):
"""
Assert that the specified directory has the expected contents.
:param dir_path: The path of the directory whose artifacts to check
:type dir_path: string
:param expected_dir_contents: A list of FSItems corresponding to the expected directory contents
:type expected_dir_contents: list[FSItem]
"""
if expected_dir_contents is not None:
dir_path = os.path.abspath(dir_path) # converts path to absolute, removes trailing slash if present
expected_dir_name = os.path.basename(dir_path)
expected_build_artifacts = Directory(expected_dir_name, expected_dir_contents)
expected_build_artifacts.assert_matches_path(dir_path, allow_extra_items=False)
def _download_and_extract_tar_results(self, master_api, build_id, download_dir):
"""
:type master_api: app.util.url_builder.UrlBuilder
:type build_id: int
:type download_dir: str
"""
download_artifacts_url = master_api.url('build', build_id, 'result')
download_filepath = os.path.join(download_dir, BuildArtifact.ARTIFACT_TARFILE_NAME)
response = self._network.get(download_artifacts_url)
if response.status_code == http.client.OK:
# save tar file to disk, decompress, and delete
with open(download_filepath, 'wb') as file:
chunk_size = 500 * 1024
for chunk in response.iter_content(chunk_size):
file.write(chunk)
fs.extract_tar(download_filepath, delete=True)
def _download_and_extract_zip_results(self, master_api: UrlBuilder, build_id: int, download_dir: str):
"""Download the artifacts.zip from the master and extract it."""
download_artifacts_url = master_api.url('build', build_id, 'artifacts.zip')
download_filepath = os.path.join(download_dir, BuildArtifact.ARTIFACT_ZIPFILE_NAME)
response = self._network.get(download_artifacts_url)
if response.status_code == http.client.OK:
# save file to disk, decompress, and delete
with open(download_filepath, 'wb') as file:
chunk_size = 500 * 1024
for chunk in response.iter_content(chunk_size):
file.write(chunk)
fs.unzip_directory(download_filepath, delete=True)