def kafka2hdfs_app(class_context, kafka_instance, hdfs_instance, kerberos_instance, api_service_admin_client):
log_fixture("kafka2hdfs: download libraries")
ingestion_repo = AppSources.get_repository(repo_name=TapGitHub.ws_kafka_hdfs, repo_owner=TapGitHub.intel_data)
kafka2hdfs_path = os.path.join(ingestion_repo.path, TapGitHub.kafka2hdfs)
log_fixture("Package kafka2hdfs app")
ingestion_repo.compile_gradle(working_directory=kafka2hdfs_path)
build_path = os.path.join(kafka2hdfs_path, "deploy")
ingestion_repo.run_build_sh(cwd=build_path, command="./pack.sh")
app_path = os.path.join(build_path, "kafka2hdfs.tar")
log_fixture("kafka2hdfs: update manifest")
p_a = PrepApp(build_path)
manifest_params = {"bindings": [kafka_instance.name, hdfs_instance.name, kerberos_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("kafka2hdfs: push application")
app = Application.push(class_context, app_path=app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("kafka2hdfs: Check the application is running")
app.ensure_running()
return app
def test_data_urls(session_context):
""" Fixture with files from tests/fixtures/data_repo.py::DATA_FILES.
They are accessible by url or filepath.
Eg.:
test_data_urls.test_transfer.url
test_data_urls['test_transfer'].url
test_data_urls.test_transfer.filepath
"""
if config.data_repo_url:
return Urls(config.data_repo_url)
else:
log_fixture("data_repo: package sample application")
p_a = PrepApp(DATA_REPO_PATH)
if os.path.exists(DATA_REPO_TAR_PATH):
gzipped_app_path = DATA_REPO_TAR_PATH
else:
gzipped_app_path = p_a.package_app(session_context)
log_fixture("data_repo: update manifest")
manifest_path = p_a.update_manifest(params={})
log_fixture("data_repo: Push data_repo application")
app = Application.push(context=session_context, app_path=gzipped_app_path, name=p_a.app_name,
manifest_path=manifest_path)
log_fixture("data_repo: Check application is running")
app.ensure_running()
return Urls(app.urls[0])
def test_cannot_push_application_twice(self, context, test_sample_apps,
sample_app,
api_service_admin_client):
"""
<b>Description:</b>
Tries to push the same application twice
<b>Input data:</b>
- Sample application that will be pushed twice
<b>Expected results:</b>
- The second push is not possible
<b>Steps:</b>
- Application is pushed
- Application is pushed again in the test itself
- The second push is blocked
"""
step("Check that pushing the same application again causes an error")
log_fixture("sample_application: update manifest")
p_a = PrepApp(test_sample_apps.tapng_python_app.filepath)
manifest_params = {
"type": TapApplicationType.PYTHON27,
"name": sample_app.name
}
manifest_path = p_a.update_manifest(params=manifest_params)
with pytest.raises(UnexpectedResponseError) as e:
Application.push(
context,
app_path=test_sample_apps.tapng_python_app.filepath,
name=sample_app.name,
manifest_path=manifest_path,
client=api_service_admin_client)
assert self.EXPECTED_MESSAGE_WHEN_APP_PUSHED_TWICE in str(e)
def test_try_push_app_with_invalid_manifest(self, class_context, tap_cli, sample_app_target_directory):
"""
<b>Description:</b>
Try to push an application with invalid manifest.
<b>Input data:</b>
1. Sample application with invalid manifest.json
<b>Expected results:</b>
Test passes when attempt to push application with invalid app name in manifest.json returns proper message.
<b>Steps:</b>
1. Prepare directory with sample application.
2. Update manifest.json with invalid app name.
3. Try to push application with wrong manifest.json.
"""
step("prepare directory with sample application")
p_a = PrepApp(sample_app_target_directory)
step("update manifest.json to contain improper app name")
manifest_params = {"name": self.INVALID_APP_NAME, "type": self.APP_TYPE}
p_a.update_manifest(params=manifest_params)
assert_raises_command_execution_exception(1, TapMessage.MANIFEST_JSON_INCORRECT_NAME_VALUE.format(self.INVALID_APP_NAME),
CliApplication.push,
class_context, tap_cli=tap_cli,
app_type=self.APP_TYPE,
app_path=p_a.app_path,
name=p_a.app_name,
instances=p_a.instances)
def app_bound_psql(module_context, psql_instance, api_service_admin_client):
log_fixture("psql_app: Download libraries")
app_src = AppSources.from_local_path(ApplicationPath.SQL_API_EXAMPLE)
p_a = PrepApp(app_src.path)
log_fixture("psql_app: package sample application")
if os.path.exists(ApplicationPath.SQL_API_EXAMPLE_TAR):
gzipped_app_path = ApplicationPath.SQL_API_EXAMPLE_TAR
else:
gzipped_app_path = p_a.package_app(module_context)
log_fixture("psql_app: update manifest")
manifest_params = {"type": TapApplicationType.PYTHON27, "bindings": [psql_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("psql_app: push sample application")
db_app = Application.push(module_context, app_path=gzipped_app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("psql_app: Check the application is running")
db_app.ensure_running()
log_fixture("psq_app: Check the application is responding")
db_app.ensure_responding()
return db_app
def test_change_app_state_in_catalog_and_delete_it(
self, context, test_sample_apps, api_service_admin_client):
"""
<b>Description:</b>
Change the application state in catalog and later delete it
<b>Input data:</b>
- Path to application
- Admin credentials
<b>Expected results:</b>
- Application state can be changed in catalog
- Application state can be set back via api service
<b>Steps:</b>
- Prepare the application and push it
- Make sure the application is running
- Change the state of the application via catalog
- Make sure the state has changed
- Stop the application via api service client and remove it
- Verify the application was removed
"""
log_fixture("sample_application: update manifest")
p_a = PrepApp(test_sample_apps.tapng_python_app.filepath)
manifest_params = {"type": TapApplicationType.PYTHON27}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("Push sample application and check it's running")
application = Application.push(
context,
app_path=test_sample_apps.tapng_python_app.filepath,
name=p_a.app_name,
manifest_path=manifest_path,
client=api_service_admin_client)
application.ensure_running()
step("Check that the application has only one instance")
instances = CatalogApplicationInstance.get_list_for_application(
application_id=application.id)
assert len(instances) == 1
updated_state = TapEntityState.FAILURE
step("Update app state to {} using catalog api".format(updated_state))
catalog_api.update_instance(instance_id=instances[0].id,
field_name="state",
value=updated_state)
step("Check that the app state was updated")
app = Application.get(app_inst_id=application.id,
client=api_service_admin_client)
assert app.state == updated_state, "Application is not in the expected state. App state: {}".format(
app.state)
step("Check that the application can be deleted")
application.delete()
step("Check that application has been removed")
apps = Application.get_list(client=api_service_admin_client)
assert application not in apps
def sample_application(self, class_context, test_sample_apps, api_service_admin_client):
log_fixture("sample_application: update manifest")
p_a = PrepApp(test_sample_apps[self.SAMPLE_APP].filepath)
manifest_params = {"type" : self.APP_TYPE}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("Push sample application and check it's running")
application = Application.push(class_context, app_path=test_sample_apps[self.SAMPLE_APP].filepath,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
application.ensure_running()
return application
def test_push_and_delete_sample_app_with_symlinks(
self, context, sample_app_target_directory_with_symlinks, tap_cli):
"""
<b>Description:</b>
Push sample application, check it's show in applications list and remove it.
<b>Input data:</b>
1. Sample application with symlinks target directory
<b>Expected results:</b>
Test passes when application is pushed, displayed in applications list and then successfully deleted.
<b>Steps:</b>
1. Update application manifest.
2. Push application.
3. Verify that application is present in applications list.
4. Check that application is running.
5. Check that application is ready.
6. Delete application.
7. Check that application is no longer presented in applications list.
"""
step("Update the manifest")
p_a = PrepApp(sample_app_target_directory_with_symlinks)
manifest_params = {"type": self.APP_TYPE}
manifest_path = p_a.update_manifest(params=manifest_params)
step("Push application")
application = CliApplication.push(
context,
tap_cli=tap_cli,
app_type=self.APP_TYPE,
app_path=sample_app_target_directory_with_symlinks,
name=p_a.app_name,
instances=p_a.instances)
step("Ensure app is on the app list")
application.ensure_on_app_list()
step("Ensure app is running")
application.ensure_app_state(state=TapEntityState.RUNNING)
step("Ensure app is ready")
application.ensure_app_is_ready()
step("Stop application")
application.stop()
application.ensure_app_state(state=TapEntityState.STOPPED)
step("Delete app")
application.delete()
step("Ensure app is not on the app list")
application.ensure_not_on_app_list()
def sample_app(self, class_context, test_sample_apps,
api_service_admin_client):
log_fixture("sample_application: update manifest")
p_a = PrepApp(test_sample_apps.tapng_python_app.filepath)
manifest_params = {"type": TapApplicationType.PYTHON27}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("Push sample application")
application = Application.push(
class_context,
app_path=test_sample_apps.tapng_python_app.filepath,
name=p_a.app_name,
manifest_path=manifest_path,
client=api_service_admin_client)
application.ensure_running()
return application
def test_no_manifest(self, class_context, tap_cli, sample_app_target_directory):
"""
<b>Description:</b>
Try to push an application, but don't provide the manifest.
<b>Input data:</b>
1. Sample application without manifest.json
<b>Expected results:</b>
Test passes when attempt to push application without providing manifest.json returns proper message.
<b>Steps:</b>
1. Prepare directory with sample application.
2. Remove manifest.json from prepared directory.
3. Try to push application.
"""
app_name = "app-test-name"
step("prepare directory with sample application")
p_a = PrepApp(sample_app_target_directory)
step("remove manifest.json from prepared directory")
manifest_path = os.path.join(sample_app_target_directory, p_a.MANIFEST_NAME)
os.remove(manifest_path)
step("try to push application")
assert_raises_command_execution_exception(1, TapMessage.CANNOT_FIND_MANIFEST,
CliApplication.push,
class_context, tap_cli=tap_cli,
app_type=self.APP_TYPE,
app_path=p_a.app_path,
name=app_name,
instances=p_a.instances)
def sample_cli_app(self, class_context, sample_app_target_directory, tap_cli):
step("Update the manifest")
p_a = PrepApp(sample_app_target_directory)
manifest_params = {"type": self.APP_TYPE}
manifest_path = p_a.update_manifest(params=manifest_params)
step("Push application")
application = CliApplication.push(class_context, tap_cli=tap_cli, app_type=self.APP_TYPE,
app_path=sample_app_target_directory,
name=p_a.app_name, instances=p_a.instances)
step("Ensure app is on the app list")
application.ensure_on_app_list()
step("Ensure app is running")
application.ensure_app_state(state=TapEntityState.RUNNING)
step("Ensure app is ready")
application.ensure_app_is_ready()
return application
def test_cascade_app_delete(self, context, instance, admin_client):
"""
<b>Description:</b>
Checks if cascade removal of an application removes bound service instance too.
<b>Input data:</b>
1. Sample service instance.
2. Sample java application.
3. Admin client
<b>Expected results:</b>
The application and the service instance are deleted.
<b>Steps:</b>
1. Push an application with bound service instance.
2. Verify the application is running.
3. Delete the application with cascade flag.
4. Verify the application is deleted.
5. Verify the service instance is deleted.
"""
step("Compile the app")
test_app_sources = AppSources.from_local_path(sources_directory=ApplicationPath.SAMPLE_JAVA_APP)
test_app_sources.compile_mvn()
step("Package the app")
p_a = PrepApp(ApplicationPath.SAMPLE_JAVA_APP)
gzipped_app_path = p_a.package_app(context)
step("Update manifest")
manifest_params = {"type" : TapApplicationType.JAVA,
"bindings" : instance.id}
manifest_path = p_a.update_manifest(params=manifest_params)
step("Push app to tap")
app = Application.push(context, app_path=gzipped_app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=admin_client)
step("Check the application is running")
app.ensure_running()
app.api_delete(cascade=True)
assertions.assert_not_in_by_id_with_retry(app.id, Application.get_list)
assertions.assert_not_in_by_id_with_retry(instance.id, ServiceInstance.get_list)
def test_push_and_delete_application(self, context, test_sample_apps, api_service_admin_client):
"""
<b>Description:</b>
Attempts to push application, stop it and then delete it.
<b>Input data:</b>
- Sample application
- Admin credentials
<b>Expected results:</b>
- Application is successfully pushed to platform
- Apllication is in running state
- It's possible to stop the application
- it's possible to remove the application
- After removal application is no longer available
<b>Steps:</b>
- Sample application is downloaded
- Manifest is updated with unique name
- Application is pushed to platform using admin
- It's verified that the application is in running state
- Application is stopped and it's verified that the application has stopped
- Remove the application and verify it's no longer available
"""
log_fixture("sample_application: update manifest")
p_a = PrepApp(test_sample_apps[self.SAMPLE_APP].filepath)
manifest_params = {"type" : self.APP_TYPE}
manifest_path = p_a.update_manifest(params=manifest_params)
step("Push sample application")
application = Application.push(context, app_path=test_sample_apps[self.SAMPLE_APP].filepath,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
step("Check application is running")
application.ensure_running()
step("Stop application")
application.stop()
application.ensure_stopped()
step("Delete application")
application.delete()
step("Check that application is not on the list")
assertions.assert_not_in_with_retry(application, Application.get_list, client=api_service_admin_client)
def ws2kafka_app(class_context, kafka_instance, api_service_admin_client):
log_fixture("ws2kafka: download libraries")
ingestion_repo = AppSources.get_repository(repo_name=TapGitHub.ws_kafka_hdfs, repo_owner=TapGitHub.intel_data)
ws2kafka_path = os.path.join(ingestion_repo.path, TapGitHub.ws2kafka)
build_path = os.path.join(ws2kafka_path, "deploy")
ingestion_repo.run_build_sh(cwd=build_path, command="./pack.sh")
app_path = os.path.join(build_path, "ws2kafka.tar.gz")
log_fixture("ws2kafka: update manifest")
p_a = PrepApp(build_path)
manifest_params = {"bindings": [kafka_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("ws2kafka: push application")
app = Application.push(class_context, app_path=app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("ws2kafka: Check the application is running")
app.ensure_running()
return app
def test_cannot_push_application_with_invalid_name(
self, context, sample_app, test_sample_apps,
api_service_admin_client):
"""
<b>Description:</b>
Tries to push the application with invalid name
<b>Input data:</b>
- Sample application that was already pushed
- Admin credentials
<b>Expected results:</b>
- Pushing the application with invalid name is forbidden
<b>Steps:</b>
- Application is pushed
- Application is prepared again and it's name is changed to invalid
- Application with the changed name is pushed
"""
step("Change the application name to invalid")
p_a = PrepApp(test_sample_apps.tapng_python_app.filepath)
app_name = "invalid_-application-_name"
manifest_params = {
"type": TapApplicationType.PYTHON27,
"name": app_name
}
manifest_path = p_a.update_manifest(params=manifest_params)
assertions.assert_raises_http_exception(
ApiServiceHttpStatus.CODE_BAD_REQUEST,
CatalogHttpStatus.MSG_INSTANCE_FORBIDDEN_CHARACTERS.format(
app_name, app_name),
Application.push,
context,
app_path=test_sample_apps.tapng_python_app.filepath,
name=app_name,
manifest_path=manifest_path,
client=api_service_admin_client)
def hbase_reader_app(class_context, kerberos_instance, api_service_admin_client):
log_fixture("hbase_reader: download libraries")
hbase_reader_repo = AppSources.get_repository(repo_name=TapGitHub.hbase_api_example, repo_owner=TapGitHub.intel_data)
log_fixture("Package hbase_reader app")
build_path = os.path.join(hbase_reader_repo.path, "deploy")
hbase_reader_repo.run_build_sh(cwd=build_path, command="./pack.sh")
app_path = os.path.join(build_path, "hbase-java-api-example-0.1.1.tar")
log_fixture("hbase_reader: update manifest")
p_a = PrepApp(build_path)
manifest_params = {"bindings": [kerberos_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("hbase_reader: push application")
app = Application.push(class_context, app_path=app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("hbase_reader: Check the application is running")
app.ensure_running()
return app
def app_bound_orientdb(module_context, orientdb_instance, api_service_admin_client):
log_fixture("orientdb_app: download libraries")
app_src = AppSources.from_local_path(ApplicationPath.ORIENTDB_API)
app_src.run_build_sh()
log_fixture("orientdb_app: package sample application")
p_a = PrepApp(app_src.path)
gzipped_app_path = p_a.package_app(module_context)
log_fixture("orientdb_app: update manifest")
manifest_params = {"type": TapApplicationType.PYTHON27, "bindings": [orientdb_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("orientdb_app: push orientdb-api application")
application = Application.push(module_context,
app_path=gzipped_app_path,
name=p_a.app_name,
manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("orientdb_app: Ensure app is running")
application.ensure_running()
log_fixture("orientdb_app: Ensure app is responding")
application.ensure_responding()
return application
def test_user_can_do_app_flow(self, test_user_clients, role, context):
"""
<b>Description:</b>
Checks if different type of users can do the application flow.
<b>Input data:</b>
1. Username.
2. User password.
3. Sample java application.
<b>Expected results:</b>
Different users can do an application flow.
<b>Steps:</b>
1. Push an application.
2. Verify is running.
3. Verify can be stopped.
4. Verify can be started.
5. Verify can be restarted.
6. Verify can be deleted.
7. Verify is doesn't exist.
"""
client = test_user_clients[role]
step("Compile the app")
test_app_sources = AppSources.from_local_path(
sources_directory=ApplicationPath.SAMPLE_JAVA_APP)
test_app_sources.compile_mvn()
step("Package the app")
p_a = PrepApp(ApplicationPath.SAMPLE_JAVA_APP)
gzipped_app_path = p_a.package_app(context)
step("Update manifest")
manifest_params = {"type": TapApplicationType.JAVA}
manifest_path = p_a.update_manifest(params=manifest_params)
step("Push app to tap")
app = Application.push(context,
app_path=gzipped_app_path,
name=p_a.app_name,
manifest_path=manifest_path,
client=client)
step("Check the application is running")
app.ensure_running()
step("Stop the application and check that it is stopped")
app.stop()
app.ensure_stopped()
step("Start the application and check that it has started")
app.start()
app.ensure_running()
step("Restart the application and check that it's running")
app.restart()
app.ensure_running()
step("Stop application before delete")
app.stop()
app.ensure_stopped()
step("Delete the application and check that it doesn't exist")
app.delete()
assertions.assert_not_in_by_id_with_retry(app.id, Application.get_list)