def kerberos_instance(session_context, api_service_admin_client):
log_fixture("create_kerberos_instance")
kerberos = ServiceInstance.create_with_name(context=session_context, offering_label=ServiceLabels.KERBEROS,
plan_name=ServicePlan.SHARED, client=api_service_admin_client)
log_fixture("Check the service instance is running")
kerberos.ensure_running()
return kerberos
def core_org():
log_fixture("core_org: Create object for core org")
ref_org_name = config.core_org_name
orgs = Organization.get_list()
core_org = next((o for o in orgs if o.name == ref_org_name), None)
assert core_org is not None, "Could not find org {}".format(ref_org_name)
return core_org
def hdfs_instance(session_context, api_service_admin_client):
log_fixture("create_hdfs_instance")
hdfs = ServiceInstance.create_with_name(context=session_context, offering_label=ServiceLabels.HDFS,
plan_name=ServicePlan.ENCRYPTED_DIR, client=api_service_admin_client)
log_fixture("Check the service instance is running")
hdfs.ensure_running()
return hdfs
def space_shuttle_application(class_context):
SPACE_SHUTTLE_DEMO_APP_NAME = "space-shuttle-demo"
log_fixture(SPACE_SHUTTLE_DEMO_APP_NAME + ": Get all information about application")
app = Application.get_by_name(SPACE_SHUTTLE_DEMO_APP_NAME)
return app
def open_tunnel(request):
log_fixture("Open SSH tunnel to jumpbox")
jump_tunnel = JumpTunnel()
jump_tunnel.open()
log_fixture("SSH tunnel to jumpbox is open")
request.addfinalizer(jump_tunnel.close)
return jump_tunnel
def model_with_artifact(request, context, core_org):
log_fixture("test_model: Create test model and add artifact")
model = ScoringEngineModel.create(context, org_guid=core_org.guid, description="Test model", revision="revision",
algorithm="algorithm", creation_tool="creationTool")
ModelArtifact.upload_artifact(model_id=model.id, filename="example_artifact.txt",
actions=[ModelArtifact.ARTIFACT_ACTIONS["publish_jar_scoring_engine"]])
return ScoringEngineModel.get(model_id=model.id)
def tap_cli(request, remote_tap_directory, jump_client):
tap_binary_name = "tap"
remote_tap_binary_path = os.path.join(remote_tap_directory,
tap_binary_name)
target_path = os.path.join(TARGET_DIRECTORY, tap_binary_name)
if config.check_tap_cli_version and os.path.isfile(target_path):
log_fixture("Deleting old tap client")
os.remove(target_path)
log_fixture("Download tap client")
return_code = jump_client.scp_from_remote(
source_path=remote_tap_binary_path, target_path=TARGET_DIRECTORY)
if config.check_tap_cli_version and return_code != 0:
pytest.fail("Latest tap client cannot be properly copied from jumpbox")
os.chmod(target_path, os.stat(target_path).st_mode | stat.S_IEXEC)
def fin():
try:
log_fixture("Deleting tap client")
os.remove(target_path)
except:
pass
request.addfinalizer(fin)
return TapCli(target_path)
def test_cannot_push_application_twice(self, context, test_sample_apps,
sample_app,
api_service_admin_client):
"""
<b>Description:</b>
Tries to push the same application twice
<b>Input data:</b>
- Sample application that will be pushed twice
<b>Expected results:</b>
- The second push is not possible
<b>Steps:</b>
- Application is pushed
- Application is pushed again in the test itself
- The second push is blocked
"""
step("Check that pushing the same application again causes an error")
log_fixture("sample_application: update manifest")
p_a = PrepApp(test_sample_apps.tapng_python_app.filepath)
manifest_params = {
"type": TapApplicationType.PYTHON27,
"name": sample_app.name
}
manifest_path = p_a.update_manifest(params=manifest_params)
with pytest.raises(UnexpectedResponseError) as e:
Application.push(
context,
app_path=test_sample_apps.tapng_python_app.filepath,
name=sample_app.name,
manifest_path=manifest_path,
client=api_service_admin_client)
assert self.EXPECTED_MESSAGE_WHEN_APP_PUSHED_TWICE in str(e)
def catalog_service(self, context, catalog_template):
log_fixture("Create sample catalog service")
return CatalogService.create(
context,
template_id=catalog_template.id,
description=self.SAMPLE_SERVICE_DESCRIPTION,
state=TapEntityState.DEPLOYING)
def kafka2hdfs_app(class_context, kafka_instance, hdfs_instance, kerberos_instance, api_service_admin_client):
log_fixture("kafka2hdfs: download libraries")
ingestion_repo = AppSources.get_repository(repo_name=TapGitHub.ws_kafka_hdfs, repo_owner=TapGitHub.intel_data)
kafka2hdfs_path = os.path.join(ingestion_repo.path, TapGitHub.kafka2hdfs)
log_fixture("Package kafka2hdfs app")
ingestion_repo.compile_gradle(working_directory=kafka2hdfs_path)
build_path = os.path.join(kafka2hdfs_path, "deploy")
ingestion_repo.run_build_sh(cwd=build_path, command="./pack.sh")
app_path = os.path.join(build_path, "kafka2hdfs.tar")
log_fixture("kafka2hdfs: update manifest")
p_a = PrepApp(build_path)
manifest_params = {"bindings": [kafka_instance.name, hdfs_instance.name, kerberos_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("kafka2hdfs: push application")
app = Application.push(class_context, app_path=app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("kafka2hdfs: Check the application is running")
app.ensure_running()
return app
def mysql_instance(session_context, api_service_admin_client):
log_fixture("create_mysql_instance")
mysql = ServiceInstance.create_with_name(context=session_context, client=api_service_admin_client,
offering_label=ServiceLabels.MYSQL, plan_name=ServicePlan.SINGLE_SMALL)
log_fixture("Check the service instance is running")
mysql.ensure_running()
return mysql
def sample_app_image(self, class_context, catalog_template, catalog_image):
log_fixture("Create sample catalog image")
step("Create application in catalog")
return CatalogApplication.create(
class_context,
template_id=catalog_template.id,
image_id=catalog_image.id,
description=self.SAMPLE_APPLICATION_DESCRIPTION)
def model_hdfs_path(request, test_org, add_admin_to_test_org):
log_fixture("Create a transfer and get hdfs path")
context = Context()
_, data_set = data_catalog.create_dataset_from_link(context,
org=test_org,
source=Urls.model_url)
request.addfinalizer(lambda: context.cleanup())
return data_set.target_uri
def service_instance(self, class_context):
log_fixture("Create sample service instance")
instance = ServiceInstance.create_with_name(
class_context,
offering_label=ServiceLabels.RABBIT_MQ,
plan_name=ServicePlan.SINGLE_SMALL)
step("Ensure that instance is running")
instance.ensure_running()
return instance
def orientdb_instance(session_context, api_service_admin_client):
log_fixture("orientdb_instance: create")
orientdb = ServiceInstance.create_with_name(context=session_context,
client=api_service_admin_client,
offering_label=ServiceLabels.ORIENT_DB,
plan_name=ServicePlan.SINGLE_SMALL)
log_fixture("orientdb_instance: ensure instance is running")
orientdb.ensure_running()
return orientdb
def test_change_app_state_in_catalog_and_delete_it(
self, context, test_sample_apps, api_service_admin_client):
"""
<b>Description:</b>
Change the application state in catalog and later delete it
<b>Input data:</b>
- Path to application
- Admin credentials
<b>Expected results:</b>
- Application state can be changed in catalog
- Application state can be set back via api service
<b>Steps:</b>
- Prepare the application and push it
- Make sure the application is running
- Change the state of the application via catalog
- Make sure the state has changed
- Stop the application via api service client and remove it
- Verify the application was removed
"""
log_fixture("sample_application: update manifest")
p_a = PrepApp(test_sample_apps.tapng_python_app.filepath)
manifest_params = {"type": TapApplicationType.PYTHON27}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("Push sample application and check it's running")
application = Application.push(
context,
app_path=test_sample_apps.tapng_python_app.filepath,
name=p_a.app_name,
manifest_path=manifest_path,
client=api_service_admin_client)
application.ensure_running()
step("Check that the application has only one instance")
instances = CatalogApplicationInstance.get_list_for_application(
application_id=application.id)
assert len(instances) == 1
updated_state = TapEntityState.FAILURE
step("Update app state to {} using catalog api".format(updated_state))
catalog_api.update_instance(instance_id=instances[0].id,
field_name="state",
value=updated_state)
step("Check that the app state was updated")
app = Application.get(app_inst_id=application.id,
client=api_service_admin_client)
assert app.state == updated_state, "Application is not in the expected state. App state: {}".format(
app.state)
step("Check that the application can be deleted")
application.delete()
step("Check that application has been removed")
apps = Application.get_list(client=api_service_admin_client)
assert application not in apps
def psql_instance(test_org, test_space):
log_fixture("create_postgres_instance")
marketplace = ServiceType.api_get_list_from_marketplace(test_space.guid)
psql = next(service for service in marketplace
if service.label == ServiceLabels.PSQL)
TestData.psql_instance = ServiceInstance.api_create(
org_guid=test_org.guid,
space_guid=test_space.guid,
service_label=ServiceLabels.PSQL,
service_plan_guid=psql.service_plan_guids[0])
return TestData.psql_instance
def _create_catalog_instance(self, context, offering):
log_fixture(
"CATALOG: Create an example service instance for offering {}".
format(offering.name))
catalog_instance = CatalogServiceInstance.create(
context,
service_id=offering.id,
plan_id=offering.plans[0].id,
state=TapEntityState.REQUESTED)
catalog_instance.ensure_in_state(expected_state=TapEntityState.RUNNING)
return catalog_instance
def sample_application(self, class_context, test_sample_apps, api_service_admin_client):
log_fixture("sample_application: update manifest")
p_a = PrepApp(test_sample_apps[self.SAMPLE_APP].filepath)
manifest_params = {"type" : self.APP_TYPE}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("Push sample application and check it's running")
application = Application.push(class_context, app_path=test_sample_apps[self.SAMPLE_APP].filepath,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
application.ensure_running()
return application
def blob_store_artifact(self, context, catalog_image, test_sample_apps):
log_fixture("BLOB-STORE: Create an artifact for application")
created_blob = Blob.create_from_file(
context,
blob_id=catalog_image.id,
file_path=test_sample_apps[
SampleAppKeys.TAPNG_NODEJS_APP].filepath)
log_fixture("BLOB-STORE Check that the blob exists")
blob = Blob.get(blob_id=catalog_image.id)
assert blob == created_blob
return catalog_image
def catalog_image(self, context):
log_fixture("CATALOG: Send application metadata - create an image")
catalog_image = CatalogImage.create(context,
image_type=self.APP_TYPE,
state=self.INITIAL_IMAGE_STATE,
blob_type=self.BLOB_TYPE)
catalog_image.ensure_in_state(TapEntityState.REQUESTED)
log_fixture("CATALOG: Check that the image is on the image list")
catalog_images = CatalogImage.get_list()
assert catalog_image in catalog_images
return catalog_image
def test_org_manager(request, test_org):
context = Context()
log_fixture("test_org_manager: Add org manager to test org")
test_org_manager = User.api_create_by_adding_to_organization(
context, org_guid=test_org.guid)
TestData.test_org_manager = test_org_manager
def fin():
log_finalizer("test_org_manager: Delete test org manager")
context.cleanup()
request.addfinalizer(fin)
return test_org_manager
def test_org(request):
context = Context()
log_fixture("test_org: Create test organization")
test_org = Organization.api_create(context)
TestData.test_org = test_org
def fin():
log_finalizer("test_org: Delete test organization")
context.cleanup()
request.addfinalizer(fin)
return test_org
def push_app_from_tar(context, app_path):
archive_path = os.path.join(app_path, 'app.tar.gz')
log_fixture('using existing archive to push app: {}'.format(archive_path))
app_name = generate_test_object_name(separator='')
manifest_path = os.path.join(app_path, 'manifest.json')
with open(manifest_path) as manifest_file:
manifest = json.load(manifest_file)
manifest['name'] = app_name
return Application.push(context, app_path=archive_path,
name=app_name, manifest=manifest,
client=api_service_admin_client())
def test_data_urls(session_context):
""" Fixture with files from tests/fixtures/data_repo.py::DATA_FILES.
They are accessible by url or filepath.
Eg.:
test_data_urls.test_transfer.url
test_data_urls['test_transfer'].url
test_data_urls.test_transfer.filepath
"""
if config.data_repo_url:
return Urls(config.data_repo_url)
else:
log_fixture("data_repo: package sample application")
p_a = PrepApp(DATA_REPO_PATH)
if os.path.exists(DATA_REPO_TAR_PATH):
gzipped_app_path = DATA_REPO_TAR_PATH
else:
gzipped_app_path = p_a.package_app(session_context)
log_fixture("data_repo: update manifest")
manifest_path = p_a.update_manifest(params={})
log_fixture("data_repo: Push data_repo application")
app = Application.push(context=session_context, app_path=gzipped_app_path, name=p_a.app_name,
manifest_path=manifest_path)
log_fixture("data_repo: Check application is running")
app.ensure_running()
return Urls(app.urls[0])
def sample_python_app(request, test_org, test_space):
context = Context()
log_fixture("sample_python_app: Push app to cf")
cf.cf_login(test_org.name, test_space.name)
app = Application.push(context=context,
space_guid=test_space.guid,
source_directory=ApplicationPath.SAMPLE_PYTHON_APP)
def fin():
log_fixture("sample_python_app: Delete sample app")
context.cleanup()
request.addfinalizer(fin)
return app
def sample_app(self, class_context, test_sample_apps,
api_service_admin_client):
log_fixture("sample_application: update manifest")
p_a = PrepApp(test_sample_apps.tapng_python_app.filepath)
manifest_params = {"type": TapApplicationType.PYTHON27}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("Push sample application")
application = Application.push(
class_context,
app_path=test_sample_apps.tapng_python_app.filepath,
name=p_a.app_name,
manifest_path=manifest_path,
client=api_service_admin_client)
application.ensure_running()
return application
def sample_service(request, test_org, test_space, sample_python_app):
log_fixture("sample_service: Register sample app in marketplace")
service = ServiceType.register_app_in_marketplace(
app_name=sample_python_app.name,
app_guid=sample_python_app.guid,
org_guid=test_org.guid,
space_guid=test_space.guid)
log_fixture("sample_service: Get service plans")
service.api_get_service_plans()
def fin():
log_fixture("sample_service: Delete service")
service.api_delete()
request.addfinalizer(fin)
return service
def space_users_clients(request, test_org, test_space, admin_client):
context = Context()
log_fixture("clients: Create clients")
clients = {}
for role, value in User.SPACE_ROLES.items():
clients[role] = User.api_create_by_adding_to_space(
context,
org_guid=test_org.guid,
space_guid=test_space.guid,
roles=value).login()
clients["admin"] = admin_client
def fin():
log_finalizer("clients: Delete test users")
context.cleanup()
request.addfinalizer(fin)
return clients
def test_push_and_delete_application(self, context, test_sample_apps, api_service_admin_client):
"""
<b>Description:</b>
Attempts to push application, stop it and then delete it.
<b>Input data:</b>
- Sample application
- Admin credentials
<b>Expected results:</b>
- Application is successfully pushed to platform
- Apllication is in running state
- It's possible to stop the application
- it's possible to remove the application
- After removal application is no longer available
<b>Steps:</b>
- Sample application is downloaded
- Manifest is updated with unique name
- Application is pushed to platform using admin
- It's verified that the application is in running state
- Application is stopped and it's verified that the application has stopped
- Remove the application and verify it's no longer available
"""
log_fixture("sample_application: update manifest")
p_a = PrepApp(test_sample_apps[self.SAMPLE_APP].filepath)
manifest_params = {"type" : self.APP_TYPE}
manifest_path = p_a.update_manifest(params=manifest_params)
step("Push sample application")
application = Application.push(context, app_path=test_sample_apps[self.SAMPLE_APP].filepath,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
step("Check application is running")
application.ensure_running()
step("Stop application")
application.stop()
application.ensure_stopped()
step("Delete application")
application.delete()
step("Check that application is not on the list")
assertions.assert_not_in_with_retry(application, Application.get_list, client=api_service_admin_client)
