def test_0_push_hdfs_hive_demo(self, dataset_target_uri, hdfs_instance, kerberos_instance, hive_instance,
cups_sso_instance, login_to_cf, class_context):
"""
<b>Description:</b>
Check that hdfs hive demo can run on platform.
<b>Input data:</b>
1. hdfs hive demo sources
2. hdfs instance
3. kerberos instance
4. hive instance
5. cups sso instance
<b>Expected results:</b>
Test passes when hdfs hive demo is successfully pushed
to platform and app has an url.
<b>Steps:</b>
1. Get hdfs hive demo sources from repository.
2. Push hdfs hive demo to platform.
3. Check that pushed app has an url address.
"""
step("Get app sources")
repo = AppSources.get_repository(repo_name=TapGitHub.hdfs_hive_demo, repo_owner=TapGitHub.intel_data)
repo.compile_mvn()
step("Push hdfs-hive-demo app to cf")
self.__class__.hdfs_reader_app = Application.push(class_context, source_directory=repo.path,
bound_services=[hdfs_instance.name, kerberos_instance.name,
hive_instance.name, cups_sso_instance.name])
step("Check hdfs-hive-demo app has url")
assert len(self.hdfs_reader_app.urls) == 1
def kafka2hdfs_app(class_context, kafka_instance, hdfs_instance, kerberos_instance, api_service_admin_client):
log_fixture("kafka2hdfs: download libraries")
ingestion_repo = AppSources.get_repository(repo_name=TapGitHub.ws_kafka_hdfs, repo_owner=TapGitHub.intel_data)
kafka2hdfs_path = os.path.join(ingestion_repo.path, TapGitHub.kafka2hdfs)
log_fixture("Package kafka2hdfs app")
ingestion_repo.compile_gradle(working_directory=kafka2hdfs_path)
build_path = os.path.join(kafka2hdfs_path, "deploy")
ingestion_repo.run_build_sh(cwd=build_path, command="./pack.sh")
app_path = os.path.join(build_path, "kafka2hdfs.tar")
log_fixture("kafka2hdfs: update manifest")
p_a = PrepApp(build_path)
manifest_params = {"bindings": [kafka_instance.name, hdfs_instance.name, kerberos_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("kafka2hdfs: push application")
app = Application.push(class_context, app_path=app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("kafka2hdfs: Check the application is running")
app.ensure_running()
return app
def test_step_0_push_ws2kafka2hdfs(self, test_space, class_context):
step("Clone and compile ingestion app sources")
github_auth = config.github_credentials()
ingestion_repo = AppSources.from_github(repo_name=TapGitHub.ws_kafka_hdfs, repo_owner=self.REPO_OWNER,
gh_auth=github_auth)
ws2kafka_path = os.path.join(ingestion_repo.path, TapGitHub.ws2kafka)
kafka2hdfs_path = os.path.join(ingestion_repo.path, TapGitHub.kafka2hdfs)
ingestion_repo.compile_gradle(working_directory=kafka2hdfs_path)
postfix = str(int(time.time()))
self.__class__.topic_name = "topic-{}".format(postfix)
step("Push application ws2kafka")
self.__class__.app_ws2kafka = Application.push(class_context, space_guid=test_space.guid,
source_directory=ws2kafka_path,
name="ws2kafka-{}".format(postfix),
bound_services=(self.KAFKA_INSTANCE_NAME,))
step("Push application kafka2hdfs")
self.__class__.app_kafka2hdfs = Application.push(class_context, space_guid=test_space.guid,
source_directory=kafka2hdfs_path,
name="kafka2hdfs-{}".format(postfix),
bound_services=(self.KAFKA_INSTANCE_NAME,
self.ZOOKEEPER_INSTANCE_NAME,
self.HDFS_INSTANCE_NAME,
self.KERBEROS_INSTANCE_NAME),
env={"TOPICS": self.topic_name,
"CONSUMER_GROUP": "group-{}".format(postfix)})
assert self.app_ws2kafka.is_started is True, "ws2kafka app is not started"
assert self.app_kafka2hdfs.is_started is True, "kafka2hdfs app is not started"
def app_bound_psql(module_context, psql_instance, api_service_admin_client):
log_fixture("psql_app: Download libraries")
app_src = AppSources.from_local_path(ApplicationPath.SQL_API_EXAMPLE)
p_a = PrepApp(app_src.path)
log_fixture("psql_app: package sample application")
if os.path.exists(ApplicationPath.SQL_API_EXAMPLE_TAR):
gzipped_app_path = ApplicationPath.SQL_API_EXAMPLE_TAR
else:
gzipped_app_path = p_a.package_app(module_context)
log_fixture("psql_app: update manifest")
manifest_params = {"type": TapApplicationType.PYTHON27, "bindings": [psql_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("psql_app: push sample application")
db_app = Application.push(module_context, app_path=gzipped_app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("psql_app: Check the application is running")
db_app.ensure_running()
log_fixture("psq_app: Check the application is responding")
db_app.ensure_responding()
return db_app
def space_shuttle_sources():
step("Get space shuttle example app sources")
example_app_sources = AppSources.from_github(
repo_name=TapGitHub.space_shuttle_demo,
repo_owner=TapGitHub.intel_data,
gh_auth=config.github_credentials()
)
return example_app_sources.path
def psql_app(psql_instance):
sql_api_sources = AppSources.from_github(
repo_name=TapGitHub.sql_api_example,
repo_owner=TapGitHub.intel_data,
gh_auth=config.github_credentials())
context = Context()
TestData.psql_app = Application.push(context=context,
space_guid=TestData.test_space.guid,
source_directory=sql_api_sources.path,
bound_services=(psql_instance.name, ))
return TestData.psql_app
def setup_psql_api_app(cls, test_space, login_to_cf, postgres_instance,
class_context):
step("Get sql api app sources")
sql_api_sources = AppSources.from_github(
repo_name=TapGitHub.sql_api_example,
repo_owner=TapGitHub.intel_data,
gh_auth=config.github_credentials())
step("Push psql api app to cf")
cls.psql_app = Application.push(
class_context,
space_guid=test_space.guid,
source_directory=sql_api_sources.path,
bound_services=(postgres_instance.name, ))
def sample_python_app(class_context):
log_fixture("sample_python_app: push sample application")
if not os.path.exists(ApplicationPath.SAMPLE_PYTHON_APP_TAR):
test_app_sources = AppSources.from_local_path(sources_directory=ApplicationPath.SAMPLE_PYTHON_APP)
test_app_sources.run_build_sh()
app = push_app_from_tar(class_context, ApplicationPath.SAMPLE_PYTHON_APP)
log_fixture("sample_python_app: Check the application is running")
app.ensure_running()
log_fixture("sample_python_app: Check the application is responding")
app.ensure_responding()
return app
def test_0_push_dataset_reader_app(self, test_space, dataset_target_uri, hdfs_instance, kerberos_instance,
login_to_cf, class_context):
step("Get app sources")
repo = AppSources.from_github(repo_name=TapGitHub.dataset_reader_sample, repo_owner=TapGitHub.trustedanalytics)
repo.compile_mvn()
step("Push dataset-reader app to cf")
self.__class__.hdfs_reader_app = Application.push(class_context, space_guid=test_space.guid,
source_directory=repo.path,
bound_services=(hdfs_instance.name, kerberos_instance.name),
env={"FILE": dataset_target_uri})
step("Check dataset reader app has url")
assert len(self.hdfs_reader_app.urls) == 1
def key_path(self):
"""
If key path was not set in configuration, download key.
Download is executed at most once.
"""
if self._key_path is None:
self._logger.info("Download repository with ssh key")
ilab_deploy = AppSources.get_repository(repo_name=TapGitHub.ilab_deploy, repo_owner=TapGitHub.intel_data)
self._ilab_deploy_path = ilab_deploy.path
self._key_path = os.path.join(self._ilab_deploy_path, RelativeRepositoryPaths.ilab_jump_key)
os.chmod(self._key_path, stat.S_IRUSR | stat.S_IWUSR)
self._key_path = os.path.expanduser(self._key_path)
assert os.path.isfile(self._key_path), "No such file {}".format(self._key_path)
return self._key_path
def test_app(self, context, test_space, instance):
test_app_sources = AppSources.from_local_path(
sources_directory=ApplicationPath.SAMPLE_JAVA_APP)
step("Compile the sources")
test_app_sources.compile_mvn()
step("Push application to cf")
application = Application.push(
context,
source_directory=ApplicationPath.SAMPLE_JAVA_APP,
space_guid=test_space.guid,
bound_services=(instance.name, ))
step("Check the application is running")
application.ensure_started()
return application
def test_1_push_app_to_cf(self, class_context, test_space, psql_instance,
login_to_cf):
step("Push application to cf")
sql_api_sources = AppSources.get_repository(
repo_name=TapGitHub.sql_api_example,
repo_owner=TapGitHub.intel_data)
self.__class__.psql_app = Application.push(
context=class_context,
source_directory=sql_api_sources.path,
space_guid=test_space.guid,
bound_services=(psql_instance.name, self.kerberos_instance.name))
step("Check the application is running")
assertions.assert_equal_with_retry(True,
self.psql_app.cf_api_app_is_running)
step("Check that application is on the list")
apps = Application.cf_api_get_list_by_space(test_space.guid)
assert self.psql_app in apps
def test_cascade_app_delete(self, context, instance, admin_client):
"""
<b>Description:</b>
Checks if cascade removal of an application removes bound service instance too.
<b>Input data:</b>
1. Sample service instance.
2. Sample java application.
3. Admin client
<b>Expected results:</b>
The application and the service instance are deleted.
<b>Steps:</b>
1. Push an application with bound service instance.
2. Verify the application is running.
3. Delete the application with cascade flag.
4. Verify the application is deleted.
5. Verify the service instance is deleted.
"""
step("Compile the app")
test_app_sources = AppSources.from_local_path(sources_directory=ApplicationPath.SAMPLE_JAVA_APP)
test_app_sources.compile_mvn()
step("Package the app")
p_a = PrepApp(ApplicationPath.SAMPLE_JAVA_APP)
gzipped_app_path = p_a.package_app(context)
step("Update manifest")
manifest_params = {"type" : TapApplicationType.JAVA,
"bindings" : instance.id}
manifest_path = p_a.update_manifest(params=manifest_params)
step("Push app to tap")
app = Application.push(context, app_path=gzipped_app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=admin_client)
step("Check the application is running")
app.ensure_running()
app.api_delete(cascade=True)
assertions.assert_not_in_by_id_with_retry(app.id, Application.get_list)
assertions.assert_not_in_by_id_with_retry(instance.id, ServiceInstance.get_list)
def ws2kafka_app(class_context, kafka_instance, api_service_admin_client):
log_fixture("ws2kafka: download libraries")
ingestion_repo = AppSources.get_repository(repo_name=TapGitHub.ws_kafka_hdfs, repo_owner=TapGitHub.intel_data)
ws2kafka_path = os.path.join(ingestion_repo.path, TapGitHub.ws2kafka)
build_path = os.path.join(ws2kafka_path, "deploy")
ingestion_repo.run_build_sh(cwd=build_path, command="./pack.sh")
app_path = os.path.join(build_path, "ws2kafka.tar.gz")
log_fixture("ws2kafka: update manifest")
p_a = PrepApp(build_path)
manifest_params = {"bindings": [kafka_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("ws2kafka: push application")
app = Application.push(class_context, app_path=app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("ws2kafka: Check the application is running")
app.ensure_running()
return app
def sample_java_app(request, test_org, test_space):
context = Context()
test_app_sources = AppSources.from_local_path(
sources_directory=ApplicationPath.SAMPLE_JAVA_APP)
log_fixture("sample_java_app: Compile the sources")
test_app_sources.compile_mvn()
log_fixture("sample_java_app: Push app to cf")
cf.cf_login(test_org.name, test_space.name)
app = Application.push(context=context,
space_guid=test_space.guid,
source_directory=ApplicationPath.SAMPLE_JAVA_APP)
app.ensure_started()
log_fixture("Check the application is running")
def fin():
log_fixture("sample_java_app: Delete sample app")
context.cleanup()
request.addfinalizer(fin)
return app
def hbase_reader_app(class_context, kerberos_instance, api_service_admin_client):
log_fixture("hbase_reader: download libraries")
hbase_reader_repo = AppSources.get_repository(repo_name=TapGitHub.hbase_api_example, repo_owner=TapGitHub.intel_data)
log_fixture("Package hbase_reader app")
build_path = os.path.join(hbase_reader_repo.path, "deploy")
hbase_reader_repo.run_build_sh(cwd=build_path, command="./pack.sh")
app_path = os.path.join(build_path, "hbase-java-api-example-0.1.1.tar")
log_fixture("hbase_reader: update manifest")
p_a = PrepApp(build_path)
manifest_params = {"bindings": [kerberos_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("hbase_reader: push application")
app = Application.push(class_context, app_path=app_path,
name=p_a.app_name, manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("hbase_reader: Check the application is running")
app.ensure_running()
return app
def app_bound_orientdb(module_context, orientdb_instance, api_service_admin_client):
log_fixture("orientdb_app: download libraries")
app_src = AppSources.from_local_path(ApplicationPath.ORIENTDB_API)
app_src.run_build_sh()
log_fixture("orientdb_app: package sample application")
p_a = PrepApp(app_src.path)
gzipped_app_path = p_a.package_app(module_context)
log_fixture("orientdb_app: update manifest")
manifest_params = {"type": TapApplicationType.PYTHON27, "bindings": [orientdb_instance.name]}
manifest_path = p_a.update_manifest(params=manifest_params)
log_fixture("orientdb_app: push orientdb-api application")
application = Application.push(module_context,
app_path=gzipped_app_path,
name=p_a.app_name,
manifest_path=manifest_path,
client=api_service_admin_client)
log_fixture("orientdb_app: Ensure app is running")
application.ensure_running()
log_fixture("orientdb_app: Ensure app is responding")
application.ensure_responding()
return application
def compiled_sample_go_app():
log_fixture("compiled_sample_go_app: download libraries")
test_app_sources = AppSources.from_local_path(sources_directory=ApplicationPath.SAMPLE_GO_APP)
test_app_sources.run_build_sh()
return test_app_sources.path
def test_user_can_do_app_flow(self, test_user_clients, role, context):
"""
<b>Description:</b>
Checks if different type of users can do the application flow.
<b>Input data:</b>
1. Username.
2. User password.
3. Sample java application.
<b>Expected results:</b>
Different users can do an application flow.
<b>Steps:</b>
1. Push an application.
2. Verify is running.
3. Verify can be stopped.
4. Verify can be started.
5. Verify can be restarted.
6. Verify can be deleted.
7. Verify is doesn't exist.
"""
client = test_user_clients[role]
step("Compile the app")
test_app_sources = AppSources.from_local_path(
sources_directory=ApplicationPath.SAMPLE_JAVA_APP)
test_app_sources.compile_mvn()
step("Package the app")
p_a = PrepApp(ApplicationPath.SAMPLE_JAVA_APP)
gzipped_app_path = p_a.package_app(context)
step("Update manifest")
manifest_params = {"type": TapApplicationType.JAVA}
manifest_path = p_a.update_manifest(params=manifest_params)
step("Push app to tap")
app = Application.push(context,
app_path=gzipped_app_path,
name=p_a.app_name,
manifest_path=manifest_path,
client=client)
step("Check the application is running")
app.ensure_running()
step("Stop the application and check that it is stopped")
app.stop()
app.ensure_stopped()
step("Start the application and check that it has started")
app.start()
app.ensure_running()
step("Restart the application and check that it's running")
app.restart()
app.ensure_running()
step("Stop application before delete")
app.stop()
app.ensure_stopped()
step("Delete the application and check that it doesn't exist")
app.delete()
assertions.assert_not_in_by_id_with_retry(app.id, Application.get_list)
def test_mqtt_demo(self, test_org, test_space, login_to_cf, class_context):
step("Clone repository")
mqtt_demo_sources = AppSources.from_github(
repo_name=self.REPO_NAME,
repo_owner=self.SOURCES_OWNER,
gh_auth=config.github_credentials())
step("Compile the sources")
mqtt_demo_sources.compile_mvn()
step("Create required service instances.")
ServiceInstance.api_create_with_plan_name(
org_guid=test_org.guid,
space_guid=test_space.guid,
service_label=ServiceLabels.INFLUX_DB,
name=self.INFLUX_INSTANCE_NAME,
service_plan_name=ServicePlan.FREE)
ServiceInstance.api_create_with_plan_name(
org_guid=test_org.guid,
space_guid=test_space.guid,
service_label=ServiceLabels.MOSQUITTO,
name=self.MQTT_INSTANCE_NAME,
service_plan_name=ServicePlan.FREE)
step("Push mqtt app to cf")
mqtt_demo_app = Application.push(
class_context,
source_directory=mqtt_demo_sources.path,
space_guid=test_space.guid)
step("Retrieve credentials for mqtt service instance")
self.credentials = mqtt_demo_app.get_credentials(
service_name=ServiceLabels.MOSQUITTO)
mqtt_port = self.credentials.get("port")
assert mqtt_port is not None
mqtt_username = self.credentials.get("username")
assert mqtt_username is not None
mqtt_pwd = self.credentials.get("password")
assert mqtt_pwd is not None
step("Connect to mqtt app with mqtt client")
mqtt_client = mqtt.Client()
mqtt_client.username_pw_set(mqtt_username, mqtt_pwd)
mqtt_client.tls_set(self.SERVER_CERTIFICATE,
tls_version=ssl.PROTOCOL_TLSv1_2)
mqtt_server_address = mqtt_demo_app.urls[0]
mqtt_client.connect(mqtt_server_address, int(mqtt_port), 20)
with open(self.TEST_DATA_FILE) as f:
expected_data = f.read().split("\n")
step("Start reading logs")
logs = subprocess.Popen(["cf", "logs", "mqtt-demo"],
stdout=subprocess.PIPE)
time.sleep(5)
step("Send {0} data vectors to {1}:{2} on topic {3}".format(
len(expected_data), mqtt_server_address, mqtt_port,
self.MQTT_TOPIC_NAME))
for line in expected_data:
mqtt_client.publish(self.MQTT_TOPIC_NAME, line)
step("Stop reading logs. Retrieve vectors from log content.")
grep = subprocess.Popen(["grep", "message:"],
stdin=logs.stdout,
stdout=subprocess.PIPE)
logs.stdout.close()
time.sleep(50)
os.kill(logs.pid, signal.SIGTERM)
cut = subprocess.Popen("cut -d ':' -f7 ",
stdin=grep.stdout,
stdout=subprocess.PIPE,
shell=True)
grep.stdout.close()
step("Check that logs display all the vectors sent")
log_result = cut.communicate()[0].decode().split("\n")
log_result = [
item.strip() for item in log_result if item not in (" ", "")
]
self.maxDiff = None # allows for full diff to be displayed
assert log_result == expected_data, "Data in logs do not match sent data"