def test_step_0_push_ws2kafka2hdfs(self, test_space, class_context):
step("Clone and compile ingestion app sources")
github_auth = config.github_credentials()
ingestion_repo = AppSources.from_github(repo_name=TapGitHub.ws_kafka_hdfs, repo_owner=self.REPO_OWNER,
gh_auth=github_auth)
ws2kafka_path = os.path.join(ingestion_repo.path, TapGitHub.ws2kafka)
kafka2hdfs_path = os.path.join(ingestion_repo.path, TapGitHub.kafka2hdfs)
ingestion_repo.compile_gradle(working_directory=kafka2hdfs_path)
postfix = str(int(time.time()))
self.__class__.topic_name = "topic-{}".format(postfix)
step("Push application ws2kafka")
self.__class__.app_ws2kafka = Application.push(class_context, space_guid=test_space.guid,
source_directory=ws2kafka_path,
name="ws2kafka-{}".format(postfix),
bound_services=(self.KAFKA_INSTANCE_NAME,))
step("Push application kafka2hdfs")
self.__class__.app_kafka2hdfs = Application.push(class_context, space_guid=test_space.guid,
source_directory=kafka2hdfs_path,
name="kafka2hdfs-{}".format(postfix),
bound_services=(self.KAFKA_INSTANCE_NAME,
self.ZOOKEEPER_INSTANCE_NAME,
self.HDFS_INSTANCE_NAME,
self.KERBEROS_INSTANCE_NAME),
env={"TOPICS": self.topic_name,
"CONSUMER_GROUP": "group-{}".format(postfix)})
assert self.app_ws2kafka.is_started is True, "ws2kafka app is not started"
assert self.app_kafka2hdfs.is_started is True, "kafka2hdfs app is not started"
def space_shuttle_sources():
step("Get space shuttle example app sources")
example_app_sources = AppSources.from_github(
repo_name=TapGitHub.space_shuttle_demo,
repo_owner=TapGitHub.intel_data,
gh_auth=config.github_credentials()
)
return example_app_sources.path
def psql_app(psql_instance):
sql_api_sources = AppSources.from_github(
repo_name=TapGitHub.sql_api_example,
repo_owner=TapGitHub.intel_data,
gh_auth=config.github_credentials())
context = Context()
TestData.psql_app = Application.push(context=context,
space_guid=TestData.test_space.guid,
source_directory=sql_api_sources.path,
bound_services=(psql_instance.name, ))
return TestData.psql_app
def setup_psql_api_app(cls, test_space, login_to_cf, postgres_instance,
class_context):
step("Get sql api app sources")
sql_api_sources = AppSources.from_github(
repo_name=TapGitHub.sql_api_example,
repo_owner=TapGitHub.intel_data,
gh_auth=config.github_credentials())
step("Push psql api app to cf")
cls.psql_app = Application.push(
class_context,
space_guid=test_space.guid,
source_directory=sql_api_sources.path,
bound_services=(postgres_instance.name, ))
def test_0_push_dataset_reader_app(self, test_space, dataset_target_uri, hdfs_instance, kerberos_instance,
login_to_cf, class_context):
step("Get app sources")
repo = AppSources.from_github(repo_name=TapGitHub.dataset_reader_sample, repo_owner=TapGitHub.trustedanalytics)
repo.compile_mvn()
step("Push dataset-reader app to cf")
self.__class__.hdfs_reader_app = Application.push(class_context, space_guid=test_space.guid,
source_directory=repo.path,
bound_services=(hdfs_instance.name, kerberos_instance.name),
env={"FILE": dataset_target_uri})
step("Check dataset reader app has url")
assert len(self.hdfs_reader_app.urls) == 1
def test_mqtt_demo(self, test_org, test_space, login_to_cf, class_context):
step("Clone repository")
mqtt_demo_sources = AppSources.from_github(
repo_name=self.REPO_NAME,
repo_owner=self.SOURCES_OWNER,
gh_auth=config.github_credentials())
step("Compile the sources")
mqtt_demo_sources.compile_mvn()
step("Create required service instances.")
ServiceInstance.api_create_with_plan_name(
org_guid=test_org.guid,
space_guid=test_space.guid,
service_label=ServiceLabels.INFLUX_DB,
name=self.INFLUX_INSTANCE_NAME,
service_plan_name=ServicePlan.FREE)
ServiceInstance.api_create_with_plan_name(
org_guid=test_org.guid,
space_guid=test_space.guid,
service_label=ServiceLabels.MOSQUITTO,
name=self.MQTT_INSTANCE_NAME,
service_plan_name=ServicePlan.FREE)
step("Push mqtt app to cf")
mqtt_demo_app = Application.push(
class_context,
source_directory=mqtt_demo_sources.path,
space_guid=test_space.guid)
step("Retrieve credentials for mqtt service instance")
self.credentials = mqtt_demo_app.get_credentials(
service_name=ServiceLabels.MOSQUITTO)
mqtt_port = self.credentials.get("port")
assert mqtt_port is not None
mqtt_username = self.credentials.get("username")
assert mqtt_username is not None
mqtt_pwd = self.credentials.get("password")
assert mqtt_pwd is not None
step("Connect to mqtt app with mqtt client")
mqtt_client = mqtt.Client()
mqtt_client.username_pw_set(mqtt_username, mqtt_pwd)
mqtt_client.tls_set(self.SERVER_CERTIFICATE,
tls_version=ssl.PROTOCOL_TLSv1_2)
mqtt_server_address = mqtt_demo_app.urls[0]
mqtt_client.connect(mqtt_server_address, int(mqtt_port), 20)
with open(self.TEST_DATA_FILE) as f:
expected_data = f.read().split("\n")
step("Start reading logs")
logs = subprocess.Popen(["cf", "logs", "mqtt-demo"],
stdout=subprocess.PIPE)
time.sleep(5)
step("Send {0} data vectors to {1}:{2} on topic {3}".format(
len(expected_data), mqtt_server_address, mqtt_port,
self.MQTT_TOPIC_NAME))
for line in expected_data:
mqtt_client.publish(self.MQTT_TOPIC_NAME, line)
step("Stop reading logs. Retrieve vectors from log content.")
grep = subprocess.Popen(["grep", "message:"],
stdin=logs.stdout,
stdout=subprocess.PIPE)
logs.stdout.close()
time.sleep(50)
os.kill(logs.pid, signal.SIGTERM)
cut = subprocess.Popen("cut -d ':' -f7 ",
stdin=grep.stdout,
stdout=subprocess.PIPE,
shell=True)
grep.stdout.close()
step("Check that logs display all the vectors sent")
log_result = cut.communicate()[0].decode().split("\n")
log_result = [
item.strip() for item in log_result if item not in (" ", "")
]
self.maxDiff = None # allows for full diff to be displayed
assert log_result == expected_data, "Data in logs do not match sent data"