def load_data(self, num_of_customers):
"""
Call the AcmeAir app REST API to populate with the given # of users
"""
cloudantUtils = CloudantDbUtils(self.test_properties)
if cloudantUtils.get_doc_count("n_customer") > 0:
raise RuntimeException("Test databases already contains unknown data so AcmeAir data loader will not run!")
r = requests.Session()
url = "{}{}/{}".format(
self._app_host, self._api_context_root, "loader/load")
if isinstance(num_of_customers, int):
num_of_customers = str(num_of_customers)
param = {"numCustomers" : num_of_customers}
headers = { "Content-Type" : self.FORM_CONTENT_TYPE}
print ("Start AcmeAir database loader with num_of_customers = ", num_of_customers)
start_time = time.time()
try:
r.get(url, headers=headers, params=param)
except requests.exceptions.ConnectionError:
# the request aborts after 2 mins due to server timeout, wait until expected # of rows is reached
cloudantUtils.wait_for_doc_count("n_customer", num_of_customers, int(num_of_customers) / 500)
cloudantUtils.wait_for_doc_count("n_airportcodemapping", 14, 5)
cloudantUtils.wait_for_doc_count("n_flightsegment", 395, 5)
cloudantUtils.wait_for_doc_count("n_flight", 1971, 20)
print ("Database load completed after {} mins\n".format(int(round((time.time() - start_time) / 60))))
def remove_AcmeDb(self, num_of_cust): """ Drop all AcmeAir databases """ acmeair = AcmeAirUtils() if acmeair.is_acmeair_running() != 0: acmeair.stop_acmeair() cloudantUtils = CloudantDbUtils(test_properties) cloudantUtils.drop_all_databases()
def test_005(self, sparksubmit):
script_name = "schema_sample_size_005.py"
returncode, out, err = self.run_test(self.get_script_path(script_name), sparksubmit)
cloudantUtils = CloudantDbUtils(test_properties)
if cloudantUtils.check_if_doc_exists("n_customer", "zzzzzzzzzzzzzz") :
print(err)
assert err.index("org.apache.spark.SparkException") > 0
assert returncode == 1
else :
assert returncode == 0
def remove_AcmeDb(self, num_of_cust):
"""
Drop all AcmeAir databases
"""
acmeair = AcmeAirUtils()
if acmeair.is_acmeair_running() != 0:
acmeair.stop_acmeair()
cloudantUtils = CloudantDbUtils(test_properties)
cloudantUtils.drop_all_databases()
def loadData(self):
cloudantUtils = CloudantDbUtils(test_properties)
for db in cloudantUtils.test_dbs:
inputJsonFilePath = self.getInputJsonFilepath(db)
if os.path.exists(inputJsonFilePath):
if cloudantUtils.db_exists(db):
with open(inputJsonFilePath, "r") as fh:
bulkDocsJson = json.loads(fh.read())
if cloudantUtils.bulk_insert(db, json.dumps(bulkDocsJson)):
print("Successfully inserted the docs into {} from ./{}".format(db, inputJsonFilePath))
else:
print("Failed to insert the docs into {}".format(db))
else:
print("{} doesn't exist".format(db))
def start_acmeair(self):
"""
Set the required env vars for cloudant and start the AcmeAir app locally
If app is already running, check if it's functioning
"""
app_status = self.is_acmeair_running()
if app_status == -1:
raise RuntimeError(
"AcmeAir is already running but malfunctioning. Please shut it down."
)
elif app_status == 1:
print(
"AcmeAir app is already running, will not attempt to start\n")
return
cloudantUtils = CloudantDbUtils(self.test_properties)
cloudantUtils.check_databases()
# set the env vars required to start the app
new_env = os.environ.copy()
new_env["dbtype"] = "cloudant"
new_env["CLOUDANT_URL"] = "https://{}:{}@{}".format(
self.test_properties["cloudantusername"],
self.test_properties["cloudantpassword"],
self.test_properties["cloudanthost"])
# start the acmeair app
os.chdir(self.acmehome)
command = ["node", "app.js"]
self.proc = subprocess.Popen(command,
env=new_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# wait at most 10 sec for app to come up
timeout = time.time() + 10
while (True):
if not self.is_acmeair_running() == 1:
time.sleep(1)
else:
print("\nAcemAir started!")
break
if time.time() > timeout:
raise RuntimeError("Cannot connect to AcmeAir!")
def loadData(self):
cloudantUtils = CloudantDbUtils(test_properties)
for db in cloudantUtils.test_dbs:
inputJsonFilePath = self.getInputJsonFilepath(db)
if os.path.exists(inputJsonFilePath):
if cloudantUtils.db_exists(db):
with open(inputJsonFilePath, "r") as fh:
bulkDocsJson = json.loads(fh.read())
if cloudantUtils.bulk_insert(db,
json.dumps(bulkDocsJson)):
print(
"Successfully inserted the docs into {} from ./{}"
.format(db, inputJsonFilePath))
else:
print(
"Failed to insert the docs into {}".format(db))
else:
print("{} doesn't exist".format(db))
def start_acmeair(self):
"""
Set the required env vars for cloudant and start the AcmeAir app locally
If app is already running, check if it's functioning
"""
app_status = self.is_acmeair_running();
if app_status == -1:
raise RuntimeError("AcmeAir is already running but malfunctioning. Please shut it down.")
elif app_status == 1:
print ("AcmeAir app is already running, will not attempt to start\n")
return
cloudantUtils = CloudantDbUtils(self.test_properties)
cloudantUtils.check_databases()
# set the env vars required to start the app
new_env = os.environ.copy()
new_env["dbtype"] = "cloudant"
new_env["CLOUDANT_URL"] = "https://{}:{}@{}".format(
self.test_properties["cloudantusername"],
self.test_properties["cloudantpassword"],
self.test_properties["cloudanthost"])
# start the acmeair app
os.chdir(self.acmehome)
command = ["node", "app.js"]
self.proc = subprocess.Popen(command, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# wait at most 10 sec for app to come up
timeout = time.time() + 10
while (True):
if not self.is_acmeair_running() == 1:
time.sleep(1)
else:
print ("\nAcemAir started!")
break
if time.time() > timeout:
raise RuntimeError("Cannot connect to AcmeAir!")
def load_AcmeData(self, num_of_cust):
"""
Reset databases and use the AcmeAir database loader to populate initial customer,
flight and airportmapping data. Does NOT generate user data like bookings.
"""
print ("num_of_cust: ", num_of_cust)
acmeair = AcmeAirUtils()
try:
if acmeair.is_acmeair_running() != 0:
raise RuntimeError("""
AcmeAir is already running which may cause unexpected results when
resetting databases. Please shut down the app and try again.
""")
else:
cloudantUtils = CloudantDbUtils(test_properties)
cloudantUtils.reset_databases()
acmeair.start_acmeair()
acmeair.load_data(num_of_cust)
finally:
acmeair.stop_acmeair()
def load_AcmeData(self, num_of_cust):
"""
Reset databases and use the AcmeAir database loader to populate initial customer,
flight and airportmapping data. Does NOT generate user data like bookings.
"""
print("num_of_cust: ", num_of_cust)
acmeair = AcmeAirUtils()
try:
if acmeair.is_acmeair_running() != 0:
raise RuntimeError("""
AcmeAir is already running which may cause unexpected results when
resetting databases. Please shut down the app and try again.
""")
else:
cloudantUtils = CloudantDbUtils(test_properties)
cloudantUtils.reset_databases()
acmeair.start_acmeair()
acmeair.load_data(num_of_cust)
finally:
acmeair.stop_acmeair()
def get_flightId_by_number(self, flightNum):
"""
Get the generated flight ID for the given flight number
"""
cloudantUtils = CloudantDbUtils(self.test_properties)
url = "https://{}/{}".format(
self.test_properties["cloudanthost"],
"n_flight/_design/view/_search/n_flights?q=flightSegmentId:" +
flightNum)
param = {"q": "flightSegmentId:" + flightNum}
response = cloudantUtils.r.get(url, params=param)
data = response.json()
if int(data["total_rows"]) > 0:
# just get one from the dict
return data["rows"][0]["id"]
else:
raise RuntimeError("n_flights has no data for ", flightNum)
def setup_test_databases(test_properties): from helpers.dbutils import CloudantDbUtils cloudantUtils = CloudantDbUtils(test_properties) cloudantUtils.check_databases()
finally:
acmeair.stop_acmeair()
if __name__ =='__main__':
"""
Utility to create test databases and load data
"""
import argparse
parser = argparse.ArgumentParser(description="Utility to load AcmeAir data required for python spark-cloudant tests")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-cleanup', action='store_true', help='Drop all test databases')
group.add_argument('-load', help='Reset and Load databases with the given # of users. -load 0 to just recreate databases and indexes.', type=int)
args = parser.parse_args()
dataloader = DataLoader()
if args.load is not None:
if args.load == 0:
cloudantUtils = CloudantDbUtils(test_properties)
cloudantUtils.reset_databases()
else:
dataloader.load_AcmeData(args.load)
dataloader.load_SpecCharValuePredicateData()
elif args.cleanup:
cloudantUtils = CloudantDbUtils(test_properties)
cloudantUtils.drop_all_databases()
def setup_test_databases(test_properties):
from helpers.dbutils import CloudantDbUtils
cloudantUtils = CloudantDbUtils(test_properties)
cloudantUtils.check_databases()
"""
import argparse
parser = argparse.ArgumentParser(
description=
"Utility to load AcmeAir data required for python spark-cloudant tests"
)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-cleanup',
action='store_true',
help='Drop all test databases')
group.add_argument(
'-load',
help=
'Reset and Load databases with the given # of users. -load 0 to just recreate databases and indexes.',
type=int)
args = parser.parse_args()
dataloader = DataLoader()
if args.load is not None:
if args.load == 0:
cloudantUtils = CloudantDbUtils(test_properties)
cloudantUtils.reset_databases()
else:
dataloader.load_AcmeData(args.load)
dataloader.load_SpecCharValuePredicateData()
elif args.cleanup:
cloudantUtils = CloudantDbUtils(test_properties)
cloudantUtils.drop_all_databases()
def load_data(self, num_of_customers):
"""
Call the AcmeAir app REST API to populate with the given # of users
"""
cloudantUtils = CloudantDbUtils(self.test_properties)
if cloudantUtils.get_doc_count("n_customer") > 0:
raise RuntimeException(
"Test databases already contains unknown data so AcmeAir data loader will not run!"
)
r = requests.Session()
url = "{}{}/{}".format(self._app_host, self._api_context_root,
"loader/load")
if isinstance(num_of_customers, int):
num_of_customers = str(num_of_customers)
param = {"numCustomers": num_of_customers}
headers = {"Content-Type": self.FORM_CONTENT_TYPE}
print("Start AcmeAir database loader with num_of_customers = ",
num_of_customers)
start_time = time.time()
try:
r.get(url, headers=headers, params=param)
except requests.exceptions.ConnectionError:
# the request aborts after 2 mins due to server timeout, wait until expected # of rows is reached
cloudantUtils.wait_for_doc_count("n_customer", num_of_customers,
int(num_of_customers) / 500)
cloudantUtils.wait_for_doc_count("n_airportcodemapping", 14, 5)
cloudantUtils.wait_for_doc_count("n_flightsegment", 395, 5)
cloudantUtils.wait_for_doc_count("n_flight", 1971, 20)
print("Database load completed after {} mins\n".format(
int(round((time.time() - start_time) / 60))))
