def load_data(self, num_of_customers): """ Call the AcmeAir app REST API to populate with the given # of users """ cloudantUtils = CloudantDbUtils(self.test_properties) if cloudantUtils.get_doc_count("n_customer") > 0: raise RuntimeException( "Test databases already contains unknown data so AcmeAir data loader will not run!" ) r = requests.Session() url = "{}{}/{}".format(self._app_host, self._api_context_root, "loader/load") if isinstance(num_of_customers, int): num_of_customers = str(num_of_customers) param = {"numCustomers": num_of_customers} headers = {"Content-Type": self.FORM_CONTENT_TYPE} print("Start AcmeAir database loader with num_of_customers = ", num_of_customers) start_time = time.time() try: r.get(url, headers=headers, params=param) except requests.exceptions.ConnectionError: # the request aborts after 2 mins due to server timeout, wait until expected # of rows is reached cloudantUtils.wait_for_doc_count("n_customer", num_of_customers, int(num_of_customers) / 500) cloudantUtils.wait_for_doc_count("n_airportcodemapping", 14, 5) cloudantUtils.wait_for_doc_count("n_flightsegment", 395, 5) cloudantUtils.wait_for_doc_count("n_flight", 1971, 20) print("Database load completed after {} mins\n".format( int(round((time.time() - start_time) / 60))))
def test_005(self, sparksubmit): script_name = "schema_sample_size_005.py" returncode, out, err = self.run_test(self.get_script_path(script_name), sparksubmit) cloudantUtils = CloudantDbUtils(test_properties) if cloudantUtils.check_if_doc_exists("n_customer", "zzzzzzzzzzzzzz") : print(err) assert err.index("org.apache.spark.SparkException") > 0 assert returncode == 1 else : assert returncode == 0
def remove_AcmeDb(self, num_of_cust): """ Drop all AcmeAir databases """ acmeair = AcmeAirUtils() if acmeair.is_acmeair_running() != 0: acmeair.stop_acmeair() cloudantUtils = CloudantDbUtils(test_properties) cloudantUtils.drop_all_databases()
def start_acmeair(self): """ Set the required env vars for cloudant and start the AcmeAir app locally If app is already running, check if it's functioning """ app_status = self.is_acmeair_running() if app_status == -1: raise RuntimeError( "AcmeAir is already running but malfunctioning. Please shut it down." ) elif app_status == 1: print( "AcmeAir app is already running, will not attempt to start\n") return cloudantUtils = CloudantDbUtils(self.test_properties) cloudantUtils.check_databases() # set the env vars required to start the app new_env = os.environ.copy() new_env["dbtype"] = "cloudant" new_env["CLOUDANT_URL"] = "https://{}:{}@{}".format( self.test_properties["cloudantusername"], self.test_properties["cloudantpassword"], self.test_properties["cloudanthost"]) # start the acmeair app os.chdir(self.acmehome) command = ["node", "app.js"] self.proc = subprocess.Popen(command, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # wait at most 10 sec for app to come up timeout = time.time() + 10 while (True): if not self.is_acmeair_running() == 1: time.sleep(1) else: print("\nAcemAir started!") break if time.time() > timeout: raise RuntimeError("Cannot connect to AcmeAir!")
def loadData(self): cloudantUtils = CloudantDbUtils(test_properties) for db in cloudantUtils.test_dbs: inputJsonFilePath = self.getInputJsonFilepath(db) if os.path.exists(inputJsonFilePath): if cloudantUtils.db_exists(db): with open(inputJsonFilePath, "r") as fh: bulkDocsJson = json.loads(fh.read()) if cloudantUtils.bulk_insert(db, json.dumps(bulkDocsJson)): print( "Successfully inserted the docs into {} from ./{}" .format(db, inputJsonFilePath)) else: print( "Failed to insert the docs into {}".format(db)) else: print("{} doesn't exist".format(db))
def get_flightId_by_number(self, flightNum): """ Get the generated flight ID for the given flight number """ cloudantUtils = CloudantDbUtils(self.test_properties) url = "https://{}/{}".format( self.test_properties["cloudanthost"], "n_flight/_design/view/_search/n_flights?q=flightSegmentId:" + flightNum) param = {"q": "flightSegmentId:" + flightNum} response = cloudantUtils.r.get(url, params=param) data = response.json() if int(data["total_rows"]) > 0: # just get one from the dict return data["rows"][0]["id"] else: raise RuntimeError("n_flights has no data for ", flightNum)
def load_AcmeData(self, num_of_cust): """ Reset databases and use the AcmeAir database loader to populate initial customer, flight and airportmapping data. Does NOT generate user data like bookings. """ print("num_of_cust: ", num_of_cust) acmeair = AcmeAirUtils() try: if acmeair.is_acmeair_running() != 0: raise RuntimeError(""" AcmeAir is already running which may cause unexpected results when resetting databases. Please shut down the app and try again. """) else: cloudantUtils = CloudantDbUtils(test_properties) cloudantUtils.reset_databases() acmeair.start_acmeair() acmeair.load_data(num_of_cust) finally: acmeair.stop_acmeair()
def setup_test_databases(test_properties): from helpers.dbutils import CloudantDbUtils cloudantUtils = CloudantDbUtils(test_properties) cloudantUtils.check_databases()
""" import argparse parser = argparse.ArgumentParser( description= "Utility to load AcmeAir data required for python spark-cloudant tests" ) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-cleanup', action='store_true', help='Drop all test databases') group.add_argument( '-load', help= 'Reset and Load databases with the given # of users. -load 0 to just recreate databases and indexes.', type=int) args = parser.parse_args() dataloader = DataLoader() if args.load is not None: if args.load == 0: cloudantUtils = CloudantDbUtils(test_properties) cloudantUtils.reset_databases() else: dataloader.load_AcmeData(args.load) dataloader.load_SpecCharValuePredicateData() elif args.cleanup: cloudantUtils = CloudantDbUtils(test_properties) cloudantUtils.drop_all_databases()