def load_data(self, num_of_customers):
		"""
		Call the AcmeAir app REST API to populate with the given # of users
		"""
		cloudantUtils = CloudantDbUtils(self.test_properties)
		if cloudantUtils.get_doc_count("n_customer") > 0:
			raise RuntimeException("Test databases already contains unknown data so AcmeAir data loader will not run!")
	
		r = requests.Session()
		url = "{}{}/{}".format(
				self._app_host, self._api_context_root, "loader/load")

		if isinstance(num_of_customers, int):
			num_of_customers = str(num_of_customers)
		param = {"numCustomers" : num_of_customers}
		headers = { "Content-Type" : self.FORM_CONTENT_TYPE}
		
		print ("Start AcmeAir database loader with num_of_customers = ", num_of_customers)
		start_time = time.time()
		try:
			r.get(url, headers=headers, params=param)
			
		except requests.exceptions.ConnectionError:
			# the request aborts after 2 mins due to server timeout, wait until expected # of rows is reached
			cloudantUtils.wait_for_doc_count("n_customer", num_of_customers, int(num_of_customers) / 500)
			cloudantUtils.wait_for_doc_count("n_airportcodemapping", 14, 5)			
			cloudantUtils.wait_for_doc_count("n_flightsegment", 395, 5)
			cloudantUtils.wait_for_doc_count("n_flight", 1971, 20)
			print ("Database load completed after {} mins\n".format(int(round((time.time() - start_time) / 60))))
示例#2
0
	def remove_AcmeDb(self, num_of_cust):
		"""
		Drop all AcmeAir databases
		"""
		acmeair = AcmeAirUtils()
		if acmeair.is_acmeair_running() != 0:
			acmeair.stop_acmeair()
			
		cloudantUtils = CloudantDbUtils(test_properties)
		cloudantUtils.drop_all_databases()		
	def test_005(self, sparksubmit):
		script_name = "schema_sample_size_005.py"
		returncode, out, err = self.run_test(self.get_script_path(script_name), sparksubmit)
		cloudantUtils = CloudantDbUtils(test_properties)
		if cloudantUtils.check_if_doc_exists("n_customer", "zzzzzzzzzzzzzz") :
			print(err)
			assert err.index("org.apache.spark.SparkException") > 0
			assert returncode == 1
		else :
			assert returncode == 0
    def remove_AcmeDb(self, num_of_cust):
        """
		Drop all AcmeAir databases
		"""
        acmeair = AcmeAirUtils()
        if acmeair.is_acmeair_running() != 0:
            acmeair.stop_acmeair()

        cloudantUtils = CloudantDbUtils(test_properties)
        cloudantUtils.drop_all_databases()
示例#5
0
 def loadData(self):
     cloudantUtils = CloudantDbUtils(test_properties)
     for db in cloudantUtils.test_dbs:
         inputJsonFilePath = self.getInputJsonFilepath(db)
         if os.path.exists(inputJsonFilePath):
             if cloudantUtils.db_exists(db):
                 with open(inputJsonFilePath, "r") as fh:
                     bulkDocsJson = json.loads(fh.read())
                     if cloudantUtils.bulk_insert(db, json.dumps(bulkDocsJson)):
                         print("Successfully inserted the docs into {} from ./{}".format(db, inputJsonFilePath))
                     else:
                         print("Failed to insert the docs into {}".format(db))
             else:
                 print("{} doesn't exist".format(db))
    def start_acmeair(self):
        """
		Set the required env vars for cloudant and start the AcmeAir app locally
		If app is already running, check if it's functioning
		"""
        app_status = self.is_acmeair_running()
        if app_status == -1:
            raise RuntimeError(
                "AcmeAir is already running but malfunctioning.  Please shut it down."
            )

        elif app_status == 1:
            print(
                "AcmeAir app is already running, will not attempt to start\n")
            return

        cloudantUtils = CloudantDbUtils(self.test_properties)
        cloudantUtils.check_databases()

        # set the env vars required to start the app
        new_env = os.environ.copy()
        new_env["dbtype"] = "cloudant"
        new_env["CLOUDANT_URL"] = "https://{}:{}@{}".format(
            self.test_properties["cloudantusername"],
            self.test_properties["cloudantpassword"],
            self.test_properties["cloudanthost"])

        # start the acmeair app
        os.chdir(self.acmehome)
        command = ["node", "app.js"]
        self.proc = subprocess.Popen(command,
                                     env=new_env,
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE)

        # wait at most 10 sec for app to come up
        timeout = time.time() + 10
        while (True):
            if not self.is_acmeair_running() == 1:
                time.sleep(1)
            else:
                print("\nAcemAir started!")
                break

            if time.time() > timeout:
                raise RuntimeError("Cannot connect to AcmeAir!")
示例#7
0
 def loadData(self):
     cloudantUtils = CloudantDbUtils(test_properties)
     for db in cloudantUtils.test_dbs:
         inputJsonFilePath = self.getInputJsonFilepath(db)
         if os.path.exists(inputJsonFilePath):
             if cloudantUtils.db_exists(db):
                 with open(inputJsonFilePath, "r") as fh:
                     bulkDocsJson = json.loads(fh.read())
                     if cloudantUtils.bulk_insert(db,
                                                  json.dumps(bulkDocsJson)):
                         print(
                             "Successfully inserted the docs into {} from ./{}"
                             .format(db, inputJsonFilePath))
                     else:
                         print(
                             "Failed to insert the docs into {}".format(db))
             else:
                 print("{} doesn't exist".format(db))
	def start_acmeair(self):
		"""
		Set the required env vars for cloudant and start the AcmeAir app locally
		If app is already running, check if it's functioning
		"""		
		app_status = self.is_acmeair_running();
		if app_status == -1:
			raise RuntimeError("AcmeAir is already running but malfunctioning.  Please shut it down.")
		
		elif app_status == 1:
			print ("AcmeAir app is already running, will not attempt to start\n")
			return
			
		cloudantUtils = CloudantDbUtils(self.test_properties)
		cloudantUtils.check_databases()
		
		# set the env vars required to start the app
		new_env = os.environ.copy()
		new_env["dbtype"] = "cloudant"
		new_env["CLOUDANT_URL"] = "https://{}:{}@{}".format(
			self.test_properties["cloudantusername"], 
			self.test_properties["cloudantpassword"],
			self.test_properties["cloudanthost"])
			
		# start the acmeair app
		os.chdir(self.acmehome)
		command = ["node", "app.js"]
		self.proc = subprocess.Popen(command, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		
		# wait at most 10 sec for app to come up
		timeout = time.time() + 10
		while (True):		
			if not self.is_acmeair_running() == 1:
				time.sleep(1)
			else:
				print ("\nAcemAir started!")
				break
				
			if time.time() > timeout:
				raise RuntimeError("Cannot connect to AcmeAir!")
示例#9
0
	def load_AcmeData(self, num_of_cust):
		"""
		Reset databases and use the AcmeAir database loader to populate initial customer,
		flight and airportmapping data. Does NOT generate user data like bookings.
		"""	
		print ("num_of_cust: ", num_of_cust)
			
		acmeair = AcmeAirUtils()	
		try:
			if acmeair.is_acmeair_running() != 0:
				raise RuntimeError("""
				AcmeAir is already running which may cause unexpected results when
				resetting databases.  Please shut down the app and try again.
				""")
			else:
				cloudantUtils = CloudantDbUtils(test_properties)
				cloudantUtils.reset_databases()
				acmeair.start_acmeair()
				acmeair.load_data(num_of_cust)
		
		finally:
			acmeair.stop_acmeair()
示例#10
0
    def load_AcmeData(self, num_of_cust):
        """
		Reset databases and use the AcmeAir database loader to populate initial customer,
		flight and airportmapping data. Does NOT generate user data like bookings.
		"""
        print("num_of_cust: ", num_of_cust)

        acmeair = AcmeAirUtils()
        try:
            if acmeair.is_acmeair_running() != 0:
                raise RuntimeError("""
				AcmeAir is already running which may cause unexpected results when
				resetting databases.  Please shut down the app and try again.
				""")
            else:
                cloudantUtils = CloudantDbUtils(test_properties)
                cloudantUtils.reset_databases()
                acmeair.start_acmeair()
                acmeair.load_data(num_of_cust)

        finally:
            acmeair.stop_acmeair()
    def get_flightId_by_number(self, flightNum):
        """
		Get the generated flight ID for the given flight number
		"""
        cloudantUtils = CloudantDbUtils(self.test_properties)

        url = "https://{}/{}".format(
            self.test_properties["cloudanthost"],
            "n_flight/_design/view/_search/n_flights?q=flightSegmentId:" +
            flightNum)
        param = {"q": "flightSegmentId:" + flightNum}

        response = cloudantUtils.r.get(url, params=param)
        data = response.json()
        if int(data["total_rows"]) > 0:
            # just get one from the dict
            return data["rows"][0]["id"]
        else:
            raise RuntimeError("n_flights has no data for ", flightNum)
示例#12
0
def setup_test_databases(test_properties):
	from helpers.dbutils import CloudantDbUtils
	
	cloudantUtils = CloudantDbUtils(test_properties)
	cloudantUtils.check_databases()
示例#13
0
		finally:
			acmeair.stop_acmeair()
	
	
if  __name__ =='__main__':
	"""
	Utility to create test databases and load data
	"""
	import argparse
	parser = argparse.ArgumentParser(description="Utility to load AcmeAir data required for python spark-cloudant tests")
	group = parser.add_mutually_exclusive_group(required=True)
	group.add_argument('-cleanup', action='store_true', help='Drop all test databases')
	group.add_argument('-load', help='Reset and Load databases with the given # of users. -load 0 to just recreate databases and indexes.', type=int)
	args = parser.parse_args()

	dataloader = DataLoader()
	
	if args.load is not None:
		if args.load == 0:
			cloudantUtils = CloudantDbUtils(test_properties)
			cloudantUtils.reset_databases()
		else:			
			dataloader.load_AcmeData(args.load)
			dataloader.load_SpecCharValuePredicateData()
		
	elif args.cleanup:
		cloudantUtils = CloudantDbUtils(test_properties)
		cloudantUtils.drop_all_databases()

示例#14
0
def setup_test_databases(test_properties):
    from helpers.dbutils import CloudantDbUtils

    cloudantUtils = CloudantDbUtils(test_properties)
    cloudantUtils.check_databases()
示例#15
0
	"""
    import argparse
    parser = argparse.ArgumentParser(
        description=
        "Utility to load AcmeAir data required for python spark-cloudant tests"
    )
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-cleanup',
                       action='store_true',
                       help='Drop all test databases')
    group.add_argument(
        '-load',
        help=
        'Reset and Load databases with the given # of users. -load 0 to just recreate databases and indexes.',
        type=int)
    args = parser.parse_args()

    dataloader = DataLoader()

    if args.load is not None:
        if args.load == 0:
            cloudantUtils = CloudantDbUtils(test_properties)
            cloudantUtils.reset_databases()
        else:
            dataloader.load_AcmeData(args.load)
            dataloader.load_SpecCharValuePredicateData()

    elif args.cleanup:
        cloudantUtils = CloudantDbUtils(test_properties)
        cloudantUtils.drop_all_databases()
    def load_data(self, num_of_customers):
        """
		Call the AcmeAir app REST API to populate with the given # of users
		"""
        cloudantUtils = CloudantDbUtils(self.test_properties)
        if cloudantUtils.get_doc_count("n_customer") > 0:
            raise RuntimeException(
                "Test databases already contains unknown data so AcmeAir data loader will not run!"
            )

        r = requests.Session()
        url = "{}{}/{}".format(self._app_host, self._api_context_root,
                               "loader/load")

        if isinstance(num_of_customers, int):
            num_of_customers = str(num_of_customers)
        param = {"numCustomers": num_of_customers}
        headers = {"Content-Type": self.FORM_CONTENT_TYPE}

        print("Start AcmeAir database loader with num_of_customers = ",
              num_of_customers)
        start_time = time.time()
        try:
            r.get(url, headers=headers, params=param)

        except requests.exceptions.ConnectionError:
            # the request aborts after 2 mins due to server timeout, wait until expected # of rows is reached
            cloudantUtils.wait_for_doc_count("n_customer", num_of_customers,
                                             int(num_of_customers) / 500)
            cloudantUtils.wait_for_doc_count("n_airportcodemapping", 14, 5)
            cloudantUtils.wait_for_doc_count("n_flightsegment", 395, 5)
            cloudantUtils.wait_for_doc_count("n_flight", 1971, 20)
            print("Database load completed after {} mins\n".format(
                int(round((time.time() - start_time) / 60))))