def setUp(self): if not self._testMethodName == 'suite_setUp': self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "git_repo") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == 'suite_setUp': self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.dataset = self.input.param("dataset", "default") self.gens_load = self.generate_docs(docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators(self.log, self._generate_full_docs_list(self.gens_load))
class QueryTests(BaseTestCase): def setUp(self): if not self._testMethodName == 'suite_setUp': self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "git_repo") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == 'suite_setUp': self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.dataset = self.input.param("dataset", "default") self.gens_load = self.generate_docs(docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators(self.log, self._generate_full_docs_list(self.gens_load)) def suite_setUp(self): try: self.load(self.gens_load, flag=self.item_flag) if not self.input.param("skip_build_tuq", False): self._build_tuq(self.master) self.skip_buckets_handle = True except: self.tearDown() def tearDown(self): if self._testMethodName == 'suite_tearDown': self.skip_buckets_handle = False super(QueryTests, self).tearDown() def suite_tearDown(self): if not self.input.param("skip_build_tuq", False): if hasattr(self, 'shell'): self.shell.execute_command("killall /tmp/tuq/cbq-engine") self.shell.execute_command("killall tuqtng") self.shell.disconnect() ############################################################################################## # # SIMPLE CHECKS ############################################################################################## def test_simple_check(self): for bucket in self.buckets: query_template = 'FROM %s select $str0, $str1 ORDER BY $str0,$str1 ASC' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_simple_negative_check(self): queries_errors = {'SELECT $str0 FROM {0} WHERE COUNT({0}.$str0)>3' : 'Aggregate function not allowed here', 'SELECT *.$str0 FROM {0}' : 'Parse Error - syntax error', 'SELECT *.* FROM {0} ... ERROR' : 'Parse Error - syntax error', 'FROM %s SELECT $str0 WHERE id=null' : 'Parse Error - syntax error',} self.negative_common_body(queries_errors) def test_consistent_simple_check(self): queries = [self.gen_results.generate_query('SELECT $str0, $int0, $int1 FROM %s ' +\ 'WHERE $str0 IS NOT NULL AND $int0<10 ' +\ 'OR $int1 = 6 ORDER BY $int0, $int1'), self.gen_results.generate_query('SELECT $str0, $int0, $int1 FROM %s ' +\ 'WHERE $int1 = 6 OR $str0 IS NOT NULL AND ' +\ '$int0<10 ORDER BY $int0, $int1')] for bucket in self.buckets: actual_result1 = self.run_cbq_query(queries[0] % bucket.name) actual_result2 = self.run_cbq_query(queries[1] % bucket.name) self.assertTrue(actual_result1['resultset'] == actual_result2['resultset'], "Results are inconsistent.Difference: %s %s %s %s" %( len(actual_result1['resultset']), len(actual_result2['resultset']), actual_result1['resultset'][100], actual_result2['resultset'][100])) def test_simple_nulls(self): queries = ['SELECT id FROM %s WHERE id=NULL or id="null"'] for bucket in self.buckets: for query in queries: actual_result = self.run_cbq_query(query % (bucket.name)) self._verify_results(actual_result['resultset'], []) ############################################################################################## # # LIMIT OFFSET CHECKS ############################################################################################## def test_limit_offset(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 10' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) def test_limit_offset_zero(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 0' % (bucket.name) self.query = self.gen_results.generate_query(query_template) actual_result = self.run_cbq_query() self.assertEquals(actual_result['resultset'], [], "Results are incorrect.Actual %s.\n Expected: %s.\n" % ( actual_result['resultset'], [])) query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 0' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self.assertEquals(actual_result['resultset'], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % ( actual_result['resultset'], expected_result)) def test_limit_offset_negative_check(self): queries_errors = {'SELECT DISTINCT $str0 FROM {0} LIMIT -1' : 'Parse Error - syntax error', 'SELECT DISTINCT $str0 FROM {0} LIMIT 1.1' : 'Parse Error - syntax error', 'SELECT DISTINCT $str0 FROM {0} OFFSET -1' : 'Parse Error - syntax error', 'SELECT DISTINCT $str0 FROM {0} OFFSET 1.1' : 'Parse Error - syntax error'} self.negative_common_body(queries_errors) ############################################################################################## # # ALIAS CHECKS ############################################################################################## def test_simple_alias(self): for bucket in self.buckets: query_template = 'SELECT COUNT($str0) AS COUNT_EMPLOYEE FROM %s' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self.assertEquals(actual_result['resultset'], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % ( actual_result['resultset'], expected_result)) query_template = 'SELECT COUNT(*) + 1 AS COUNT_EMPLOYEE FROM %s' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) expected_result = [ { "COUNT_EMPLOYEE": expected_result[0]['COUNT_EMPLOYEE'] + 1 } ] self.assertEquals(actual_result['resultset'], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % ( actual_result['resultset'], expected_result)) def test_simple_negative_alias(self): queries_errors = {'SELECT $str0._last_name as *' : 'Parse Error - syntax error', 'SELECT $str0._last_name as DATABASE ?' : 'Parse Error - syntax error', 'SELECT $str0 AS NULL FROM {0}' : 'Parse Error - syntax error', 'SELECT $str1 as $str0, $str0 FROM {0}' : 'alias name is defined more than once', 'SELECT $obj0 AS points, points.task1 FROM {0}' : 'Alias points cannot be referenced', 'SELECT $obj0.task1 AS points_new FROM {0} AS test ' + 'WHERE points_new >0' : "Alias points_new cannot be referenced", 'SELECT DISTINCT $obj0 AS points_new FROM {0} AS test ' + 'ORDER BY points_new': 'Expression points_new is not in the list', 'SELECT $obj0 AS points FROM {0} AS test GROUP BY points': 'Alias points cannot be referenced', 'SELECT test.$obj0 as points FROM {0} AS TEST ' + 'GROUP BY TEST.points' : 'The expression TEST is not satisfied by these dependencies', 'SELECT test.$obj0 as points FROM {0} AS TEST ' + 'GROUP BY $obj0 AS GROUP_POINT' : 'parse_error', 'SELECT COUNT($obj0) as COUNT_NEW_POINT, COUNT($str0) ' + 'as COUNT_EMP FROM {0} AS TEST GROUP BY $str0 ' + 'HAVING COUNT_NEW_POINT >0' : 'Alias COUNT_NEW_POINT cannot be referenced', 'SELECT * FROM {0} emp UNNEST {0}.$list_obj0' : 'Invalid Bucket in UNNEST clause'} self.negative_common_body(queries_errors) def test_alias_from_clause(self): queries_templates = ['SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ORDER BY points' % (bucket.name), 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test WHERE test.$int0 >0' % (bucket.name) +\ ' ORDER BY points', 'SELECT tasks_points.task1 AS points FROM %s AS test ' % (bucket.name) +\ 'WHERE FLOOR(test.test_rate) >0 ORDER BY points', 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ' % (bucket.name) +\ 'GROUP BY test.$obj0.$_obj0_int0 ORDER BY points'] for bucket in self.buckets: for query_template in queries_templates: actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_alias_from_clause_group(self): for bucket in self.buckets: query_template = 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ' %(bucket.name) +\ 'GROUP BY $obj0.$_obj0_int0 ORDER BY points' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_alias_order_desc(self): for bucket in self.buckets: query_template = 'SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new DESC' %( bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_alias_order_asc(self): for bucket in self.buckets: query_template = 'SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new ASC' %( bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_alias_aggr_fn(self): for bucket in self.buckets: query_template = 'SELECT COUNT(TEST.$str0) from %s AS TEST' %(bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_alias_unnest(self): for bucket in self.buckets: query_template = 'SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 AS skill' %( bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) query_template = 'SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 skill' %( bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) ############################################################################################## # # ORDER BY CHECKS ############################################################################################## def test_order_by_check(self): for bucket in self.buckets: query_template = 'SELECT $str0, $str1, $obj0.$_obj0_int0 points FROM %s' % (bucket.name) +\ ' ORDER BY $str1, $str0, $obj0.$_obj0_int0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\ ' ORDER BY $obj0.$_obj0_int0, $str0, $str1' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_order_by_alias(self): for bucket in self.buckets: query_template = 'SELECT $str1, $obj0 AS points FROM %s' % (bucket.name) +\ ' AS test ORDER BY $str1 DESC, points DESC' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_order_by_alias_arrays(self): for bucket in self.buckets: query_template = 'SELECT $str1, $obj0, $list_str0[0] AS SKILL FROM %s' % ( bucket.name) +\ ' AS TEST ORDER BY SKILL, $str1, TEST.$obj0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_order_by_alias_aggr_fn(self): for bucket in self.buckets: query_template = 'SELECT $int0, $int1, count(*) AS emp_per_month from %s'% ( bucket.name) +\ ' WHERE $int1 >7 GROUP BY $int0, $int1 ORDER BY emp_per_month, $int1, $int0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_order_by_aggr_fn(self): for bucket in self.buckets: query_template = 'SELECT $str1 AS TITLE FROM %s GROUP' % (bucket.name) +\ ' BY $str1 ORDER BY MIN($int1), $str1' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_order_by_precedence(self): for bucket in self.buckets: query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\ ' ORDER BY $str0, $str1' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\ ' ORDER BY $str1, $str0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_order_by_satisfy(self): for bucket in self.buckets: query_template = 'SELECT $str0, $list_obj0 FROM %s AS employee ' % (bucket.name) +\ 'WHERE ANY vm IN employee.$list_obj0 SATISFIES vm.$_list_obj0_int0 > 5 AND' +\ ' vm.$_list_obj0_str0 = "ubuntu" END ORDER BY $str0, $list_obj0[0].$_list_obj0_int0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) ############################################################################################## # # DISTINCT ############################################################################################## def test_distinct(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $str1 FROM %s ORDER BY $str1' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_distinct_nested(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $obj0.$_obj0_int0 as VAR FROM %s ' % (bucket.name) +\ 'ORDER BY $obj0.$_obj0_int0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) query_template = 'SELECT DISTINCT $list_str0[0] as skill' +\ ' FROM %s ORDER BY $list_str0[0]' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) self.query = 'SELECT DISTINCT $obj0.* FROM %s' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) ############################################################################################## # # COMPLEX PATHS ############################################################################################## def test_simple_complex_paths(self): for bucket in self.buckets: query_template = 'SELECT $_obj0_int0 FROM %s.$obj0' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_alias_complex_paths(self): for bucket in self.buckets: query_template = 'SELECT $_obj0_int0 as new_attribute FROM %s.$obj0' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) def test_where_complex_paths(self): for bucket in self.buckets: query_template = 'SELECT $_obj0_int0 FROM %s.$obj0 WHERE $_obj0_int0 = 1' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['resultset'], expected_result) ############################################################################################## # # COMMON FUNCTIONS ############################################################################################## def run_query_from_template(self, query_template): self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def negative_common_body(self, queries_errors={}): if not queries_errors: self.fail("No queries to run!") for bucket in self.buckets: for query_template, error in queries_errors.iteritems(): try: query = self.gen_results.generate_query(query_template) actual_result = self.run_cbq_query(query.format(bucket.name)) except CBQError as ex: self.log.error(ex) self.assertTrue(str(ex).find(error) != -1, "Error is incorrect.Actual %s.\n Expected: %s.\n" %( str(ex).split(':')[-1], error)) else: self.fail("There was no errors. Error expected: %s" % error) def run_cbq_query(self, query=None, min_output_size=10, server=None): if query is None: query = self.query if server is None: server = self.master if self.input.tuq_client and "client" in self.input.tuq_client: server = self.tuq_client if self.use_rest: result = RestConnection(server).query_tool(query) else: if self.version == "git_repo": output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbaselabs/tuqtng/" +\ "tuq_client/tuq_client " +\ "-engine=http://%s:8093/" % server.ip, subcommands=[query,], min_output_size=20, end_msg='tuq_client>') else: output = self.shell.execute_commands_inside("/tmp/tuq/cbq -engine=http://%s:8093/" % server.ip, subcommands=[query,], min_output_size=20, end_msg='cbq>') result = self._parse_query_output(output) if 'error' in result: raise CBQError(result["error"], server.ip) self.log.info("TOTAL ELAPSED TIME: %s" % [param["message"] for param in result["info"] if param["key"] == "total_elapsed_time"]) return result def build_url(self, version): info = self.shell.extract_remote_info() type = info.distribution_type.lower() if type in ["ubuntu", "centos", "red hat"]: url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/" url += "couchbase-query_%s_%s_linux.tar.gz" %( version, info.architecture_type) #TODO for windows return url def _build_tuq(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': goroot = testconstants.LINUX_GOROOT gopath = testconstants.LINUX_GOPATH else: goroot = testconstants.WINDOWS_GOROOT gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if self.input.tuq_client and "goroot" in self.input.tuq_client: goroot = self.input.tuq_client["goroot"] cmd = "rm -rf {0}/src/github.com".format(gopath) self.shell.execute_command(cmd) cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'go get github.com/couchbaselabs/tuqtng;' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\ 'go get -d -v ./...; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .' self.shell.execute_command(cmd) else: cbq_url = self.build_url(self.version) #TODO for windows cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format(cbq_url) cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz" self.shell.execute_command(cmd) def _start_command_line_query(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': gopath = testconstants.LINUX_GOPATH else: gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if os == 'windows': cmd = "cd %s/src/github.com/couchbaselabs/tuqtng/; " % (gopath) +\ "./tuqtng.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s/src/github.com/couchbaselabs/tuqtng/; " % (gopath) +\ "./tuqtng -couchbase http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) self.shell.execute_command(cmd) else: os = self.shell.extract_remote_info().type.lower() if os != 'windows': cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) self.shell.execute_command(cmd) def _parse_query_output(self, output): if output.find("cbq>") == 0: output = output[output.find("cbq>") + 4:].strip() if output.find("tuq_client>") == 0: output = output[output.find("tuq_client>") + 11:].strip() if output.find("cbq>") != -1: output = output[:output.find("cbq>")].strip() if output.find("tuq_client>") != -1: output = output[:output.find("tuq_client>")].strip() return json.loads(output) def generate_docs(self, num_items, start=0): try: return getattr(self, 'generate_docs_' + self.dataset)(num_items, start) except: self.fail("There is no dataset %s, please enter a valid one" % self.dataset) def generate_docs_default(self, docs_per_day, start=0): generators = [] types = ['Engineer', 'Sales', 'Support'] join_yr = [2010, 2011] join_mo = xrange(1, 12 + 1) join_day = xrange(1, 28 + 1) template = '{{ "name":"{0}", "join_yr":{1}, "join_mo":{2}, "join_day":{3},' template += ' "email":"{4}", "job_title":"{5}", "test_rate":{8}, "skills":{9},' template += '"VMs": {10},' template += ' "tasks_points" : {{"task1" : {6}, "task2" : {7}}}}}' for info in types: for year in join_yr: for month in join_mo: for day in join_day: prefix = str(uuid.uuid4())[:7] name = ["employee-%s" % (str(day))] email = ["*****@*****.**" % (str(day))] vms = [{"RAM": month, "os": "ubuntu", "name": "vm_%s" % month, "memory": month}, {"RAM": month, "os": "windows", "name": "vm_%s"% (month + 1), "memory": month}] generators.append(DocumentGenerator("query-test" + prefix, template, name, [year], [month], [day], email, [info], range(1,10), range(1,10), [float("%s.%s" % (month, month))], [["skill%s" % y for y in join_yr]], [vms], start=start, end=docs_per_day)) return generators def generate_docs_sabre(self, docs_per_day, start=0): generators = [] dests = ['BOS', 'MIA', 'SFO'] join_yr = [2010, 2011] join_mo = xrange(1, 12 + 1) join_day = xrange(1, 28 + 1) template = '{{ "Amount":{0}, "CurrencyCode":"{1}",' template += ' "TotalTax":{{"DecimalPlaces" : {2}, "Amount" : {3}, "CurrencyCode" : "{4}",}}, ,' template += ' "Tax":{5}, "FareBasisCode":{6}, "PassengerTypeQuantity":{7}, "TicketType":"{8}",' template += '"SequenceNumber": {9},' template += ' "DirectionInd" : "{10}", "Itinerary" : {11}, "Destination" : "{12}",' template += '"join_yr":{13}, "join_mo":{14}, "join_day":{15}, "Codes" :{16}}}' for dest in dests: for year in join_yr: for month in join_mo: for day in join_day: prefix = '%s_%s-%s-%s' % (dest, year, month, day) amount = [float("%s.%s" % (month, month))] currency = [("USD", "EUR")[month in [1,3,5]]] decimal_tax = [1,2] amount_tax = [day] currency_tax = currency taxes = [{"DecimalPlaces": 2, "Amount": float(amount_tax)/3, "TaxCode": "US1", "CurrencyCode": currency}, {"DecimalPlaces": 2, "Amount": float(amount_tax)/4, "TaxCode": "US2", "CurrencyCode": currency}, {"DecimalPlaces": 2, "Amount": amount_tax - float(amount_tax)/4-\ float(amount_tax)/3, "TaxCode": "US2", "CurrencyCode": currency}] fare_basis = [{"content": "XA21A0NY", "DepartureAirportCode": dest, "BookingCode": "X", "ArrivalAirportCode": "MSP"}, {"content": "XA21A0NY", "DepartureAirportCode": "MSP", "AvailabilityBreak": True, "BookingCode": "X", "ArrivalAirportCode": "BOS"}] pass_amount = [day] ticket_type = [("eTicket", "testType")[month in [1,3,5]]] sequence = [year] direction = [("oneWay", "return")[month in [2,6,10]]] itinerary = {"OriginDestinationOptions": {"OriginDestinationOption": [ {"FlightSegment": [ {"TPA_Extensions": {"eTicket": {"Ind": True}}, "MarketingAirline": {"Code": dest}, "StopQuantity": month, "DepartureTimeZone": {"GMTOffset": -7}, "OperatingAirline": {"Code": "DL", "FlightNumber": year + month}, "DepartureAirport": {"LocationCode": "SFO"}, "ArrivalTimeZone": {"GMTOffset": -5}, "ResBookDesigCode": "X", "FlightNumber": year + day, "ArrivalDateTime": "2014-07-12T06:07:00", "ElapsedTime": 212, "Equipment": {"AirEquipType": 763}, "DepartureDateTime": "2014-07-12T00:35:00", "MarriageGrp": "O", "ArrivalAirport": {"LocationCode": "MSP"}}, {"TPA_Extensions": {"eTicket": {"Ind": False}}, "MarketingAirline": {"Code": dest}, "StopQuantity": month, "DepartureTimeZone": {"GMTOffset": -7}, "OperatingAirline": {"Code": "DL", "FlightNumber": year + month + 1}, "DepartureAirport": {"LocationCode": "SFO"}, "ArrivalTimeZone": {"GMTOffset": -3}, "ResBookDesigCode": "X", "FlightNumber": year + day, "ArrivalDateTime": "2014-07-12T06:07:00", "ElapsedTime": 212, "Equipment": {"AirEquipType": 764}, "DepartureDateTime": "2014-07-12T00:35:00", "MarriageGrp": "1", "ArrivalAirport": {"LocationCode": "MSP"}}], "ElapsedTime": 619}, {"FlightSegment": [ {"TPA_Extensions": {"eTicket": {"Ind": True}}, "MarketingAirline": {"Code": dest}, "StopQuantity": month, "DepartureTimeZone": {"GMTOffset": -7}, "OperatingAirline": {"Code": "DL", "FlightNumber": year + month}, "DepartureAirport": {"LocationCode": "SFO"}, "ArrivalTimeZone": {"GMTOffset": -5}, "ResBookDesigCode": "X", "FlightNumber": year + day, "ArrivalDateTime": "2014-07-12T06:07:00", "ElapsedTime": 212, "Equipment": {"AirEquipType": 763}, "DepartureDateTime": "2014-07-12T00:35:00", "MarriageGrp": "O", "ArrivalAirport": {"LocationCode": "MSP"}}, {"TPA_Extensions": {"eTicket": {"Ind": False}}, "MarketingAirline": {"Code": dest}, "StopQuantity": month, "DepartureTimeZone": {"GMTOffset": -7}, "OperatingAirline": {"Code": "DL", "FlightNumber": year + month + 1}, "DepartureAirport": {"LocationCode": "SFO"}, "ArrivalTimeZone": {"GMTOffset": -3}, "ResBookDesigCode": "X", "FlightNumber": year + day, "ArrivalDateTime": "2014-07-12T06:07:00", "ElapsedTime": 212, "Equipment": {"AirEquipType": 764}, "DepartureDateTime": "2014-07-12T00:35:00", "MarriageGrp": "1", "ArrivalAirport": {"LocationCode": "MSP"}}]}]}, "DirectionInd": "Return"} generators.append(DocumentGenerator(prefix, template, amount, currency, decimal_tax, amount_tax, currency_tax, [taxes], [fare_basis], pass_amount, ticket_type, sequence, direction, itinerary, [dest], [year], [month], [day], [[dest, dest]], start=start, end=docs_per_day)) return generators def load(self, generators_load, exp=0, flag=0, kv_store=1, only_store_hash=True, batch_size=1, pause_secs=1, timeout_secs=30, op_type='create', start_items=0): gens_load = {} for bucket in self.buckets: tmp_gen = [] for generator_load in generators_load: tmp_gen.append(copy.deepcopy(generator_load)) gens_load[bucket] = copy.deepcopy(tmp_gen) tasks = [] items = 0 for gen_load in gens_load[self.buckets[0]]: items += (gen_load.end - gen_load.start) for bucket in self.buckets: self.log.info("%s %s to %s documents..." % (op_type, items, bucket.name)) tasks.append(self.cluster.async_load_gen_docs(self.master, bucket.name, gens_load[bucket], bucket.kvs[kv_store], op_type, exp, flag, only_store_hash, batch_size, pause_secs, timeout_secs)) for task in tasks: task.result() self.num_items = items + start_items self.verify_cluster_stats(self.servers[:self.nodes_init]) self.log.info("LOAD IS FINISHED") def _generate_full_docs_list(self, gens_load, keys=[]): all_docs_list = [] for gen_load in gens_load: doc_gen = copy.deepcopy(gen_load) while doc_gen.has_next(): key, val = doc_gen.next() try: val = json.loads(val) val['mutated'] = 0 except TypeError: pass if keys: if not (key in keys): continue all_docs_list.append(val) return all_docs_list def _verify_results(self, actual_result, expected_result): if len(actual_result) != len(expected_result): missing, extra = self.check_missing_and_extra(actual_result, expected_result) self.log.error("Missing items: %s.\n Extra items: %s" % (missing[:100], extra[:100])) self.fail("Results are incorrect.Actual num %s. Expected num: %s.\n" % ( len(actual_result), len(expected_result))) if self.max_verify is not None: actual_result = actual_result[:self.max_verify] expected_result = expected_result[:self.max_verify] msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\ "Expected first and last 100: %s.\n ... \n %s" self.assertTrue(actual_result == expected_result, msg % (actual_result[:100],actual_result[-100:], expected_result[:100],expected_result[-100:])) def check_missing_and_extra(self, actual, expected): missing = [] extra = [] for item in actual: if not (item in expected): extra.append(item) for item in expected: if not (item in actual): missing.append(item) return missing, extra def sort_nested_list(self, result): actual_result = [] for item in result: curr_item = {} for key, value in item.iteritems(): if isinstance(value, list) or isinstance(value, set): curr_item[key] = sorted(value) else: curr_item[key] = value actual_result.append(curr_item) return actual_result def configure_gomaxprocs(self): max_proc = self.input.param("gomaxprocs", None) cmd = "export GOMAXPROCS=%s" % max_proc for server in self.servers: shell_connection = RemoteMachineShellConnection(self.master) shell_connection.execute_command(cmd)