def _test_interface_returns_as_expected(self): """Integration test for the bandit endpoints.""" for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]: for historical_info in self._historical_infos: json_payload = self._build_json_payload( subtype, historical_info) arm_names = set([ arm_name for arm_name in historical_info.arms_sampled.iterkeys() ]) resp = self.testapp.post(self._moe_route.endpoint, json_payload) resp_schema = BanditResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) resp_arm_names = set([ arm_name for arm_name in resp_dict['arm_allocations'].iterkeys() ]) T.assert_sets_equal(arm_names, resp_arm_names) # The allocations should be in range [0, 1] # The sum of all allocations should be 1.0. total_allocation = 0 for allocation in resp_dict['arm_allocations'].itervalues(): T.assert_gte(allocation, 0) T.assert_lte(allocation, 1) total_allocation += allocation T.assert_equal(total_allocation, 1.0)
def test_latin_hypercube_equally_spaced(self): """Test that generate_latin_hypercube_points returns properly spaced points. Sampling from a latin hypercube results in a set of points that in each dimension are drawn uniformly from sub-intervals of the domain this tests that every sub-interval in each dimension contains exactly one point. """ for domain in self.domains_to_test: for num_points in self.num_points_to_test: domain_bounds = domain._domain_bounds points = generate_latin_hypercube_points( num_points, domain_bounds) for dim in xrange(domain.dim): # This size of each slice sub_domain_width = domain_bounds[dim].length / float( num_points) # Sort in dim dimension points = sorted(points, key=lambda points: points[dim]) for i, point in enumerate(points): # This point must fall somewhere within the slice min_val = domain_bounds[dim].min + sub_domain_width * i max_val = min_val + sub_domain_width T.assert_gte(point[dim], min_val) T.assert_lte(point[dim], max_val)
def test(self): # If we schedule a job for later today, it should run today run_time = self.scheduler.next_run_time(self.now) next_run_date = run_time.date() assert_equal(next_run_date, self.now.date()) earlier_time = datetime.datetime( self.now.year, self.now.month, self.now.day, hour=13) assert_lte(earlier_time, run_time)
def test(self): # If we schedule a job for later today, it should run today run_time = self.scheduler.next_run_time(self.now) next_run_date = run_time.date() tomorrow = self.now.date() + datetime.timedelta(days=1) assert_equal(next_run_date, tomorrow) earlier_time = datetime.datetime(year=tomorrow.year, month=tomorrow.month, day=tomorrow.day, hour=13) assert_lte(earlier_time, run_time)
def test(self): # If we schedule a job for later today, it should run today run_time = self.scheduler.next_run_time(self.now) next_run_date = run_time.date() assert_equal(next_run_date, self.now.date()) earlier_time = datetime.datetime(self.now.year, self.now.month, self.now.day, hour=13) assert_lte(earlier_time, run_time)
def test_streaming(self): """Cleverish test to check that vimap is really streaming. Essentially we make the input generator that emits, [0, 1, 2, 3, ..., 99] # variable inputs_which_must_be_processed and then emits [None, None, ...] until each of the numerical inputs have been processed (fed through the worker, and retrieved as output). """ inputs_which_must_be_processed = frozenset(xrange(100)) already_processed = set() num_elements_total = 0 def input_generator(): for i in sorted(inputs_which_must_be_processed): yield i while not already_processed.issuperset( inputs_which_must_be_processed): yield None pool = self.fork_pool() for in_, _ in pool.imap(input_generator()).zip_in_out(): already_processed.add(in_) num_elements_total += 1 # NOTE: streaming_lookahead is the number of None elements emitted by # input_generator(). It can be greater than zero, when the worker # hasn't finished processing the first 100 numerical inputs, but our # main thread wants to enqueue more inputs (to keep the workers busy). streaming_lookahead = num_elements_total - len( inputs_which_must_be_processed) T.assert_gte( streaming_lookahead, 0, "Sanity check failed.") # Note: This can *very* occasionally flake, since we can feed a bunch # of stuff to the input queue, pull a bunch to the temporary output # buffer (in the queue manager), but only yield one element from the # zip_in_out() function. # # We may refine streaming properties to make this impossible, but in # general vimap works under the assumption that the input may be an # infinte stream, but should be something we can do some limited # non-blocking read-ahead with. T.assert_lte( streaming_lookahead, pool.qm.max_total_in_flight, "max_total_in_flight is a hard upper bound, but was violated.")
def test_auto_everything(self): test_start = datetime.datetime.utcnow() os.environ['USER'] = '******' runner = MRTwoStepJob(['--no-conf']).make_runner() match = JOB_NAME_RE.match(runner.get_job_name()) assert_equal(match.group(1), 'mr_two_step_job') assert_equal(match.group(2), 'mcp') job_start = datetime.datetime.strptime( match.group(3) + match.group(4), '%Y%m%d%H%M%S') job_start = job_start.replace(microsecond=int(match.group(5))) assert_gte(job_start, test_start) assert_lte(job_start - test_start, datetime.timedelta(seconds=5))
def assert_scalar_within_absolute(self, value, truth, tol): """Check whether a scalar ``value`` is equal to ``truth``: ``|value - truth| <= tol``. :param value: scalar to check :type value: float64 :param truth: exact/desired result :type value: float64 :param tol: max permissible absolute difference :type tol: float64 :raise: AssertionError value, truth are not equal to within tolerance """ diff = numpy.fabs(value - truth) T.assert_lte( diff, tol, message='value = {0:.18E}, truth = {1:.18E}, diff = {2:.18E}, tol = {3:.18E}'.format(value, truth, diff, tol), )
def _test_interface_returns_as_expected(self): """Integration test for the bandit endpoints.""" for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]: for historical_info in self._historical_infos: json_payload = self._build_json_payload(subtype, historical_info) arm_names = set([arm_name for arm_name in historical_info.arms_sampled.iterkeys()]) resp = self.testapp.post(self._moe_route.endpoint, json_payload) resp_schema = BanditResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) resp_arm_names = set([arm_name for arm_name in resp_dict['arm_allocations'].iterkeys()]) T.assert_sets_equal(arm_names, resp_arm_names) # The allocations should be in range [0, 1] # The sum of all allocations should be 1.0. total_allocation = 0 for allocation in resp_dict['arm_allocations'].itervalues(): T.assert_gte(allocation, 0) T.assert_lte(allocation, 1) total_allocation += allocation T.assert_equal(total_allocation, 1.0)
def assert_scalar_within_absolute(self, value, truth, tol): """Check whether a scalar ``value`` is equal to ``truth``: ``|value - truth| <= tol``. :param value: scalar to check :type value: float64 :param truth: exact/desired result :type value: float64 :param tol: max permissible absolute difference :type tol: float64 :raise: AssertionError value, truth are not equal to within tolerance """ diff = numpy.fabs(value - truth) T.assert_lte( diff, tol, message= 'value = {0:.18E}, truth = {1:.18E}, diff = {2:.18E}, tol = {3:.18E}' .format(value, truth, diff, tol), )
def test_interface_returns_as_expected(self): """Integration test for the /bandit/epsilon endpoint.""" moe_route = BANDIT_EPSILON_MOE_ROUTE for subtype in EPSILON_SUBTYPES: for historical_info in self.historical_infos_to_test: json_payload = self._build_json_payload(subtype, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[subtype]) arm_names = set([arm_name for arm_name in historical_info.arms_sampled.iterkeys()]) resp = self.testapp.post(moe_route.endpoint, json_payload) resp_schema = BanditEpsilonResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) resp_arm_names = set([arm_name for arm_name in resp_dict['arm_allocations'].iterkeys()]) T.assert_sets_equal(arm_names, resp_arm_names) # The allocations should be in range [0, 1] # The sum of all allocations should be 1.0. total_allocation = 0 for allocation in resp_dict['arm_allocations'].itervalues(): T.assert_gte(allocation, 0) T.assert_lte(allocation, 1) total_allocation += allocation T.assert_equal(total_allocation, 1.0)
def assert_scalar_within_relative(self, value, truth, tol): """Check whether a scalar ``value`` is relatively equal to ``truth``: ``|value - truth|/|truth| <= tol``. :param value: scalar to check :type value: float64 :param truth: exact/desired result :type value: float64 :param tol: max permissible absolute difference :type tol: float64 :raise: AssertionError value, truth are not relatively equal """ denom = numpy.fabs(truth) if denom < numpy.finfo(numpy.float64).tiny: denom = 1.0 # do not divide by 0 diff = numpy.fabs((value - truth) / denom) T.assert_lte( diff, tol, message='value = {0:.18E}, truth = {1:.18E}, diff = {2:.18E}, tol = {3:.18E}'.format(value, truth, diff, tol), )
def assert_scalar_within_relative(self, value, truth, tol): """Check whether a scalar ``value`` is relatively equal to ``truth``: ``|value - truth|/|truth| <= tol``. :param value: scalar to check :type value: float64 :param truth: exact/desired result :type value: float64 :param tol: max permissible relative difference :type tol: float64 :raise: AssertionError value, truth are not relatively equal """ denom = numpy.fabs(truth) if denom < numpy.finfo(numpy.float64).tiny: denom = 1.0 # do not divide by 0 diff = numpy.fabs((value - truth) / denom) T.assert_lte( diff, tol, message= 'value = {0:.18E}, truth = {1:.18E}, diff = {2:.18E}, tol = {3:.18E}' .format(value, truth, diff, tol), )
def test_latin_hypercube_equally_spaced(self): """Test that generate_latin_hypercube_points returns properly spaced points. Sampling from a latin hypercube results in a set of points that in each dimension are drawn uniformly from sub-intervals of the domain this tests that every sub-interval in each dimension contains exactly one point. """ for domain in self.domains_to_test: for num_points in self.num_points_to_test: domain_bounds = domain._domain_bounds points = generate_latin_hypercube_points(num_points, domain_bounds) for dim in xrange(domain.dim): # This size of each slice sub_domain_width = domain_bounds[dim].length / float(num_points) # Sort in dim dimension points = sorted(points, key=lambda points: points[dim]) for i, point in enumerate(points): # This point must fall somewhere within the slice min_val = domain_bounds[dim].min + sub_domain_width * i max_val = min_val + sub_domain_width T.assert_gte(point[dim], min_val) T.assert_lte(point[dim], max_val)