def test_latin_hypercube_equally_spaced(self): """Test that generate_latin_hypercube_points returns properly spaced points. Sampling from a latin hypercube results in a set of points that in each dimension are drawn uniformly from sub-intervals of the domain this tests that every sub-interval in each dimension contains exactly one point. """ for domain in self.domains_to_test: for num_points in self.num_points_to_test: domain_bounds = domain._domain_bounds points = generate_latin_hypercube_points( num_points, domain_bounds) for dim in xrange(domain.dim): # This size of each slice sub_domain_width = domain_bounds[dim].length / float( num_points) # Sort in dim dimension points = sorted(points, key=lambda points: points[dim]) for i, point in enumerate(points): # This point must fall somewhere within the slice min_val = domain_bounds[dim].min + sub_domain_width * i max_val = min_val + sub_domain_width T.assert_gte(point[dim], min_val) T.assert_lte(point[dim], max_val)
def _test_interface_returns_as_expected(self): """Integration test for the bandit endpoints.""" for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]: for historical_info in self._historical_infos: json_payload = self._build_json_payload( subtype, historical_info) arm_names = set([ arm_name for arm_name in historical_info.arms_sampled.iterkeys() ]) resp = self.testapp.post(self._moe_route.endpoint, json_payload) resp_schema = BanditResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) resp_arm_names = set([ arm_name for arm_name in resp_dict['arm_allocations'].iterkeys() ]) T.assert_sets_equal(arm_names, resp_arm_names) # The allocations should be in range [0, 1] # The sum of all allocations should be 1.0. total_allocation = 0 for allocation in resp_dict['arm_allocations'].itervalues(): T.assert_gte(allocation, 0) T.assert_lte(allocation, 1) total_allocation += allocation T.assert_equal(total_allocation, 1.0)
def test_all_constant_liar_methods_function(self): """Test that each contant liar ``lie_method`` runs to completion. This is an integration test.""" for test_case in self.gp_test_environments: python_domain, python_gp = test_case python_cov, historical_data = python_gp.get_core_data_copy() for constant_liar_method in CONSTANT_LIAR_METHODS: json_payload = self._build_json_payload( python_domain, python_cov, historical_data, 2, # num_to_sample lie_method=constant_liar_method, ) resp = self.testapp.post(GP_NEXT_POINTS_CONSTANT_LIAR_ENDPOINT, json_payload) resp_schema = GpNextPointsResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) T.assert_in('points_to_sample', resp_dict) T.assert_equal(len(resp_dict['points_to_sample']), 2) # num_to_sample T.assert_equal(len(resp_dict['points_to_sample'][0]), python_gp.dim) T.assert_in('status', resp_dict) T.assert_in('expected_improvement', resp_dict['status']) T.assert_gte(resp_dict['status']['expected_improvement'], 0.0)
def test_interface_returns_same_as_cpp(self): """Integration test for the /gp/next_points/* endpoints.""" for moe_route in ALL_NEXT_POINTS_MOE_ROUTES: for test_case in self.gp_test_environments: for num_to_sample in (1, 2, 4): python_domain, python_gp = test_case python_cov, historical_data = python_gp.get_core_data_copy() # Next point from REST if moe_route.route_name == GP_NEXT_POINTS_CONSTANT_LIAR_ROUTE_NAME: json_payload = self._build_json_payload(python_domain, python_cov, historical_data, num_to_sample, lie_value=0.0) elif moe_route.route_name == GP_NEXT_POINTS_EPI_ROUTE_NAME and num_to_sample > 1: json_payload = self._build_json_payload(python_domain, python_cov, historical_data, num_to_sample, l_bfgs_b=True) else: json_payload = self._build_json_payload(python_domain, python_cov, historical_data, num_to_sample) resp = self.testapp.post(moe_route.endpoint, json_payload) resp_schema = GpNextPointsResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) T.assert_in('points_to_sample', resp_dict) T.assert_equal(len(resp_dict['points_to_sample']), num_to_sample) T.assert_equal(len(resp_dict['points_to_sample'][0]), python_gp.dim) T.assert_in('status', resp_dict) T.assert_in('expected_improvement', resp_dict['status']) T.assert_gte(resp_dict['status']['expected_improvement'], 0.0)
def test_all_fds_cleaned_up(self): initial_open_fds = get_open_fds() pool = vimap.pool.fork_identical(basic_worker, num_workers=1) after_fork_open_fds = get_open_fds() list(pool.imap([1, 2, 3]).zip_in_out()) after_finish_open_fds = get_open_fds() # Check that some FDs were opened after forking after_fork = difference_open_fds(initial_open_fds, after_fork_open_fds) # T.assert_equal(after_fork['closed'], []) T.assert_gte(len(after_fork['opened']), 2) # should have at least 3 open fds # All opened files should be FIFOs if not all(info.modes == ['fifo'] for info in after_fork['opened'].values()): print("Infos: {0}".format(after_fork['opened'])) T.assert_not_reached("Some infos are not FIFOs") after_cleanup = difference_open_fds(after_fork_open_fds, after_finish_open_fds) T.assert_gte(len(after_cleanup['closed']), 2) left_around = difference_open_fds(initial_open_fds, after_finish_open_fds) if len(left_around['opened']) != 0: queue_fds_left_around = dict(item for item in self.queue_fds.items() if item[0] in left_around['opened']) print("Queue FDs left around: {0}".format(queue_fds_left_around)) T.assert_equal(len(left_around['opened']), 0)
def test_all_fds_cleaned_up(self): initial_open_fds = get_open_fds() pool = vimap.pool.fork_identical(basic_worker, num_workers=1) after_fork_open_fds = get_open_fds() list(pool.imap([1, 2, 3]).zip_in_out()) after_finish_open_fds = get_open_fds() # Check that some FDs were opened after forking after_fork = difference_open_fds(initial_open_fds, after_fork_open_fds) # T.assert_equal(after_fork['closed'], []) T.assert_gte(len(after_fork['opened']), 2) # should have at least 3 open fds # All opened files should be FIFOs if not all(info.modes == ['fifo'] for info in after_fork['opened'].values()): print("Infos: {0}".format(after_fork['opened'])) T.assert_not_reached("Some infos are not FIFOs") after_cleanup = difference_open_fds(after_fork_open_fds, after_finish_open_fds) T.assert_gte(len(after_cleanup['closed']), 2) left_around = difference_open_fds(initial_open_fds, after_finish_open_fds) if len(left_around['opened']) != 0: queue_fds_left_around = dict( item for item in self.queue_fds.items() if item[0] in left_around['opened']) print( "Queue FDs left around: {0}".format(queue_fds_left_around)) T.assert_equal(len(left_around['opened']), 0)
def test_weekly(self): sch = scheduler_from_config('every monday at 01:00') next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(calendar.weekday(next_run_date.year, next_run_date.month, next_run_date.day), 0)
def test_daily(self): sch = scheduler_from_config('every day') next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(next_run_date.month, 6) assert_equal(next_run_date.day, 2) assert_equal(next_run_date.hour, 0)
def test_weekly(self): sch = scheduler_from_config('every monday at 01:00') next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal( calendar.weekday(next_run_date.year, next_run_date.month, next_run_date.day), 0)
def test_daily(self): cfg = parse_daily('every day') sch = scheduler.GeneralScheduler(**cfg._asdict()) next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(next_run_date.month, 6) assert_equal(next_run_date.day, 2) assert_equal(next_run_date.hour, 0)
def test_weekly(self): cfg = parse_daily('every monday at 01:00') sch = scheduler.GeneralScheduler(**cfg._asdict()) next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(calendar.weekday(next_run_date.year, next_run_date.month, next_run_date.day), 0)
def test_daily_with_time(self): sch = scheduler_from_config('every day at 02:00') next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(next_run_date.year, self.now.year) assert_equal(next_run_date.month, 6) assert_equal(next_run_date.day, 1) assert_equal(next_run_date.hour, 2) assert_equal(next_run_date.minute, 0)
def test_daily_with_time(self): cfg = parse_daily('every day at 02:00') sch = scheduler.GeneralScheduler(**cfg._asdict()) next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(next_run_date.year, self.now.year) assert_equal(next_run_date.month, 6) assert_equal(next_run_date.day, 1) assert_equal(next_run_date.hour, 2) assert_equal(next_run_date.minute, 0)
def test_weekly_in_month(self): sch = scheduler_from_config('every monday of january at 00:01') next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(next_run_date.year, self.now.year+1) assert_equal(next_run_date.month, 1) assert_equal(next_run_date.hour, 0) assert_equal(next_run_date.minute, 1) assert_equal(calendar.weekday(next_run_date.year, next_run_date.month, next_run_date.day), 0)
def test_weekly_in_month(self): sch = scheduler_from_config('every monday of january at 00:01') next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(next_run_date.year, self.now.year + 1) assert_equal(next_run_date.month, 1) assert_equal(next_run_date.hour, 0) assert_equal(next_run_date.minute, 1) assert_equal( calendar.weekday(next_run_date.year, next_run_date.month, next_run_date.day), 0)
def test_pushes_order(self): self.insert_pushes() pushes, _ = self.api_call("pushes") T.assert_length(pushes, 6) lastpush = None for push in pushes: if lastpush is not None: if push['state'] == 'accepting': T.assert_equal('accepting', lastpush['state']) elif lastpush['state'] != 'accepting': T.assert_gte(push['modified'], lastpush['modified']) lastpush = push
def test_pushes_order(self): self.insert_pushes() pushes, _ = self.api_call("pushes") T.assert_length(pushes, 6) lastpush = None for push in pushes: if lastpush is not None: if push["state"] == "accepting": T.assert_equal("accepting", lastpush["state"]) elif lastpush["state"] != "accepting": T.assert_gte(push["modified"], lastpush["modified"]) lastpush = push
def test_weekly_in_month(self): cfg = parse_daily('every monday of january at 00:01') sch = scheduler.GeneralScheduler(**cfg._asdict()) next_run_date = sch.next_run_time(None) assert_gte(next_run_date, self.now) assert_equal(next_run_date.year, self.now.year+1) assert_equal(next_run_date.month, 1) assert_equal(next_run_date.hour, 0) assert_equal(next_run_date.minute, 1) assert_equal(calendar.weekday(next_run_date.year, next_run_date.month, next_run_date.day), 0)
def test_streaming(self): """Cleverish test to check that vimap is really streaming. Essentially we make the input generator that emits, [0, 1, 2, 3, ..., 99] # variable inputs_which_must_be_processed and then emits [None, None, ...] until each of the numerical inputs have been processed (fed through the worker, and retrieved as output). """ inputs_which_must_be_processed = frozenset(xrange(100)) already_processed = set() num_elements_total = 0 def input_generator(): for i in sorted(inputs_which_must_be_processed): yield i while not already_processed.issuperset( inputs_which_must_be_processed): yield None pool = self.fork_pool() for in_, _ in pool.imap(input_generator()).zip_in_out(): already_processed.add(in_) num_elements_total += 1 # NOTE: streaming_lookahead is the number of None elements emitted by # input_generator(). It can be greater than zero, when the worker # hasn't finished processing the first 100 numerical inputs, but our # main thread wants to enqueue more inputs (to keep the workers busy). streaming_lookahead = num_elements_total - len( inputs_which_must_be_processed) T.assert_gte( streaming_lookahead, 0, "Sanity check failed.") # Note: This can *very* occasionally flake, since we can feed a bunch # of stuff to the input queue, pull a bunch to the temporary output # buffer (in the queue manager), but only yield one element from the # zip_in_out() function. # # We may refine streaming properties to make this impossible, but in # general vimap works under the assumption that the input may be an # infinte stream, but should be something we can do some limited # non-blocking read-ahead with. T.assert_lte( streaming_lookahead, pool.qm.max_total_in_flight, "max_total_in_flight is a hard upper bound, but was violated.")
def test_auto_everything(self): test_start = datetime.datetime.utcnow() os.environ['USER'] = '******' runner = MRTwoStepJob(['--no-conf']).make_runner() match = JOB_NAME_RE.match(runner.get_job_name()) assert_equal(match.group(1), 'mr_two_step_job') assert_equal(match.group(2), 'mcp') job_start = datetime.datetime.strptime( match.group(3) + match.group(4), '%Y%m%d%H%M%S') job_start = job_start.replace(microsecond=int(match.group(5))) assert_gte(job_start, test_start) assert_lte(job_start - test_start, datetime.timedelta(seconds=5))
def test_all_fds_cleaned_up(self): initial_open_fds = get_open_fds() pool = vimap.pool.fork_identical(basic_worker, num_workers=1) after_fork_open_fds = get_open_fds() list(pool.imap([1, 2, 3]).zip_in_out()) after_finish_open_fds = get_open_fds() # Check that some FDs were opened after forking after_fork = difference_open_fds(initial_open_fds, after_fork_open_fds) # T.assert_equal(after_fork['closed'], []) T.assert_gte(len(after_fork['opened']), 2) # should have at least 3 open fds # All opened files should be FIFOs T.assert_equal(all(typ == ['fifo'] for typ in after_fork['opened'].values()), True) after_cleanup = difference_open_fds(after_fork_open_fds, after_finish_open_fds) T.assert_gte(len(after_cleanup['closed']), 2) left_around = difference_open_fds(initial_open_fds, after_finish_open_fds) T.assert_equal(len(left_around['opened']), 0)
def _test_interface_returns_as_expected(self): """Integration test for the bandit endpoints.""" for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]: for historical_info in self._historical_infos: json_payload = self._build_json_payload(subtype, historical_info) arm_names = set([arm_name for arm_name in historical_info.arms_sampled.iterkeys()]) resp = self.testapp.post(self._moe_route.endpoint, json_payload) resp_schema = BanditResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) resp_arm_names = set([arm_name for arm_name in resp_dict['arm_allocations'].iterkeys()]) T.assert_sets_equal(arm_names, resp_arm_names) # The allocations should be in range [0, 1] # The sum of all allocations should be 1.0. total_allocation = 0 for allocation in resp_dict['arm_allocations'].itervalues(): T.assert_gte(allocation, 0) T.assert_lte(allocation, 1) total_allocation += allocation T.assert_equal(total_allocation, 1.0)
def test_interface_returns_same_as_cpp(self): """Integration test for the /gp/next_points/* endpoints.""" for moe_route in ALL_NEXT_POINTS_MOE_ROUTES: for test_case in self.gp_test_environments: for num_to_sample in (1, 2, 4): python_domain, python_gp = test_case python_cov, historical_data = python_gp.get_core_data_copy( ) # Next point from REST if moe_route.route_name == GP_NEXT_POINTS_CONSTANT_LIAR_ROUTE_NAME: json_payload = self._build_json_payload( python_domain, python_cov, historical_data, num_to_sample, lie_value=0.0) elif moe_route.route_name == GP_NEXT_POINTS_EPI_ROUTE_NAME and num_to_sample > 1: json_payload = self._build_json_payload( python_domain, python_cov, historical_data, num_to_sample, l_bfgs_b=True) else: json_payload = self._build_json_payload( python_domain, python_cov, historical_data, num_to_sample) resp = self.testapp.post(moe_route.endpoint, json_payload) resp_schema = GpNextPointsResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) T.assert_in('points_to_sample', resp_dict) T.assert_equal(len(resp_dict['points_to_sample']), num_to_sample) T.assert_equal(len(resp_dict['points_to_sample'][0]), python_gp.dim) T.assert_in('status', resp_dict) T.assert_in('expected_improvement', resp_dict['status']) T.assert_gte(resp_dict['status']['expected_improvement'], 0.0)
def test_interface_returns_as_expected(self): """Integration test for the /bandit/epsilon endpoint.""" moe_route = BANDIT_EPSILON_MOE_ROUTE for subtype in EPSILON_SUBTYPES: for historical_info in self.historical_infos_to_test: json_payload = self._build_json_payload(subtype, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[subtype]) arm_names = set([arm_name for arm_name in historical_info.arms_sampled.iterkeys()]) resp = self.testapp.post(moe_route.endpoint, json_payload) resp_schema = BanditEpsilonResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) resp_arm_names = set([arm_name for arm_name in resp_dict['arm_allocations'].iterkeys()]) T.assert_sets_equal(arm_names, resp_arm_names) # The allocations should be in range [0, 1] # The sum of all allocations should be 1.0. total_allocation = 0 for allocation in resp_dict['arm_allocations'].itervalues(): T.assert_gte(allocation, 0) T.assert_lte(allocation, 1) total_allocation += allocation T.assert_equal(total_allocation, 1.0)
def test_latin_hypercube_equally_spaced(self): """Test that generate_latin_hypercube_points returns properly spaced points. Sampling from a latin hypercube results in a set of points that in each dimension are drawn uniformly from sub-intervals of the domain this tests that every sub-interval in each dimension contains exactly one point. """ for domain in self.domains_to_test: for num_points in self.num_points_to_test: domain_bounds = domain._domain_bounds points = generate_latin_hypercube_points(num_points, domain_bounds) for dim in xrange(domain.dim): # This size of each slice sub_domain_width = domain_bounds[dim].length / float(num_points) # Sort in dim dimension points = sorted(points, key=lambda points: points[dim]) for i, point in enumerate(points): # This point must fall somewhere within the slice min_val = domain_bounds[dim].min + sub_domain_width * i max_val = min_val + sub_domain_width T.assert_gte(point[dim], min_val) T.assert_lte(point[dim], max_val)
def check_died_prematurely_warning(self, print_warning_mock): T.assert_gte(print_warning_mock.call_args_list, 1) for (args, kwargs) in print_warning_mock.call_args_list: T.assert_equal(args, ('All processes died prematurely!',))
def check_died_prematurely_warning(self, print_warning_mock): T.assert_gte(print_warning_mock.call_args_list, 1) for (args, kwargs) in print_warning_mock.call_args_list: T.assert_equal(args, ('All processes died prematurely!', ))