def _test_interface_returns_as_expected(self):
"""Integration test for the bandit endpoints."""
for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]:
for historical_info in self._historical_infos:
json_payload = self._build_json_payload(
subtype, historical_info)
arm_names = set([
arm_name
for arm_name in historical_info.arms_sampled.iterkeys()
])
resp = self.testapp.post(self._moe_route.endpoint,
json_payload)
resp_schema = BanditResponse()
resp_dict = resp_schema.deserialize(json.loads(resp.body))
resp_arm_names = set([
arm_name
for arm_name in resp_dict['arm_allocations'].iterkeys()
])
T.assert_sets_equal(arm_names, resp_arm_names)
# The allocations should be in range [0, 1]
# The sum of all allocations should be 1.0.
total_allocation = 0
for allocation in resp_dict['arm_allocations'].itervalues():
T.assert_gte(allocation, 0)
T.assert_lte(allocation, 1)
total_allocation += allocation
T.assert_equal(total_allocation, 1.0)
def test_latin_hypercube_equally_spaced(self):
"""Test that generate_latin_hypercube_points returns properly spaced points.
Sampling from a latin hypercube results in a set of points that in each dimension are drawn
uniformly from sub-intervals of the domain this tests that every sub-interval in each dimension
contains exactly one point.
"""
for domain in self.domains_to_test:
for num_points in self.num_points_to_test:
domain_bounds = domain._domain_bounds
points = generate_latin_hypercube_points(
num_points, domain_bounds)
for dim in xrange(domain.dim):
# This size of each slice
sub_domain_width = domain_bounds[dim].length / float(
num_points)
# Sort in dim dimension
points = sorted(points, key=lambda points: points[dim])
for i, point in enumerate(points):
# This point must fall somewhere within the slice
min_val = domain_bounds[dim].min + sub_domain_width * i
max_val = min_val + sub_domain_width
T.assert_gte(point[dim], min_val)
T.assert_lte(point[dim], max_val)
def test(self):
# If we schedule a job for later today, it should run today
run_time = self.scheduler.next_run_time(self.now)
next_run_date = run_time.date()
assert_equal(next_run_date, self.now.date())
earlier_time = datetime.datetime(
self.now.year, self.now.month, self.now.day, hour=13)
assert_lte(earlier_time, run_time)
def test(self):
# If we schedule a job for later today, it should run today
run_time = self.scheduler.next_run_time(self.now)
next_run_date = run_time.date()
tomorrow = self.now.date() + datetime.timedelta(days=1)
assert_equal(next_run_date, tomorrow)
earlier_time = datetime.datetime(year=tomorrow.year, month=tomorrow.month,
day=tomorrow.day, hour=13)
assert_lte(earlier_time, run_time)
def test(self):
# If we schedule a job for later today, it should run today
run_time = self.scheduler.next_run_time(self.now)
next_run_date = run_time.date()
assert_equal(next_run_date, self.now.date())
earlier_time = datetime.datetime(self.now.year,
self.now.month,
self.now.day,
hour=13)
assert_lte(earlier_time, run_time)
def test(self):
# If we schedule a job for later today, it should run today
run_time = self.scheduler.next_run_time(self.now)
next_run_date = run_time.date()
tomorrow = self.now.date() + datetime.timedelta(days=1)
assert_equal(next_run_date, tomorrow)
earlier_time = datetime.datetime(year=tomorrow.year,
month=tomorrow.month,
day=tomorrow.day,
hour=13)
assert_lte(earlier_time, run_time)
def test_streaming(self):
"""Cleverish test to check that vimap is really streaming. Essentially
we make the input generator that emits,
[0, 1, 2, 3, ..., 99] # variable inputs_which_must_be_processed
and then emits [None, None, ...] until each of the numerical inputs
have been processed (fed through the worker, and retrieved as output).
"""
inputs_which_must_be_processed = frozenset(xrange(100))
already_processed = set()
num_elements_total = 0
def input_generator():
for i in sorted(inputs_which_must_be_processed):
yield i
while not already_processed.issuperset(
inputs_which_must_be_processed):
yield None
pool = self.fork_pool()
for in_, _ in pool.imap(input_generator()).zip_in_out():
already_processed.add(in_)
num_elements_total += 1
# NOTE: streaming_lookahead is the number of None elements emitted by
# input_generator(). It can be greater than zero, when the worker
# hasn't finished processing the first 100 numerical inputs, but our
# main thread wants to enqueue more inputs (to keep the workers busy).
streaming_lookahead = num_elements_total - len(
inputs_which_must_be_processed)
T.assert_gte(
streaming_lookahead,
0,
"Sanity check failed.")
# Note: This can *very* occasionally flake, since we can feed a bunch
# of stuff to the input queue, pull a bunch to the temporary output
# buffer (in the queue manager), but only yield one element from the
# zip_in_out() function.
#
# We may refine streaming properties to make this impossible, but in
# general vimap works under the assumption that the input may be an
# infinte stream, but should be something we can do some limited
# non-blocking read-ahead with.
T.assert_lte(
streaming_lookahead,
pool.qm.max_total_in_flight,
"max_total_in_flight is a hard upper bound, but was violated.")
def test_auto_everything(self):
test_start = datetime.datetime.utcnow()
os.environ['USER'] = '******'
runner = MRTwoStepJob(['--no-conf']).make_runner()
match = JOB_NAME_RE.match(runner.get_job_name())
assert_equal(match.group(1), 'mr_two_step_job')
assert_equal(match.group(2), 'mcp')
job_start = datetime.datetime.strptime(
match.group(3) + match.group(4), '%Y%m%d%H%M%S')
job_start = job_start.replace(microsecond=int(match.group(5)))
assert_gte(job_start, test_start)
assert_lte(job_start - test_start, datetime.timedelta(seconds=5))
def test_auto_everything(self):
test_start = datetime.datetime.utcnow()
os.environ['USER'] = '******'
runner = MRTwoStepJob(['--no-conf']).make_runner()
match = JOB_NAME_RE.match(runner.get_job_name())
assert_equal(match.group(1), 'mr_two_step_job')
assert_equal(match.group(2), 'mcp')
job_start = datetime.datetime.strptime(
match.group(3) + match.group(4), '%Y%m%d%H%M%S')
job_start = job_start.replace(microsecond=int(match.group(5)))
assert_gte(job_start, test_start)
assert_lte(job_start - test_start, datetime.timedelta(seconds=5))
def assert_scalar_within_absolute(self, value, truth, tol):
"""Check whether a scalar ``value`` is equal to ``truth``: ``|value - truth| <= tol``.
:param value: scalar to check
:type value: float64
:param truth: exact/desired result
:type value: float64
:param tol: max permissible absolute difference
:type tol: float64
:raise: AssertionError value, truth are not equal to within tolerance
"""
diff = numpy.fabs(value - truth)
T.assert_lte(
diff,
tol,
message='value = {0:.18E}, truth = {1:.18E}, diff = {2:.18E}, tol = {3:.18E}'.format(value, truth, diff, tol),
)
def _test_interface_returns_as_expected(self):
"""Integration test for the bandit endpoints."""
for subtype in BANDIT_ENDPOINTS_TO_SUBTYPES[self._endpoint]:
for historical_info in self._historical_infos:
json_payload = self._build_json_payload(subtype, historical_info)
arm_names = set([arm_name for arm_name in historical_info.arms_sampled.iterkeys()])
resp = self.testapp.post(self._moe_route.endpoint, json_payload)
resp_schema = BanditResponse()
resp_dict = resp_schema.deserialize(json.loads(resp.body))
resp_arm_names = set([arm_name for arm_name in resp_dict['arm_allocations'].iterkeys()])
T.assert_sets_equal(arm_names, resp_arm_names)
# The allocations should be in range [0, 1]
# The sum of all allocations should be 1.0.
total_allocation = 0
for allocation in resp_dict['arm_allocations'].itervalues():
T.assert_gte(allocation, 0)
T.assert_lte(allocation, 1)
total_allocation += allocation
T.assert_equal(total_allocation, 1.0)
def assert_scalar_within_absolute(self, value, truth, tol):
"""Check whether a scalar ``value`` is equal to ``truth``: ``|value - truth| <= tol``.
:param value: scalar to check
:type value: float64
:param truth: exact/desired result
:type value: float64
:param tol: max permissible absolute difference
:type tol: float64
:raise: AssertionError value, truth are not equal to within tolerance
"""
diff = numpy.fabs(value - truth)
T.assert_lte(
diff,
tol,
message=
'value = {0:.18E}, truth = {1:.18E}, diff = {2:.18E}, tol = {3:.18E}'
.format(value, truth, diff, tol),
)
def test_interface_returns_as_expected(self):
"""Integration test for the /bandit/epsilon endpoint."""
moe_route = BANDIT_EPSILON_MOE_ROUTE
for subtype in EPSILON_SUBTYPES:
for historical_info in self.historical_infos_to_test:
json_payload = self._build_json_payload(subtype, historical_info, EPSILON_SUBTYPES_TO_DEFAULT_HYPERPARAMETER_INFOS[subtype])
arm_names = set([arm_name for arm_name in historical_info.arms_sampled.iterkeys()])
resp = self.testapp.post(moe_route.endpoint, json_payload)
resp_schema = BanditEpsilonResponse()
resp_dict = resp_schema.deserialize(json.loads(resp.body))
resp_arm_names = set([arm_name for arm_name in resp_dict['arm_allocations'].iterkeys()])
T.assert_sets_equal(arm_names, resp_arm_names)
# The allocations should be in range [0, 1]
# The sum of all allocations should be 1.0.
total_allocation = 0
for allocation in resp_dict['arm_allocations'].itervalues():
T.assert_gte(allocation, 0)
T.assert_lte(allocation, 1)
total_allocation += allocation
T.assert_equal(total_allocation, 1.0)
def assert_scalar_within_relative(self, value, truth, tol):
"""Check whether a scalar ``value`` is relatively equal to ``truth``: ``|value - truth|/|truth| <= tol``.
:param value: scalar to check
:type value: float64
:param truth: exact/desired result
:type value: float64
:param tol: max permissible absolute difference
:type tol: float64
:raise: AssertionError value, truth are not relatively equal
"""
denom = numpy.fabs(truth)
if denom < numpy.finfo(numpy.float64).tiny:
denom = 1.0 # do not divide by 0
diff = numpy.fabs((value - truth) / denom)
T.assert_lte(
diff,
tol,
message='value = {0:.18E}, truth = {1:.18E}, diff = {2:.18E}, tol = {3:.18E}'.format(value, truth, diff, tol),
)
def assert_scalar_within_relative(self, value, truth, tol):
"""Check whether a scalar ``value`` is relatively equal to ``truth``: ``|value - truth|/|truth| <= tol``.
:param value: scalar to check
:type value: float64
:param truth: exact/desired result
:type value: float64
:param tol: max permissible relative difference
:type tol: float64
:raise: AssertionError value, truth are not relatively equal
"""
denom = numpy.fabs(truth)
if denom < numpy.finfo(numpy.float64).tiny:
denom = 1.0 # do not divide by 0
diff = numpy.fabs((value - truth) / denom)
T.assert_lte(
diff,
tol,
message=
'value = {0:.18E}, truth = {1:.18E}, diff = {2:.18E}, tol = {3:.18E}'
.format(value, truth, diff, tol),
)
def test_latin_hypercube_equally_spaced(self):
"""Test that generate_latin_hypercube_points returns properly spaced points.
Sampling from a latin hypercube results in a set of points that in each dimension are drawn
uniformly from sub-intervals of the domain this tests that every sub-interval in each dimension
contains exactly one point.
"""
for domain in self.domains_to_test:
for num_points in self.num_points_to_test:
domain_bounds = domain._domain_bounds
points = generate_latin_hypercube_points(num_points, domain_bounds)
for dim in xrange(domain.dim):
# This size of each slice
sub_domain_width = domain_bounds[dim].length / float(num_points)
# Sort in dim dimension
points = sorted(points, key=lambda points: points[dim])
for i, point in enumerate(points):
# This point must fall somewhere within the slice
min_val = domain_bounds[dim].min + sub_domain_width * i
max_val = min_val + sub_domain_width
T.assert_gte(point[dim], min_val)
T.assert_lte(point[dim], max_val)
