def test_parse_no_weekday(self):
cfg = parse_daily('1st,2nd,3rd,10th day of march,apr,September at 00:00')
assert_equal(cfg.ordinals, None)
assert_equal(cfg.monthdays, set((1,2,3,10)))
assert_equal(cfg.weekdays, None)
assert_equal(cfg.months, set((3, 4, 9)))
assert_equal(cfg.timestr, '00:00')
def test_streaming(self):
input_iter = iter(xrange(int(10000)))
doubled_stream = vimap.ext.sugar.imap_ordered(
lambda x: 2 * x,
input_iter
)
# take a few from the doubled output stream
consumed = tuple(itertools.islice(doubled_stream, 40))
# exhaust the input
unspooled_input = tuple(input_iter)
# now take the rest from the output stream
rest = tuple(doubled_stream)
num_processed = len(consumed) + len(rest)
T.assert_gt(
len(unspooled_input),
9000,
message="Most inputs should not be processed (too much spooling / "
"not lazy). Only {0} remained.".format(len(unspooled_input))
)
assert num_processed + len(unspooled_input) == 10000, "Something got dropped"
T.assert_equal(
consumed + rest,
tuple(2 * i for i in xrange(num_processed)),
message="Processed inputs weren't the first in the stream, or are out of order."
)
def test_chunking(self):
"""
Makes sure we do chunk the data and each process gets a chunk.
"""
# For each input value, each worker process should return the input
# itself along with the process pid so that we know which process gets
# which input value.
def input_with_pid(input):
# Delay a bit here to prevent the test being flaky when one worker
# finishing with one chunk of input and grabbing next before we assign
# that next chunk to another worker
time.sleep(0.1)
return (input, multiprocessing.current_process().pid)
input_with_pids = tuple(vimap.ext.sugar.imap_ordered_chunked(
input_with_pid,
range(8),
chunk_size=3
))
# By grouping return values by pid, we could get all the input values for
# each worker process. Make sure the input value groups are the same as
# the result of chunking (expected_input_chunks).
expected_input_chunks = [(0, 1, 2), (3, 4, 5), (6, 7)]
actual_input_chunks = []
for pid, group in itertools.groupby(input_with_pids, key=lambda (x, pid): pid):
input_chunk, pids = zip(*group)
actual_input_chunks.append(input_chunk)
T.assert_equal(expected_input_chunks, actual_input_chunks)
def test_parse_no_month(self):
cfg = parse_daily('1st,2nd,3rd,10th day at 00:00')
assert_equal(cfg.ordinals, None)
assert_equal(cfg.monthdays, set((1,2,3,10)))
assert_equal(cfg.weekdays, None)
assert_equal(cfg.months, None)
assert_equal(cfg.timestr, '00:00')
def test_basic(self):
def fcn(i, to_add):
return i + to_add
T.assert_equal(
set(vimap.ext.sugar.imap_unordered(fcn, [1, 2, 3], to_add=1)),
set([2, 3, 4])
)
def test_getitem(self):
self.smap['jugglers'] = 'awesomesauce'
testify.assert_equal(self.smap['jugglers'], 'awesomesauce')
testify.assert_raises(
KeyError,
lambda: self.smap['unicyclers']
)
def test_get_api_page(self):
MockedSettings['api_app'] = {'port': 8043, 'servername': 'push.test.com'}
with mock.patch.dict(Settings, MockedSettings):
T.assert_equal(
RequestHandler.get_api_page("pushes"),
"http://push.test.com:8043/api/pushes"
)
def test_iter(self):
# __iter__ should behave like iterkeys()
d = {'1': 'a', '2': 'b', '3': 'c'}
self.prepopulate_map_test(d, self.smap)
expected_keys = set(d.iterkeys())
real_keys = set(self.smap)
testify.assert_equal(expected_keys, real_keys)
def test_default_mode(self):
# Turn off umask for inital testing of modes
os.umask(0000)
# Create a db, check default mode is 0666
sqlite3dbm.dbm.SqliteMap(self.path, flag='c')
testify.assert_equal(self.get_perm_mask(self.path), 0666)
def test_class_teardown_counted_as_failure_after_limit_reached(self):
assert_equal(self.server.failure_count, 0)
get_test(self.server, 'runner')
# The following behavior is bad because it doesn't allow clients to
# report class_teardown failures (which they are contractually
# obligated to run regardless of any failure limit). See
# https://github.com/Yelp/Testify/issues/120 for ideas about how to fix
# this.
#
# For now, we write this test to pin down the existing behavior and
# notice if it changes.
test_case_name = self.dummy_test_case.__name__
assert_raises_and_contains(
ValueError,
'%s not checked out.' % test_case_name,
self.run_test,
'runner',
)
# Verify that only N failing tests are run, where N is the server's
# failure_limit.
#
# Once issue #120 is fixed, the failure count should (probably) be
# TEST_RUNNER_SERVER_FAILURE_LIMIT + CLASS_TEARDOWN_FAILURES.
assert_equal(self.server.failure_count, self.TEST_RUNNER_SERVER_FAILURE_LIMIT)
def test_end_to_end(self):
n_file_path = os.path.join(self.tmp_dir, 'n_file')
with open(n_file_path, 'w') as f:
f.write('3')
os.environ['LOCAL_N_FILE_PATH'] = n_file_path
stdin = ['0\n', '1\n', '2\n']
mr_job = MRTowerOfPowers(['--no-conf', '-v', '--cleanup=NONE', '--n-file', n_file_path])
assert_equal(len(mr_job.steps()), 3)
mr_job.sandbox(stdin=stdin)
with mr_job.make_runner() as runner:
assert isinstance(runner, LocalMRJobRunner)
# make sure our file gets "uploaded"
assert [fd for fd in runner._files if fd['path'] == n_file_path]
runner.run()
output = set()
for line in runner.stream_output():
_, value = mr_job.parse_output_line(line)
output.add(value)
assert_equal(set(output), set([0, 1, ((2**3)**3)**3]))
def test_run(self):
self.task.run()
self.task.notify.assert_called_with(self.task.NOTIFY_START)
self.task.node.submit_command.assert_called_with(self.task.action)
self.task.hang_check_callback.start.assert_called_with()
assert_equal(self.task.action.command, self.task.command)
def test_get_value_cached(self):
expected = "the other stars"
validator = mock.Mock()
value_proxy = proxy.ValueProxy(validator, self.value_cache, 'something.string')
value_proxy._value = expected
assert_equal(value_proxy.value, expected)
validator.assert_not_called()
def test_handle_action_event_failstart(self):
action = mock.create_autospec(ActionCommand)
event = ActionCommand.FAILSTART
patcher = mock.patch('tron.core.serviceinstance.log', autospec=True)
with patcher as mock_log:
self.task.handle_action_event(action, event)
assert_equal(mock_log.warn.call_count, 1)
def test_create_tasks(self):
self.instance.create_tasks()
assert_equal(self.instance.watch.mock_calls, [
mock.call(self.instance.monitor_task),
mock.call(self.instance.start_task),
mock.call(self.instance.stop_task),
])
def test_is_sqliteurl(self):
assert is_sqlite_filepath("sqlite:///")
assert is_sqlite_filepath("sqlite:///test.db")
assert is_sqlite_filepath("sqlite:////tmp/test-database.sqlite")
sa_engine_url = SA.engine.url.URL(drivername='mysql', host='fakehost', database='fakedb')
T.assert_equal(is_sqlite_filepath(sa_engine_url), False)
def test_handle_complete_failed(self):
action = mock.create_autospec(ActionCommand, is_failed=True)
with mock.patch('tron.core.serviceinstance.log', autospec=True) as mock_log:
self.task._handle_complete(action)
assert_equal(mock_log.error.call_count, 1)
self.task.notify.assert_called_with(self.task.NOTIFY_SUCCESS)
def test_checklist_duplicate(self):
with fake_checklist_request():
# insert fake data from FakeDataMixin
fake_pushid = 2
self.insert_pushes()
self.insert_requests()
test1_request = self.get_requests_by_user('testuser1')[0]
test2_request = self.get_requests_by_user('testuser2')[0]
self.insert_pushcontent(test1_request['id'], fake_pushid)
self.insert_pushcontent(test2_request['id'], fake_pushid)
# insert fake checklist data
checklist_queries = []
for req in (test1_request, test2_request):
checklist_queries.append(db.push_checklist.insert({
'request': req['id'],
'type': 'search',
'target': 'prod'
}))
checklist_queries.append(db.push_checklist.insert({
'request': req['id'],
'type': 'search-cleanup',
'target': 'post-verify-prod'
}))
db.execute_transaction_cb(checklist_queries, on_db_return)
uri = "/checklist?id=%d" % fake_pushid
response = self.fetch(uri)
T.assert_equal(response.error, None)
T.assert_not_in("No checklist items for this push", response.body)
T.assert_not_equal(re.search("for testuser\d,testuser\d", response.body), None)
T.assert_in("Before Certifying - Do In Prod", response.body)
def test_checklist_single_search_tag(self):
with fake_checklist_request():
# insert fake data from FakeDataMixin
fake_pushid = 2
self.insert_pushes()
self.insert_requests()
test1_request = self.get_requests_by_user('testuser1')[0]
self.insert_pushcontent(test1_request['id'], fake_pushid)
# insert fake checklist data
checklist_queries = [
db.push_checklist.insert({
'request': test1_request['id'],
'type': 'search',
'target': 'prod'
}),
db.push_checklist.insert({
'request': test1_request['id'],
'type': 'search-cleanup',
'target': 'post-verify-prod'
}),
]
db.execute_transaction_cb(checklist_queries, on_db_return)
uri = "/checklist?id=%d" % fake_pushid
response = self.fetch(uri)
T.assert_equal(response.error, None)
T.assert_not_in("No checklist items for this push", response.body)
T.assert_not_in("multiple requests", response.body)
T.assert_in("for testuser1", response.body)
T.assert_in("Before Certifying - Do In Prod", response.body)
def test_load_response_content_success(self):
content = 'not:valid:json'
http_response = build_file_mock(content)
response = client.load_response_content(http_response)
assert_equal(response.error, client.DECODE_ERROR)
assert_in('No JSON object', response.msg)
assert_equal(response.content, content)
def test_read_raw_config(self):
name = 'name'
path = os.path.join(self.temp_dir, name)
manager.write(path, self.content)
self.manifest.get_file_name.return_value = path
config = self.manager.read_raw_config(name)
assert_equal(config, yaml.dump(self.content))
def test_get_file_mapping(self):
file_mapping = {
'one': 'a.yaml',
'two': 'b.yaml',
}
manager.write(self.manifest.filename, file_mapping)
assert_equal(self.manifest.get_file_mapping(), file_mapping)
def test_display_scheduler_with_jitter(self):
source = {
'value': '5 minutes',
'type': 'interval',
'jitter': ' (+/- 2 min)'}
result = display.display_scheduler(source)
assert_equal(result, 'interval 5 minutes%s' % (source['jitter']))
def test_multistarted_gradient_descent_optimizer_crippled_start(self):
"""Check that multistarted GD is finding the best result from GD."""
# Only allow 1 GD iteration.
gd_parameters_crippled = GradientDescentParameters(
1,
1,
self.gd_parameters.num_steps_averaged,
self.gd_parameters.gamma,
self.gd_parameters.pre_mult,
self.gd_parameters.max_relative_change,
self.gd_parameters.tolerance,
)
gradient_descent_optimizer_crippled = GradientDescentOptimizer(self.domain, self.polynomial, gd_parameters_crippled)
num_points = 15
points = self.domain.generate_uniform_random_points_in_domain(num_points)
multistart_optimizer = MultistartOptimizer(gradient_descent_optimizer_crippled, num_points)
test_best_point, _ = multistart_optimizer.optimize(random_starts=points)
# This point set won't include the optimum so multistart GD won't find it.
for value in (test_best_point - self.polynomial.optimum_point):
T.assert_not_equal(value, 0.0)
points_with_opt = numpy.append(points, self.polynomial.optimum_point.reshape((1, self.polynomial.dim)), axis=0)
test_best_point, _ = multistart_optimizer.optimize(random_starts=points_with_opt)
# This point set will include the optimum so multistart GD will find it.
for value in (test_best_point - self.polynomial.optimum_point):
T.assert_equal(value, 0.0)
def test_wildcards(self):
cfg = parse_daily('every day')
assert_equal(cfg.ordinals, None)
assert_equal(cfg.monthdays, None)
assert_equal(cfg.weekdays, None)
assert_equal(cfg.months, None)
assert_equal(cfg.timestr, '00:00')
def test_monthly(self):
cfg = parse_daily('1st day')
sch = scheduler.GeneralScheduler(**cfg._asdict())
next_run_date = sch.next_run_time(None)
assert_gt(next_run_date, self.now)
assert_equal(next_run_date.month, 7)
def test_hoods_checklists(self):
with fake_checklist_request():
# insert fake data from FakeDataMixin
fake_pushid = 2
self.insert_pushes()
self.insert_requests()
req = self.get_requests_by_user('testuser1')[0]
self.insert_pushcontent(req['id'], fake_pushid)
# insert fake checklist data
checklist_queries = []
checklist_items = (
{'request': req['id'], 'type': 'hoods', 'target': 'stage'},
{'request': req['id'], 'type': 'hoods', 'target': 'prod'},
{'request': req['id'], 'type': 'hoods-cleanup', 'target': 'post-verify-stage'},
)
for checklist_item in checklist_items:
checklist_queries.append(db.push_checklist.insert(checklist_item))
db.execute_transaction_cb(checklist_queries, on_db_return)
uri = "/checklist?id=%d" % fake_pushid
response = self.fetch(uri)
T.assert_equal(response.error, None)
T.assert_not_in("No checklist items for this push", response.body)
T.assert_in("Notify testuser1 to deploy Geoservices to stage", response.body)
T.assert_in("Notify testuser1 to deploy Geoservices to prod", response.body)
def test_persist_objects(self):
contents_to_load = {
'api1': {
'key1': 'value1',
'key2': 11,
'key3': {'some': 'dict'},
'key4': ['a', 'list']
},
'api2': {
'key1': 'value42',
'key4': 'lavash bread'
}
}
# Open an empty cache
api_cache = self._open_cache()
# Load the cache
for api_name in contents_to_load.keys():
for key in contents_to_load[api_name]:
api_cache.cache_value(api_name, key, contents_to_load[api_name][key])
# Verify the cache
for api_name in contents_to_load.keys():
for key in contents_to_load[api_name]:
expected_val = contents_to_load[api_name][key]
actual_val = api_cache.lookup_value(api_name, key)
T.assert_equal(expected_val, actual_val)
# Close the cache
final_contents = self._close_cache(api_cache)
T.assert_equal(contents_to_load, final_contents)
def test_cron_scheduler(self):
line = "cron */5 * * 7,8 *"
config = schedule_parse.valid_schedule('test', line)
sched = scheduler.scheduler_from_config(config, mock.Mock())
start_time = datetime.datetime(2012, 3, 14, 15, 9, 26)
next_time = sched.next_run_time(start_time)
assert_equal(next_time, datetime.datetime(2012, 7, 1, 0))
def test_valid_original_config(self):
test_config = self.BASE_CONFIG + """
jobs:
-
name: "test_job0"
node: node0
schedule: "interval 20s"
actions:
"""
expected_result = {'MASTER':
{'nodes':
[{'hostname': 'localhost',
'name': 'local'}],
'config_name': 'MASTER',
'jobs':
[{'node': 'node0',
'name': 'test_job0',
'actions': None,
'schedule': 'interval 20s'}],
'ssh_options': {'agent': True},
'state_persistence': {'store_type': 'shelve',
'name': 'state_data.shelve'}}}
fd = open(self.filename,'w')
fd.write(test_config)
fd.close()
assert_equal(expected_result, _initialize_original_config(self.filename))
def just_class_teardown(self):
assert_equal(self.counter, 18)
self.counter += 1
def test_first_call_is_not_cached(self):
assert_equal(self.counter.next(), 0)
def __class_teardown_postrun_2(self):
assert_equal(self.counter, 20)
def __class_teardown_postrun_1(self):
assert_equal(self.counter, 19)
self.counter += 1
def __zteardown_postrun_1(self):
assert_equal(self.counter, 14)
self.counter += 1
def test_fixture_registration_order(self):
assert_equal(self.counter, 10)
self.counter += 1
def __teardown_postrun_2(self):
assert_equal(self.counter, 15)
self.counter += 1
def __context_manager_1(self):
assert_equal(self.counter, 8)
self.counter += 1
yield
assert_equal(self.counter, 12)
self.counter += 1
def do_some_teardown(self):
assert_equal(self.counter, 13)
self.counter += 1
def __setup_prerun_2(self):
assert_equal(self.counter, 6)
self.counter += 1
def __context_manager_2(self):
assert_equal(self.counter, 9)
self.counter += 1
yield
assert_equal(self.counter, 11)
self.counter += 1
def __class_context_manager_1(self):
assert_equal(self.counter, 3)
self.counter += 1
yield
assert_equal(self.counter, 17)
self.counter += 1
def real_setup(self):
assert_equal(self.counter, 7)
self.counter += 1
def __class_setup_prerun_2(self):
assert_equal(self.counter, 1)
self.counter += 1
def __class_context_manager_2(self):
assert_equal(self.counter, 4)
self.counter += 1
yield
assert_equal(self.counter, 16)
self.counter += 1
def assert_submit_request(self, request, edit=False, redirect=None):
results = []
def on_db_return(success, db_results):
assert success
results.extend(db_results.fetchall())
results = []
db.execute_cb(db.push_requests.select(), on_db_return)
num_results_before = len(results)
if redirect:
with mock.patch.object(NewRequestServlet, "redirect") as redir:
response = self.fetch("/newrequest",
method="POST",
body=urllib.urlencode(request))
T.assert_equal(response.error, None)
T.assert_equal(redir.call_args[0][0], redirect)
else:
response = self.fetch("/newrequest",
method="POST",
body=urllib.urlencode(request))
T.assert_equal(response.error, None)
results = []
db.execute_cb(db.push_requests.select(), on_db_return)
num_results_after = len(results)
if not edit:
T.assert_equal(num_results_after, num_results_before + 1)
last_req = self.get_requests()[-1]
T.assert_equal(len(results), last_req['id'])
return last_req
else:
T.assert_equal(num_results_after, num_results_before)
last_req = self.get_requests()[-1]
return last_req
def third_setup(self):
assert_equal(self.counter, 2)
self.counter += 1
def test_search_no_change(self):
tag = ['search-backend']
orig_reqid = self.assert_checklist_for_tags(tag)
new_reqid = self.assert_checklist_for_tags(tag, orig_reqid)
T.assert_equal(orig_reqid, new_reqid)
def assert_request(self, expected, actual):
T.assert_equal(expected['user'], actual['user'])
T.assert_equal(expected['request-repo'], actual['repo'])
T.assert_equal(expected['request-branch'], actual['branch'])
T.assert_equal(expected['request-tags'], actual['tags'])
T.assert_equal(expected['request-comments'], actual['comments'])
T.assert_equal(expected['request-description'], actual['description'])
T.assert_equal(expected['request-watchers'], actual['watchers'])
def test_supports_combiners(self):
assert_equal(supports_combiners_in_hadoop_streaming('0.19'), False)
assert_equal(supports_combiners_in_hadoop_streaming('0.19.2'), False)
assert_equal(supports_combiners_in_hadoop_streaming('0.20'), True)
assert_equal(supports_combiners_in_hadoop_streaming('0.20.203'), True)
def test_hoods_no_change(self):
tag = ['hoods']
orig_reqid = self.assert_checklist_for_tags(tag)
new_reqid = self.assert_checklist_for_tags(tag, orig_reqid)
T.assert_equal(orig_reqid, new_reqid)
def test_restore_file_missing(self):
state_data = self.store.restore(['some', 'keys'])
assert_equal(state_data, {})
def test_uses_generic_jobconf(self):
assert_equal(uses_generic_jobconf('0.18'), False)
assert_equal(uses_generic_jobconf('0.20'), True)
assert_equal(uses_generic_jobconf('0.21'), True)
def test_flatten(self):
for input, expected_output in self.inputs_to_expected_outputs:
T.assert_equal(list(flatten(input)), expected_output)
def test_get_jobconf_value_2(self):
os.environ['mapreduce_job_user_name'] = 'Edsger W. Dijkstra'
assert_equal(get_jobconf_value('user.name'), 'Edsger W. Dijkstra')
assert_equal(get_jobconf_value('mapreduce.job.user.name'),
'Edsger W. Dijkstra')
