def test_services_that_run_on_should_properly_read_configuration(self):
     expected = ['fake_service1', 'fake_service2']
     fake_hostname = 'fake_hostname2'
     fake_service_configuration = self.fake_service_configuration
     actual = service_configuration_lib.services_that_run_on(
         fake_hostname, fake_service_configuration)
     T.assert_sorted_equal(expected, actual)
Пример #2
0
    def test_exception_before_iteration(self, print_exc_mock,
                                        print_warning_mock):
        """For workers that raise exceptions before they even look at their
        input generators, make sure we return exceptions appropriately.

        NOTE: For the overridden ChunkedExceptionsTest, there will be only 1
        in-flight job, containing all of the inputs. So, we only expect to see
        one exception from the pool, and the others may be printed when the
        pool is shut down.
        """
        res_to_compare = [(inp, serialize_error(ec.value), typ)
                          for inp, ec, typ in self.run_test_pool(
                              worker_raise_exc_immediately)]
        # Each worker will stop processing once an exception makes it to
        # the top so we only get that number of exceptions back out.
        expected_res_to_compare = [
            (vimap.pool.NO_INPUT, serialize_error(
                ValueError("hello")), 'exception'),
        ] * self.chunk_adjusted_num_output_exceptions(3)
        T.assert_sorted_equal(res_to_compare, expected_res_to_compare)
        self.check_died_prematurely_warning(print_warning_mock)
        # No matter how many exceptions are returned (3 in the default case, 1
        # in the chunked case), there should always be an exception printed for
        # each worker before the pool is shut down.
        self.check_printed_exceptions(print_exc_mock, ValueError("hello"), 3)
Пример #3
0
 def test_exception_with_curleys(self, print_exc_mock,
                                 print_warning_mock):
     '''Dumb test ... I aim to write tests for most every bug that had existed,
     but this is kinda 1-off ... (.format() got a curley brace).
     '''
     res_to_compare = [
         (serialize_error(ec.value), typ)
         for _, ec, typ
         in self.run_test_pool(worker_raise_exc_with_curleys)
         ]
     # Each worker will stop processing once an exception makes it to
     # the top so we only get that number of exceptions back out.
     expected_res_to_compare = [
                                   # We're not sure which inputs
                                   #  will get picked, but all
                                   # should return this exception.
                                   (serialize_error(
                                       ValueError(
                                           "{0} curley braces!")),
                                    'exception'),
                               ] * self.\
         chunk_adjusted_num_output_exceptions(
         3)
     T.assert_sorted_equal(res_to_compare, expected_res_to_compare)
     self.check_died_prematurely_warning(print_warning_mock)
Пример #4
0
    def test_exception_before_iteration(self, print_exc_mock, print_warning_mock):
        """For workers that raise exceptions before they even look at their
        input generators, make sure we return exceptions appropriately.

        NOTE: For the overridden ChunkedExceptionsTest, there will be only 1
        in-flight job, containing all of the inputs. So, we only expect to see
        one exception from the pool, and the others may be printed when the
        pool is shut down.
        """
        res_to_compare = [
            (inp, serialize_error(ec.value), typ)
            for inp, ec, typ
            in self.run_test_pool(worker_raise_exc_immediately)
        ]
        # Each worker will stop processing once an exception makes it to
        # the top so we only get that number of exceptions back out.
        expected_res_to_compare = [
            (vimap.pool.NO_INPUT, serialize_error(ValueError("hello")), 'exception'),
        ] * self.chunk_adjusted_num_output_exceptions(3)
        T.assert_sorted_equal(res_to_compare, expected_res_to_compare)
        self.check_died_prematurely_warning(print_warning_mock)
        # No matter how many exceptions are returned (3 in the default case, 1
        # in the chunked case), there should always be an exception printed for
        # each worker before the pool is shut down.
        self.check_printed_exceptions(print_exc_mock, ValueError("hello"), 3)
Пример #5
0
    def test_fail_after_a_while(self, print_exc_mock, print_warning_mock):
        processes = vimap.pool.fork(
            (worker_raise_exc_with_curleys.init_args(init=i) for i in xrange(100)), in_queue_size_factor=2
        )
        processes.imap([-1] * 3000 + list(range(50)))

        # Check yielded output.
        res_to_compare = []
        for inp, out, typ in processes.zip_in_out_typ():
            if typ == "exception":
                res_to_compare.append((inp, serialize_error(out.value), typ))
            else:
                res_to_compare.append((inp, out, typ))
        # All the -1s will produce None output.
        expected_res_to_compare = [(-1, None, "output")] * 3000
        # Once we get to the positive numbers, we start causing 50 of
        # the 100 workers to throw exceptions.
        expected_res_to_compare.extend(
            [(i, serialize_error(ValueError("{0} curley braces!")), "exception") for i in range(50)]
        )
        T.assert_sorted_equal(res_to_compare, expected_res_to_compare)

        # Check out exception logging.
        calls = print_exc_mock.call_args_list
        errors = [serialize_error(call_args[0].value) for call_args, _ in calls]
        T.assert_equal(errors, [serialize_error(ValueError("{0} curley braces!"))] * 50)

        # NOTE: Sometimes, the weakref in the pool is deleted, so 'has_exceptions' is
        # not set, and the pool prints warnings we don't actually care about. Make
        # sure that this is the only warning printed.
        if print_warning_mock.call_args_list:
            T.assert_equal(len(print_warning_mock.call_args_list), 1)
            [warning] = print_warning_mock.call_args_list
            T.assert_in("Pool disposed before input was consumed", warning[0][0])
Пример #6
0
 def test_exception_after_iteration_not_returned(self, print_exc_mock):
     res_to_compare = [(inp, out, typ) for inp, out, typ in
                       self.run_test_pool(worker_raise_exc_after_iteration)]
     # The pool notices that all output has been returned, so doesn't
     # wait for any more responses. We shouldn't see exceptions.
     expected_res_to_compare = [(inp, inp, 'output')
                                for inp in range(1, 10)]
     T.assert_sorted_equal(res_to_compare, expected_res_to_compare)
 def test_services_that_run_on_should_return_an_empty_array_when_the_hostname_isnt_anywhere(
         self):
     expected = []
     fake_hostname = 'non_existent_fake_hostname2'
     fake_service_configuration = self.fake_service_configuration
     actual = service_configuration_lib.services_that_run_on(
         fake_hostname, fake_service_configuration)
     T.assert_sorted_equal(expected, actual)
Пример #8
0
 def test_exception_before_iteration(self, print_exc_mock, print_warning_mock):
     res_to_compare = [
         (inp, serialize_error(ec.value), typ) for inp, ec, typ in run_test_pool(worker_raise_exc_immediately)
     ]
     # Each worker will stop processing once an exception makes it to
     # the top so we only get that number of exceptions back out.
     expected_res_to_compare = [(vimap.pool.NO_INPUT, serialize_error(ValueError("hello")), "exception")] * 3
     T.assert_sorted_equal(res_to_compare, expected_res_to_compare)
     self.check_died_prematurely_warning(print_warning_mock)
Пример #9
0
    def test_request_quicktags(self):
        tree = self.render_etree(self.newrequest_page)

        found_tags = []
        for span in tree.iter('span'):
            if span.attrib['class'] == 'tag-suggestion':
                found_tags.append(span.text)

        T.assert_sorted_equal(self.tags, found_tags)
Пример #10
0
    def verify_database_state(self, data, success, db_results):
        self.check_db_results(success, db_results)

        # id, user, state, repo, branch, *tags*, created, modified, etc...
        data_tags = [d[5] for d in data]
        #id, user, state, repo, branch, revision, *tags*, created, etc...
        tags = [result[6] for result in db_results.fetchall()]

        T.assert_sorted_equal(data_tags, tags)
    def verify_database_state(self, data, success, db_results):
        self.check_db_results(success, db_results)

        # id, push, *type*, status, target
        data_types = [d[2] for d in data]
        # id, push, *type*, status, target
        types = [result[2] for result in db_results.fetchall()]

        T.assert_sorted_equal(data_types, types)
Пример #12
0
    def verify_database_state(self, data, success, db_results):
        self.check_db_results(success, db_results)

        # id, user, state, repo, branch, *tags*, created, modified, etc...
        data_tags = [d[5] for d in data]
        # id, user, state, repo, branch, revision, *tags*, created, etc...
        tags = [result[6] for result in db_results.fetchall()]

        T.assert_sorted_equal(data_tags, tags)
    def test_request_quicktags(self):
        tree = self.render_etree(self.newrequest_page)

        found_tags = []
        for span in tree.iter('span'):
            if span.attrib['class'] == 'tag-suggestion':
                found_tags.append(span.text)

        T.assert_sorted_equal(self.tags, found_tags)
Пример #14
0
    def test_review_mapper(self):
        """Test the review_mapper function to make sure that based on a mock input,
        it produces the correct calculated output
        """
        biz_review_positivity = (BIZ_NAME, (TEXT, 3))

        job = WeightedPositiveWords()
        review_results = list(job.review_mapper(CATEGORY, biz_review_positivity))
        results = [((CATEGORY, u'world'), (BIZ_NAME, 3)), ((CATEGORY, u'hello'), (BIZ_NAME, 3))]
        T.assert_sorted_equal(review_results, results)
Пример #15
0
    def test_review_mapper(self):
        """Test the review_mapper function to make sure that based on a mock input,
        it produces the correct calculated output
        """
        biz_review_positivity = (BIZ_NAME, (TEXT, 3))

        job = WeightedPositiveWords()
        review_results = list(
            job.review_mapper(CATEGORY, biz_review_positivity))
        results = [((CATEGORY, u'world'), (BIZ_NAME, 3)),
                   ((CATEGORY, u'hello'), (BIZ_NAME, 3))]
        T.assert_sorted_equal(review_results, results)
    def test_request_form_labels(self):
        tree = self.render_etree(self.newrequest_page)

        form_attr = ['request-form-%s' % elem for elem in self.form_elements]
        form_attr_with_id = ['takeover']

        found_labels = []
        for label in tree.iter('label'):
            found_labels.append(label.attrib['for'])
            if label.attrib['for'] in form_attr_with_id:
                T.assert_equal(label.attrib['id'], '%s-label' % label.attrib['for'])

        T.assert_sorted_equal(form_attr, found_labels)
Пример #17
0
    def test_request_form_labels(self):
        tree = self.render_etree(self.newrequest_page)

        form_attr = ['request-form-%s' % elem for elem in self.form_elements]
        form_attr_with_id = ['takeover']

        found_labels = []
        for label in tree.iter('label'):
            found_labels.append(label.attrib['for'])
            if label.attrib['for'] in form_attr_with_id:
                T.assert_equal(label.attrib['id'], '%s-label' % label.attrib['for'])

        T.assert_sorted_equal(form_attr, found_labels)
Пример #18
0
    def test_fail_after_a_while(self, print_exc_mock, print_warning_mock):
        processes = self.fork_pool(
            (worker_raise_exc_with_curleys.init_args(init=i) for i in
             xrange(100)),
            max_real_in_flight_factor=2
        )
        processes.imap([-1] * 3000 + list(range(50)))

        # Check yielded output.
        res_to_compare = []
        for inp, out, typ in processes.zip_in_out_typ():
            if typ == 'exception':
                res_to_compare.append(
                    (inp, serialize_error(out.value), typ))
            else:
                res_to_compare.append((inp, out, typ))
        # All the -1s will produce None output.
        expected_res_to_compare = [
                                      (-1, None, 'output')
                                  ] * 3000
        # Once we get to the positive numbers, we start causing 50 of
        # the 100 workers to throw exceptions.
        expected_res_to_compare\
            .extend([
                           (i, serialize_error(ValueError(
                               "{0} curley braces!")),
                            'exception')
                           for i in range(
                            self
                            .chunk_adjusted_num_output_exceptions(50))
                           ])
        T.assert_sorted_equal(res_to_compare, expected_res_to_compare)

        # Check out exception logging.
        calls = print_exc_mock.call_args_list
        errors = [serialize_error(call_args[0].value) for call_args, _ in
                  calls]
        T.assert_equal(errors, (
            [serialize_error(ValueError("{0} curley braces!"))] *
            self.chunk_adjusted_num_output_exceptions(50)))

        # NOTE: Sometimes, the weakref in the pool is deleted,
        #  so 'has_exceptions' is
        # not set, and the pool prints warnings
        # we don't actually care about. Make
        # sure that this is the only warning printed.
        if print_warning_mock.call_args_list:
            T.assert_equal(len(print_warning_mock.call_args_list), 1)
            [warning] = print_warning_mock.call_args_list
            T.assert_in('Pool disposed before input was consumed',
                        warning[0][0])
Пример #19
0
 def test_exception_after_iteration_not_returned(self, print_exc_mock):
     res_to_compare = [
         (inp, out, typ)
         for inp, out, typ
         in self.run_test_pool(worker_raise_exc_after_iteration)
     ]
     # The pool notices that all output has been returned, so doesn't
     # wait for any more responses. We shouldn't see exceptions.
     expected_res_to_compare = [
         (inp, inp, 'output')
         for inp
         in range(1, 10)
     ]
     T.assert_sorted_equal(res_to_compare, expected_res_to_compare)
Пример #20
0
    def test_generation(self, write_file_mock, shutil_mock, fileinput_mock):
        """
        The closest thing to a unit test right now.
        """
        main.main(["basic", "-b"])
        written_files = dict(
            (os.path.basename(filename), writer) for ((filename, writer), _) in write_file_mock.call_args_list
        )
        testify.assert_sorted_equal(written_files.keys(), ["data.json", "index.html"])
        testify.assert_subset(["copytree", "copy"], [c[0] for c in shutil_mock.method_calls])

        # hacky stuff to make sure data.json is written correctly
        stringio = cStringIO.StringIO()
        written_files["data.json"](stringio)
        data_reread = simplejson.loads(stringio.getvalue())
        testify.assert_equal(data_reread, [33])
Пример #21
0
    def test_generation(self, write_file_mock, shutil_mock, fileinput_mock):
        """
        The closest thing to a unit test right now.
        """
        main.main(['basic', '-b'])
        written_files = dict(
            (os.path.basename(filename), writer)
            for ((filename, writer), _) in write_file_mock.call_args_list)
        testify.assert_sorted_equal(written_files.keys(),
                                    ['data.json', 'index.html'])
        testify.assert_subset(['copytree', 'copy'],
                              [c[0] for c in shutil_mock.method_calls])

        # hacky stuff to make sure data.json is written correctly
        stringio = cStringIO.StringIO()
        written_files['data.json'](stringio)
        data_reread = simplejson.loads(stringio.getvalue())
        testify.assert_equal(data_reread, [33])
    def test_request_form_input(self):
        tree = self.render_etree(self.newrequest_page)

        id_attr = ['request-form-%s' % elem for elem in self.form_elements]
        name_attr = ['request-%s' % elem for elem in self.form_elements]

        found_id = []
        found_name = []
        for field in tree.iter('input'):
            if 'type' not in field.attrib or field.attrib['type'] in ['checkbox']:  # ignore hidden/submit
                found_id.append(field.attrib['id'])
                found_name.append(field.attrib['name'])

        for textarea in tree.iter('textarea'):
            found_id.append(textarea.attrib['id'])
            found_name.append(textarea.attrib['name'])

        T.assert_sorted_equal(id_attr, found_id)
        T.assert_sorted_equal(name_attr, found_name)
Пример #23
0
    def test_request_form_input(self):
        tree = self.render_etree(self.newrequest_page)

        id_attr = ['request-form-%s' % elem for elem in self.form_elements]
        name_attr = ['request-%s' % elem for elem in self.form_elements]

        found_id = []
        found_name = []
        for field in tree.iter('input'):
            if 'type' not in field.attrib or field.attrib['type'] in ['checkbox']:  # ignore hidden/submit
                found_id.append(field.attrib['id'])
                found_name.append(field.attrib['name'])

        for textarea in tree.iter('textarea'):
            found_id.append(textarea.attrib['id'])
            found_name.append(textarea.attrib['name'])

        T.assert_sorted_equal(id_attr, found_id)
        T.assert_sorted_equal(name_attr, found_name)