def test_adding_filter_ignores_argument_validation_error_and_adds_to_self(self, mock_query, mock_load, mock_jira_client): mock_query.side_effect = ArgumentValidationError('1st', 'append', 'pyccata.core.filter.Filter', 'object') mock_jira_client.return_value = DataProviders._get_client() with patch('pyccata.core.configuration.Configuration.manager', new_callable=PropertyMock) as mock_manager: with patch('pyccata.core.configuration.Configuration._configuration', new_callable=PropertyMock) as mock_config: mock_config.return_value = DataProviders._get_config_for_test() mock_manager.return_value = 'jira' manager = ThreadManager() manager.append(TestObservableThread()) self.assertEquals(1, len(manager))
def test_execute_batches_pool_size_and_logs_on_error(self, mock_query, mock_load, mock_jira_client): mock_query.side_effect = ArgumentValidationError('1st', 'append', 'pyccata.core.filter.Filter', 'object') mock_jira_client.return_value = DataProviders._get_client() with patch('pyccata.core.configuration.Configuration.manager', new_callable=PropertyMock) as mock_manager: with patch('pyccata.core.configuration.Configuration._configuration', new_callable=PropertyMock) as mock_config: mock_config.return_value = DataProviders._get_config_for_test() mock_manager.return_value = 'jira' manager = ThreadManager() test_threads = DataProviders.some_threads_explode() for thread in test_threads: manager.append(thread) manager.start()
def test_execute_batches_pool_size_and_fills_on_complete(self, mock_query, mock_load, mock_jira_client): mock_query.side_effect = ArgumentValidationError('1st', 'append', 'pyccata.core.filter.Filter', 'object') mock_jira_client.return_value = DataProviders._get_client() with patch('pyccata.core.configuration.Configuration.manager', new_callable=PropertyMock) as mock_manager: with patch('pyccata.core.configuration.Configuration._configuration', new_callable=PropertyMock) as mock_config: mock_config.return_value = DataProviders._get_config_for_test() mock_manager.return_value = 'jira' manager = ThreadManager() # start with 100 threads... for i in range(100): manager.append(ViableTestThread()) manager.start()
def test_adding_filter_adds_project_manager(self, mock_load, mock_jira_client): mock_jira_client.return_value = DataProviders._get_client() with patch('pyccata.core.configuration.Configuration.manager', new_callable=PropertyMock) as mock_manager: with patch('pyccata.core.configuration.Configuration._configuration', new_callable=PropertyMock) as mock_config: mock_config.return_value = DataProviders._get_config_for_test() mock_manager.return_value = 'jira' manager = ThreadManager() self.assertIsInstance(manager.projectmanager, ProjectManager) self.assertIsInstance(manager.querymanager, QueryManager) self.assertIsInstance(manager.configuration, Configuration) mock_filter = Filter('assignee = "Foo"') manager.append(mock_filter) self.assertIsInstance(mock_filter.projectmanager, ProjectManager)
def test_execute_adds_to_failures_when_all_observers_fail(self, mock_load, mock_jira_client): mock_jira_client.return_value = None with patch('pyccata.core.configuration.Configuration.manager', new_callable=PropertyMock) as mock_manager: with patch('pyccata.core.configuration.Configuration._configuration', new_callable=PropertyMock) as mock_config: mock_config.return_value = DataProviders._get_config_for_test() mock_manager.return_value = 'jira' mock_broken = BrokenConnectionFilter('assignee = "Bob"') mock_filter = BrokenConnectionFilter('assignee = "Bob"') manager = ThreadManager() manager.append(mock_broken) manager.append(mock_filter) self.assertEquals(1, len(manager)) self.assertEquals(1, len(manager[0]._observers)) manager.start() self.assertEquals(1, len(manager._failed_threads))
def test_we_only_accept_threadable_objects(self, mock_load, mock_jira_client): with self.assertRaises(ArgumentValidationError): manager = ThreadManager() manager.append(object())
class TestThreadableCommand(TestCase): @patch('pyccata.core.log.Logger.log') @patch('argparse.ArgumentParser.parse_args') @patch('pyccata.core.configuration.Configuration._get_locations') def setUp(self, mock_config, mock_parser, mock_log): path = os.path.dirname(os.path.realpath(__file__ + '../../../')) self._path = os.path.join(path, os.path.join('tests', 'conf')) mock_config.return_value = [self._path] mock_parser.return_value = [] mock_log.return_value = None Logger._instance = mock_log Configuration(filename='config_sections.json') self._thread_manager = ThreadManager() def tearDown(self): if Configuration._instance is not None: Configuration._instance = None Configuration.NAMESPACE = 'pyccata.core' self._thread_manager.clear() self._thread_manager._instance = None del self._thread_manager @data( [ "sed 's/24/25/g'", { 'command':'sed', 'args':['s/24/25/g'] } ], [ "sed 's/24/25/g' &>/dev/null", { 'command':'sed', 'args':['s/24/25/g'], 'redirects': {'0': '/dev/null', '2':'/dev/null'} } ], [ "sed 's/24/25/g' 1>&2", { 'command':'sed', 'args':['s/24/25/g'], 'redirects': {'0':'1'} } ], [ "sed 's/24/25/g' 2>&1 1>/dev/null", { 'command':'sed', 'args':['s/24/25/g'], 'redirects': {'0': '/dev/null', '1':'0'} } ], [ "sed 's/24/25/g' &>/dev/null < infile.txt", { 'command':'sed', 'args':['s/24/25/g'], 'redirects': {'0': '/dev/null', '1':'/dev/null'} } ], [ "grep -rin --col 'i < 24\|b>19' > /dev/null", { 'command':'grep', 'args':['-rin', '--col', 'i < 24\|b>19'], 'redirects': {'0': '/dev/null'} } ], [ "grep -rin --col 'i < 24\|b>19' 3>/dev/null", { 'command':'grep', 'args':['-rin', '--col', 'i < 24\|b>19'], 'redirects': {'0': '0'} } ], [ "grep -rin --col 'i < 24\|b>19' 3>4", { 'command':'grep', 'args':['-rin', '--col', 'i < 24\|b>19'], 'redirects': {'0': '0'} } ] ) @unpack def test_build_command_with_simple_commands(self, command, expected_return): configuration = namedtuple('Config', 'name command input_directory output_directory wait_for') config = configuration( name='Test command', command=command, input_directory='/tmp', output_directory='/tmp', wait_for=None ) threadable = ThreadableCommand(self._thread_manager, config) commands = threadable._commands for command in commands: self.assertEquals(command.command, expected_return['command']) self.assertEquals(command.arguments, expected_return['args']) if not 'redirects' in expected_return.keys(): continue for key in expected_return['redirects'].keys(): for redirect in command.redirects: if str(redirect.redirect_input) == str(int(key)-1): out = redirect.redirect_output if isinstance(redirect.redirect_output, str) else str(redirect.redirect_output) self.assertEquals(out, expected_return['redirects'][key]) @data( "sed 's/24/25/g", "sed 's/24/ > /g' &>/dev/null" ) def test_build_command_raises_error_if_command_wont_parse(self, command): configuration = namedtuple('Config', 'name command input_directory output_directory wait_for') config = configuration( name='Test command', command=command, input_directory='/tmp', output_directory='/tmp', wait_for=None ) with self.assertRaises(ValueError): threadable = ThreadableCommand(self._thread_manager, config) """ def test_build_command_assigns_observer_to_threadable(self): configuration = namedtuple('Config', 'name command input_directory output_directory wait_for') config_observing = configuration( name='Observing', command="sed 's/24/25/g'", input_directory='/tmp', output_directory='/tmp', wait_for=None ) observing = ThreadableCommand(self._thread_manager, config_observing) config_observer = configuration( name='Observer', command="grep -rin --col 'bobjones'", input_directory='/tmp', output_directory='/tmp', wait_for=observing ) observer = ThreadableCommand(self._thread_manager, config_observer) self.assertTrue(observing.hasobservers) self.assertEquals(1, len(observing.observers)) """ def test_run_method_forms_a_pipe_and_reads_output(self): configuration = namedtuple('Config', 'name command input_directory output_directory wait_for') config = configuration( name='GrepForTestsAndReplaceWithBuild', command="grep -rin --col 'def test_*' tests | sed 's/test/build/g'", input_directory=os.getcwd(), output_directory='/tmp', wait_for=None ) thread = ThreadableCommand(self._thread_manager, config) self._thread_manager.append(thread) self._thread_manager.execute() self.assertGreater(len(thread.results), 0) def test_run_with_redirect(self): configuration = namedtuple('Config', 'name command input_directory output_directory wait_for') config = configuration( name='GrepForTestsAndReplaceWithBuild', command="grep 2>&1", input_directory=os.getcwd(), output_directory='/tmp', wait_for=None ) thread = ThreadableCommand(self._thread_manager, config) self._thread_manager.append(thread) self._thread_manager.execute() self.assertGreater(len(thread.results), 0) def test_run_raises_error_on_stderr(self): configuration = namedtuple('Config', 'name command input_directory output_directory wait_for') config = configuration( name='GrepForTestsAndReplaceWithBuild', command="grep", input_directory=os.getcwd(), output_directory='/tmp', wait_for=None ) thread = ThreadableCommand(self._thread_manager, config) self._thread_manager.append(thread) with self.assertRaises(ThreadFailedError): self._thread_manager.execute() @patch('builtins.open', create=True) @data( "grep -rin --col 'def test_*' tests | sed 's/test/build/g' 1>/tmp/test", "grep -rin --col 'def test_*' tests | sed 's/test/build/g' &>/tmp/test", "grep -rin --col 'def test_*' tests | sed 's/test/build/g' 2>/tmp/test", "grep -rin --col 'def test_*' tests | sed 's/test/build/g' 2>/tmp/test 1>&2" ) def test_run_with_redirect_to_file(self, command, mock_open): mock_open.return_value = None configuration = namedtuple('Config', 'name command input_directory output_directory wait_for') config = configuration( name='GrepForTestsAndReplaceWithBuild', command=command, input_directory=os.getcwd(), output_directory='/tmp', wait_for=None ) Configuration._instance = None thread = ThreadableCommand(self._thread_manager, config) self._thread_manager.append(thread) self._thread_manager.execute() if (len(thread._commands[1].redirects) == 2): self.assertEquals(thread._commands[1].stdout, thread._commands[1].stderr) self.assertGreater(mock_open.call_count, 0) mock_open.assert_called_with('/tmp/test', mode='a')
class TestAttachments(TestCase): _report_manager = None _thread_manager = None @patch('pyccata.core.log.Logger.log') @patch('argparse.ArgumentParser.parse_args') @patch('pyccata.core.configuration.Configuration._get_locations') def setUp(self, mock_config, mock_parser, mock_log): self.tearDown() mock_log.return_value = None Logger._instance = mock_log path = os.path.dirname(os.path.realpath(__file__ + '../../../../')) self._path = os.path.join(path, os.path.join('tests', 'conf')) mock_config.return_value = [self._path] mock_parser.return_value = [] config = None with patch('argparse.ArgumentParser.add_argument'): config = Configuration(filename='valid_config.json') config.check = True self._report_manager = ReportManager() self._thread_manager = ThreadManager() def tearDown(self): if ThreadManager._instance is not None: ThreadManager._instance = None if Configuration._instance is not None: del Configuration._instance Configuration._instance = None Configuration.NAMESPACE = 'pyccata.core' if Replacements._instance is not None: del Replacements._instance Replacements._instance = None @data( ('zip', 1, '/tmp/28/Jul/2016/AnotherTestApplication.zip'), ('sql', 1, '/tmp/28/Jul/2016/TestApplication.sql'), ('zip,sql', 2, ['/tmp/28/Jul/2016/AnotherTestApplication.zip', '/tmp/28/Jul/2016/TestApplication.sql']) ) @patch('builtins.open', create=True) @patch('jira.client.JIRA.__init__') @patch('jira.client.JIRA.search_issues') @patch('pyccata.core.configuration.Configuration._get_locations') @unpack def test_setup_and_run( self, collation, result_count, result_filename, mock_config_list, mock_results, mock_jira_client, mock_open ): mock_jira_client.return_value = None mock_config_list.return_value = [self._path] Configuration.NAMESPACE = 'pyccata.core' report = ReportManager() mock_results.return_value = DataProviders._test_data_for_attachments() report.add_callback('attachments', getattr(DataProviders, 'test_callback')) self.assertIsInstance(Configuration().replacements, Replacements) Config = namedtuple('Config', 'query fields collate output_path') config = Config( query='project=test and attachments is not empty', fields=[ 'key', 'attachments' ], collate=collation, output_path='/tmp/{FIX_VERSION}' ) Replacements().find('FIX_VERSION').value = '28/Jul/2016' attachments = None with patch('os.makedirs') as mock_os: attachments = Attachments(self._thread_manager, config) mock_os.assert_called_with('/tmp/28/Jul/2016') self._thread_manager.append(attachments) with patch('pycurl.Curl') as mock_curl: with patch('pycurl.Curl.setopt') as mock_setopt: with patch('pycurl.Curl.perform') as mock_perform: with patch('pycurl.Curl.close') as mock_close: Curl = namedtuple('Curl', 'URL WRITEDATA setopt perform close getinfo') mock_curl.return_value = Curl( URL=None, WRITEDATA=None, setopt=mock_setopt, perform=mock_perform, close=mock_close, getinfo=lambda x: 200 ) self._thread_manager.execute() self.assertEquals(result_count, len(attachments._content)) self.assertEquals((3 * result_count), mock_setopt.call_count) self.assertEquals((1 * result_count), mock_perform.call_count) self.assertEquals((1 * result_count), mock_close.call_count) self.assertEquals((1 * result_count), mock_open.call_count) calls = [] if isinstance(result_filename, list): for filename in result_filename: calls.append(call(filename, 'wb')) else: calls.append(call(result_filename, 'wb')) mock_open.assert_has_calls(calls, any_order=True) with patch('pyccata.core.managers.report.ReportManager.add_paragraph') as mock_paragraph: with patch('pyccata.core.managers.report.ReportManager.add_list') as mock_list: attachments.render(report) mock_paragraph.assert_called_with('The following file(s) have been attached to this document:') mock_list.assert_called_with('TestFile.zip') @data( ('zip', 1, '/tmp/28/Jul/2016/AnotherTestApplication.zip'), ('sql', 1, '/tmp/28/Jul/2016/TestApplication.sql'), ('zip,sql', 2, ['/tmp/28/Jul/2016/AnotherTestApplication.zip', '/tmp/28/Jul/2016/TestApplication.sql']) ) @patch('builtins.open', create=True) @patch('jira.client.JIRA.__init__') @patch('jira.client.JIRA.search_issues') @patch('pyccata.core.configuration.Configuration._get_locations') @unpack def test_setup_and_run_without_callback( self, collation, result_count, result_filename, mock_config_list, mock_results, mock_jira_client, mock_open ): mock_jira_client.return_value = None mock_config_list.return_value = [self._path] Configuration.NAMESPACE = 'pyccata.core' report = ReportManager() report.add_callback('attachments', None) mock_results.return_value = DataProviders._test_data_for_attachments() self.assertIsInstance(Configuration().replacements, Replacements) Config = namedtuple('Config', 'query fields collate output_path') config = Config( query='project=test and attachments is not empty', fields=[ 'key', 'attachments' ], collate=collation, output_path='/tmp/{FIX_VERSION}' ) Replacements().find('FIX_VERSION').value = '28/Jul/2016' attachments = None with patch('os.makedirs') as mock_os: attachments = Attachments(self._thread_manager, config) mock_os.assert_called_with('/tmp/28/Jul/2016') self._thread_manager.append(attachments) with patch('pycurl.Curl') as mock_curl: with patch('pycurl.Curl.setopt') as mock_setopt: with patch('pycurl.Curl.perform') as mock_perform: with patch('pycurl.Curl.close') as mock_close: Curl = namedtuple('Curl', 'URL WRITEDATA setopt perform close getinfo') mock_curl.return_value = Curl( URL=None, WRITEDATA=None, setopt=mock_setopt, perform=mock_perform, close=mock_close, getinfo=lambda x: 200 ) self._thread_manager.execute() self.assertEquals(result_count, len(attachments._content)) self.assertEquals((3 * result_count), mock_setopt.call_count) self.assertEquals((1 * result_count), mock_perform.call_count) self.assertEquals((1 * result_count), mock_close.call_count) self.assertEquals((1 * result_count), mock_open.call_count) calls = [] if isinstance(result_filename, list): for filename in result_filename: calls.append(call(filename, 'wb')) else: calls.append(call(result_filename, 'wb')) mock_open.assert_has_calls(calls, any_order=True) with patch('pyccata.core.managers.report.ReportManager.add_paragraph') as mock_paragraph: with patch('pyccata.core.managers.report.ReportManager.add_list') as mock_list: attachments.render(report) mock_paragraph.assert_not_called() mock_list.assert_not_called() @data( ('zip', 1, '/tmp/28/Jul/2016/AnotherTestApplication.zip'), ('sql', 1, '/tmp/28/Jul/2016/TestApplication.sql'), ('zip,sql', 2, ['/tmp/28/Jul/2016/AnotherTestApplication.zip', '/tmp/28/Jul/2016/TestApplication.sql']) ) @patch('builtins.open', create=True) @patch('jira.client.JIRA.__init__') @patch('jira.client.JIRA.search_issues') @patch('pyccata.core.configuration.Configuration._get_locations') @unpack def test_setup_and_run_where_callback_returns_string( self, collation, result_count, result_filename, mock_config_list, mock_results, mock_jira_client, mock_open ): mock_jira_client.return_value = None mock_config_list.return_value = [self._path] Configuration.NAMESPACE = 'pyccata.core' report = ReportManager() mock_results.return_value = DataProviders._test_data_for_attachments() report.add_callback('attachments', lambda x,y: 'TestFile.zip') self.assertIsInstance(Configuration().replacements, Replacements) Config = namedtuple('Config', 'query fields collate output_path') config = Config( query='project=test and attachments is not empty', fields=[ 'key', 'attachments' ], collate=collation, output_path='/tmp/{FIX_VERSION}' ) Replacements().find('FIX_VERSION').value = '28/Jul/2016' attachments = None with patch('os.makedirs') as mock_os: attachments = Attachments(self._thread_manager, config) mock_os.assert_called_with('/tmp/28/Jul/2016') self._thread_manager.append(attachments) with patch('pycurl.Curl') as mock_curl: with patch('pycurl.Curl.setopt') as mock_setopt: with patch('pycurl.Curl.perform') as mock_perform: with patch('pycurl.Curl.close') as mock_close: Curl = namedtuple('Curl', 'URL WRITEDATA setopt perform close getinfo') mock_curl.return_value = Curl( URL=None, WRITEDATA=None, setopt=mock_setopt, perform=mock_perform, close=mock_close, getinfo=lambda x: 200 ) self._thread_manager.execute() self.assertEquals(result_count, len(attachments._content)) self.assertEquals((3 * result_count), mock_setopt.call_count) self.assertEquals((1 * result_count), mock_perform.call_count) self.assertEquals((1 * result_count), mock_close.call_count) self.assertEquals((1 * result_count), mock_open.call_count) calls = [] if isinstance(result_filename, list): for filename in result_filename: calls.append(call(filename, 'wb')) else: calls.append(call(result_filename, 'wb')) mock_open.assert_has_calls(calls, any_order=True) with patch('pyccata.core.managers.report.ReportManager.add_paragraph') as mock_paragraph: with patch('pyccata.core.managers.report.ReportManager.add_list') as mock_list: attachments.render(report) mock_paragraph.assert_called_with('The following file(s) have been attached to this document:') mock_list.assert_called_with('TestFile.zip') @data( ('zip,sql', 2, []) ) @patch('builtins.open', create=True) @patch('jira.client.JIRA.__init__') @patch('jira.client.JIRA.search_issues') @patch('pyccata.core.configuration.Configuration._get_locations') @unpack def test_setup_and_run_doesnt_download_if_attachments_is_empty( self, collation, result_count, result_filename, mock_config_list, mock_results, mock_jira_client, mock_open ): mock_jira_client.return_value = None mock_config_list.return_value = [self._path] Configuration.NAMESPACE = 'pyccata.core' report = ReportManager() mock_results.return_value = [] report.add_callback('attachments', lambda x,y: '') self.assertIsInstance(Configuration().replacements, Replacements) Config = namedtuple('Config', 'query fields collate output_path') config = Config( query='project=test and attachments is not empty', fields=[ 'key', 'attachments' ], collate=collation, output_path='/tmp/{FIX_VERSION}' ) Replacements().find('FIX_VERSION').value = '28/Jul/2016' attachments = None with patch('os.makedirs') as mock_os: attachments = Attachments(self._thread_manager, config) mock_os.assert_called_with('/tmp/28/Jul/2016') with patch('pyccata.core.parts.attachments.Attachments._download_attachments') as mock_download: self._thread_manager.execute() mock_download.assert_not_called() @patch('pyccata.core.filter.Filter.failure', new_callable=PropertyMock) @patch('argparse.ArgumentParser.parse_args') @patch('pyccata.core.configuration.Configuration._get_locations') def test_setup_and_run_handles_exception(self, mock_config_list, mock_parser, mock_failure): mock_parser.return_value = [] mock_config_list.return_value = [self._path] Configuration.NAMESPACE = 'pyccata.core' report = ReportManager() # never actually thrown from the filter but useful for testing and coverage ;-) mock_failure.return_value = InvalidFilenameError('The specified file does not exist') report.add_callback('test', 'test_callback') self.assertEquals('test_callback', report.get_callback('test')) self.assertIsInstance(Configuration().replacements, Replacements) Config = namedtuple('Config', 'query fields collate output_path') config = Config( query='project=test and attachments is not empty', fields=[ 'key', 'attachments' ], collate='zip', output_path='/tmp/{FIX_VERSION}' ) Replacements().find('FIX_VERSION').value = '28/Jul/2016' attachments = None with patch('os.makedirs') as mock_os: attachments = Attachments(self._thread_manager, config) mock_os.assert_called_with('/tmp/28/Jul/2016') with patch('pyccata.core.filter.Filter.failed', return_value=True): attachments.run() self.assertEquals(str(attachments._content.failure), 'The specified file does not exist') @patch('builtins.open', create=True) @patch('jira.client.JIRA.__init__') @patch('jira.client.JIRA.search_issues') @patch('pyccata.core.managers.clients.jira.Jira.server', new_callable=PropertyMock) @patch('pyccata.core.configuration.Configuration._get_locations') @unpack def test_run_raises_type_error_if_attachments_callback_function_is_not_set( self, mock_config_list, mock_results, mock_jira_results, mock_jira_client, mock_open ): mock_jira_client.return_value = None mock_config_list.return_value = [self._path] Configuration.NAMESPACE = 'pyccata.core' report = ReportManager() report.add_callback('attachments', None) mock_jira_results.return_value = DataProviders._test_data_for_attachments() server = namedtuple('Server', 'server_address attachments') mock_results.return_value = server(server_address=None, attachments=None) self.assertIsInstance(Configuration().replacements, Replacements) Config = namedtuple('Config', 'query fields collate output_path') config = Config( query='project=test and attachments is not empty', fields=[ 'key', 'attachments' ], collate='zip', output_path='/tmp/{FIX_VERSION}' ) attachments = None with patch('os.path.exists') as mock_exists: mock_exists.return_value = False with patch('os.makedirs') as mock_os: attachments = Attachments(self._thread_manager, config) mock_os.assert_called_with('/tmp/' + Configuration().replacements.replace('{FIX_VERSION}')) self._thread_manager.append(attachments) with patch('pycurl.Curl') as mock_curl: with patch('pycurl.Curl.setopt') as mock_setopt: with patch('pycurl.Curl.perform') as mock_perform: with patch('pycurl.Curl.close') as mock_close: Curl = namedtuple('Curl', 'URL WRITEDATA setopt perform close getinfo') mock_curl.return_value = Curl( URL=None, WRITEDATA=None, setopt=mock_setopt, perform=mock_perform, close=mock_close, getinfo=lambda x: 200 ) with self.assertRaises(InvalidCallbackError): self._thread_manager.execute() self.assertIsInstance(attachments.failure, InvalidCallbackError) mock_setopt.assert_not_called() mock_perform.assert_not_called() mock_close.assert_not_called() mock_open.assert_not_called()