def store_binaries(session, task_id, path): """ Store the resulting binaries from a Brew scratch build (really wish this was part of the official koji api...) """ task = session.getTaskInfo(task_id) assert task['method'] == 'build' subtasks = session.getTaskChildren(task_id) # Assemble a list of tasks/files to download. downloads = [] for task in subtasks: if task['method'] != 'buildArch': continue files = list_task_output_all_volumes(session, task['id']) for filename in files: if not filename.endswith('.rpm'): continue if filename.endswith('.src.rpm'): continue for volume in files[filename]: # I'm keeping it simple for now by only supporting the default # volume. I haven't found a buildArch task that blows up yet... assert volume == 'DEFAULT' downloads.append((task, filename)) # Do the downloads. pathinfo = get_koji_pathinfo() number = 0 for (task, filename) in downloads: number += 1 work = pathinfo.work('DEFAULT') taskrelpath = pathinfo.taskrelpath(task['id']) url = '%s/%s/%s' % (work, taskrelpath, filename) filename = os.path.join(path, 'rpms', filename) download_file(url, filename, size=len(downloads), num=number)
def test_handle_download_file_error_500(self, m): m.get("http://url", text='Internal Server Error\n', status_code=500) with self.assertRaises(requests.HTTPError): download_file("http://url", self.filename) try: os.unlink(self.filename) except Exception: pass
def test_handle_download_file_dir(self): with self.assertRaises(IOError) as cm: download_file("http://url", self.tempdir) actual = self.stdout.getvalue() expected = 'Downloading: %s\n' % self.tempdir self.assertMultiLineEqual(actual, expected) self.assertEqual(cm.exception.args, (21, 'Is a directory')) self.requests_get.assert_called_once()
def test_handle_download_file_error_404(self, m): m.get("http://url", text='Not Found\n', status_code=404) with self.assertRaises(requests.HTTPError): download_file("http://url", self.filename) try: os.unlink(self.filename) except Exception: pass
def test_handle_download_file_error_500(self, m): m.get("http://url", text='Internal Server Error\n', status_code=500) with self.assertRaises(requests.HTTPError): download_file("http://url", self.filename) try: self.assertFalse(os.path.exists(self.filename)) except AssertionError: os.unlink(self.filename) raise
def test_handle_download_file_error_404(self, m): m.get("http://url", text='Not Found\n', status_code=404) with self.assertRaises(requests.HTTPError): download_file("http://url", self.filename) try: self.assertFalse(os.path.exists(self.filename)) except AssertionError: os.unlink(self.filename) raise
def test_handle_download_file_dir(self): with self.assertRaises(IOError) as cm: download_file("http://url", self.tempdir) actual = self.stdout.getvalue() expected = 'Downloading: %s\n' % self.tempdir self.assertMultiLineEqual(actual, expected) self.assertEqual(cm.exception.args, (21, 'Is a directory')) self.curlClass.assert_called_once() self.assertEqual(self.curl.setopt.call_count, 2) self.curl.perform.assert_not_called()
def test_handle_download_file_dir(self): with self.assertRaises(IOError) as cm: download_file("http://url", self.tempdir) actual = self.stdout.getvalue() expected = 'Downloading: %s\n' % self.tempdir self.assertMultiLineEqual(actual, expected) if isinstance(cm.exception, tuple): self.assertEqual(cm.exception[0], 21) self.assertEqual(cm.exception[1], 'Is a directory') else: self.assertEqual(cm.exception.args, (21, 'Is a directory'))
def test_handle_download_file_dir(self): with self.assertRaises(IOError) as cm: download_file("http://url", self.tempdir) actual = self.stdout.getvalue() expected = 'Downloading: %s\n' % self.tempdir self.assertMultiLineEqual(actual, expected) if isinstance(cm.exception, tuple): self.assertEqual(cm.exception[0], 21) self.assertEqual(cm.exception[1], 'Is a directory') else: self.assertEqual(cm.exception.args, (21, 'Is a directory')) self.requests_get.assert_called_once()
def download_srpm(session, build, destination): # Download the SRPM from this Koji build into "destination" dir. url = srpm_url(session, build) filename = posixpath.basename(url) destfile = os.path.join(destination, filename) if os.path.exists(destfile): if verify_srpm(destfile): return destfile print('existing %s does not verify, re-downloading.' % destfile) print('downloading %s to %s' % (url, destination)) download_file(url, destfile, size=1, num=1) verify_srpm(destfile) return destfile
def test_handle_download_file_with_size(self): rv = download_file("http://url", self.filename, size=10, num=8) actual = self.stdout.getvalue() expected = 'Downloading [8/10]: %s\n\n' % self.filename self.assertMultiLineEqual(actual, expected) self.requests_get.assert_called_once() self.assertIsNone(rv)
def test_handle_download_file_with_size(self): rv = download_file("http://url", self.filename, size=10, num=8) actual = self.stdout.getvalue() expected = 'Downloading [8/10]: %s\n\n' % self.filename self.assertMultiLineEqual(actual, expected) self.curlClass.assert_called_once() self.assertEqual(self.curl.setopt.call_count, 5) self.curl.perform.assert_called_once() self.curl.close.assert_called_once() self.assertIsNone(rv)
def test_handle_download_file_curl_version(self): self.curl.XFERINFOFUNCTION = None download_file("http://url", self.filename, quiet=False, noprogress=False) actual = self.stdout.getvalue() expected = 'Downloading: %s\n\n' % self.filename self.assertMultiLineEqual(actual, expected) self.assertEqual(self.curl.setopt.call_count, 5) self.curl.setopt.assert_has_calls([call(self.curl.PROGRESSFUNCTION, _download_progress)]) self.reset_mock() self.curl.PROGRESSFUNCTION = None with self.assertRaises(SystemExit) as cm: download_file("http://url", self.filename, quiet=False, noprogress=False) actual = self.stdout.getvalue() expected = 'Downloading: %s\n' % self.filename self.assertMultiLineEqual(actual, expected) actual = self.stderr.getvalue() expected = 'Error: XFERINFOFUNCTION and PROGRESSFUNCTION are not supported by pyCurl. Quit download progress\n' self.assertMultiLineEqual(actual, expected) self.assertEqual(self.curl.setopt.call_count, 3) self.assertEqual(cm.exception.code, 1)
def test_handle_download_file_undefined_length(self, m_open): self.reset_mock() response = mock.MagicMock() self.requests_get.return_value = response response.headers.get.return_value = None # content-length response.iter_content.return_value = ['a' * 65536, 'b' * 65536] rv = download_file("http://url", self.filename) actual = self.stdout.getvalue() expected = 'Downloading: %s\n[ ] ???%% 64.00 KiB\r[ ] ???%% 128.00 KiB\r[====================================] 100%% 128.00 KiB\r\n' % self.filename self.assertMultiLineEqual(actual, expected) self.requests_get.assert_called_once() m_open.assert_called_once() response.headers.get.assert_called_once() response.iter_content.assert_called_once() self.assertIsNone(rv)
def test_handle_download_file(self, m_open): self.reset_mock() response = mock.MagicMock() self.requests_get.return_value = response response.headers.get.return_value = '5' # content-length response.iter_content.return_value = ['abcde'] rv = download_file("http://url", self.filename) actual = self.stdout.getvalue() expected = 'Downloading: %s\n[====================================] 100%% 5.00 B\r\n' % self.filename self.assertMultiLineEqual(actual, expected) self.requests_get.assert_called_once() m_open.assert_called_once() response.headers.get.assert_called_once() response.iter_content.assert_called_once() self.assertIsNone(rv)
def test_handle_download_file_undefined_length(self, m_open): self.reset_mock() response = mock.MagicMock() self.requests_get.return_value = response response.headers.get.return_value = None # content-length response.content = 'abcdef' rv = download_file("http://url", self.filename) actual = self.stdout.getvalue() expected = 'Downloading: %s\n[====================================] 100%% 6.00 B\r\n' % self.filename self.assertMultiLineEqual(actual, expected) self.requests_get.assert_called_once() m_open.assert_called_once() response.headers.get.assert_called_once() response.iter_content.assert_not_called() self.assertIsNone(rv)
def test_handle_download_file_quiet_noprogress(self): download_file("http://url", self.filename, quiet=True, noprogress=False) actual = self.stdout.getvalue() expected = '' self.assertMultiLineEqual(actual, expected) self.reset_mock() download_file("http://url", self.filename, quiet=True, noprogress=True) actual = self.stdout.getvalue() expected = '' self.assertMultiLineEqual(actual, expected) self.reset_mock() download_file("http://url", self.filename, quiet=False, noprogress=True) actual = self.stdout.getvalue() expected = 'Downloading: %s\n' % self.filename self.assertMultiLineEqual(actual, expected)