def store_binaries(session, task_id, path):
"""
Store the resulting binaries from a Brew scratch build
(really wish this was part of the official koji api...)
"""
task = session.getTaskInfo(task_id)
assert task['method'] == 'build'
subtasks = session.getTaskChildren(task_id)
# Assemble a list of tasks/files to download.
downloads = []
for task in subtasks:
if task['method'] != 'buildArch':
continue
files = list_task_output_all_volumes(session, task['id'])
for filename in files:
if not filename.endswith('.rpm'):
continue
if filename.endswith('.src.rpm'):
continue
for volume in files[filename]:
# I'm keeping it simple for now by only supporting the default
# volume. I haven't found a buildArch task that blows up yet...
assert volume == 'DEFAULT'
downloads.append((task, filename))
# Do the downloads.
pathinfo = get_koji_pathinfo()
number = 0
for (task, filename) in downloads:
number += 1
work = pathinfo.work('DEFAULT')
taskrelpath = pathinfo.taskrelpath(task['id'])
url = '%s/%s/%s' % (work, taskrelpath, filename)
filename = os.path.join(path, 'rpms', filename)
download_file(url, filename, size=len(downloads), num=number)
def test_handle_download_file_error_500(self, m):
m.get("http://url", text='Internal Server Error\n', status_code=500)
with self.assertRaises(requests.HTTPError):
download_file("http://url", self.filename)
try:
os.unlink(self.filename)
except Exception:
pass
def test_handle_download_file_dir(self):
with self.assertRaises(IOError) as cm:
download_file("http://url", self.tempdir)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n' % self.tempdir
self.assertMultiLineEqual(actual, expected)
self.assertEqual(cm.exception.args, (21, 'Is a directory'))
self.requests_get.assert_called_once()
def test_handle_download_file_error_404(self, m):
m.get("http://url", text='Not Found\n', status_code=404)
with self.assertRaises(requests.HTTPError):
download_file("http://url", self.filename)
try:
os.unlink(self.filename)
except Exception:
pass
def test_handle_download_file_error_500(self, m):
m.get("http://url", text='Internal Server Error\n', status_code=500)
with self.assertRaises(requests.HTTPError):
download_file("http://url", self.filename)
try:
self.assertFalse(os.path.exists(self.filename))
except AssertionError:
os.unlink(self.filename)
raise
def test_handle_download_file_error_404(self, m):
m.get("http://url", text='Not Found\n', status_code=404)
with self.assertRaises(requests.HTTPError):
download_file("http://url", self.filename)
try:
self.assertFalse(os.path.exists(self.filename))
except AssertionError:
os.unlink(self.filename)
raise
def test_handle_download_file_dir(self):
with self.assertRaises(IOError) as cm:
download_file("http://url", self.tempdir)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n' % self.tempdir
self.assertMultiLineEqual(actual, expected)
self.assertEqual(cm.exception.args, (21, 'Is a directory'))
self.curlClass.assert_called_once()
self.assertEqual(self.curl.setopt.call_count, 2)
self.curl.perform.assert_not_called()
def test_handle_download_file_dir(self):
with self.assertRaises(IOError) as cm:
download_file("http://url", self.tempdir)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n' % self.tempdir
self.assertMultiLineEqual(actual, expected)
if isinstance(cm.exception, tuple):
self.assertEqual(cm.exception[0], 21)
self.assertEqual(cm.exception[1], 'Is a directory')
else:
self.assertEqual(cm.exception.args, (21, 'Is a directory'))
def test_handle_download_file_dir(self):
with self.assertRaises(IOError) as cm:
download_file("http://url", self.tempdir)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n' % self.tempdir
self.assertMultiLineEqual(actual, expected)
if isinstance(cm.exception, tuple):
self.assertEqual(cm.exception[0], 21)
self.assertEqual(cm.exception[1], 'Is a directory')
else:
self.assertEqual(cm.exception.args, (21, 'Is a directory'))
self.requests_get.assert_called_once()
def download_srpm(session, build, destination):
# Download the SRPM from this Koji build into "destination" dir.
url = srpm_url(session, build)
filename = posixpath.basename(url)
destfile = os.path.join(destination, filename)
if os.path.exists(destfile):
if verify_srpm(destfile):
return destfile
print('existing %s does not verify, re-downloading.' % destfile)
print('downloading %s to %s' % (url, destination))
download_file(url, destfile, size=1, num=1)
verify_srpm(destfile)
return destfile
def test_handle_download_file_with_size(self):
rv = download_file("http://url", self.filename, size=10, num=8)
actual = self.stdout.getvalue()
expected = 'Downloading [8/10]: %s\n\n' % self.filename
self.assertMultiLineEqual(actual, expected)
self.requests_get.assert_called_once()
self.assertIsNone(rv)
def test_handle_download_file_with_size(self):
rv = download_file("http://url", self.filename, size=10, num=8)
actual = self.stdout.getvalue()
expected = 'Downloading [8/10]: %s\n\n' % self.filename
self.assertMultiLineEqual(actual, expected)
self.requests_get.assert_called_once()
self.assertIsNone(rv)
def test_handle_download_file_with_size(self):
rv = download_file("http://url", self.filename, size=10, num=8)
actual = self.stdout.getvalue()
expected = 'Downloading [8/10]: %s\n\n' % self.filename
self.assertMultiLineEqual(actual, expected)
self.curlClass.assert_called_once()
self.assertEqual(self.curl.setopt.call_count, 5)
self.curl.perform.assert_called_once()
self.curl.close.assert_called_once()
self.assertIsNone(rv)
def test_handle_download_file_curl_version(self):
self.curl.XFERINFOFUNCTION = None
download_file("http://url", self.filename, quiet=False, noprogress=False)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n\n' % self.filename
self.assertMultiLineEqual(actual, expected)
self.assertEqual(self.curl.setopt.call_count, 5)
self.curl.setopt.assert_has_calls([call(self.curl.PROGRESSFUNCTION, _download_progress)])
self.reset_mock()
self.curl.PROGRESSFUNCTION = None
with self.assertRaises(SystemExit) as cm:
download_file("http://url", self.filename, quiet=False, noprogress=False)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n' % self.filename
self.assertMultiLineEqual(actual, expected)
actual = self.stderr.getvalue()
expected = 'Error: XFERINFOFUNCTION and PROGRESSFUNCTION are not supported by pyCurl. Quit download progress\n'
self.assertMultiLineEqual(actual, expected)
self.assertEqual(self.curl.setopt.call_count, 3)
self.assertEqual(cm.exception.code, 1)
def test_handle_download_file_undefined_length(self, m_open):
self.reset_mock()
response = mock.MagicMock()
self.requests_get.return_value = response
response.headers.get.return_value = None # content-length
response.iter_content.return_value = ['a' * 65536, 'b' * 65536]
rv = download_file("http://url", self.filename)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n[ ] ???%% 64.00 KiB\r[ ] ???%% 128.00 KiB\r[====================================] 100%% 128.00 KiB\r\n' % self.filename
self.assertMultiLineEqual(actual, expected)
self.requests_get.assert_called_once()
m_open.assert_called_once()
response.headers.get.assert_called_once()
response.iter_content.assert_called_once()
self.assertIsNone(rv)
def test_handle_download_file(self, m_open):
self.reset_mock()
response = mock.MagicMock()
self.requests_get.return_value = response
response.headers.get.return_value = '5' # content-length
response.iter_content.return_value = ['abcde']
rv = download_file("http://url", self.filename)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n[====================================] 100%% 5.00 B\r\n' % self.filename
self.assertMultiLineEqual(actual, expected)
self.requests_get.assert_called_once()
m_open.assert_called_once()
response.headers.get.assert_called_once()
response.iter_content.assert_called_once()
self.assertIsNone(rv)
def test_handle_download_file_undefined_length(self, m_open):
self.reset_mock()
response = mock.MagicMock()
self.requests_get.return_value = response
response.headers.get.return_value = None # content-length
response.content = 'abcdef'
rv = download_file("http://url", self.filename)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n[====================================] 100%% 6.00 B\r\n' % self.filename
self.assertMultiLineEqual(actual, expected)
self.requests_get.assert_called_once()
m_open.assert_called_once()
response.headers.get.assert_called_once()
response.iter_content.assert_not_called()
self.assertIsNone(rv)
def test_handle_download_file_quiet_noprogress(self):
download_file("http://url", self.filename, quiet=True, noprogress=False)
actual = self.stdout.getvalue()
expected = ''
self.assertMultiLineEqual(actual, expected)
self.reset_mock()
download_file("http://url", self.filename, quiet=True, noprogress=True)
actual = self.stdout.getvalue()
expected = ''
self.assertMultiLineEqual(actual, expected)
self.reset_mock()
download_file("http://url", self.filename, quiet=False, noprogress=True)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n' % self.filename
self.assertMultiLineEqual(actual, expected)
