def _upload_files(self, file_urls, files, output, retry, retry_wait):
t1 = time.time()
failed = []
uploader = FileUploader(self.requester, output, self.verify_ssl)
# conan_package.tgz and conan_export.tgz are uploaded first to avoid uploading conaninfo.txt
# or conanamanifest.txt with missing files due to a network failure
for filename, resource_url in sorted(file_urls.items()):
if output and not output.is_terminal:
output.rewrite_line("Uploading %s" % filename)
auth, dedup = self._file_server_capabilities(resource_url)
try:
headers = self._artifacts_properties if not self._matrix_params else {}
uploader.upload(resource_url,
files[filename],
auth=auth,
dedup=dedup,
retry=retry,
retry_wait=retry_wait,
headers=headers)
except Exception as exc:
output.error("\nError uploading file: %s, '%s'" %
(filename, exc))
failed.append(filename)
if failed:
raise ConanException(
"Execute upload again to retry upload the failed files: %s" %
", ".join(failed))
else:
logger.debug("UPLOAD: \nAll uploaded! Total time: %s\n" %
str(time.time() - t1))
def _upload_files(self, files, urls, retry, retry_wait):
t1 = time.time()
failed = []
uploader = FileUploader(self.requester, self._output, self.verify_ssl)
# conan_package.tgz and conan_export.tgz are uploaded first to avoid uploading conaninfo.txt
# or conanamanifest.txt with missing files due to a network failure
for filename in sorted(files):
if self._output and not self._output.is_terminal:
self._output.rewrite_line("Uploading %s" % filename)
resource_url = urls[filename]
try:
uploader.upload(resource_url, files[filename], auth=self.auth,
dedup=self._checksum_deploy, retry=retry,
retry_wait=retry_wait,
headers=self._put_headers)
except (AuthenticationException, ForbiddenException):
raise
except Exception as exc:
self._output.error("\nError uploading file: %s, '%s'" % (filename, exc))
failed.append(filename)
if failed:
raise ConanException("Execute upload again to retry upload the failed files: %s"
% ", ".join(failed))
else:
logger.debug("\nUPLOAD: All uploaded! Total time: %s\n" % str(time.time() - t1))
def test_error_401(self):
output = TestBufferConanOutput()
uploader = FileUploader(requester=_RequesterMock(401, "content"),
output=output,
verify=False)
with six.assertRaisesRegex(self, AuthenticationException, "content"):
uploader.upload(url="fake", abs_path=self.filename, retry=2)
output_lines = str(output).splitlines()
counter = Counter(output_lines)
self.assertEqual(counter["ERROR: content"], 0)
self.assertEqual(counter["Waiting 0 seconds to retry..."], 0)
def test_error_500(self):
output = TestBufferConanOutput()
uploader = FileUploader(requester=_RequesterMock(500, "content"),
output=output,
verify=False,
config=_ConfigMock())
with six.assertRaisesRegex(self, Exception,
"500 Server Error: content"):
uploader.upload(url="fake", abs_path=self.filename, retry=2)
output_lines = str(output).splitlines()
counter = Counter(output_lines)
self.assertEqual(counter["ERROR: 500 Server Error: content"], 2)
self.assertEqual(counter["Waiting 0 seconds to retry..."], 2)
def test_401_raises_unauthoirzed_exception(self):
class MockRequester(object):
retry = 0
retry_wait = 0
def put(self, *args, **kwargs):
return namedtuple("response", "status_code content")(401,
"tururu")
out = TestBufferConanOutput()
uploader = FileUploader(MockRequester(), out, verify=False)
f = tempfile.mktemp()
save(f, "some contents")
with six.assertRaisesRegex(self, AuthenticationException, "tururu"):
uploader.upload("fake_url", f)
def test_error_403_forbidden(self):
output = TestBufferConanOutput()
uploader = FileUploader(requester=_RequesterMock(403, "content"),
output=output,
verify=False)
with six.assertRaisesRegex(self, ForbiddenException, "content"):
auth = namedtuple("auth", "token")
uploader.upload(url="fake",
abs_path=self.filename,
retry=2,
auth=auth("token"))
output_lines = str(output).splitlines()
counter = Counter(output_lines)
self.assertEqual(counter["ERROR: content"], 0)
self.assertEqual(counter["Waiting 0 seconds to retry..."], 0)
def test_403_raises_forbidden_exception_if_token(self):
class MockRequester(object):
retry = 0
retry_wait = 0
def put(self, *args, **kwargs):
return namedtuple("response", "status_code content")(403,
"tururu")
out = TestBufferConanOutput()
auth = namedtuple("auth", "token")("SOMETOKEN")
uploader = FileUploader(MockRequester(), out, verify=False)
f = tempfile.mktemp()
save(f, "some contents")
with six.assertRaisesRegex(self, ForbiddenException, "tururu"):
uploader.upload("fake_url", f, auth=auth)
def test_error_requests(self):
class _RequesterMock:
def put(self, *args, **kwargs):
raise Exception("any exception")
output = TestBufferConanOutput()
uploader = FileUploader(requester=_RequesterMock(),
output=output,
verify=False,
config=_ConfigMock())
with six.assertRaisesRegex(self, Exception, "any exception"):
uploader.upload(url="fake", abs_path=self.filename, retry=2)
output_lines = str(output).splitlines()
counter = Counter(output_lines)
self.assertEqual(counter["ERROR: any exception"], 2)
self.assertEqual(counter["Waiting 0 seconds to retry..."], 2)
def test_403_raises_forbidden_exception_if_token(self):
auth = namedtuple("auth", "token")("SOMETOKEN")
uploader = FileUploader(MockRequester(403), self.out, verify=False)
with six.assertRaisesRegex(self, ForbiddenException, "tururu"):
uploader.upload("fake_url", self.f, auth=auth)
def test_403_raises_unauthorized_exception_if_no_auth(self):
uploader = FileUploader(MockRequester(403), self.out, verify=False)
with six.assertRaisesRegex(self, AuthenticationException, "tururu"):
uploader.upload("fake_url", self.f)
def test_403_raises_unauthoirzed_exception_if_no_token(self):
auth = namedtuple("auth", "token")(None)
uploader = FileUploader(MockRequester(403), self.out, verify=False)
with six.assertRaisesRegex(self, AuthenticationException, "tururu"):
uploader.upload("fake_url", self.f, auth=auth)