def TestingSymUpload(upload_url, sym_item, _product_name):
"""A stub version of SymUpload for --testing usage"""
cmd = ['sym_upload', sym_item.sym_file, upload_url]
# Randomly fail 80% of the time (the retry logic makes this 80%/3 per file).
returncode = random.randint(1, 100) <= 80
logging.debug('would run (and return %i): %s', returncode,
cros_build_lib.CmdToStr(cmd))
if returncode:
output = 'Failed to send the symbol file.'
else:
output = 'Successfully sent the symbol file.'
result = cros_build_lib.CommandResult(cmd=cmd,
error=None,
output=output,
returncode=returncode)
if returncode:
exceptions = (
socket.error('[socket.error] forced test fail'),
httplib.BadStatusLine('[BadStatusLine] forced test fail'),
urllib2.HTTPError(upload_url, 400, '[HTTPError] forced test fail',
{}, None),
urllib2.URLError('[URLError] forced test fail'),
)
raise random.choice(exceptions)
else:
return result
def test_jsonrpc_call_bad_status_line(self):
self.rpc._connection.request = mock.Mock()
self.rpc._connection.getresponse = mock.Mock(
side_effect=httplib.BadStatusLine("fake_line"))
self.assertRaises(exception.QBException, self.rpc.call, 'method',
{'param': 'value'})
def request(self, query, headers):
# httplib would really help but there isn't any trivial way
# that I know of to use your own socket with it.
request = Message()
for k, v in headers.iteritems():
request[k] = v
self.sock.send(query + "\r\n" + request.as_string())
buffer = self.sock.recv(4096)
try:
[result, data] = buffer.split('\r\n', 1)
except ValueError:
traceback.print_exc()
print >> sys.stderr, 'Buffer:', buffer
print >> sys.stderr, 'Query:', query
print >> sys.stderr, 'Headers:', headers
return False
result = result.split(' ')
if int(result[1]) != 200:
self.log.info('Request failed: %s', ' '.join(result))
raise httplib.BadStatusLine(' '.join(result))
response = FeedParser()
response.feed(data)
return response.close()
def _read_status(self, line):
version = None
status = None
reason = None
# Initialize with Simple-Response defaults
if not line:
# Presumably, the server closed the connection before
# sending a valid response.
raise httplib.BadStatusLine(line)
try:
[version, status, reason] = line.split(None, 2)
except ValueError:
try:
[version, status] = line.split(None, 1)
reason = ""
except ValueError:
# empty version will cause next test to fail and status
# will be treated as 0.9 response.
version = ""
if not version.startswith('HTTP/'):
print repr(line)
errinfo = "error http status"
raise socket.error(errinfo)
# The status code is a three-digit number
try:
status = int(status)
if status < 100 or status > 999:
raise httplib.BadStatusLine(line)
except ValueError:
raise httplib.BadStatusLine(line)
self.status = status
self.reason = reason.strip()
if version == 'HTTP/1.0':
self.version = 10
elif version.startswith('HTTP/1.'):
self.version = 11
elif version == 'HTTP/0.9':
self.version = 9
else:
raise httplib.UnknownProtocol(version)
def _get_ec2_user_data():
"""
Recursive call in _get_ec2_hostinfo() does not retrieve user-data.
"""
response = _call_aws("/latest/user-data")
# _call_aws returns None for all non '200' reponses,
# catching that here would rule out AWS resource
if response:
data = json.loads(response)
return _snake_caseify_dict(data)
else:
raise httplib.BadStatusLine("Could not read EC2 user-data")
def testRetryBadStatusLine(self):
sequence = HttpMockSequence([
('discovery/v1/apis/storage/v1/rest', {
'status': '200'
}, self.DISCOVERY),
('storage/v1/b/makani/o?prefix=misc&alt=json',
httplib.BadStatusLine('junk'), None),
('storage/v1/b/makani/o?prefix=misc&alt=json',
httplib.BadStatusLine('junk'), None),
('storage/v1/b/makani/o?prefix=misc&alt=json', {
'status': '200'
}, json.dumps({'items': [{
'name': 'a.txt'
}]})),
])
mock_sleep = mock.MagicMock()
gcs = self.GetService(sequence)
with mock.patch('time.sleep', mock_sleep), test_util.DisableWarnings():
names = gcs.List(self.Path('misc'))
self.assertEqual(2, mock_sleep.call_count)
self.assertEqual(['a.txt'], names)
def _get_ec2_additional():
"""
Recursive call in _get_ec2_hostinfo() does not retrieve some of
the hosts information like region, availability zone or
architecture.
"""
response = _call_aws("/latest/dynamic/instance-identity/document")
# _call_aws returns None for all non '200' reponses,
# catching that here would rule out AWS resource
if response:
data = json.loads(response)
return _snake_caseify_dict(data)
else:
raise httplib.BadStatusLine("Could not read EC2 metadata")
def test_20_preauth(self):
""" Test various levels of preauth checks """
# a _preauth without users configured must fail hard:
with self.assertRaises(Exception):
self.library._preauth()
self.library.user_config = self.user_data['push']
with mock.patch.object(self.library, 'log') as mock_log:
with mock.patch.object(self.library,
'preauth',
side_effect=socket.error):
res = self.library._preauth()
self.assertFalse(res, "Broken check must return False")
# Check the call_args - [1] is the kwargs.
self.assertEqual(mock_log.call_args[1]['details']['error'], 'true')
self.assertEqual(mock_log.call_args[1]['details']['success'],
'false')
with mock.patch.object(self.library, 'log') as mock_log:
with mock.patch.object(self.library,
'preauth',
side_effect=RuntimeError):
res = self.library._preauth()
self.assertFalse(res, "Broken check must return False")
# Check the call_args - [1] is the kwargs.
self.assertEqual(mock_log.call_args[1]['details']['error'], 'true')
self.assertEqual(mock_log.call_args[1]['details']['success'],
'false')
with mock.patch.object(self.library, 'log') as mock_log:
with mock.patch.object(self.library,
'preauth',
side_effect=httplib.BadStatusLine('')):
res = self.library._preauth()
self.assertFalse(res, "Broken check must return False")
# Check the call_args - [1] is the kwargs.
self.assertEqual(mock_log.call_args[1]['details']['error'], 'true')
self.assertEqual(mock_log.call_args[1]['details']['success'],
'false')
upstream_return = ['some', 14, 'thing']
with mock.patch.object(self.library,
'preauth',
return_value=upstream_return):
res = self.library._preauth()
self.assertEqual(res, upstream_return,
"Good preauth returns whatever upstream sent")
def _get_ec2_user_data():
"""
Recursive call in _get_ec2_hostinfo() does not retrieve user-data.
"""
response = _call_aws("/latest/user-data")
# _call_aws returns None for all non '200' reponses,
# catching that here would rule out AWS resource
if response.status == 200:
response_data = response.read()
try:
return json.loads(response_data)
except ValueError as e:
return response_data
elif response.status == 404:
return ''
else:
raise httplib.BadStatusLine("Could not read EC2 user-data")
def upload_cloudinary(user_id, filename):
if not user_id:
raise httplib.CannotSendRequest(
"Cloudinary upload called with no user_id.")
if not filename:
raise httplib.CannotSendRequest(
"Cloudinary upload called with no filename.")
req_url = "{}/{}/{}".format(CLOUDINARY_METHOD, user_id, filename)
try:
# Use HEAD since no data is required
httpConn.request("HEAD", req_url)
except Exception as e:
raise httplib.CannotSendRequest(
"Cloudinary request failed for {}.".format(req_url))
cloudinary_return = False
cloudinary_attempts = 0
# Retry Cloudinary trigger until success or at most 3 times
while not cloudinary_return and cloudinary_attempts < 3:
cloudinary_attempts += 1
try:
resp = httpConn.getresponse()
if resp.status != 200:
# Cloudinary failed, return from method
raise httplib.BadStatusLine(
"Cloudinary HTTP HEAD status {} reason {} for {}.".format(
resp.status, resp.reason, req_url))
else:
# Read the response to enabled next request
resp.read()
cloudinary_return = True
except httplib.ResponseNotReady as e:
logger.info(
"Cloudinary HTTP response for {} not ready after attempt {}, retrying..."
.format(req_url, cloudinary_attempts))
# The request never returned
if not cloudinary_return:
raise httplib.IncompleteRead(
"Cloudinary HTTP request for {} never returned.".format(req_url))
def test_get_releases_throws_exception(self):
rb = self._get_rb()
# mock content_connection so we can verify it's calls
with mock.patch.object(rb, 'content_connection') as mock_cc:
mock_cc.get_versions.side_effect = \
httplib.BadStatusLine("some bogus status")
releases = rb.get_releases()
self.assertEquals([], releases)
mock_cc.get_versions.side_effect = \
socket.error()
releases = rb.get_releases()
self.assertEquals([], releases)
mock_cc.get_versions.side_effect = \
SSLError()
releases = rb.get_releases()
self.assertEquals([], releases)
def test_event_with_vidly_token_badstatusline(self, p_urlopen):
# based on https://bugzilla.mozilla.org/show_bug.cgi?id=842588
event = Event.objects.get(title='Test event')
# first we need a template that uses `vidly_tokenize()`
template = event.template
template.content = """
{% set token = vidly_tokenize(tag, 90) %}
<iframe src="http://s.vid.ly/embeded.html?
link={{ tag }}{% if token %}&token={{ token }}{% endif %}"></iframe>
"""
template.save()
event.template_environment = "tag=abc123"
event.save()
p_urlopen.side_effect = httplib.BadStatusLine('TroubleX')
url = reverse('main:event', args=(event.slug, ))
response = self.client.get(url)
eq_(response.status_code, 200)
ok_('Temporary network error' in response.content)
ok_('TroubleX' not in response.content)
def connection_error(*args, **kwargs):
raise ConnectionError('Connection aborted.',
httplib.BadStatusLine("''", ))
def test_bad_status_repr(self):
exc = httplib.BadStatusLine('')
self.assertEquals(repr(exc), '''BadStatusLine("\'\'",)''')
def __init__(self, *_, **__):
raise httplib.BadStatusLine('Test')
def test_bad_status_line_error(self):
error = httplib.BadStatusLine('')
e = exc.HTTPClientError(self.req, error)
expected = "GET on http://localhost/path failed " \
"with '%s'" % error.__class__.__name__
self.assertEquals(str(e), expected)
client.repos.c['localhost'].checkVersion()
except errors.OpenError, e:
if 'A particular gai error' not in e.args[0]:
raise
# this should fail right away
self.theException = socket.gaierror('bogus error',
'bogus gai error')
self.mockMax = 100
try:
client.repos.c['localhost'].checkVersion()
except errors.OpenError, e:
self.assertEquals(self.mockMax, 99)
# test bad status errors
self.theException = httplib.BadStatusLine(
'A particular bad status line error')
self.mockMax = 2
try:
client.repos.c['localhost'].checkVersion()
except errors.OpenError, e:
if 'successException' not in e.args[0]:
raise
self.mockMax = 6
try:
client.repos.c['localhost'].checkVersion()
except errors.OpenError, e:
if 'A particular bad status line error' not in e.args[0]:
raise
finally:
class TestOpenshift(object):
def test_set_labels_on_build(self, openshift): # noqa
labels = openshift.set_labels_on_build(TEST_BUILD,
{TEST_LABEL: TEST_LABEL_VALUE})
assert labels.json() is not None
@pytest.mark.parametrize(
'exc',
[ # noqa
ConnectionError('Connection aborted.', httplib.BadStatusLine(
"''", )),
])
def test_stream_logs_bad_initial_connection(self, openshift, exc):
response = flexmock(status_code=httplib.OK)
(response.should_receive('iter_lines').and_return(
[b"{'stream': 'foo\n'}"]).and_raise(StopIteration))
wrapped_exc = OsbsNetworkException('http://spam.com',
str(exc),
status_code=None,
cause=exc)
(flexmock(openshift).should_receive('_get')
# First: simulate initial connection problem
.and_raise(wrapped_exc)
# Next: return a real response
.and_return(response))
(flexmock(time).should_receive('time').and_return(0).and_return(100))
logs = openshift.stream_logs(TEST_BUILD)
assert len([log for log in logs]) == 1
def test_stream_logs_utf8(self, openshift): # noqa
response = flexmock(status_code=httplib.OK)
(response.should_receive('iter_lines').and_return([
u"{'stream': 'Uňícode íš hářd\n'}".encode('utf-8')
]).and_raise(StopIteration))
(flexmock(openshift).should_receive('_get').and_return(response))
logs = openshift.stream_logs(TEST_BUILD)
assert len([log for log in logs]) == 1
def test_stream_logs_not_decoded(self, caplog):
server = Openshift('http://oapi/v1/',
'v1',
'http://oauth/authorize',
k8s_api_url='http://api/v1/')
logs = (
u'Lógs'.encode('utf-8'),
u'Lðgs'.encode('utf-8'),
)
fake_response = flexmock(status_code=httplib.OK, headers={})
(fake_response.should_receive('iter_lines').and_yield(*logs).with_args(
decode_unicode=False))
(flexmock(requests).should_receive('request').and_return(fake_response)
)
with caplog.atLevel(logging.ERROR):
for result in server.stream_logs('anything'):
assert isinstance(result, six.binary_type)
def test_list_builds(self, openshift): # noqa
list_builds = openshift.list_builds()
assert list_builds is not None
assert bool(list_builds.json()) # is there at least something
def test_list_pods(self, openshift): # noqa
response = openshift.list_pods(label="openshift.io/build.name=%s" %
TEST_BUILD)
assert isinstance(response, HttpResponse)
def test_get_oauth_token(self, openshift): # noqa
token = openshift.get_oauth_token()
assert token is not None
def test_get_user(self, openshift): # noqa
l = openshift.get_user()
assert l.json() is not None
def test_watch_build(self, openshift): # noqa
response = openshift.wait_for_build_to_finish(TEST_BUILD)
status_lower = response["status"]["phase"].lower()
assert response["metadata"]["name"] == TEST_BUILD
assert status_lower in BUILD_FINISHED_STATES
assert isinstance(TEST_BUILD, six.text_type)
assert isinstance(status_lower, six.text_type)
def test_create_build(self, openshift): # noqa
response = openshift.create_build({})
assert response is not None
assert response.json()["metadata"]["name"] == TEST_BUILD
assert response.json()["status"]["phase"].lower(
) in BUILD_FINISHED_STATES
def test_cancel_build(self, openshift): # noqa
response = openshift.cancel_build(TEST_CANCELLED_BUILD)
assert response is not None
assert response.json()["metadata"]["name"] == TEST_CANCELLED_BUILD
assert response.json()["status"]["phase"].lower(
) in BUILD_CANCELLED_STATE
def test_get_build_config(self, openshift): # noqa
mock_response = {"spam": "maps"}
build_config_name = 'some-build-config-name'
expected_url = openshift._build_url("buildconfigs/%s/" %
build_config_name)
(flexmock(openshift).should_receive("_get").with_args(
expected_url).once().and_return(make_json_response(mock_response)))
response = openshift.get_build_config(build_config_name)
assert response['spam'] == 'maps'
def test_get_missing_build_config(self, openshift): # noqa
build_config_name = 'some-build-config-name'
expected_url = openshift._build_url("buildconfigs/%s/" %
build_config_name)
(flexmock(openshift).should_receive("_get").with_args(
expected_url).once().and_return(HttpResponse(404, {}, b'')))
with pytest.raises(OsbsResponseException):
openshift.get_build_config(build_config_name)
def test_get_build_config_by_labels(self, openshift): # noqa
mock_response = {"items": [{"spam": "maps"}]}
label_selectors = (
('label-1', 'value-1'),
('label-2', 'value-2'),
)
expected_url = openshift._build_url(
"buildconfigs/?labelSelector=label-1%3Dvalue-1%2Clabel-2%3Dvalue-2"
)
(flexmock(openshift).should_receive("_get").with_args(
expected_url).once().and_return(make_json_response(mock_response)))
response = openshift.get_build_config_by_labels(label_selectors)
assert response['spam'] == 'maps'
def test_get_missing_build_config_by_labels(self, openshift): # noqa
mock_response = {"items": []}
label_selectors = (
('label-1', 'value-1'),
('label-2', 'value-2'),
)
expected_url = openshift._build_url(
"buildconfigs/?labelSelector=label-1%3Dvalue-1%2Clabel-2%3Dvalue-2"
)
(flexmock(openshift).should_receive("_get").with_args(
expected_url).once().and_return(make_json_response(mock_response)))
with pytest.raises(OsbsException) as exc:
openshift.get_build_config_by_labels(label_selectors)
assert str(exc.value).startswith('Build config not found')
def test_get_multiple_build_config_by_labels(self, openshift): # noqa
mock_response = {"items": [{"spam": "maps"}, {"eggs": "sgge"}]}
label_selectors = (
('label-1', 'value-1'),
('label-2', 'value-2'),
)
expected_url = openshift._build_url(
"buildconfigs/?labelSelector=label-1%3Dvalue-1%2Clabel-2%3Dvalue-2"
)
(flexmock(openshift).should_receive("_get").with_args(
expected_url).once().and_return(make_json_response(mock_response)))
with pytest.raises(OsbsException) as exc:
openshift.get_build_config_by_labels(label_selectors)
assert str(exc.value).startswith('More than one build config found')
@pytest.mark.parametrize(
('status_codes', 'should_raise'),
[ # noqa
([httplib.OK], False),
([httplib.CONFLICT, httplib.CONFLICT, httplib.OK], False),
([httplib.CONFLICT, httplib.OK], False),
([httplib.CONFLICT, httplib.CONFLICT, httplib.UNAUTHORIZED], True),
([httplib.UNAUTHORIZED], True),
([httplib.CONFLICT for _ in range(10)], True),
])
@pytest.mark.parametrize('update_or_set', ['update', 'set'])
@pytest.mark.parametrize('attr_type', ['labels', 'annotations'])
@pytest.mark.parametrize('object_type', ['build', 'build_config'])
def test_retry_update_attributes(self, openshift, status_codes,
should_raise, update_or_set, attr_type,
object_type):
try:
fn = getattr(
openshift,
"{update}_{attr}_on_{object}".format(update=update_or_set,
attr=attr_type,
object=object_type))
except AttributeError:
return # not every combination is implemented
get_expectation = (flexmock(openshift).should_receive('_get').times(
len(status_codes)))
put_expectation = (flexmock(openshift).should_receive('_put').times(
len(status_codes)))
for status_code in status_codes:
get_response = make_json_response({"metadata": {}})
put_response = HttpResponse(status_code, headers={}, content=b'')
get_expectation = get_expectation.and_return(get_response)
put_expectation = put_expectation.and_return(put_response)
(flexmock(time).should_receive('sleep').with_args(0.5))
args = ('any-object-id', {'key': 'value'})
if should_raise:
with pytest.raises(OsbsResponseException):
fn(*args)
else:
fn(*args)
def test_put_image_stream_tag(self, openshift): # noqa
tag_name = 'spam'
tag_id = 'maps:' + tag_name
mock_data = {
'kind': 'ImageStreamTag',
'apiVersion': 'v1',
'tag': {
'name': tag_name
}
}
expected_url = openshift._build_url('imagestreamtags/' + tag_id)
(flexmock(openshift).should_receive("_put").with_args(
expected_url,
data=json.dumps(mock_data),
headers={
"Content-Type": "application/json"
}).once().and_return(make_json_response(mock_data)))
openshift.put_image_stream_tag(tag_id, mock_data)
def _make_tag_template(self):
# TODO: Just read from inputs folder
return json.loads(
dedent('''\
{
"kind": "ImageStreamTag",
"apiVersion": "v1",
"metadata": {
"name": "{{IMAGE_STREAM_ID}}:{{TAG_ID}}"
},
"tag": {
"name": "{{TAG_ID}}",
"from": {
"kind": "DockerImage",
"name": "{{REPOSITORY}}:{{TAG_ID}}"
},
"importPolicy": {}
}
}
'''))
@pytest.mark.parametrize('existing_scheduled', (True, False, None)) # noqa
@pytest.mark.parametrize('existing_insecure', (True, False, None))
@pytest.mark.parametrize('expected_scheduled', (True, False))
@pytest.mark.parametrize(('s_annotations', 'expected_insecure'), (
({
'openshift.io/image.insecureRepository': 'true'
}, True),
({
'openshift.io/image.insecureRepository': 'false'
}, False),
({}, False),
(None, False),
))
@pytest.mark.parametrize('status_code', (200, 404, 500))
def test_ensure_image_stream_tag(self, existing_scheduled,
existing_insecure, expected_scheduled,
s_annotations, expected_insecure,
status_code, openshift):
stream_name = 'spam'
stream_repo = 'some.registry.com/spam'
stream = {
'metadata': {
'name': stream_name
},
'spec': {
'dockerImageRepository': stream_repo
}
}
if s_annotations is not None:
stream['metadata']['annotations'] = s_annotations
tag_name = 'maps'
tag_id = '{0}:{1}'.format(stream_name, tag_name)
expected_url = openshift._build_url('imagestreamtags/' + tag_id)
def verify_image_stream_tag(*args, **kwargs):
data = json.loads(kwargs['data'])
assert (bool(data['tag']['importPolicy'].get('insecure')) ==
expected_insecure)
assert (bool(data['tag']['importPolicy'].get('scheduled')) ==
expected_scheduled)
# Also verify new image stream tags are created properly.
if status_code == 404:
assert data['metadata']['name'] == tag_id
assert data['tag']['name'] == tag_name
assert (data['tag']['from']['name'] == '{0}:{1}'.format(
stream_repo, tag_name))
return make_json_response({})
expected_change = False
expected_error = status_code == 500
mock_response = {}
expectation = (flexmock(openshift).should_receive("_get").with_args(
expected_url).once())
if status_code == 200:
existing_image_stream_tag = {'tag': {'importPolicy': {}}}
if existing_insecure is not None:
existing_image_stream_tag['tag']['importPolicy']['insecure'] = \
existing_insecure
if existing_scheduled is not None:
existing_image_stream_tag['tag']['importPolicy']['scheduled'] = \
existing_scheduled
mock_response = existing_image_stream_tag
if expected_insecure != bool(existing_insecure) or \
expected_scheduled != bool(existing_scheduled):
expected_change = True
expectation.and_return(make_json_response(mock_response))
else:
expectation.and_return(
HttpResponse(status_code, headers={}, content=b''))
if status_code == 404:
expected_change = True
if expected_change:
(flexmock(openshift).should_receive("_put").with_args(
expected_url,
data=str,
headers={
"Content-Type": "application/json"
}).replace_with(verify_image_stream_tag).once())
if expected_error:
with pytest.raises(OsbsResponseException):
openshift.ensure_image_stream_tag(stream, tag_name,
self._make_tag_template(),
expected_scheduled)
else:
assert (openshift.ensure_image_stream_tag(
stream, tag_name, self._make_tag_template(),
expected_scheduled) == expected_change)
@pytest.mark.parametrize(('kwargs', 'called'), (
({
'use_auth': True,
'use_kerberos': True
}, False),
({
'use_auth': True,
'username': '******',
'password': '******'
}, False),
({
'use_auth': True,
'token': 'foo'
}, False),
({
'use_auth': False,
'use_kerberos': True
}, False),
({
'use_auth': False,
'username': '******',
'password': '******'
}, False),
({
'use_auth': False,
'token': 'foo'
}, False),
({
'use_kerberos': True
}, False),
({
'username': '******',
'password': '******'
}, False),
({
'token': 'foo'
}, False),
({
'use_auth': False
}, True),
({}, True),
))
def test_use_service_account_token(self, kwargs, called):
openshift_mock = flexmock(Openshift).should_receive(
'can_use_serviceaccount_token')
if called:
openshift_mock.once()
else:
openshift_mock.never()
Openshift(OAPI_PREFIX, API_VER, "/oauth/authorize", **kwargs)
@pytest.mark.parametrize('modify', [True, False]) # noqa
def test_import_image(self, openshift, modify):
"""
tests that import_image return True
regardless if tags were changed
"""
this_file = inspect.getfile(TestCheckResponse)
this_dir = os.path.dirname(this_file)
json_path = os.path.join(this_dir, "mock_jsons",
openshift._con.version, 'imagestream.json')
resource_json = json.load(open(json_path))
# keep just 1 tag, so it will be different from oldtags (3 tags)
if modify:
resource_json['status']['tags'] = [
resource_json['status']['tags'][0]
]
(flexmock(Openshift).should_receive('watch_resource').and_yield(
(WATCH_MODIFIED, resource_json)))
assert openshift.import_image(TEST_IMAGESTREAM)
def _urlopen(*args):
import httplib
raise httplib.BadStatusLine('line')