def testNotifyURLError(self, mock_jsonRequest):
mock_jsonRequest.side_effect = urllib2.URLError("hello")
Slack(token=None, channels=['123', '123',
'456']).notify({'text': "Test message"})
self.assertEquals(mock_jsonRequest.call_count, 0)
def raise_urllib2_error(*_unused, **_unused2):
"""Always raise a urllib2.URLError"""
raise urllib2.URLError("test failure")
def do_open(self, req):
'''
Called by handler's url_open method.
'''
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
try:
resp_statuses = self._hostresp.setdefault(host,
self._get_tail_filter())
# Check if all our last 'resp_statuses' were timeouts and raise
# a w3afMustStopException if this is the case.
if len(resp_statuses) == self._curr_check_failures and \
all(st == RESP_TIMEOUT for st in resp_statuses):
msg = (
'w3af found too many consecutive timeouts. The remote '
'webserver seems to be unresponsive; please verify manually.'
)
reason = 'Timeout while trying to reach target.'
raise w3afMustStopByKnownReasonExc(msg, reason=reason)
conn_factory = self._get_connection
conn = self._cm.get_available_connection(host, conn_factory)
if conn.is_fresh:
# First of all, call the request method. This is needed for
# HTTPS Proxy
if isinstance(conn, ProxyHTTPConnection):
conn.proxy_setup(req.get_full_url())
conn.is_fresh = False
self._start_transaction(conn, req)
resp = conn.getresponse()
else:
# We'll try to use a previously created connection
resp = self._reuse_connection(conn, req, host)
# If the resp is None it means that connection is bad. It was
# possibly closed by the server. Replace it with a new one.
if resp is None:
conn.close()
conn = self._cm.replace_connection(conn, host,
conn_factory)
# First of all, call the request method. This is needed for
# HTTPS Proxy
if isinstance(conn, ProxyHTTPConnection):
conn.proxy_setup(req.get_full_url())
# Try again with the fresh one
conn.is_fresh = False
self._start_transaction(conn, req)
resp = conn.getresponse()
except (socket.error, httplib.HTTPException), err:
# We better discard this connection
self._cm.remove_connection(conn, host)
if isinstance(err, socket.timeout):
resp_statuses.append(RESP_TIMEOUT)
_err = URLTimeoutError()
else:
resp_statuses.append(RESP_BAD)
if isinstance(err, httplib.HTTPException):
err = repr(err)
_err = urllib2.URLError(err)
raise _err
def do_open(self, http_class, req):
data = req.get_data()
v_files = []
v_vars = []
# mapping object (dict)
if req.has_data() and type(data) != str:
if hasattr(data, 'items'):
data = data.items()
else:
try:
if len(data) and not isinstance(data[0], tuple):
raise TypeError
except TypeError:
ty, va, tb = sys.exc_info()
raise TypeError, "not a valid non-string sequence or mapping object", tb
for (k, v) in data:
# if fd is provided with a filename
if isinstance(v, dict):
if not v.has_key('fd'):
raise TypeError("if value is dict, it must have keys 'fd' and 'filename")
if not v.has_key('filename'):
raise TypeError("if value is dict, it must have keys 'fd' and 'filename")
v_files.append( (k, v) )
elif hasattr(v, 'read'):
v_files.append( (k, v) )
else:
v_vars.append( (k, v) )
# no file ? convert to string
if len(v_vars) > 0 and len(v_files) == 0:
data = urllib.urlencode(v_vars)
v_files = []
v_vars = []
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
h = http_class(host) # will parse host:port
if req.has_data():
h.putrequest(req.get_method(), req.get_selector())
if not 'Content-type' in req.headers:
if len(v_files) > 0:
boundary = mimetools.choose_boundary()
l = send_data(v_vars, v_files, boundary)
h.putheader('Content-Type',
'multipart/form-data; boundary=%s' % boundary)
h.putheader('Content-length', str(l))
else:
h.putheader('Content-type',
'application/x-www-form-urlencoded')
if not 'Content-length' in req.headers:
h.putheader('Content-length', '%d' % len(data))
else:
h.putrequest(req.get_method(), req.get_selector())
scheme, sel = urllib.splittype(req.get_selector())
sel_host, sel_path = urllib.splithost(sel)
h.putheader('Host', sel_host or host)
for name, value in self.parent.addheaders:
name = name.capitalize()
if name not in req.headers:
h.putheader(name, value)
for k, v in req.headers.items():
h.putheader(k, v)
# httplib will attempt to connect() here. be prepared
# to convert a socket error to a URLError.
try:
h.endheaders()
except socket.error, err:
raise urllib2.URLError(err)
def RaiseError(self, request, timeout=0):
raise urllib2.URLError("Not a real connection.")
class Tier0DasInterface:
"""
Class handling common Tier0-DAS queries and connected utilities
"""
def __init__(self,
url='https://cmsweb.cern.ch/t0wmadatasvc/prod/',
proxy=None):
"""
Need base url for Tier0-DAS as input
"""
self._t0DasBaseUrl = url
self._debug = False
self._retry = 0
self._maxretry = 5
self._proxy = proxy
def getData(self, src, tout=5):
"""
Get the JSON file for a give query specified via the Tier0-DAS url.
Timeout can be set via paramter.
"""
# actually get the json file from the given url of the T0-Das service
# and returns the data
try:
if self._proxy:
print "setting proxy"
opener = urllib2.build_opener(
urllib2.HTTPHandler(), urllib2.HTTPSHandler(),
urllib2.ProxyHandler({
'http': self._proxy,
'https': self._proxy
}))
urllib2.install_opener(opener)
req = urllib2.Request(src)
req.add_header(
"User-Agent",
"DQMIntegration/1.0 python/%d.%d.%d" % sys.version_info[:3])
req.add_header("Accept", "application/json")
jsonCall = urllib2.urlopen(req, timeout=tout)
url = jsonCall.geturl()
except urllib2.HTTPError, error:
#print error.url
errStr = "Cannot retrieve Tier-0 DAS data from URL \"" + error.url + "\""
if self._proxy:
errStr += " using proxy \"" + self._proxy + "\""
print errStr
print error
raise urllib2.HTTPError("FIXME: handle exceptions")
except urllib2.URLError, error:
if self._retry < self._maxretry:
print 'Try # ' + str(
self._retry) + " connection to Tier-0 DAS timed-out"
self._retry += 1
newtout = tout * self._retry
time.sleep(3 * self._retry)
return self.getData(src, newtout)
else:
errStr = "Cannot retrieve Tier-0 DAS data from URL \"" + src + "\""
if self._proxy:
errStr += " using proxy \"" + self._proxy + "\""
self._retry = 0
print errStr
print error
raise urllib2.URLError('TimeOut reading ' + src)
def Get(self, request, timeout=None):
# if gcimagebundle is not run on GCE the metadata server will be unreachable
raise urllib2.URLError("urlopen error timed out")
def download_file(remote_url, cache=False, show_progress=True):
"""
Accepts a URL, downloads and optionally caches the result
returning the filename, with a name determined by the file's MD5
hash. If ``cache=True`` and the file is present in the cache, just
returns the filename.
Parameters
----------
remote_url : str
The URL of the file to download
cache : bool, optional
Whether to use the cache
show_progress : bool, optional
Whether to display a progress bar during the download (default
is `True`)
Returns
-------
local_path : str
Returns the local path that the file was download to.
Raises
------
`urllib2.URLError`
Whenever there's a problem getting the remote file.
"""
import hashlib
import socket
from contextlib import closing
from tempfile import NamedTemporaryFile, gettempdir
from shutil import move
from warnings import warn
from ..utils.console import ProgressBarOrSpinner
missing_cache = False
if cache:
try:
dldir, urlmapfn = _get_download_cache_locs()
except (IOError, OSError) as e:
msg = 'Remote data cache could not be accessed due to '
estr = '' if len(e.args) < 1 else (': ' + str(e))
warn(CacheMissingWarning(msg + e.__class__.__name__ + estr))
cache = False
missing_cache = True # indicates that the cache is missing to raise a warning later
try:
if cache:
# We don't need to acquire the lock here, since we are only reading
with _open_shelve(urlmapfn, True) as url2hash:
if str(remote_url) in url2hash:
return url2hash[str(remote_url)]
with closing(urllib2.urlopen(remote_url, timeout=REMOTE_TIMEOUT())) as remote:
#keep a hash to rename the local file to the hashed name
hash = hashlib.md5()
info = remote.info()
if 'Content-Length' in info:
try:
size = int(info['Content-Length'])
except ValueError:
size = None
else:
size = None
if size is not None:
check_free_space_in_dir(gettempdir(), size)
if cache:
check_free_space_in_dir(dldir, size)
if show_progress:
progress_stream = sys.stdout
else:
progress_stream = io.StringIO()
dlmsg = "Downloading {0}".format(remote_url)
with ProgressBarOrSpinner(size, dlmsg, file=progress_stream) as p:
with NamedTemporaryFile(delete=False) as f:
try:
bytes_read = 0
block = remote.read(DOWNLOAD_CACHE_BLOCK_SIZE())
while block:
f.write(block)
hash.update(block)
bytes_read += len(block)
p.update(bytes_read)
block = remote.read(DOWNLOAD_CACHE_BLOCK_SIZE())
except:
if os.path.exists(f.name):
os.remove(f.name)
raise
if cache:
_acquire_download_cache_lock()
try:
with _open_shelve(urlmapfn, True) as url2hash:
# We check now to see if another process has
# inadvertently written the file underneath us
# already
if str(remote_url) in url2hash:
return url2hash[str(remote_url)]
local_path = os.path.join(dldir, hash.hexdigest())
move(f.name, local_path)
url2hash[str(remote_url)] = local_path
finally:
_release_download_cache_lock()
else:
local_path = f.name
if missing_cache:
msg = ('File downloaded to temporary location due to problem '
'with cache directory and will not be cached.')
warn(CacheMissingWarning(msg, local_path))
if DELETE_TEMPORARY_DOWNLOADS_AT_EXIT():
global _tempfilestodel
_tempfilestodel.append(local_path)
except urllib2.URLError as e:
if hasattr(e, 'reason') and hasattr(e.reason, 'errno') and e.reason.errno == 8:
e.reason.strerror = e.reason.strerror + '. requested URL: ' + remote_url
e.reason.args = (e.reason.errno, e.reason.strerror)
raise e
except socket.timeout as e:
# this isn't supposed to happen, but occasionally a socket.timeout gets
# through. It's supposed to be caught in `urrlib2` and raised in this
# way, but for some reason in mysterious circumstances it doesn't. So
# we'll just re-raise it here instead
raise urllib2.URLError(e)
return local_path
def do_open(self, http_class, req):
data = req.get_data()
v_files = []
v_vars = []
# mapping object (dict)
if req.has_data() and not isinstance(data, str):
if hasattr(data, 'items'):
data = data.items()
else:
try:
if len(data) and not isinstance(data[0], tuple):
raise TypeError
except TypeError:
ty, va, tb = sys.exc_info()
raise TypeError, "not a valid non-string sequence or mapping object", tb
for (k, v) in data:
if hasattr(v, 'read'):
v_files.append((k, v))
else:
v_vars.append((k, v))
# no file ? convert to string
if len(v_vars) > 0 and len(v_files) == 0:
data = urllib.urlencode(v_vars)
v_files = []
v_vars = []
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
h = http_class(host) # will parse host:port
if req.has_data():
h.putrequest('POST', req.get_selector())
if 'Content-type' not in req.headers:
if len(v_files) > 0:
boundary = mimetools.choose_boundary()
length = send_data(v_vars, v_files, boundary)
h.putheader('Content-Type',
'multipart/form-data; boundary=%s' % boundary)
h.putheader('Content-length', str(length))
else:
h.putheader('Content-type',
'application/x-www-form-urlencoded')
if 'Content-length' not in req.headers:
h.putheader('Content-length', '%d' % len(data))
else:
h.putrequest('GET', req.get_selector())
scheme, sel = urllib.splittype(req.get_selector())
sel_host, sel_path = urllib.splithost(sel)
h.putheader('Host', sel_host or host)
for name, value in self.parent.addheaders:
name = name.capitalize()
if name not in req.headers:
h.putheader(name, value)
for k, v in req.headers.items():
h.putheader(k, v)
# httplib will attempt to connect() here. be prepared
# to convert a socket error to a URLError.
try:
h.endheaders()
except socket.error as err:
raise urllib2.URLError(err)
if req.has_data():
print v_files
if len(v_files) > 0:
length = send_data(v_vars, v_files, boundary, h)
elif len(v_vars) > 0:
# if data is passed as dict ...
data = urllib.urlencode(v_vars)
h.send(data)
else:
# "normal" urllib2.urlopen()
h.send(data)
code, msg, hdrs = h.getreply()
fp = h.getfile()
if code == 200:
resp = urllib.addinfourl(fp, hdrs, req.get_full_url())
resp.code = code
resp.msg = msg
return resp
else:
return self.parent.error('http', req, fp, code, msg, hdrs)
def test_url_timeout(self):
reason = socket.timeout('too slow')
self.urlopen_mock.side_effect = urllib2.URLError(reason)
tasks.fetch_manifest('url', self.upload.pk)
self.check_validation('Connection to "url" timed out.')
def test_other_url_error(self):
reason = Exception('Some other failure.')
self.urlopen_mock.side_effect = urllib2.URLError(reason)
tasks.fetch_manifest('url', self.upload.pk)
self.check_validation('Some other failure.')
def test_connection_error(self):
reason = socket.gaierror(8, 'nodename nor servname provided')
self.urlopen_mock.side_effect = urllib2.URLError(reason)
tasks.fetch_manifest('url', self.upload.pk)
self.check_validation('Could not contact host at "url".')
def set_url_error(self, url):
self.set_exception(url, urllib2.URLError((errno.ECONNREFUSED, )))
def load(self, url):
raise urllib2.URLError("Disconnected")
def smb_open(self, req):
global USE_NTLM, MACHINE_NAME
host = req.get_host()
if not host:
raise urllib2.URLError('SMB error: no host given')
host, port = splitport(host)
if port is None:
port = 139
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = user or ''
domain = ''
if ';' in user:
domain, user = user.split(';', 1)
passwd = passwd or ''
myname = MACHINE_NAME or self.generateClientMachineName()
server_name,host = host.split(',') if ',' in host else [None,host]
if server_name is None:
n = NetBIOS()
names = n.queryIPForName(host)
if names:
server_name = names[0]
else:
raise urllib2.URLError('SMB error: Hostname does not reply back with its machine name')
path, attrs = splitattr(req.get_selector())
if path.startswith('/'):
path = path[1:]
dirs = path.split('/')
dirs = map(unquote, dirs)
service, path = dirs[0], '/'.join(dirs[1:])
try:
conn = SMBConnection(user, passwd, myname, server_name, domain=domain, use_ntlm_v2 = USE_NTLM)
conn.connect(host, port)
if req.has_data():
data_fp = req.get_data()
filelen = conn.storeFile(service, path, data_fp)
headers = "Content-length: 0\n"
fp = StringIO("")
else:
fp = self.createTempFile()
file_attrs, retrlen = conn.retrieveFile(service, path, fp)
fp.seek(0)
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-length: %d\n" % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except Exception, ex:
raise urllib2.URLError, ('smb error: %s' % ex), sys.exc_info()[2]
def raise_error(_):
raise urllib2.URLError('Some request error message')
def _request(cls,
url,
post_data=None,
timeout=REQUEST_TIMEOUT,
attempts=REQUEST_ATTEMPTS):
# change fb__explicitly_shared to fb:explicitly_shared
if post_data:
post_data = dict(
(k.replace('__', ':'), v) for k, v in post_data.items())
logger.info('requesting url %s with post data %s', url, post_data)
post_request = (post_data is not None or 'method=post' in url)
if post_request and facebook_settings.FACEBOOK_READ_ONLY:
logger.info('running in readonly mode')
response = dict(id=123456789, setting_read_only=True)
return response
# nicely identify ourselves before sending the request
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Open Facebook Python')]
# get the statsd path to track response times with
path = urlparse(url).path
statsd_path = path.replace('.', '_')
# give it a few shots, connection is buggy at times
timeout_mp = 0
while attempts:
# gradually increase the timeout upon failure
timeout_mp += 1
extended_timeout = timeout * timeout_mp
response_file = None
encoded_params = encode_params(post_data) if post_data else None
post_string = (urllib.urlencode(encoded_params)
if post_data else None)
try:
start_statsd('facebook.%s' % statsd_path)
try:
response_file = opener.open(url,
post_string,
timeout=extended_timeout)
response = response_file.read().decode('utf8')
except (urllib2.HTTPError, ), e:
response_file = e
response = response_file.read().decode('utf8')
# Facebook sents error codes for many of their flows
# we still want the json to allow for proper handling
msg_format = 'FB request, error type %s, code %s'
logger.warn(msg_format, type(e), getattr(e, 'code', None))
# detect if its a server or application error
server_error = cls.is_server_error(e, response)
if server_error:
# trigger a retry
raise urllib2.URLError('Facebook is down %s' %
response)
break
except (urllib2.HTTPError, urllib2.URLError, ssl.SSLError), e:
# These are often temporary errors, so we will retry before
# failing
error_format = 'Facebook encountered a timeout (%ss) or error %s'
logger.warn(error_format, extended_timeout, unicode(e))
attempts -= 1
if not attempts:
# if we have no more attempts actually raise the error
error_instance = facebook_exceptions.convert_unreachable_exception(
e)
error_msg = 'Facebook request failed after several retries, raising error %s'
logger.warn(error_msg, error_instance)
raise error_instance
def test_get_httperror(self, urlopen_mock):
urlopen_mock.side_effect = urllib2.URLError('BOOM')
result = self.collector._get('/path')
self.assertFalse(result)
def timeout_catcher(signnum, _):
raise urllib2.URLError("read timeout: max 2 sec, is the image large?")
class FunctestUtilsTesting(unittest.TestCase):
logging.disable(logging.CRITICAL)
def setUp(self):
self.url = 'http://www.opnfv.org/'
self.timeout = 5
self.dest_path = 'test_path'
self.repo_path = 'test_repo_path'
self.installer = 'test_installer'
self.scenario = 'test_scenario'
self.build_tag = 'jenkins-functest-fuel-opnfv-jump-2-daily-master-190'
self.build_tag_week = 'jenkins-functest-fuel-baremetal-weekly-master-8'
self.version = 'master'
self.node_name = 'test_node_name'
self.project = 'test_project'
self.case_name = 'test_case_name'
self.status = 'test_status'
self.details = 'test_details'
self.db_url = 'test_db_url'
self.success_rate = 2.0
self.criteria = 'test_criteria==2.0'
self.start_date = 1482624000
self.stop_date = 1482624000
self.start_time = time.time()
self.stop_time = time.time()
self.readline = -1
self.test_ip = ['10.1.23.4', '10.1.14.15', '10.1.16.15']
self.test_file = 'test_file'
self.error_msg = 'test_error_msg'
self.cmd = 'test_cmd'
self.output_file = 'test_output_file'
self.testname = 'testname'
self.testcase_dict = {'name': 'testname', 'criteria': self.criteria}
self.parameter = 'general.openstack.image_name'
self.config_yaml = 'test_config_yaml-'
self.db_url_env = 'http://foo/testdb'
self.file_yaml = {'general': {'openstack': {'image_name':
'test_image_name'}}}
@mock.patch('urllib2.urlopen',
side_effect=urllib2.URLError('no host given'))
def test_check_internet_connectivity_failed(self, mock_method):
self.assertFalse(functest_utils.check_internet_connectivity())
mock_method.assert_called_once_with(self.url, timeout=self.timeout)
@mock.patch('urllib2.urlopen')
def test_check_internet_connectivity_default(self, mock_method):
self.assertTrue(functest_utils.check_internet_connectivity())
mock_method.assert_called_once_with(self.url, timeout=self.timeout)
@mock.patch('urllib2.urlopen')
def test_check_internet_connectivity_debian(self, mock_method):
self.url = "https://www.debian.org/"
self.assertTrue(functest_utils.check_internet_connectivity(self.url))
mock_method.assert_called_once_with(self.url, timeout=self.timeout)
@mock.patch('urllib2.urlopen',
side_effect=urllib2.URLError('no host given'))
def test_download_url_failed(self, mock_url):
self.assertFalse(functest_utils.download_url(self.url, self.dest_path))
@mock.patch('urllib2.urlopen')
def test_download_url_default(self, mock_url):
with mock.patch("__builtin__.open", mock.mock_open()) as m, \
mock.patch('functest.utils.functest_utils.shutil.copyfileobj')\
as mock_sh:
name = self.url.rsplit('/')[-1]
dest = self.dest_path + "/" + name
self.assertTrue(functest_utils.download_url(self.url,
self.dest_path))
m.assert_called_once_with(dest, 'wb')
self.assertTrue(mock_sh.called)
def test_get_git_branch(self):
with mock.patch('functest.utils.functest_utils.Repo') as mock_repo:
mock_obj2 = mock.Mock()
attrs = {'name': 'test_branch'}
mock_obj2.configure_mock(**attrs)
mock_obj = mock.Mock()
attrs = {'active_branch': mock_obj2}
mock_obj.configure_mock(**attrs)
mock_repo.return_value = mock_obj
self.assertEqual(functest_utils.get_git_branch(self.repo_path),
'test_branch')
@mock.patch('functest.utils.functest_utils.Repo',
side_effect=NoSuchPathError)
def test_get_git_branch_failed(self, mock_repo):
self.assertRaises(NoSuchPathError,
lambda: functest_utils.get_git_branch(self.repo_path
))
@mock.patch('functest.utils.functest_utils.logger.error')
def test_get_installer_type_failed(self, mock_logger_error):
with mock.patch.dict(os.environ,
{},
clear=True):
self.assertEqual(functest_utils.get_installer_type(),
"Unknown_installer")
mock_logger_error.assert_called_once_with("Impossible to retrieve"
" the installer type")
def test_get_installer_type_default(self):
with mock.patch.dict(os.environ,
{'INSTALLER_TYPE': 'test_installer'},
clear=True):
self.assertEqual(functest_utils.get_installer_type(),
self.installer)
@mock.patch('functest.utils.functest_utils.logger.info')
def test_get_scenario_failed(self, mock_logger_info):
with mock.patch.dict(os.environ,
{},
clear=True):
self.assertEqual(functest_utils.get_scenario(),
"os-nosdn-nofeature-noha")
mock_logger_info.assert_called_once_with("Impossible to retrieve "
"the scenario.Use "
"default "
"os-nosdn-nofeature-noha")
def test_get_scenario_default(self):
with mock.patch.dict(os.environ,
{'DEPLOY_SCENARIO': 'test_scenario'},
clear=True):
self.assertEqual(functest_utils.get_scenario(),
self.scenario)
@mock.patch('functest.utils.functest_utils.get_build_tag')
def test_get_version_daily_job(self, mock_get_build_tag):
mock_get_build_tag.return_value = self.build_tag
self.assertEqual(functest_utils.get_version(), self.version)
@mock.patch('functest.utils.functest_utils.get_build_tag')
def test_get_version_weekly_job(self, mock_get_build_tag):
mock_get_build_tag.return_value = self.build_tag_week
self.assertEqual(functest_utils.get_version(), self.version)
@mock.patch('functest.utils.functest_utils.get_build_tag')
def test_get_version_with_dummy_build_tag(self, mock_get_build_tag):
mock_get_build_tag.return_value = 'whatever'
self.assertEqual(functest_utils.get_version(), 'unknown')
@mock.patch('functest.utils.functest_utils.get_build_tag')
def test_get_version_unknown(self, mock_get_build_tag):
mock_get_build_tag.return_value = "unknown_build_tag"
self.assertEqual(functest_utils.get_version(), "unknown")
@mock.patch('functest.utils.functest_utils.logger.info')
def test_get_pod_name_failed(self, mock_logger_info):
with mock.patch.dict(os.environ,
{},
clear=True):
self.assertEqual(functest_utils.get_pod_name(),
"unknown-pod")
mock_logger_info.assert_called_once_with("Unable to retrieve "
"the POD name from "
"environment. Using "
"pod name 'unknown-pod'")
def test_get_pod_name_default(self):
with mock.patch.dict(os.environ,
{'NODE_NAME': 'test_node_name'},
clear=True):
self.assertEqual(functest_utils.get_pod_name(),
self.node_name)
@mock.patch('functest.utils.functest_utils.logger.info')
def test_get_build_tag_failed(self, mock_logger_info):
with mock.patch.dict(os.environ,
{},
clear=True):
self.assertEqual(functest_utils.get_build_tag(),
"none")
mock_logger_info.assert_called_once_with("Impossible to retrieve"
" the build tag")
def test_get_build_tag_default(self):
with mock.patch.dict(os.environ,
{'BUILD_TAG': self.build_tag},
clear=True):
self.assertEqual(functest_utils.get_build_tag(),
self.build_tag)
def test_get_db_url_env_var(self):
with mock.patch.dict(os.environ,
{'TEST_DB_URL': self.db_url_env,
'CONFIG_FUNCTEST_YAML':
"./functest/ci/config_functest.yaml"},
clear=True):
self.assertEqual(functest_utils.get_db_url(),
self.db_url_env)
@mock.patch('functest.utils.functest_utils.get_functest_config')
def test_get_db_url_default(self, mock_get_functest_config):
mock_get_functest_config.return_value = self.db_url
self.assertEqual(functest_utils.get_db_url(), self.db_url)
mock_get_functest_config.assert_called_once_with('results.test_db_url')
@mock.patch('functest.utils.functest_utils.logger.info')
def test_logger_test_results(self, mock_logger_info):
with mock.patch('functest.utils.functest_utils.get_pod_name',
return_value=self.node_name), \
mock.patch('functest.utils.functest_utils.get_scenario',
return_value=self.scenario), \
mock.patch('functest.utils.functest_utils.get_version',
return_value=self.version), \
mock.patch('functest.utils.functest_utils.get_build_tag',
return_value=self.build_tag), \
mock.patch('functest.utils.functest_utils.get_db_url',
return_value=self.db_url):
functest_utils.logger_test_results(self.project, self.case_name,
self.status, self.details)
mock_logger_info.assert_called_once_with(
"\n"
"****************************************\n"
"\t %(p)s/%(n)s results \n\n"
"****************************************\n"
"DB:\t%(db)s\n"
"pod:\t%(pod)s\n"
"version:\t%(v)s\n"
"scenario:\t%(s)s\n"
"status:\t%(c)s\n"
"build tag:\t%(b)s\n"
"details:\t%(d)s\n"
% {'p': self.project,
'n': self.case_name,
'db': self.db_url,
'pod': self.node_name,
'v': self.version,
's': self.scenario,
'c': self.status,
'b': self.build_tag,
'd': self.details})
def _get_env_dict(self, var):
dic = {'INSTALLER_TYPE': self.installer,
'DEPLOY_SCENARIO': self.scenario,
'NODE_NAME': self.node_name,
'BUILD_TAG': self.build_tag}
dic.pop(var, None)
return dic
def _test_push_results_to_db_missing_env(self, env_var):
dic = self._get_env_dict(env_var)
with mock.patch('functest.utils.functest_utils.get_db_url',
return_value=self.db_url), \
mock.patch.dict(os.environ,
dic,
clear=True), \
mock.patch('functest.utils.functest_utils.logger.error') \
as mock_logger_error:
functest_utils.push_results_to_db(self.project, self.case_name,
self.start_date, self.stop_date,
self.criteria, self.details)
mock_logger_error.assert_called_once_with("Please set env var: " +
str("\'" + env_var +
"\'"))
def test_push_results_to_db_missing_installer(self):
self._test_push_results_to_db_missing_env('INSTALLER_TYPE')
def test_push_results_to_db_missing_scenario(self):
self._test_push_results_to_db_missing_env('DEPLOY_SCENARIO')
def test_push_results_to_db_missing_nodename(self):
self._test_push_results_to_db_missing_env('NODE_NAME')
def test_push_results_to_db_missing_buildtag(self):
self._test_push_results_to_db_missing_env('BUILD_TAG')
def test_push_results_to_db_request_post_failed(self):
dic = self._get_env_dict(None)
with mock.patch('functest.utils.functest_utils.get_db_url',
return_value=self.db_url), \
mock.patch.dict(os.environ,
dic,
clear=True), \
mock.patch('functest.utils.functest_utils.logger.error') \
as mock_logger_error, \
mock.patch('functest.utils.functest_utils.requests.post',
side_effect=requests.RequestException):
self.assertFalse(functest_utils.
push_results_to_db(self.project, self.case_name,
self.start_date,
self.stop_date,
self.criteria, self.details))
mock_logger_error.assert_called_once_with(test_utils.
RegexMatch("Pushing "
"Result to"
" DB"
"(\S+\s*) "
"failed:"))
def test_push_results_to_db_request_post_exception(self):
dic = self._get_env_dict(None)
with mock.patch('functest.utils.functest_utils.get_db_url',
return_value=self.db_url), \
mock.patch.dict(os.environ,
dic,
clear=True), \
mock.patch('functest.utils.functest_utils.logger.error') \
as mock_logger_error, \
mock.patch('functest.utils.functest_utils.requests.post',
side_effect=Exception):
self.assertFalse(functest_utils.
push_results_to_db(self.project, self.case_name,
self.start_date,
self.stop_date,
self.criteria, self.details))
self.assertTrue(mock_logger_error.called)
def test_push_results_to_db_default(self):
dic = self._get_env_dict(None)
with mock.patch('functest.utils.functest_utils.get_db_url',
return_value=self.db_url), \
mock.patch.dict(os.environ,
dic,
clear=True), \
mock.patch('functest.utils.functest_utils.requests.post'):
self.assertTrue(functest_utils.
push_results_to_db(self.project, self.case_name,
self.start_date,
self.stop_date,
self.criteria, self.details))
readline = 0
test_ip = ['10.1.23.4', '10.1.14.15', '10.1.16.15']
@staticmethod
def readline_side():
if FunctestUtilsTesting.readline == \
len(FunctestUtilsTesting.test_ip) - 1:
return False
FunctestUtilsTesting.readline += 1
return FunctestUtilsTesting.test_ip[FunctestUtilsTesting.readline]
# TODO: get_resolvconf_ns
@mock.patch('functest.utils.functest_utils.dns.resolver.Resolver')
def test_get_resolvconf_ns_default(self, mock_dns_resolve):
attrs = {'query.return_value': ["test"]}
mock_dns_resolve.configure_mock(**attrs)
m = mock.Mock()
attrs = {'readline.side_effect': self.readline_side}
m.configure_mock(**attrs)
with mock.patch("__builtin__.open") as mo:
mo.return_value = m
self.assertEqual(functest_utils.get_resolvconf_ns(),
self.test_ip[1:])
def _get_environ(self, var):
if var == 'INSTALLER_TYPE':
return self.installer
elif var == 'DEPLOY_SCENARIO':
return self.scenario
return var
def test_get_ci_envvars_default(self):
with mock.patch('os.environ.get',
side_effect=self._get_environ):
dic = {"installer": self.installer,
"scenario": self.scenario}
self.assertDictEqual(functest_utils.get_ci_envvars(), dic)
def cmd_readline(self):
return 'test_value\n'
@mock.patch('functest.utils.functest_utils.logger.error')
@mock.patch('functest.utils.functest_utils.logger.info')
def test_execute_command_args_present_with_error(self, mock_logger_info,
mock_logger_error):
with mock.patch('functest.utils.functest_utils.subprocess.Popen') \
as mock_subproc_open, \
mock.patch('__builtin__.open', mock.mock_open()) as mopen:
FunctestUtilsTesting.readline = 0
mock_obj = mock.Mock()
attrs = {'readline.side_effect': self.cmd_readline()}
mock_obj.configure_mock(**attrs)
mock_obj2 = mock.Mock()
attrs = {'stdout': mock_obj, 'wait.return_value': 1}
mock_obj2.configure_mock(**attrs)
mock_subproc_open.return_value = mock_obj2
resp = functest_utils.execute_command(self.cmd, info=True,
error_msg=self.error_msg,
verbose=True,
output_file=self.output_file)
self.assertEqual(resp, 1)
msg_exec = ("Executing command: '%s'" % self.cmd)
mock_logger_info.assert_called_once_with(msg_exec)
mopen.assert_called_once_with(self.output_file, "w")
mock_logger_error.assert_called_once_with(self.error_msg)
@mock.patch('functest.utils.functest_utils.logger.info')
def test_execute_command_args_present_with_success(self, mock_logger_info,
):
with mock.patch('functest.utils.functest_utils.subprocess.Popen') \
as mock_subproc_open, \
mock.patch('__builtin__.open', mock.mock_open()) as mopen:
FunctestUtilsTesting.readline = 0
mock_obj = mock.Mock()
attrs = {'readline.side_effect': self.cmd_readline()}
mock_obj.configure_mock(**attrs)
mock_obj2 = mock.Mock()
attrs = {'stdout': mock_obj, 'wait.return_value': 0}
mock_obj2.configure_mock(**attrs)
mock_subproc_open.return_value = mock_obj2
resp = functest_utils.execute_command(self.cmd, info=True,
error_msg=self.error_msg,
verbose=True,
output_file=self.output_file)
self.assertEqual(resp, 0)
msg_exec = ("Executing command: '%s'" % self.cmd)
mock_logger_info.assert_called_once_with(msg_exec)
mopen.assert_called_once_with(self.output_file, "w")
@mock.patch('functest.utils.functest_utils.logger.info')
def test_execute_command_args_missing_with_success(self, mock_logger_info,
):
with mock.patch('functest.utils.functest_utils.subprocess.Popen') \
as mock_subproc_open:
FunctestUtilsTesting.readline = 2
mock_obj = mock.Mock()
attrs = {'readline.side_effect': self.cmd_readline()}
mock_obj.configure_mock(**attrs)
mock_obj2 = mock.Mock()
attrs = {'stdout': mock_obj, 'wait.return_value': 0}
mock_obj2.configure_mock(**attrs)
mock_subproc_open.return_value = mock_obj2
resp = functest_utils.execute_command(self.cmd, info=False,
error_msg="",
verbose=False,
output_file=None)
self.assertEqual(resp, 0)
@mock.patch('functest.utils.functest_utils.logger.error')
def test_execute_command_args_missing_with_error(self, mock_logger_error,
):
with mock.patch('functest.utils.functest_utils.subprocess.Popen') \
as mock_subproc_open:
FunctestUtilsTesting.readline = 2
mock_obj = mock.Mock()
attrs = {'readline.side_effect': self.cmd_readline()}
mock_obj.configure_mock(**attrs)
mock_obj2 = mock.Mock()
attrs = {'stdout': mock_obj, 'wait.return_value': 1}
mock_obj2.configure_mock(**attrs)
mock_subproc_open.return_value = mock_obj2
resp = functest_utils.execute_command(self.cmd, info=False,
error_msg="",
verbose=False,
output_file=None)
self.assertEqual(resp, 1)
def _get_functest_config(self, var):
return var
@mock.patch('functest.utils.functest_utils.logger.error')
def test_get_dict_by_test(self, mock_logger_error):
with mock.patch('__builtin__.open', mock.mock_open()), \
mock.patch('functest.utils.functest_utils.yaml.safe_load') \
as mock_yaml, \
mock.patch('functest.utils.functest_utils.get_testcases_'
'file_dir'):
mock_obj = mock.Mock()
attrs = {'get.return_value': [{'testcases': [self.testcase_dict]}]}
mock_obj.configure_mock(**attrs)
mock_yaml.return_value = mock_obj
self.assertDictEqual(functest_utils.
get_dict_by_test(self.testname),
self.testcase_dict)
@mock.patch('functest.utils.functest_utils.get_dict_by_test')
def test_get_criteria_by_test_default(self, mock_get_dict_by_test):
mock_get_dict_by_test.return_value = self.testcase_dict
self.assertEqual(functest_utils.get_criteria_by_test(self.testname),
self.criteria)
@mock.patch('functest.utils.functest_utils.get_dict_by_test')
def test_get_criteria_by_test_failed(self, mock_get_dict_by_test):
mock_get_dict_by_test.return_value = None
self.assertIsNone(functest_utils.get_criteria_by_test(self.testname))
def test_get_parameter_from_yaml_failed(self):
self.file_yaml['general'] = None
with mock.patch('__builtin__.open', mock.mock_open()), \
mock.patch('functest.utils.functest_utils.yaml.safe_load') \
as mock_yaml, \
self.assertRaises(ValueError) as excep:
mock_yaml.return_value = self.file_yaml
functest_utils.get_parameter_from_yaml(self.parameter,
self.test_file)
self.assertTrue(("The parameter %s is not"
" defined in config_functest.yaml" %
self.parameter) in excep.exception)
def test_get_parameter_from_yaml_default(self):
with mock.patch('__builtin__.open', mock.mock_open()), \
mock.patch('functest.utils.functest_utils.yaml.safe_load') \
as mock_yaml:
mock_yaml.return_value = self.file_yaml
self.assertEqual(functest_utils.
get_parameter_from_yaml(self.parameter,
self.test_file),
'test_image_name')
@mock.patch('functest.utils.functest_utils.get_parameter_from_yaml')
def test_get_functest_config_default(self, mock_get_parameter_from_yaml):
with mock.patch.dict(os.environ,
{'CONFIG_FUNCTEST_YAML': self.config_yaml}):
functest_utils.get_functest_config(self.parameter)
mock_get_parameter_from_yaml. \
assert_called_once_with(self.parameter,
self.config_yaml)
def test_check_success_rate_default(self):
with mock.patch('functest.utils.functest_utils.get_criteria_by_test') \
as mock_criteria:
mock_criteria.return_value = self.criteria
resp = functest_utils.check_success_rate(self.case_name,
self.success_rate)
self.assertEqual(resp, 'PASS')
def test_check_success_rate_failed(self):
with mock.patch('functest.utils.functest_utils.get_criteria_by_test') \
as mock_criteria:
mock_criteria.return_value = self.criteria
resp = functest_utils.check_success_rate(self.case_name,
3.0)
self.assertEqual(resp, 'FAIL')
# TODO: merge_dicts
def test_get_testcases_file_dir(self):
resp = functest_utils.get_testcases_file_dir()
self.assertEqual(resp,
"/home/opnfv/repos/functest/"
"functest/ci/testcases.yaml")
def test_get_functest_yaml(self):
with mock.patch('__builtin__.open', mock.mock_open()), \
mock.patch('functest.utils.functest_utils.yaml.safe_load') \
as mock_yaml:
mock_yaml.return_value = self.file_yaml
resp = functest_utils.get_functest_yaml()
self.assertEqual(resp, self.file_yaml)
@mock.patch('functest.utils.functest_utils.logger.info')
def test_print_separator(self, mock_logger_info):
functest_utils.print_separator()
mock_logger_info.assert_called_once_with("======================="
"=======================")
def throw_errors(self, num):
for x in range(num):
yield urllib2.URLError('Bang!')
yield True
def fetch(self, url, timeout=None):
"""Attempts to fetch the URL requested which should refer to a
robots.txt file, e.g. http://example.com/robots.txt.
"""
# ISO-8859-1 is the default encoding for text files per the specs for
# HTTP 1.0 (RFC 1945 sec 3.6.1) and HTTP 1.1 (RFC 2616 sec 3.7.1).
# ref: http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1
encoding = "iso-8859-1"
content = ""
expires_header = None
content_type_header = None
self._response_code = 0
self._source_url = url
if self.user_agent:
req = urllib_request.Request(url, None,
{'User-Agent': self.user_agent})
else:
req = urllib_request.Request(url)
try:
if timeout:
f = urllib_request.urlopen(req, timeout=timeout)
else:
f = urllib_request.urlopen(req)
content = f.read(MAX_FILESIZE)
# As of Python 2.5, f.info() looks like it returns the HTTPMessage
# object created during the connection.
expires_header = f.info().get("expires")
content_type_header = f.info().get("Content-Type")
# As of Python 2.4, this file-like object reports the response
# code, too.
if hasattr(f, "code"):
self._response_code = f.code
else:
self._response_code = 200
f.close()
except urllib_error.URLError:
# This is a slightly convoluted way to get the error instance,
# but it works under Python 2 & 3.
error_instance = sys.exc_info()
if len(error_instance) > 1:
error_instance = error_instance[1]
if hasattr(error_instance, "code"):
self._response_code = error_instance.code
# MK1996 section 3.4 says, "...robots should take note of Expires
# header set by the origin server. If no cache-control directives
# are present robots should default to an expiry of 7 days".
# This code is lazy and looks at the Expires header but not
# Cache-Control directives.
self.expiration_date = None
if self._response_code >= 200 and self._response_code < 300:
# All's well.
if expires_header:
self.expiration_date = email_utils.parsedate_tz(expires_header)
if self.expiration_date:
# About time zones -- the call to parsedate_tz() returns a
# 10-tuple with the time zone offset in the 10th element.
# There are 3 valid formats for HTTP dates, and one of
# them doesn't contain time zone information. (UTC is
# implied since all HTTP header dates are UTC.) When given
# a date that lacks time zone information, parsedate_tz()
# returns None in the 10th element. mktime_tz() interprets
# None in the 10th (time zone) element to mean that the
# date is *local* time, not UTC.
# Therefore, if the HTTP timestamp lacks time zone info
# and I run that timestamp through parsedate_tz() and pass
# it directly to mktime_tz(), I'll get back a local
# timestamp which isn't what I want. To fix this, I simply
# convert a time zone of None to zero. It's much more
# difficult to explain than to fix. =)
# ref: http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3.1
if self.expiration_date[9] is None:
self.expiration_date = self.expiration_date[:9] + (0, )
self.expiration_date = email_utils.mktime_tz(
self.expiration_date)
if self.use_local_time:
# I have to do a little more converting to get this
# UTC timestamp into localtime.
self.expiration_date = time.mktime(
time.gmtime(self.expiration_date))
# else:
# The expires header was garbage.
if not self.expiration_date:
self.expiration_date = self._now() + SEVEN_DAYS
if (self._response_code >= 200) and (self._response_code < 300):
# All's well.
media_type, encoding = _parse_content_type_header(
content_type_header)
# RFC 2616 sec 3.7.1 --
# When no explicit charset parameter is provided by the sender,
# media subtypes of the "text" type are defined to have a default
# charset value of "ISO-8859-1" when received via HTTP.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1
if not encoding:
encoding = "iso-8859-1"
elif self._response_code in (401, 403):
# 401 or 403 ==> Go away or I will taunt you a second time!
# (according to MK1996)
content = "User-agent: *\nDisallow: /\n"
elif self._response_code == 404:
# No robots.txt ==> everyone's welcome
content = ""
else:
# Uh-oh. I punt this up to the caller.
raise urllib_error.URLError(self._response_code)
if ((PY_MAJOR_VERSION == 2) and isinstance(content, str)) or \
((PY_MAJOR_VERSION > 2) and (not isinstance(content, str))):
# This ain't Unicode yet! It needs to be.
# Unicode decoding errors are another point of failure that I punt
# up to the caller.
try:
content = content.decode(encoding)
except UnicodeError:
msg = "Robots.txt contents are not in the encoding expected (%s)." % encoding
raise UnicodeError(msg)
except (LookupError, ValueError):
# LookupError ==> Python doesn't have a decoder for that encoding.
# One can also get a ValueError here if the encoding starts with
# a dot (ASCII 0x2e). See Python bug 1446043 for details. This
# bug was supposedly fixed in Python 2.5.
msg = """I don't understand the encoding "%s".""" % encoding
raise UnicodeError(msg)
# Now that I've fetched the content and turned it into Unicode, I
# can parse it.
self.parse(content)
def _authenticate_(self,
account,
user,
passwd,
new_passwd=None,
timeout=15,
duration=3600):
if user == 'admin' and duration > 3600: # admin cannot have more than 1 hour duration
duration = 3600
# because of the variability, we need to keep this here, not in __init__
auth_path = self.TEMPLATE.format(dur=duration)
if not self.dns_enabled:
auth_path = self.NON_DNS_QUERY_PATH + auth_path
else:
auth_path = '/' + auth_path
host = self.host
if self.dns_enabled:
host = 'tokens.{0}'.format(host)
if self.validate_certs:
conn = CertValidatingHTTPSConnection(host,
self.port,
timeout=timeout,
**self.kwargs)
else:
conn = httplib.HTTPSConnection(host, self.port, timeout=timeout)
if new_passwd:
auth_string = u"{user}@{account};{pw}@{new_pw}".format(
user=base64.b64encode(user),
account=base64.b64encode(account),
pw=base64.b64encode(passwd),
new_pw=new_passwd)
else:
auth_string = u"{user}@{account}:{pw}".format(
user=base64.b64encode(user),
account=base64.b64encode(account),
pw=passwd)
encoded_auth = base64.b64encode(auth_string)
headers = {'Authorization': "Basic %s" % encoded_auth}
try:
conn.request('GET', auth_path, '', headers)
response = conn.getresponse()
if response.status != 200:
raise urllib2.HTTPError(url='',
code=response.status,
msg=response.reason,
hdrs=None,
fp=None)
body = response.read()
# parse AccessKeyId, SecretAccessKey and SessionToken
creds = Credentials()
h = BotoXmlHandler(creds, None)
parseString(body, h)
return creds
except SSLError as err:
if err.message != '':
raise urllib2.URLError(str(err))
else:
raise urllib2.URLError(err[1])
except socket.error as err:
# when dns enabled, but path cloud, we get here with
# err=gaierror(8, 'nodename nor servname provided, or not known')
# when dns disabled, but path cloud, we get here with
# err=gaierror(8, 'nodename nor servname provided, or not known')
raise urllib2.URLError(str(err))
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm,
headers):
user, pw = self.passwd.find_user_password(realm, req.get_full_url())
if pw is not None:
# ntlm secures a socket, so we must use the same socket for the complete handshake
headers = dict(req.headers)
headers.update(req.unredirected_hdrs)
auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(user)
if req.headers.get(self.auth_header, None) == auth:
return None
headers[self.auth_header] = auth
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
h = None
if req.get_full_url().startswith('https://'):
h = httplib.HTTPSConnection(host) # will parse host:port
else:
h = httplib.HTTPConnection(host) # will parse host:port
h.set_debuglevel(self._debuglevel)
# we must keep the connection because NTLM authenticates the connection, not single requests
headers["Connection"] = "Keep-Alive"
headers = dict(
(name.title(), val) for name, val in headers.items())
h.request(req.get_method(), req.get_selector(), req.data, headers)
r = h.getresponse()
r.begin()
r._safe_read(int(r.getheader('content-length')))
if r.getheader('set-cookie'):
# this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out)
headers['Cookie'] = r.getheader('set-cookie')
r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open)
auth_header_value = r.getheader(auth_header_field, None)
(ServerChallenge,
NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(
auth_header_value[5:])
user_parts = user.split('\\', 1)
DomainName = user_parts[0].upper()
UserName = user_parts[1]
auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
ServerChallenge, UserName, DomainName, pw, NegotiateFlags)
headers[self.auth_header] = auth
headers["Connection"] = "Close"
headers = dict(
(name.title(), val) for name, val in headers.items())
try:
h.request(req.get_method(), req.get_selector(), req.data,
headers)
# none of the configured handlers are triggered, for example redirect-responses are not handled!
response = h.getresponse()
def notimplemented():
raise NotImplementedError
response.readline = notimplemented
infourl = addinfourl(response, response.msg,
req.get_full_url())
infourl.code = response.status
infourl.msg = response.reason
return infourl
except socket.error, err:
raise urllib2.URLError(err)
from test_resources.general_search_results import search_results
from test_resources.tracker_results import tracker_results
from test_resources.fake_tracker import Tracker
from BeautifulSoup import BeautifulSoup
#to mock out urlopen
FakeSearchSock = Mock()
FakeSearchSock.read = Mock(return_value=search_results)
FakeSearchUrlOpen = Mock(return_value=FakeSearchSock)
FakeTrackerSock = Mock()
FakeTrackerSock.read = Mock(return_value=tracker_results)
FakeTrackerUrlOpen = Mock(return_value=FakeTrackerSock)
FakeEmptySock = Mock()
FakeEmptySock.read = Mock(return_value='')
FakeEmptyUrlOpen = Mock(return_value=FakeEmptySock)
FakeUrlOpenWithError = Mock(side_effect=urllib2.URLError(''))
class FakeChangingUrlOpen(object):
def __init__(self):
self._call_count = 0
self.args = []
def call(self, *args):
self.args = args
if self._call_count == 0:
self._call_count += 1
raise urllib2.URLError('')
else:
return FakeEmptySock
class GenerateReportTesting(unittest.TestCase):
logging.disable(logging.CRITICAL)
def test_init(self):
test_array = gen_report.init()
self.assertEqual(test_array, [])
@mock.patch('functest.ci.generate_report.urllib2.urlopen',
side_effect=urllib2.URLError('no host given'))
def test_get_results_from_db_fail(self, mock_method):
url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(),
CONST.BUILD_TAG)
self.assertIsNone(gen_report.get_results_from_db())
mock_method.assert_called_once_with(url)
@mock.patch('functest.ci.generate_report.urllib2.urlopen',
return_value={'results': []})
def test_get_results_from_db_success(self, mock_method):
url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(),
CONST.BUILD_TAG)
self.assertEqual(gen_report.get_results_from_db(), None)
mock_method.assert_called_once_with(url)
def test_get_data(self):
self.assertIsInstance(gen_report.get_data({'result': ''}, ''), dict)
def test_print_line_with_ci_run(self):
CONST.IS_CI_RUN = True
w1 = 'test_print_line'
test_str = ("| %s| %s| %s| %s| %s|\n"
% (w1.ljust(gen_report.COL_1_LEN - 1),
''.ljust(gen_report.COL_2_LEN - 1),
''.ljust(gen_report.COL_3_LEN - 1),
''.ljust(gen_report.COL_4_LEN - 1),
''.ljust(gen_report.COL_5_LEN - 1)))
self.assertEqual(gen_report.print_line(w1), test_str)
def test_print_line_without_ci_run(self):
CONST.IS_CI_RUN = False
w1 = 'test_print_line'
test_str = ("| %s| %s| %s| %s|\n"
% (w1.ljust(gen_report.COL_1_LEN - 1),
''.ljust(gen_report.COL_2_LEN - 1),
''.ljust(gen_report.COL_3_LEN - 1),
''.ljust(gen_report.COL_4_LEN - 1)))
self.assertEqual(gen_report.print_line(w1), test_str)
def test_print_line_no_column_with_ci_run(self):
CONST.IS_CI_RUN = True
TOTAL_LEN = gen_report.COL_1_LEN + gen_report.COL_2_LEN
TOTAL_LEN += gen_report.COL_3_LEN + gen_report.COL_4_LEN + 2
TOTAL_LEN += gen_report.COL_5_LEN + 1
test_str = ("| %s|\n" % 'test'.ljust(TOTAL_LEN))
self.assertEqual(gen_report.print_line_no_columns('test'), test_str)
def test_print_line_no_column_without_ci_run(self):
CONST.IS_CI_RUN = False
TOTAL_LEN = gen_report.COL_1_LEN + gen_report.COL_2_LEN
TOTAL_LEN += gen_report.COL_3_LEN + gen_report.COL_4_LEN + 2
test_str = ("| %s|\n" % 'test'.ljust(TOTAL_LEN))
self.assertEqual(gen_report.print_line_no_columns('test'), test_str)
def test_print_separator_with_ci_run(self):
CONST.IS_CI_RUN = True
test_str = ("+" + "=" * gen_report.COL_1_LEN +
"+" + "=" * gen_report.COL_2_LEN +
"+" + "=" * gen_report.COL_3_LEN +
"+" + "=" * gen_report.COL_4_LEN +
"+" + "=" * gen_report.COL_5_LEN)
test_str += '+\n'
self.assertEqual(gen_report.print_separator(), test_str)
def test_print_separator_without_ci_run(self):
CONST.IS_CI_RUN = False
test_str = ("+" + "=" * gen_report.COL_1_LEN +
"+" + "=" * gen_report.COL_2_LEN +
"+" + "=" * gen_report.COL_3_LEN +
"+" + "=" * gen_report.COL_4_LEN)
test_str += "+\n"
self.assertEqual(gen_report.print_separator(), test_str)
@mock.patch('functest.ci.generate_report.logger.info')
def test_main_with_ci_run(self, mock_method):
CONST.IS_CI_RUN = True
gen_report.main()
mock_method.assert_called_once_with(test_utils.SubstrMatch('URL'))
@mock.patch('functest.ci.generate_report.logger.info')
def test_main_with_ci_loop(self, mock_method):
CONST.CI_LOOP = 'daily'
gen_report.main()
mock_method.assert_called_once_with(test_utils.SubstrMatch('CI LOOP'))
@mock.patch('functest.ci.generate_report.logger.info')
def test_main_with_scenario(self, mock_method):
CONST.DEPLOY_SCENARIO = 'test_scenario'
gen_report.main()
mock_method.assert_called_once_with(test_utils.SubstrMatch('SCENARIO'))
@mock.patch('functest.ci.generate_report.logger.info')
def test_main_with_build_tag(self, mock_method):
CONST.BUILD_TAG = 'test_build_tag'
gen_report.main()
mock_method.assert_called_once_with(test_utils.
SubstrMatch('BUILD TAG'))
def failSome(_url, symbol, _product):
if symbol.file_name == fail.file_name:
raise urllib2.URLError('network failure')
# the socket has been closed by the server since we
# last used the connection.
if DEBUG: print "failed to re-use connection to %s" % host
h.close()
else:
if DEBUG: print "re-using connection to %s" % host
need_new_connection = 0
if need_new_connection:
if DEBUG: print "creating new connection to %s" % host
h = http_class(host)
self._connections[key] = h
self._start_connection(h, req)
r = h.getresponse()
except socket.error, err:
if h: h.close()
raise urllib2.URLError(err)
# if not a persistent connection, don't try to reuse it
if r.will_close: self._remove_connection(host)
if DEBUG:
print "STATUS: %s, %s" % (r.status, r.reason)
r._handler = self
r._host = host
r._url = req.get_full_url()
#if r.status == 200 or not HANDLE_ERRORS:
#return r
if r.status == 200 or not HANDLE_ERRORS:
# [speedplane] Must return an adinfourl object
resp = urllib2.addinfourl(r, r.msg, req.get_full_url())
def urlopener(url_or_request, log, **kwargs):
"""
Utility function for pulling back a url, with a retry of 3 times, increasing the timeout, etc.
Re-raises any errors as URLError.
.. warning:: This is being replaced by requests library.
flexget.utils.requests should be used going forward.
:param str url_or_request: URL or Request object to get.
:param log: Logger to log debug info and errors to
:param kwargs: Keyword arguments to be passed to urlopen
:return: The file-like object returned by urlopen
"""
from flexget.utils.requests import is_unresponsive, set_unresponsive
if isinstance(url_or_request, urllib2.Request):
url = url_or_request.get_host()
else:
url = url_or_request
if is_unresponsive(url):
msg = '%s is known to be unresponsive, not trying again.' % urlparse(
url).hostname
log.warning(msg)
raise urllib2.URLError(msg)
retries = kwargs.get('retries', 3)
timeout = kwargs.get('timeout', 15.0)
# get the old timeout for sockets, so we can set it back to that when done. This is NOT threadsafe by the way.
# In order to avoid requiring python 2.6, we're not using the urlopen timeout parameter. That really should be used
# after checking for python 2.6.
oldtimeout = socket.getdefaulttimeout()
try:
socket.setdefaulttimeout(timeout)
handlers = [SmartRedirectHandler()]
if urllib2._opener:
handlers.extend(urllib2._opener.handlers)
if kwargs.get('handlers'):
handlers.extend(kwargs['handlers'])
if len(handlers) > 1:
handler_names = [h.__class__.__name__ for h in handlers]
log.debug(
'Additional handlers have been specified for this urlopen: %s'
% ', '.join(handler_names))
opener = urllib2.build_opener(*handlers).open
for i in range(retries): # retry getting the url up to 3 times.
if i > 0:
time.sleep(3)
try:
retrieved = opener(url_or_request, kwargs.get('data'))
except urllib2.HTTPError as e:
if e.code < 500:
# If it was not a server error, don't keep retrying.
log.warning('Could not retrieve url (HTTP %s error): %s' %
(e.code, e.url))
raise
log.debug('HTTP error (try %i/%i): %s' %
(i + 1, retries, e.code))
except (urllib2.URLError, socket.timeout) as e:
if hasattr(e, 'reason'):
reason = str(e.reason)
else:
reason = 'N/A'
if reason == 'timed out':
set_unresponsive(url)
log.debug('Failed to retrieve url (try %i/%i): %s' %
(i + 1, retries, reason))
except httplib.IncompleteRead as e:
log.critical('Incomplete read - see python bug 6312')
break
else:
# make the returned instance usable in a with statement by adding __enter__ and __exit__ methods
def enter(self):
return self
def exit(self, exc_type, exc_val, exc_tb):
self.close()
retrieved.__class__.__enter__ = enter
retrieved.__class__.__exit__ = exit
return retrieved
log.warning('Could not retrieve url: %s' % url_or_request)
raise urllib2.URLError('Could not retrieve url after %s tries.' %
retries)
finally:
socket.setdefaulttimeout(oldtimeout)
def fake_check_call(self, argv, **kwargs):
if argv[-2] == TestRsyncHandler.fail_url:
raise urllib2.URLError('fail url')
if not os.path.exists(argv[-1]):
open(argv[-1], 'w').write(TestRsyncHandler.file_data)
