def testNotifyURLError(self, mock_jsonRequest): mock_jsonRequest.side_effect = urllib2.URLError("hello") Slack(token=None, channels=['123', '123', '456']).notify({'text': "Test message"}) self.assertEquals(mock_jsonRequest.call_count, 0)
def raise_urllib2_error(*_unused, **_unused2): """Always raise a urllib2.URLError""" raise urllib2.URLError("test failure")
def do_open(self, req): ''' Called by handler's url_open method. ''' host = req.get_host() if not host: raise urllib2.URLError('no host given') try: resp_statuses = self._hostresp.setdefault(host, self._get_tail_filter()) # Check if all our last 'resp_statuses' were timeouts and raise # a w3afMustStopException if this is the case. if len(resp_statuses) == self._curr_check_failures and \ all(st == RESP_TIMEOUT for st in resp_statuses): msg = ( 'w3af found too many consecutive timeouts. The remote ' 'webserver seems to be unresponsive; please verify manually.' ) reason = 'Timeout while trying to reach target.' raise w3afMustStopByKnownReasonExc(msg, reason=reason) conn_factory = self._get_connection conn = self._cm.get_available_connection(host, conn_factory) if conn.is_fresh: # First of all, call the request method. This is needed for # HTTPS Proxy if isinstance(conn, ProxyHTTPConnection): conn.proxy_setup(req.get_full_url()) conn.is_fresh = False self._start_transaction(conn, req) resp = conn.getresponse() else: # We'll try to use a previously created connection resp = self._reuse_connection(conn, req, host) # If the resp is None it means that connection is bad. It was # possibly closed by the server. Replace it with a new one. if resp is None: conn.close() conn = self._cm.replace_connection(conn, host, conn_factory) # First of all, call the request method. This is needed for # HTTPS Proxy if isinstance(conn, ProxyHTTPConnection): conn.proxy_setup(req.get_full_url()) # Try again with the fresh one conn.is_fresh = False self._start_transaction(conn, req) resp = conn.getresponse() except (socket.error, httplib.HTTPException), err: # We better discard this connection self._cm.remove_connection(conn, host) if isinstance(err, socket.timeout): resp_statuses.append(RESP_TIMEOUT) _err = URLTimeoutError() else: resp_statuses.append(RESP_BAD) if isinstance(err, httplib.HTTPException): err = repr(err) _err = urllib2.URLError(err) raise _err
def do_open(self, http_class, req): data = req.get_data() v_files = [] v_vars = [] # mapping object (dict) if req.has_data() and type(data) != str: if hasattr(data, 'items'): data = data.items() else: try: if len(data) and not isinstance(data[0], tuple): raise TypeError except TypeError: ty, va, tb = sys.exc_info() raise TypeError, "not a valid non-string sequence or mapping object", tb for (k, v) in data: # if fd is provided with a filename if isinstance(v, dict): if not v.has_key('fd'): raise TypeError("if value is dict, it must have keys 'fd' and 'filename") if not v.has_key('filename'): raise TypeError("if value is dict, it must have keys 'fd' and 'filename") v_files.append( (k, v) ) elif hasattr(v, 'read'): v_files.append( (k, v) ) else: v_vars.append( (k, v) ) # no file ? convert to string if len(v_vars) > 0 and len(v_files) == 0: data = urllib.urlencode(v_vars) v_files = [] v_vars = [] host = req.get_host() if not host: raise urllib2.URLError('no host given') h = http_class(host) # will parse host:port if req.has_data(): h.putrequest(req.get_method(), req.get_selector()) if not 'Content-type' in req.headers: if len(v_files) > 0: boundary = mimetools.choose_boundary() l = send_data(v_vars, v_files, boundary) h.putheader('Content-Type', 'multipart/form-data; boundary=%s' % boundary) h.putheader('Content-length', str(l)) else: h.putheader('Content-type', 'application/x-www-form-urlencoded') if not 'Content-length' in req.headers: h.putheader('Content-length', '%d' % len(data)) else: h.putrequest(req.get_method(), req.get_selector()) scheme, sel = urllib.splittype(req.get_selector()) sel_host, sel_path = urllib.splithost(sel) h.putheader('Host', sel_host or host) for name, value in self.parent.addheaders: name = name.capitalize() if name not in req.headers: h.putheader(name, value) for k, v in req.headers.items(): h.putheader(k, v) # httplib will attempt to connect() here. be prepared # to convert a socket error to a URLError. try: h.endheaders() except socket.error, err: raise urllib2.URLError(err)
def RaiseError(self, request, timeout=0): raise urllib2.URLError("Not a real connection.")
class Tier0DasInterface: """ Class handling common Tier0-DAS queries and connected utilities """ def __init__(self, url='https://cmsweb.cern.ch/t0wmadatasvc/prod/', proxy=None): """ Need base url for Tier0-DAS as input """ self._t0DasBaseUrl = url self._debug = False self._retry = 0 self._maxretry = 5 self._proxy = proxy def getData(self, src, tout=5): """ Get the JSON file for a give query specified via the Tier0-DAS url. Timeout can be set via paramter. """ # actually get the json file from the given url of the T0-Das service # and returns the data try: if self._proxy: print "setting proxy" opener = urllib2.build_opener( urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.ProxyHandler({ 'http': self._proxy, 'https': self._proxy })) urllib2.install_opener(opener) req = urllib2.Request(src) req.add_header( "User-Agent", "DQMIntegration/1.0 python/%d.%d.%d" % sys.version_info[:3]) req.add_header("Accept", "application/json") jsonCall = urllib2.urlopen(req, timeout=tout) url = jsonCall.geturl() except urllib2.HTTPError, error: #print error.url errStr = "Cannot retrieve Tier-0 DAS data from URL \"" + error.url + "\"" if self._proxy: errStr += " using proxy \"" + self._proxy + "\"" print errStr print error raise urllib2.HTTPError("FIXME: handle exceptions") except urllib2.URLError, error: if self._retry < self._maxretry: print 'Try # ' + str( self._retry) + " connection to Tier-0 DAS timed-out" self._retry += 1 newtout = tout * self._retry time.sleep(3 * self._retry) return self.getData(src, newtout) else: errStr = "Cannot retrieve Tier-0 DAS data from URL \"" + src + "\"" if self._proxy: errStr += " using proxy \"" + self._proxy + "\"" self._retry = 0 print errStr print error raise urllib2.URLError('TimeOut reading ' + src)
def Get(self, request, timeout=None): # if gcimagebundle is not run on GCE the metadata server will be unreachable raise urllib2.URLError("urlopen error timed out")
def download_file(remote_url, cache=False, show_progress=True): """ Accepts a URL, downloads and optionally caches the result returning the filename, with a name determined by the file's MD5 hash. If ``cache=True`` and the file is present in the cache, just returns the filename. Parameters ---------- remote_url : str The URL of the file to download cache : bool, optional Whether to use the cache show_progress : bool, optional Whether to display a progress bar during the download (default is `True`) Returns ------- local_path : str Returns the local path that the file was download to. Raises ------ `urllib2.URLError` Whenever there's a problem getting the remote file. """ import hashlib import socket from contextlib import closing from tempfile import NamedTemporaryFile, gettempdir from shutil import move from warnings import warn from ..utils.console import ProgressBarOrSpinner missing_cache = False if cache: try: dldir, urlmapfn = _get_download_cache_locs() except (IOError, OSError) as e: msg = 'Remote data cache could not be accessed due to ' estr = '' if len(e.args) < 1 else (': ' + str(e)) warn(CacheMissingWarning(msg + e.__class__.__name__ + estr)) cache = False missing_cache = True # indicates that the cache is missing to raise a warning later try: if cache: # We don't need to acquire the lock here, since we are only reading with _open_shelve(urlmapfn, True) as url2hash: if str(remote_url) in url2hash: return url2hash[str(remote_url)] with closing(urllib2.urlopen(remote_url, timeout=REMOTE_TIMEOUT())) as remote: #keep a hash to rename the local file to the hashed name hash = hashlib.md5() info = remote.info() if 'Content-Length' in info: try: size = int(info['Content-Length']) except ValueError: size = None else: size = None if size is not None: check_free_space_in_dir(gettempdir(), size) if cache: check_free_space_in_dir(dldir, size) if show_progress: progress_stream = sys.stdout else: progress_stream = io.StringIO() dlmsg = "Downloading {0}".format(remote_url) with ProgressBarOrSpinner(size, dlmsg, file=progress_stream) as p: with NamedTemporaryFile(delete=False) as f: try: bytes_read = 0 block = remote.read(DOWNLOAD_CACHE_BLOCK_SIZE()) while block: f.write(block) hash.update(block) bytes_read += len(block) p.update(bytes_read) block = remote.read(DOWNLOAD_CACHE_BLOCK_SIZE()) except: if os.path.exists(f.name): os.remove(f.name) raise if cache: _acquire_download_cache_lock() try: with _open_shelve(urlmapfn, True) as url2hash: # We check now to see if another process has # inadvertently written the file underneath us # already if str(remote_url) in url2hash: return url2hash[str(remote_url)] local_path = os.path.join(dldir, hash.hexdigest()) move(f.name, local_path) url2hash[str(remote_url)] = local_path finally: _release_download_cache_lock() else: local_path = f.name if missing_cache: msg = ('File downloaded to temporary location due to problem ' 'with cache directory and will not be cached.') warn(CacheMissingWarning(msg, local_path)) if DELETE_TEMPORARY_DOWNLOADS_AT_EXIT(): global _tempfilestodel _tempfilestodel.append(local_path) except urllib2.URLError as e: if hasattr(e, 'reason') and hasattr(e.reason, 'errno') and e.reason.errno == 8: e.reason.strerror = e.reason.strerror + '. requested URL: ' + remote_url e.reason.args = (e.reason.errno, e.reason.strerror) raise e except socket.timeout as e: # this isn't supposed to happen, but occasionally a socket.timeout gets # through. It's supposed to be caught in `urrlib2` and raised in this # way, but for some reason in mysterious circumstances it doesn't. So # we'll just re-raise it here instead raise urllib2.URLError(e) return local_path
def do_open(self, http_class, req): data = req.get_data() v_files = [] v_vars = [] # mapping object (dict) if req.has_data() and not isinstance(data, str): if hasattr(data, 'items'): data = data.items() else: try: if len(data) and not isinstance(data[0], tuple): raise TypeError except TypeError: ty, va, tb = sys.exc_info() raise TypeError, "not a valid non-string sequence or mapping object", tb for (k, v) in data: if hasattr(v, 'read'): v_files.append((k, v)) else: v_vars.append((k, v)) # no file ? convert to string if len(v_vars) > 0 and len(v_files) == 0: data = urllib.urlencode(v_vars) v_files = [] v_vars = [] host = req.get_host() if not host: raise urllib2.URLError('no host given') h = http_class(host) # will parse host:port if req.has_data(): h.putrequest('POST', req.get_selector()) if 'Content-type' not in req.headers: if len(v_files) > 0: boundary = mimetools.choose_boundary() length = send_data(v_vars, v_files, boundary) h.putheader('Content-Type', 'multipart/form-data; boundary=%s' % boundary) h.putheader('Content-length', str(length)) else: h.putheader('Content-type', 'application/x-www-form-urlencoded') if 'Content-length' not in req.headers: h.putheader('Content-length', '%d' % len(data)) else: h.putrequest('GET', req.get_selector()) scheme, sel = urllib.splittype(req.get_selector()) sel_host, sel_path = urllib.splithost(sel) h.putheader('Host', sel_host or host) for name, value in self.parent.addheaders: name = name.capitalize() if name not in req.headers: h.putheader(name, value) for k, v in req.headers.items(): h.putheader(k, v) # httplib will attempt to connect() here. be prepared # to convert a socket error to a URLError. try: h.endheaders() except socket.error as err: raise urllib2.URLError(err) if req.has_data(): print v_files if len(v_files) > 0: length = send_data(v_vars, v_files, boundary, h) elif len(v_vars) > 0: # if data is passed as dict ... data = urllib.urlencode(v_vars) h.send(data) else: # "normal" urllib2.urlopen() h.send(data) code, msg, hdrs = h.getreply() fp = h.getfile() if code == 200: resp = urllib.addinfourl(fp, hdrs, req.get_full_url()) resp.code = code resp.msg = msg return resp else: return self.parent.error('http', req, fp, code, msg, hdrs)
def test_url_timeout(self): reason = socket.timeout('too slow') self.urlopen_mock.side_effect = urllib2.URLError(reason) tasks.fetch_manifest('url', self.upload.pk) self.check_validation('Connection to "url" timed out.')
def test_other_url_error(self): reason = Exception('Some other failure.') self.urlopen_mock.side_effect = urllib2.URLError(reason) tasks.fetch_manifest('url', self.upload.pk) self.check_validation('Some other failure.')
def test_connection_error(self): reason = socket.gaierror(8, 'nodename nor servname provided') self.urlopen_mock.side_effect = urllib2.URLError(reason) tasks.fetch_manifest('url', self.upload.pk) self.check_validation('Could not contact host at "url".')
def set_url_error(self, url): self.set_exception(url, urllib2.URLError((errno.ECONNREFUSED, )))
def load(self, url): raise urllib2.URLError("Disconnected")
def smb_open(self, req): global USE_NTLM, MACHINE_NAME host = req.get_host() if not host: raise urllib2.URLError('SMB error: no host given') host, port = splitport(host) if port is None: port = 139 else: port = int(port) # username/password handling user, host = splituser(host) if user: user, passwd = splitpasswd(user) else: passwd = None host = unquote(host) user = user or '' domain = '' if ';' in user: domain, user = user.split(';', 1) passwd = passwd or '' myname = MACHINE_NAME or self.generateClientMachineName() server_name,host = host.split(',') if ',' in host else [None,host] if server_name is None: n = NetBIOS() names = n.queryIPForName(host) if names: server_name = names[0] else: raise urllib2.URLError('SMB error: Hostname does not reply back with its machine name') path, attrs = splitattr(req.get_selector()) if path.startswith('/'): path = path[1:] dirs = path.split('/') dirs = map(unquote, dirs) service, path = dirs[0], '/'.join(dirs[1:]) try: conn = SMBConnection(user, passwd, myname, server_name, domain=domain, use_ntlm_v2 = USE_NTLM) conn.connect(host, port) if req.has_data(): data_fp = req.get_data() filelen = conn.storeFile(service, path, data_fp) headers = "Content-length: 0\n" fp = StringIO("") else: fp = self.createTempFile() file_attrs, retrlen = conn.retrieveFile(service, path, fp) fp.seek(0) headers = "" mtype = mimetypes.guess_type(req.get_full_url())[0] if mtype: headers += "Content-type: %s\n" % mtype if retrlen is not None and retrlen >= 0: headers += "Content-length: %d\n" % retrlen sf = StringIO(headers) headers = mimetools.Message(sf) return addinfourl(fp, headers, req.get_full_url()) except Exception, ex: raise urllib2.URLError, ('smb error: %s' % ex), sys.exc_info()[2]
def raise_error(_): raise urllib2.URLError('Some request error message')
def _request(cls, url, post_data=None, timeout=REQUEST_TIMEOUT, attempts=REQUEST_ATTEMPTS): # change fb__explicitly_shared to fb:explicitly_shared if post_data: post_data = dict( (k.replace('__', ':'), v) for k, v in post_data.items()) logger.info('requesting url %s with post data %s', url, post_data) post_request = (post_data is not None or 'method=post' in url) if post_request and facebook_settings.FACEBOOK_READ_ONLY: logger.info('running in readonly mode') response = dict(id=123456789, setting_read_only=True) return response # nicely identify ourselves before sending the request opener = urllib2.build_opener() opener.addheaders = [('User-agent', 'Open Facebook Python')] # get the statsd path to track response times with path = urlparse(url).path statsd_path = path.replace('.', '_') # give it a few shots, connection is buggy at times timeout_mp = 0 while attempts: # gradually increase the timeout upon failure timeout_mp += 1 extended_timeout = timeout * timeout_mp response_file = None encoded_params = encode_params(post_data) if post_data else None post_string = (urllib.urlencode(encoded_params) if post_data else None) try: start_statsd('facebook.%s' % statsd_path) try: response_file = opener.open(url, post_string, timeout=extended_timeout) response = response_file.read().decode('utf8') except (urllib2.HTTPError, ), e: response_file = e response = response_file.read().decode('utf8') # Facebook sents error codes for many of their flows # we still want the json to allow for proper handling msg_format = 'FB request, error type %s, code %s' logger.warn(msg_format, type(e), getattr(e, 'code', None)) # detect if its a server or application error server_error = cls.is_server_error(e, response) if server_error: # trigger a retry raise urllib2.URLError('Facebook is down %s' % response) break except (urllib2.HTTPError, urllib2.URLError, ssl.SSLError), e: # These are often temporary errors, so we will retry before # failing error_format = 'Facebook encountered a timeout (%ss) or error %s' logger.warn(error_format, extended_timeout, unicode(e)) attempts -= 1 if not attempts: # if we have no more attempts actually raise the error error_instance = facebook_exceptions.convert_unreachable_exception( e) error_msg = 'Facebook request failed after several retries, raising error %s' logger.warn(error_msg, error_instance) raise error_instance
def test_get_httperror(self, urlopen_mock): urlopen_mock.side_effect = urllib2.URLError('BOOM') result = self.collector._get('/path') self.assertFalse(result)
def timeout_catcher(signnum, _): raise urllib2.URLError("read timeout: max 2 sec, is the image large?")
class FunctestUtilsTesting(unittest.TestCase): logging.disable(logging.CRITICAL) def setUp(self): self.url = 'http://www.opnfv.org/' self.timeout = 5 self.dest_path = 'test_path' self.repo_path = 'test_repo_path' self.installer = 'test_installer' self.scenario = 'test_scenario' self.build_tag = 'jenkins-functest-fuel-opnfv-jump-2-daily-master-190' self.build_tag_week = 'jenkins-functest-fuel-baremetal-weekly-master-8' self.version = 'master' self.node_name = 'test_node_name' self.project = 'test_project' self.case_name = 'test_case_name' self.status = 'test_status' self.details = 'test_details' self.db_url = 'test_db_url' self.success_rate = 2.0 self.criteria = 'test_criteria==2.0' self.start_date = 1482624000 self.stop_date = 1482624000 self.start_time = time.time() self.stop_time = time.time() self.readline = -1 self.test_ip = ['10.1.23.4', '10.1.14.15', '10.1.16.15'] self.test_file = 'test_file' self.error_msg = 'test_error_msg' self.cmd = 'test_cmd' self.output_file = 'test_output_file' self.testname = 'testname' self.testcase_dict = {'name': 'testname', 'criteria': self.criteria} self.parameter = 'general.openstack.image_name' self.config_yaml = 'test_config_yaml-' self.db_url_env = 'http://foo/testdb' self.file_yaml = {'general': {'openstack': {'image_name': 'test_image_name'}}} @mock.patch('urllib2.urlopen', side_effect=urllib2.URLError('no host given')) def test_check_internet_connectivity_failed(self, mock_method): self.assertFalse(functest_utils.check_internet_connectivity()) mock_method.assert_called_once_with(self.url, timeout=self.timeout) @mock.patch('urllib2.urlopen') def test_check_internet_connectivity_default(self, mock_method): self.assertTrue(functest_utils.check_internet_connectivity()) mock_method.assert_called_once_with(self.url, timeout=self.timeout) @mock.patch('urllib2.urlopen') def test_check_internet_connectivity_debian(self, mock_method): self.url = "https://www.debian.org/" self.assertTrue(functest_utils.check_internet_connectivity(self.url)) mock_method.assert_called_once_with(self.url, timeout=self.timeout) @mock.patch('urllib2.urlopen', side_effect=urllib2.URLError('no host given')) def test_download_url_failed(self, mock_url): self.assertFalse(functest_utils.download_url(self.url, self.dest_path)) @mock.patch('urllib2.urlopen') def test_download_url_default(self, mock_url): with mock.patch("__builtin__.open", mock.mock_open()) as m, \ mock.patch('functest.utils.functest_utils.shutil.copyfileobj')\ as mock_sh: name = self.url.rsplit('/')[-1] dest = self.dest_path + "/" + name self.assertTrue(functest_utils.download_url(self.url, self.dest_path)) m.assert_called_once_with(dest, 'wb') self.assertTrue(mock_sh.called) def test_get_git_branch(self): with mock.patch('functest.utils.functest_utils.Repo') as mock_repo: mock_obj2 = mock.Mock() attrs = {'name': 'test_branch'} mock_obj2.configure_mock(**attrs) mock_obj = mock.Mock() attrs = {'active_branch': mock_obj2} mock_obj.configure_mock(**attrs) mock_repo.return_value = mock_obj self.assertEqual(functest_utils.get_git_branch(self.repo_path), 'test_branch') @mock.patch('functest.utils.functest_utils.Repo', side_effect=NoSuchPathError) def test_get_git_branch_failed(self, mock_repo): self.assertRaises(NoSuchPathError, lambda: functest_utils.get_git_branch(self.repo_path )) @mock.patch('functest.utils.functest_utils.logger.error') def test_get_installer_type_failed(self, mock_logger_error): with mock.patch.dict(os.environ, {}, clear=True): self.assertEqual(functest_utils.get_installer_type(), "Unknown_installer") mock_logger_error.assert_called_once_with("Impossible to retrieve" " the installer type") def test_get_installer_type_default(self): with mock.patch.dict(os.environ, {'INSTALLER_TYPE': 'test_installer'}, clear=True): self.assertEqual(functest_utils.get_installer_type(), self.installer) @mock.patch('functest.utils.functest_utils.logger.info') def test_get_scenario_failed(self, mock_logger_info): with mock.patch.dict(os.environ, {}, clear=True): self.assertEqual(functest_utils.get_scenario(), "os-nosdn-nofeature-noha") mock_logger_info.assert_called_once_with("Impossible to retrieve " "the scenario.Use " "default " "os-nosdn-nofeature-noha") def test_get_scenario_default(self): with mock.patch.dict(os.environ, {'DEPLOY_SCENARIO': 'test_scenario'}, clear=True): self.assertEqual(functest_utils.get_scenario(), self.scenario) @mock.patch('functest.utils.functest_utils.get_build_tag') def test_get_version_daily_job(self, mock_get_build_tag): mock_get_build_tag.return_value = self.build_tag self.assertEqual(functest_utils.get_version(), self.version) @mock.patch('functest.utils.functest_utils.get_build_tag') def test_get_version_weekly_job(self, mock_get_build_tag): mock_get_build_tag.return_value = self.build_tag_week self.assertEqual(functest_utils.get_version(), self.version) @mock.patch('functest.utils.functest_utils.get_build_tag') def test_get_version_with_dummy_build_tag(self, mock_get_build_tag): mock_get_build_tag.return_value = 'whatever' self.assertEqual(functest_utils.get_version(), 'unknown') @mock.patch('functest.utils.functest_utils.get_build_tag') def test_get_version_unknown(self, mock_get_build_tag): mock_get_build_tag.return_value = "unknown_build_tag" self.assertEqual(functest_utils.get_version(), "unknown") @mock.patch('functest.utils.functest_utils.logger.info') def test_get_pod_name_failed(self, mock_logger_info): with mock.patch.dict(os.environ, {}, clear=True): self.assertEqual(functest_utils.get_pod_name(), "unknown-pod") mock_logger_info.assert_called_once_with("Unable to retrieve " "the POD name from " "environment. Using " "pod name 'unknown-pod'") def test_get_pod_name_default(self): with mock.patch.dict(os.environ, {'NODE_NAME': 'test_node_name'}, clear=True): self.assertEqual(functest_utils.get_pod_name(), self.node_name) @mock.patch('functest.utils.functest_utils.logger.info') def test_get_build_tag_failed(self, mock_logger_info): with mock.patch.dict(os.environ, {}, clear=True): self.assertEqual(functest_utils.get_build_tag(), "none") mock_logger_info.assert_called_once_with("Impossible to retrieve" " the build tag") def test_get_build_tag_default(self): with mock.patch.dict(os.environ, {'BUILD_TAG': self.build_tag}, clear=True): self.assertEqual(functest_utils.get_build_tag(), self.build_tag) def test_get_db_url_env_var(self): with mock.patch.dict(os.environ, {'TEST_DB_URL': self.db_url_env, 'CONFIG_FUNCTEST_YAML': "./functest/ci/config_functest.yaml"}, clear=True): self.assertEqual(functest_utils.get_db_url(), self.db_url_env) @mock.patch('functest.utils.functest_utils.get_functest_config') def test_get_db_url_default(self, mock_get_functest_config): mock_get_functest_config.return_value = self.db_url self.assertEqual(functest_utils.get_db_url(), self.db_url) mock_get_functest_config.assert_called_once_with('results.test_db_url') @mock.patch('functest.utils.functest_utils.logger.info') def test_logger_test_results(self, mock_logger_info): with mock.patch('functest.utils.functest_utils.get_pod_name', return_value=self.node_name), \ mock.patch('functest.utils.functest_utils.get_scenario', return_value=self.scenario), \ mock.patch('functest.utils.functest_utils.get_version', return_value=self.version), \ mock.patch('functest.utils.functest_utils.get_build_tag', return_value=self.build_tag), \ mock.patch('functest.utils.functest_utils.get_db_url', return_value=self.db_url): functest_utils.logger_test_results(self.project, self.case_name, self.status, self.details) mock_logger_info.assert_called_once_with( "\n" "****************************************\n" "\t %(p)s/%(n)s results \n\n" "****************************************\n" "DB:\t%(db)s\n" "pod:\t%(pod)s\n" "version:\t%(v)s\n" "scenario:\t%(s)s\n" "status:\t%(c)s\n" "build tag:\t%(b)s\n" "details:\t%(d)s\n" % {'p': self.project, 'n': self.case_name, 'db': self.db_url, 'pod': self.node_name, 'v': self.version, 's': self.scenario, 'c': self.status, 'b': self.build_tag, 'd': self.details}) def _get_env_dict(self, var): dic = {'INSTALLER_TYPE': self.installer, 'DEPLOY_SCENARIO': self.scenario, 'NODE_NAME': self.node_name, 'BUILD_TAG': self.build_tag} dic.pop(var, None) return dic def _test_push_results_to_db_missing_env(self, env_var): dic = self._get_env_dict(env_var) with mock.patch('functest.utils.functest_utils.get_db_url', return_value=self.db_url), \ mock.patch.dict(os.environ, dic, clear=True), \ mock.patch('functest.utils.functest_utils.logger.error') \ as mock_logger_error: functest_utils.push_results_to_db(self.project, self.case_name, self.start_date, self.stop_date, self.criteria, self.details) mock_logger_error.assert_called_once_with("Please set env var: " + str("\'" + env_var + "\'")) def test_push_results_to_db_missing_installer(self): self._test_push_results_to_db_missing_env('INSTALLER_TYPE') def test_push_results_to_db_missing_scenario(self): self._test_push_results_to_db_missing_env('DEPLOY_SCENARIO') def test_push_results_to_db_missing_nodename(self): self._test_push_results_to_db_missing_env('NODE_NAME') def test_push_results_to_db_missing_buildtag(self): self._test_push_results_to_db_missing_env('BUILD_TAG') def test_push_results_to_db_request_post_failed(self): dic = self._get_env_dict(None) with mock.patch('functest.utils.functest_utils.get_db_url', return_value=self.db_url), \ mock.patch.dict(os.environ, dic, clear=True), \ mock.patch('functest.utils.functest_utils.logger.error') \ as mock_logger_error, \ mock.patch('functest.utils.functest_utils.requests.post', side_effect=requests.RequestException): self.assertFalse(functest_utils. push_results_to_db(self.project, self.case_name, self.start_date, self.stop_date, self.criteria, self.details)) mock_logger_error.assert_called_once_with(test_utils. RegexMatch("Pushing " "Result to" " DB" "(\S+\s*) " "failed:")) def test_push_results_to_db_request_post_exception(self): dic = self._get_env_dict(None) with mock.patch('functest.utils.functest_utils.get_db_url', return_value=self.db_url), \ mock.patch.dict(os.environ, dic, clear=True), \ mock.patch('functest.utils.functest_utils.logger.error') \ as mock_logger_error, \ mock.patch('functest.utils.functest_utils.requests.post', side_effect=Exception): self.assertFalse(functest_utils. push_results_to_db(self.project, self.case_name, self.start_date, self.stop_date, self.criteria, self.details)) self.assertTrue(mock_logger_error.called) def test_push_results_to_db_default(self): dic = self._get_env_dict(None) with mock.patch('functest.utils.functest_utils.get_db_url', return_value=self.db_url), \ mock.patch.dict(os.environ, dic, clear=True), \ mock.patch('functest.utils.functest_utils.requests.post'): self.assertTrue(functest_utils. push_results_to_db(self.project, self.case_name, self.start_date, self.stop_date, self.criteria, self.details)) readline = 0 test_ip = ['10.1.23.4', '10.1.14.15', '10.1.16.15'] @staticmethod def readline_side(): if FunctestUtilsTesting.readline == \ len(FunctestUtilsTesting.test_ip) - 1: return False FunctestUtilsTesting.readline += 1 return FunctestUtilsTesting.test_ip[FunctestUtilsTesting.readline] # TODO: get_resolvconf_ns @mock.patch('functest.utils.functest_utils.dns.resolver.Resolver') def test_get_resolvconf_ns_default(self, mock_dns_resolve): attrs = {'query.return_value': ["test"]} mock_dns_resolve.configure_mock(**attrs) m = mock.Mock() attrs = {'readline.side_effect': self.readline_side} m.configure_mock(**attrs) with mock.patch("__builtin__.open") as mo: mo.return_value = m self.assertEqual(functest_utils.get_resolvconf_ns(), self.test_ip[1:]) def _get_environ(self, var): if var == 'INSTALLER_TYPE': return self.installer elif var == 'DEPLOY_SCENARIO': return self.scenario return var def test_get_ci_envvars_default(self): with mock.patch('os.environ.get', side_effect=self._get_environ): dic = {"installer": self.installer, "scenario": self.scenario} self.assertDictEqual(functest_utils.get_ci_envvars(), dic) def cmd_readline(self): return 'test_value\n' @mock.patch('functest.utils.functest_utils.logger.error') @mock.patch('functest.utils.functest_utils.logger.info') def test_execute_command_args_present_with_error(self, mock_logger_info, mock_logger_error): with mock.patch('functest.utils.functest_utils.subprocess.Popen') \ as mock_subproc_open, \ mock.patch('__builtin__.open', mock.mock_open()) as mopen: FunctestUtilsTesting.readline = 0 mock_obj = mock.Mock() attrs = {'readline.side_effect': self.cmd_readline()} mock_obj.configure_mock(**attrs) mock_obj2 = mock.Mock() attrs = {'stdout': mock_obj, 'wait.return_value': 1} mock_obj2.configure_mock(**attrs) mock_subproc_open.return_value = mock_obj2 resp = functest_utils.execute_command(self.cmd, info=True, error_msg=self.error_msg, verbose=True, output_file=self.output_file) self.assertEqual(resp, 1) msg_exec = ("Executing command: '%s'" % self.cmd) mock_logger_info.assert_called_once_with(msg_exec) mopen.assert_called_once_with(self.output_file, "w") mock_logger_error.assert_called_once_with(self.error_msg) @mock.patch('functest.utils.functest_utils.logger.info') def test_execute_command_args_present_with_success(self, mock_logger_info, ): with mock.patch('functest.utils.functest_utils.subprocess.Popen') \ as mock_subproc_open, \ mock.patch('__builtin__.open', mock.mock_open()) as mopen: FunctestUtilsTesting.readline = 0 mock_obj = mock.Mock() attrs = {'readline.side_effect': self.cmd_readline()} mock_obj.configure_mock(**attrs) mock_obj2 = mock.Mock() attrs = {'stdout': mock_obj, 'wait.return_value': 0} mock_obj2.configure_mock(**attrs) mock_subproc_open.return_value = mock_obj2 resp = functest_utils.execute_command(self.cmd, info=True, error_msg=self.error_msg, verbose=True, output_file=self.output_file) self.assertEqual(resp, 0) msg_exec = ("Executing command: '%s'" % self.cmd) mock_logger_info.assert_called_once_with(msg_exec) mopen.assert_called_once_with(self.output_file, "w") @mock.patch('functest.utils.functest_utils.logger.info') def test_execute_command_args_missing_with_success(self, mock_logger_info, ): with mock.patch('functest.utils.functest_utils.subprocess.Popen') \ as mock_subproc_open: FunctestUtilsTesting.readline = 2 mock_obj = mock.Mock() attrs = {'readline.side_effect': self.cmd_readline()} mock_obj.configure_mock(**attrs) mock_obj2 = mock.Mock() attrs = {'stdout': mock_obj, 'wait.return_value': 0} mock_obj2.configure_mock(**attrs) mock_subproc_open.return_value = mock_obj2 resp = functest_utils.execute_command(self.cmd, info=False, error_msg="", verbose=False, output_file=None) self.assertEqual(resp, 0) @mock.patch('functest.utils.functest_utils.logger.error') def test_execute_command_args_missing_with_error(self, mock_logger_error, ): with mock.patch('functest.utils.functest_utils.subprocess.Popen') \ as mock_subproc_open: FunctestUtilsTesting.readline = 2 mock_obj = mock.Mock() attrs = {'readline.side_effect': self.cmd_readline()} mock_obj.configure_mock(**attrs) mock_obj2 = mock.Mock() attrs = {'stdout': mock_obj, 'wait.return_value': 1} mock_obj2.configure_mock(**attrs) mock_subproc_open.return_value = mock_obj2 resp = functest_utils.execute_command(self.cmd, info=False, error_msg="", verbose=False, output_file=None) self.assertEqual(resp, 1) def _get_functest_config(self, var): return var @mock.patch('functest.utils.functest_utils.logger.error') def test_get_dict_by_test(self, mock_logger_error): with mock.patch('__builtin__.open', mock.mock_open()), \ mock.patch('functest.utils.functest_utils.yaml.safe_load') \ as mock_yaml, \ mock.patch('functest.utils.functest_utils.get_testcases_' 'file_dir'): mock_obj = mock.Mock() attrs = {'get.return_value': [{'testcases': [self.testcase_dict]}]} mock_obj.configure_mock(**attrs) mock_yaml.return_value = mock_obj self.assertDictEqual(functest_utils. get_dict_by_test(self.testname), self.testcase_dict) @mock.patch('functest.utils.functest_utils.get_dict_by_test') def test_get_criteria_by_test_default(self, mock_get_dict_by_test): mock_get_dict_by_test.return_value = self.testcase_dict self.assertEqual(functest_utils.get_criteria_by_test(self.testname), self.criteria) @mock.patch('functest.utils.functest_utils.get_dict_by_test') def test_get_criteria_by_test_failed(self, mock_get_dict_by_test): mock_get_dict_by_test.return_value = None self.assertIsNone(functest_utils.get_criteria_by_test(self.testname)) def test_get_parameter_from_yaml_failed(self): self.file_yaml['general'] = None with mock.patch('__builtin__.open', mock.mock_open()), \ mock.patch('functest.utils.functest_utils.yaml.safe_load') \ as mock_yaml, \ self.assertRaises(ValueError) as excep: mock_yaml.return_value = self.file_yaml functest_utils.get_parameter_from_yaml(self.parameter, self.test_file) self.assertTrue(("The parameter %s is not" " defined in config_functest.yaml" % self.parameter) in excep.exception) def test_get_parameter_from_yaml_default(self): with mock.patch('__builtin__.open', mock.mock_open()), \ mock.patch('functest.utils.functest_utils.yaml.safe_load') \ as mock_yaml: mock_yaml.return_value = self.file_yaml self.assertEqual(functest_utils. get_parameter_from_yaml(self.parameter, self.test_file), 'test_image_name') @mock.patch('functest.utils.functest_utils.get_parameter_from_yaml') def test_get_functest_config_default(self, mock_get_parameter_from_yaml): with mock.patch.dict(os.environ, {'CONFIG_FUNCTEST_YAML': self.config_yaml}): functest_utils.get_functest_config(self.parameter) mock_get_parameter_from_yaml. \ assert_called_once_with(self.parameter, self.config_yaml) def test_check_success_rate_default(self): with mock.patch('functest.utils.functest_utils.get_criteria_by_test') \ as mock_criteria: mock_criteria.return_value = self.criteria resp = functest_utils.check_success_rate(self.case_name, self.success_rate) self.assertEqual(resp, 'PASS') def test_check_success_rate_failed(self): with mock.patch('functest.utils.functest_utils.get_criteria_by_test') \ as mock_criteria: mock_criteria.return_value = self.criteria resp = functest_utils.check_success_rate(self.case_name, 3.0) self.assertEqual(resp, 'FAIL') # TODO: merge_dicts def test_get_testcases_file_dir(self): resp = functest_utils.get_testcases_file_dir() self.assertEqual(resp, "/home/opnfv/repos/functest/" "functest/ci/testcases.yaml") def test_get_functest_yaml(self): with mock.patch('__builtin__.open', mock.mock_open()), \ mock.patch('functest.utils.functest_utils.yaml.safe_load') \ as mock_yaml: mock_yaml.return_value = self.file_yaml resp = functest_utils.get_functest_yaml() self.assertEqual(resp, self.file_yaml) @mock.patch('functest.utils.functest_utils.logger.info') def test_print_separator(self, mock_logger_info): functest_utils.print_separator() mock_logger_info.assert_called_once_with("=======================" "=======================")
def throw_errors(self, num): for x in range(num): yield urllib2.URLError('Bang!') yield True
def fetch(self, url, timeout=None): """Attempts to fetch the URL requested which should refer to a robots.txt file, e.g. http://example.com/robots.txt. """ # ISO-8859-1 is the default encoding for text files per the specs for # HTTP 1.0 (RFC 1945 sec 3.6.1) and HTTP 1.1 (RFC 2616 sec 3.7.1). # ref: http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1 encoding = "iso-8859-1" content = "" expires_header = None content_type_header = None self._response_code = 0 self._source_url = url if self.user_agent: req = urllib_request.Request(url, None, {'User-Agent': self.user_agent}) else: req = urllib_request.Request(url) try: if timeout: f = urllib_request.urlopen(req, timeout=timeout) else: f = urllib_request.urlopen(req) content = f.read(MAX_FILESIZE) # As of Python 2.5, f.info() looks like it returns the HTTPMessage # object created during the connection. expires_header = f.info().get("expires") content_type_header = f.info().get("Content-Type") # As of Python 2.4, this file-like object reports the response # code, too. if hasattr(f, "code"): self._response_code = f.code else: self._response_code = 200 f.close() except urllib_error.URLError: # This is a slightly convoluted way to get the error instance, # but it works under Python 2 & 3. error_instance = sys.exc_info() if len(error_instance) > 1: error_instance = error_instance[1] if hasattr(error_instance, "code"): self._response_code = error_instance.code # MK1996 section 3.4 says, "...robots should take note of Expires # header set by the origin server. If no cache-control directives # are present robots should default to an expiry of 7 days". # This code is lazy and looks at the Expires header but not # Cache-Control directives. self.expiration_date = None if self._response_code >= 200 and self._response_code < 300: # All's well. if expires_header: self.expiration_date = email_utils.parsedate_tz(expires_header) if self.expiration_date: # About time zones -- the call to parsedate_tz() returns a # 10-tuple with the time zone offset in the 10th element. # There are 3 valid formats for HTTP dates, and one of # them doesn't contain time zone information. (UTC is # implied since all HTTP header dates are UTC.) When given # a date that lacks time zone information, parsedate_tz() # returns None in the 10th element. mktime_tz() interprets # None in the 10th (time zone) element to mean that the # date is *local* time, not UTC. # Therefore, if the HTTP timestamp lacks time zone info # and I run that timestamp through parsedate_tz() and pass # it directly to mktime_tz(), I'll get back a local # timestamp which isn't what I want. To fix this, I simply # convert a time zone of None to zero. It's much more # difficult to explain than to fix. =) # ref: http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3.1 if self.expiration_date[9] is None: self.expiration_date = self.expiration_date[:9] + (0, ) self.expiration_date = email_utils.mktime_tz( self.expiration_date) if self.use_local_time: # I have to do a little more converting to get this # UTC timestamp into localtime. self.expiration_date = time.mktime( time.gmtime(self.expiration_date)) # else: # The expires header was garbage. if not self.expiration_date: self.expiration_date = self._now() + SEVEN_DAYS if (self._response_code >= 200) and (self._response_code < 300): # All's well. media_type, encoding = _parse_content_type_header( content_type_header) # RFC 2616 sec 3.7.1 -- # When no explicit charset parameter is provided by the sender, # media subtypes of the "text" type are defined to have a default # charset value of "ISO-8859-1" when received via HTTP. # http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1 if not encoding: encoding = "iso-8859-1" elif self._response_code in (401, 403): # 401 or 403 ==> Go away or I will taunt you a second time! # (according to MK1996) content = "User-agent: *\nDisallow: /\n" elif self._response_code == 404: # No robots.txt ==> everyone's welcome content = "" else: # Uh-oh. I punt this up to the caller. raise urllib_error.URLError(self._response_code) if ((PY_MAJOR_VERSION == 2) and isinstance(content, str)) or \ ((PY_MAJOR_VERSION > 2) and (not isinstance(content, str))): # This ain't Unicode yet! It needs to be. # Unicode decoding errors are another point of failure that I punt # up to the caller. try: content = content.decode(encoding) except UnicodeError: msg = "Robots.txt contents are not in the encoding expected (%s)." % encoding raise UnicodeError(msg) except (LookupError, ValueError): # LookupError ==> Python doesn't have a decoder for that encoding. # One can also get a ValueError here if the encoding starts with # a dot (ASCII 0x2e). See Python bug 1446043 for details. This # bug was supposedly fixed in Python 2.5. msg = """I don't understand the encoding "%s".""" % encoding raise UnicodeError(msg) # Now that I've fetched the content and turned it into Unicode, I # can parse it. self.parse(content)
def _authenticate_(self, account, user, passwd, new_passwd=None, timeout=15, duration=3600): if user == 'admin' and duration > 3600: # admin cannot have more than 1 hour duration duration = 3600 # because of the variability, we need to keep this here, not in __init__ auth_path = self.TEMPLATE.format(dur=duration) if not self.dns_enabled: auth_path = self.NON_DNS_QUERY_PATH + auth_path else: auth_path = '/' + auth_path host = self.host if self.dns_enabled: host = 'tokens.{0}'.format(host) if self.validate_certs: conn = CertValidatingHTTPSConnection(host, self.port, timeout=timeout, **self.kwargs) else: conn = httplib.HTTPSConnection(host, self.port, timeout=timeout) if new_passwd: auth_string = u"{user}@{account};{pw}@{new_pw}".format( user=base64.b64encode(user), account=base64.b64encode(account), pw=base64.b64encode(passwd), new_pw=new_passwd) else: auth_string = u"{user}@{account}:{pw}".format( user=base64.b64encode(user), account=base64.b64encode(account), pw=passwd) encoded_auth = base64.b64encode(auth_string) headers = {'Authorization': "Basic %s" % encoded_auth} try: conn.request('GET', auth_path, '', headers) response = conn.getresponse() if response.status != 200: raise urllib2.HTTPError(url='', code=response.status, msg=response.reason, hdrs=None, fp=None) body = response.read() # parse AccessKeyId, SecretAccessKey and SessionToken creds = Credentials() h = BotoXmlHandler(creds, None) parseString(body, h) return creds except SSLError as err: if err.message != '': raise urllib2.URLError(str(err)) else: raise urllib2.URLError(err[1]) except socket.error as err: # when dns enabled, but path cloud, we get here with # err=gaierror(8, 'nodename nor servname provided, or not known') # when dns disabled, but path cloud, we get here with # err=gaierror(8, 'nodename nor servname provided, or not known') raise urllib2.URLError(str(err))
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm, headers): user, pw = self.passwd.find_user_password(realm, req.get_full_url()) if pw is not None: # ntlm secures a socket, so we must use the same socket for the complete handshake headers = dict(req.headers) headers.update(req.unredirected_hdrs) auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(user) if req.headers.get(self.auth_header, None) == auth: return None headers[self.auth_header] = auth host = req.get_host() if not host: raise urllib2.URLError('no host given') h = None if req.get_full_url().startswith('https://'): h = httplib.HTTPSConnection(host) # will parse host:port else: h = httplib.HTTPConnection(host) # will parse host:port h.set_debuglevel(self._debuglevel) # we must keep the connection because NTLM authenticates the connection, not single requests headers["Connection"] = "Keep-Alive" headers = dict( (name.title(), val) for name, val in headers.items()) h.request(req.get_method(), req.get_selector(), req.data, headers) r = h.getresponse() r.begin() r._safe_read(int(r.getheader('content-length'))) if r.getheader('set-cookie'): # this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out) headers['Cookie'] = r.getheader('set-cookie') r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open) auth_header_value = r.getheader(auth_header_field, None) (ServerChallenge, NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE( auth_header_value[5:]) user_parts = user.split('\\', 1) DomainName = user_parts[0].upper() UserName = user_parts[1] auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE( ServerChallenge, UserName, DomainName, pw, NegotiateFlags) headers[self.auth_header] = auth headers["Connection"] = "Close" headers = dict( (name.title(), val) for name, val in headers.items()) try: h.request(req.get_method(), req.get_selector(), req.data, headers) # none of the configured handlers are triggered, for example redirect-responses are not handled! response = h.getresponse() def notimplemented(): raise NotImplementedError response.readline = notimplemented infourl = addinfourl(response, response.msg, req.get_full_url()) infourl.code = response.status infourl.msg = response.reason return infourl except socket.error, err: raise urllib2.URLError(err)
from test_resources.general_search_results import search_results from test_resources.tracker_results import tracker_results from test_resources.fake_tracker import Tracker from BeautifulSoup import BeautifulSoup #to mock out urlopen FakeSearchSock = Mock() FakeSearchSock.read = Mock(return_value=search_results) FakeSearchUrlOpen = Mock(return_value=FakeSearchSock) FakeTrackerSock = Mock() FakeTrackerSock.read = Mock(return_value=tracker_results) FakeTrackerUrlOpen = Mock(return_value=FakeTrackerSock) FakeEmptySock = Mock() FakeEmptySock.read = Mock(return_value='') FakeEmptyUrlOpen = Mock(return_value=FakeEmptySock) FakeUrlOpenWithError = Mock(side_effect=urllib2.URLError('')) class FakeChangingUrlOpen(object): def __init__(self): self._call_count = 0 self.args = [] def call(self, *args): self.args = args if self._call_count == 0: self._call_count += 1 raise urllib2.URLError('') else: return FakeEmptySock
class GenerateReportTesting(unittest.TestCase): logging.disable(logging.CRITICAL) def test_init(self): test_array = gen_report.init() self.assertEqual(test_array, []) @mock.patch('functest.ci.generate_report.urllib2.urlopen', side_effect=urllib2.URLError('no host given')) def test_get_results_from_db_fail(self, mock_method): url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(), CONST.BUILD_TAG) self.assertIsNone(gen_report.get_results_from_db()) mock_method.assert_called_once_with(url) @mock.patch('functest.ci.generate_report.urllib2.urlopen', return_value={'results': []}) def test_get_results_from_db_success(self, mock_method): url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(), CONST.BUILD_TAG) self.assertEqual(gen_report.get_results_from_db(), None) mock_method.assert_called_once_with(url) def test_get_data(self): self.assertIsInstance(gen_report.get_data({'result': ''}, ''), dict) def test_print_line_with_ci_run(self): CONST.IS_CI_RUN = True w1 = 'test_print_line' test_str = ("| %s| %s| %s| %s| %s|\n" % (w1.ljust(gen_report.COL_1_LEN - 1), ''.ljust(gen_report.COL_2_LEN - 1), ''.ljust(gen_report.COL_3_LEN - 1), ''.ljust(gen_report.COL_4_LEN - 1), ''.ljust(gen_report.COL_5_LEN - 1))) self.assertEqual(gen_report.print_line(w1), test_str) def test_print_line_without_ci_run(self): CONST.IS_CI_RUN = False w1 = 'test_print_line' test_str = ("| %s| %s| %s| %s|\n" % (w1.ljust(gen_report.COL_1_LEN - 1), ''.ljust(gen_report.COL_2_LEN - 1), ''.ljust(gen_report.COL_3_LEN - 1), ''.ljust(gen_report.COL_4_LEN - 1))) self.assertEqual(gen_report.print_line(w1), test_str) def test_print_line_no_column_with_ci_run(self): CONST.IS_CI_RUN = True TOTAL_LEN = gen_report.COL_1_LEN + gen_report.COL_2_LEN TOTAL_LEN += gen_report.COL_3_LEN + gen_report.COL_4_LEN + 2 TOTAL_LEN += gen_report.COL_5_LEN + 1 test_str = ("| %s|\n" % 'test'.ljust(TOTAL_LEN)) self.assertEqual(gen_report.print_line_no_columns('test'), test_str) def test_print_line_no_column_without_ci_run(self): CONST.IS_CI_RUN = False TOTAL_LEN = gen_report.COL_1_LEN + gen_report.COL_2_LEN TOTAL_LEN += gen_report.COL_3_LEN + gen_report.COL_4_LEN + 2 test_str = ("| %s|\n" % 'test'.ljust(TOTAL_LEN)) self.assertEqual(gen_report.print_line_no_columns('test'), test_str) def test_print_separator_with_ci_run(self): CONST.IS_CI_RUN = True test_str = ("+" + "=" * gen_report.COL_1_LEN + "+" + "=" * gen_report.COL_2_LEN + "+" + "=" * gen_report.COL_3_LEN + "+" + "=" * gen_report.COL_4_LEN + "+" + "=" * gen_report.COL_5_LEN) test_str += '+\n' self.assertEqual(gen_report.print_separator(), test_str) def test_print_separator_without_ci_run(self): CONST.IS_CI_RUN = False test_str = ("+" + "=" * gen_report.COL_1_LEN + "+" + "=" * gen_report.COL_2_LEN + "+" + "=" * gen_report.COL_3_LEN + "+" + "=" * gen_report.COL_4_LEN) test_str += "+\n" self.assertEqual(gen_report.print_separator(), test_str) @mock.patch('functest.ci.generate_report.logger.info') def test_main_with_ci_run(self, mock_method): CONST.IS_CI_RUN = True gen_report.main() mock_method.assert_called_once_with(test_utils.SubstrMatch('URL')) @mock.patch('functest.ci.generate_report.logger.info') def test_main_with_ci_loop(self, mock_method): CONST.CI_LOOP = 'daily' gen_report.main() mock_method.assert_called_once_with(test_utils.SubstrMatch('CI LOOP')) @mock.patch('functest.ci.generate_report.logger.info') def test_main_with_scenario(self, mock_method): CONST.DEPLOY_SCENARIO = 'test_scenario' gen_report.main() mock_method.assert_called_once_with(test_utils.SubstrMatch('SCENARIO')) @mock.patch('functest.ci.generate_report.logger.info') def test_main_with_build_tag(self, mock_method): CONST.BUILD_TAG = 'test_build_tag' gen_report.main() mock_method.assert_called_once_with(test_utils. SubstrMatch('BUILD TAG'))
def failSome(_url, symbol, _product): if symbol.file_name == fail.file_name: raise urllib2.URLError('network failure')
# the socket has been closed by the server since we # last used the connection. if DEBUG: print "failed to re-use connection to %s" % host h.close() else: if DEBUG: print "re-using connection to %s" % host need_new_connection = 0 if need_new_connection: if DEBUG: print "creating new connection to %s" % host h = http_class(host) self._connections[key] = h self._start_connection(h, req) r = h.getresponse() except socket.error, err: if h: h.close() raise urllib2.URLError(err) # if not a persistent connection, don't try to reuse it if r.will_close: self._remove_connection(host) if DEBUG: print "STATUS: %s, %s" % (r.status, r.reason) r._handler = self r._host = host r._url = req.get_full_url() #if r.status == 200 or not HANDLE_ERRORS: #return r if r.status == 200 or not HANDLE_ERRORS: # [speedplane] Must return an adinfourl object resp = urllib2.addinfourl(r, r.msg, req.get_full_url())
def urlopener(url_or_request, log, **kwargs): """ Utility function for pulling back a url, with a retry of 3 times, increasing the timeout, etc. Re-raises any errors as URLError. .. warning:: This is being replaced by requests library. flexget.utils.requests should be used going forward. :param str url_or_request: URL or Request object to get. :param log: Logger to log debug info and errors to :param kwargs: Keyword arguments to be passed to urlopen :return: The file-like object returned by urlopen """ from flexget.utils.requests import is_unresponsive, set_unresponsive if isinstance(url_or_request, urllib2.Request): url = url_or_request.get_host() else: url = url_or_request if is_unresponsive(url): msg = '%s is known to be unresponsive, not trying again.' % urlparse( url).hostname log.warning(msg) raise urllib2.URLError(msg) retries = kwargs.get('retries', 3) timeout = kwargs.get('timeout', 15.0) # get the old timeout for sockets, so we can set it back to that when done. This is NOT threadsafe by the way. # In order to avoid requiring python 2.6, we're not using the urlopen timeout parameter. That really should be used # after checking for python 2.6. oldtimeout = socket.getdefaulttimeout() try: socket.setdefaulttimeout(timeout) handlers = [SmartRedirectHandler()] if urllib2._opener: handlers.extend(urllib2._opener.handlers) if kwargs.get('handlers'): handlers.extend(kwargs['handlers']) if len(handlers) > 1: handler_names = [h.__class__.__name__ for h in handlers] log.debug( 'Additional handlers have been specified for this urlopen: %s' % ', '.join(handler_names)) opener = urllib2.build_opener(*handlers).open for i in range(retries): # retry getting the url up to 3 times. if i > 0: time.sleep(3) try: retrieved = opener(url_or_request, kwargs.get('data')) except urllib2.HTTPError as e: if e.code < 500: # If it was not a server error, don't keep retrying. log.warning('Could not retrieve url (HTTP %s error): %s' % (e.code, e.url)) raise log.debug('HTTP error (try %i/%i): %s' % (i + 1, retries, e.code)) except (urllib2.URLError, socket.timeout) as e: if hasattr(e, 'reason'): reason = str(e.reason) else: reason = 'N/A' if reason == 'timed out': set_unresponsive(url) log.debug('Failed to retrieve url (try %i/%i): %s' % (i + 1, retries, reason)) except httplib.IncompleteRead as e: log.critical('Incomplete read - see python bug 6312') break else: # make the returned instance usable in a with statement by adding __enter__ and __exit__ methods def enter(self): return self def exit(self, exc_type, exc_val, exc_tb): self.close() retrieved.__class__.__enter__ = enter retrieved.__class__.__exit__ = exit return retrieved log.warning('Could not retrieve url: %s' % url_or_request) raise urllib2.URLError('Could not retrieve url after %s tries.' % retries) finally: socket.setdefaulttimeout(oldtimeout)
def fake_check_call(self, argv, **kwargs): if argv[-2] == TestRsyncHandler.fail_url: raise urllib2.URLError('fail url') if not os.path.exists(argv[-1]): open(argv[-1], 'w').write(TestRsyncHandler.file_data)