def test_load_swagger_resource_aggregate_raises_url_error(
self, mock_perform_request, mock_utils, mock_app):
# create extra service
crm_lm = factories.LogicModule(
name='Contacts and Appointments Service',
endpoint='http://crm.example.com',
endpoint_name='crm',
relationships={})
# mock schema urls and app
product_schema_url = {self.lm.endpoint_name: self.lm.endpoint}
crm_schema_url = {'crm': crm_lm.endpoint}
mock_utils.get_swagger_urls.side_effect = [
product_schema_url, crm_schema_url
]
msg = f'Make sure that {crm_lm.endpoint} is accessible.'
exception_obj = error.URLError(msg)
mock_app.side_effect = exception_obj
# mock service response
service_response = Mock(PySwaggerResponse)
service_response.status = 200
service_response.data = self.response_data
mock_perform_request.return_value = service_response
# make api request and validate error
path = '/old/{}/{}/'.format(self.lm.endpoint_name, 'products')
with self.assertRaises(error.URLError) as context:
self.client.get(path, {'aggregate': 'true'})
self.assertTrue(msg in context.exception)
def _get_fake_responses(
self, url: str
) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, str]]]:
"""Return response and headers if faked for this URL in uaclient.conf.
:return: A tuple of response and header dicts if the URL has an overlay
response defined. Return (None, {}) otherwise.
:raises exceptions.URLError: When faked response "code" is != 200.
URLError reason will be "response" value and any optional
"headers" provided.
"""
responses = self._get_response_overlay(url)
if not responses:
return None, {}
if len(responses) == 1:
# When only one respose is defined, repeat it for all calls
response = responses[0]
else:
# When multiple responses defined pop the first one off the list.
response = responses.pop(0)
if response["code"] == 200:
return response["response"], response.get("headers", {})
# Must be an error
e = error.URLError(response["response"])
raise exceptions.UrlError(
e,
code=response["code"],
headers=response.get("headers", {}),
url=url,
)
def fetch_url(self, url: str) -> HTTPResponse:
req = request.Request(url, data=None, headers={'User-Agent': self.UA})
res = request.urlopen(req)
if isinstance(res, response.addinfourl):
raise error.URLError("Got FTP handle instead of HTTP response")
return res # type: ignore
def test_articles_skips_unreachable_articles(self):
from urllib import error
self.create_patch('argos.core.membrane.extractor.extract_entry_data',
side_effect=[error.URLError('unreachable')])
articles = []
collector.get_articles(self.feed, lambda a: articles.append(a))
self.assertEquals(len(articles), 0)
def test_monitor_call_for_failure(self, mock_urlopen):
mock_urlopen.side_effect = urlerr.URLError("MOCK Error")
test_vnf = {}
test_kwargs = {'mgmt_ip': 'a.b.c.d'}
monitor_return = self.monitor_http_ping.monitor_call(
test_vnf, test_kwargs)
self.assertEqual('failure', monitor_return)
def open(self, url, conn_timeout=None):
if conn_timeout == 0:
raise urllib_error.URLError('Could not reach %s within deadline.' % url)
if url.startswith('http'):
self.opened.set()
if self.error:
raise urllib_error.HTTPError(url, self.error, None, None, Compatibility.BytesIO(b'glhglhg'))
return urllib_request.addinfourl(Compatibility.BytesIO(self.rv), url, None, self.code)
def open(self, url, conn_timeout=None, **kw):
"""
Wrapper in front of urlopen that more gracefully handles odd network environments.
"""
url = self.maybe_local_url(url)
with TRACER.timed('Fetching', V=1):
if not self.reachable(url, conn_timeout=conn_timeout):
raise urllib_error.URLError('Could not reach %s within deadline.' % url)
return urllib_request.urlopen(url, **kw)
def get_api_token(email: str, password: str) -> str:
try:
api_token_response = hit_api('/API/GetAPIToken', {
'Email': email,
'Password': password
})
api_token: str = loads(api_token_response)['APIToken']
return api_token
except error.URLError as e:
raise error.URLError('Error getting API Token')
def encode_url(self, url, conn_timeout=None):
target, target_tmp, headers, headers_tmp = self.translate_all(url)
with contextlib.closing(self.really_open(url, conn_timeout=conn_timeout)) as http_fp:
if http_fp.getcode() != 200:
raise urllib_error.URLError('Non-200 response code from %s' % url)
with TRACER.timed('Caching', V=2):
with open(target_tmp, 'wb') as disk_fp:
disk_fp.write(http_fp.read())
with open(headers_tmp, 'wb') as headers_fp:
headers_fp.write(struct.pack('>h', http_fp.code or 0))
headers_fp.write(str(http_fp.headers).encode('utf8'))
os.rename(target_tmp, target)
os.rename(headers_tmp, headers)
def do_open(self, http_class, req, **http_conn_args):
host = req.host
if not host:
raise error.URLError('no host given')
con = self.get_connection(host, client.HTTPConnection, req)
headers = dict(req.headers)
headers.update(req.unredirected_hdrs)
headers['Connection'] = 'keep-alive'
headers = dict((name.title(), val) for name, val in headers.items())
if req._tunnel_host:
tunnel_headers = {}
proxy_auth_hdr = "Proxy-Authorization"
if proxy_auth_hdr in headers:
tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
# Proxy-Authorization should not be sent to origin
# server.
del headers[proxy_auth_hdr]
con.set_tunnel(req._tunnel_host, headers=tunnel_headers)
try:
try:
con.request(req.get_method(), req.selector, req.data, headers)
except OSError as err: # timeout error
raise error.URLError(err)
r = con.getresponse()
except:
con.close()
raise
if con.sock:
self.cache_connection(host, con)
r.url = req.get_full_url()
r.msg = r.reason
return r
def init_page(self, host_name, innerURL, url):
if self.hosts.get(host_name) is None:
self.hosts[host_name] = {
'host_count': 0,
'host_links_count': 0,
'innerURL': {innerURL},
'externalURL': set(),
'robot': RobotFileParser('http://%s/robots.txt' % host_name)
}
try:
info = request.urlopen(
'http://%s/robots.txt' % host_name,
timeout=self.robot_wait).info()['Content-Type']
if info is None or 'text/plain' not in info:
raise error.URLError('robots.txt is invalid')
self.hosts[host_name]['robot'].read(
) # may throw error if its url is not valid
except Exception:
self.hosts[host_name]['robot'].allow_all = True
self.hosts[host_name]['host_count'] += 1
def make_request(self, url, http_method):
"""
make HTTP or HTTPS requests
@param url: site URL address
@param http_method: HTTP protocol type
@return dictionary with headers, content and encoding
"""
request = Request(url, method=http_method)
try:
with urlopen(request, timeout=self.timeout) as conn:
encoding = conn.headers.get_content_charset() or 'UTF-8'
return {"url": url,
"code": conn.code,
"type": conn.headers['Content-Type'],
"headers": conn.headers,
"content": conn.read(),
"encoding": encoding}
except error.HTTPError:
return {"url": url, "code": 500}
except error.URLError as e:
raise error.URLError(f'bad connection {e}')
def init_page(self, host_name, innerURL, url):
if self.hosts.get(host_name) is None:
self.hosts[host_name] = {
'host_count': 0,
'host_links_count': 0,
'innerURL': {innerURL},
'externalURL': set(),
'robot': RobotFileParser('http://%s/robots.txt' % host_name)
}
try:
info = request.urlopen(
'http://%s/robots.txt' % host_name,
timeout=self.robot_wait).info()['Content-Type']
if info is None or 'text/plain' not in info:
raise error.URLError('robots.txt is invalid')
self.hosts[host_name]['robot'].read() # read robots.txt
except (error.URLError, error.HTTPError, UnicodeDecodeError,
RemoteDisconnected, ConnectionResetError, socket.timeout,
Exception):
self.hosts[host_name]['robot'].allow_all = True
self.hosts[host_name]['innerURL'].add(innerURL)
self.hosts[host_name]['host_count'] += 1
def test_download_image_handles_os_and_image_provider_errors(self):
aircraft = Aircraft(
[None, 'bar', 'bar.png', self.seen_datetime, self.seen_datetime])
image.request.urlretrieve = mock.MagicMock(side_effect=OSError)
filename = self.image_service.download_image('http://foo.com/bar.png',
aircraft)
image.request.urlretrieve.assert_called_once_with(
'http://foo.com/bar.png',
'/acars-server/acarsserver/service/../app/assets/img/aircrafts/large/bar.png'
)
self.assertIsNone(filename)
image.request.urlretrieve = mock.MagicMock(
side_effect=error.URLError('Not Found'))
filename = self.image_service.download_image('http://foo.com/bar.png',
aircraft)
image.request.urlretrieve.assert_called_once_with(
'http://foo.com/bar.png',
'/acars-server/acarsserver/service/../app/assets/img/aircrafts/large/bar.png'
)
self.assertIsNone(filename)
def urlopen(*args, **kwargs):
if raise_url_error:
raise error.URLError('FAKE ERROR')
if raise_http_error:
raise error.HTTPError(url='',
code=raise_http_error,
msg='FAKE ERROR',
hdrs={},
fp=None)
class Response:
@staticmethod
def read():
return json.dumps({
'transId': returned_token,
'cardNumber': '1111-2222-3333-4444',
'status': returned_status,
'amount': returned_amount,
'errorMessage': error_message,
'errorCode': error_code
}).encode()
return Response()
def __init__(self, tr_server, user_name, password):
"""
This method will initialize the TestRail server using the user name and password provided
:param tr_server: Name of the Test Rail server
:param user_name: TestRail user id
:param password: TestRail password
"""
file_dir = os.path.split(os.path.realpath(__file__))[0]
logging.config.fileConfig(os.path.join(file_dir, "trlogger.ini"))
# Configure the logger
self._log = logging.getLogger('testrail')
self._log.info("Starting TestRail application ")
# TestRail Connection status
# Note: This variable is used to ensure we have a valid TestRail Instance
self._connection_status = False
try:
# Check if the URL is valid
self._client = APIClient(tr_server)
self._client.password = password
self._client.user = user_name
self._connection_status = True
# Check if the url, user name and password is set correctly by accessing an API
self._client.send_get(Defines.TR_API_GET_PROJ + "/" +
str(Defines.TR_PROJ_ID_OTG))
self._log.info(
"Connected to TestRail server {} with user-id {}".format(
tr_server, user_name))
except err.URLError:
self._log.exception("Url: {} is not valid".format(tr_server))
raise err.URLError("Error: Please check the URL")
except APIError:
self._log.critical(
"User-id or Password is not correct. Failed to connect with TestRail url: {}"
.format(tr_server))
raise APIError("Error: Please check user id and password")
def test_articles_skips_unreachable_articles(self):
from urllib import error
self.create_patch('argos.core.membrane.feed.extract_entry_data',
side_effect=error.URLError('unreachable'))
articles = feed.articles(self.source)
self.assertEquals(len(articles), 0)
def __init__(self, url):
if not is_url(url):
raise error.URLError(url)
else:
self._parsed = urllib.parse.urlparse(url)
def test_returns_false_when_cra_not_running(self, mock_urlopen):
mock_urlopen.side_effect = url_error.URLError('CRA not running')
self.assertFalse(server_check.hosted_by_liveserver(CRA_URL))
def mock_urlopen(url, timeout=None):
raise urlerror.URLError("Mock network error")
def hotel_url_content(url):
ntries = 10
for tries in range(ntries):
try:
response = urlopen(url, timeout=20)
break
except error.URLError as err:
print('ReTry: %s' % url)
time.sleep(30)
if tries == ntries - 1:
raise error.URLError('')
soup = BeautifulSoup(response, "html5lib")
## extract the general information
general_information = json.loads(
soup.find('script', type="application/ld+json").text)
#return general_information
try:
review_date = general_information['datePublished']
except:
try:
review_date = soup.find('div', {
'class':
"ui_column is-10-desktop is-12-tablet is-12-mobile"
}).find('span', {'class': "ratingDate"})['title']
except:
review_date = 'unknown'
try:
title = general_information['name']
except:
title = 'unknown'
try:
content = general_information['reviewBody']
except:
content = 'unknown'
try:
overall_rating = int(
general_information['reviewRating']['ratingValue'])
except:
overall_rating = 'unknown'
##return review_date, title, content, overall_rating
## extract the traveling type and date
stay_info = soup.find('div', {'class': "recommend-titleInline"}).text
stay_hardcode = 'Stayed: '
comma_hardcode = ', '
if 'travelled' in stay_info:
stay_date = stay_info[len(stay_hardcode):stay_info.find(',')]
else:
stay_date = stay_info[len(stay_hardcode):]
traveling_type = ''
traveling_type_table = ['family', 'couple', 'solo', 'business', 'friends']
for type_ in traveling_type_table:
if type_ in stay_info:
traveling_type = type_
break
if traveling_type == '': ## save the whole crwl info if the traveling type cannot be detacted
traveling_type = 'unknown'
#try:
# print ('type cannot be detected')
# traveling_type = stay_info[len('travelled as a '):]
#except:
# traveling_type = stay_info
##return stay_date, traveling_type
## ranking part
star_info = soup.findAll(
'div', {'class': "ui_column is-10-desktop is-12-tablet is-12-mobile"})
star_info = star_info[0].findAll('li', {'class': "recommend-answer"})
description = 'ui_bubble_rating bubble_'
ranking_dict = dict()
if len(
star_info
) > 0: #only do this when the reviewer provides the rating of each area
for rate in star_info:
rate_area = rate.find('div', {
'class': "recommend-description"
}).text
pointer = str(rate).find(description) + len(
'ui_bubble_rating bubble_')
try:
rate_mark = int(str(rate)[pointer:pointer + 2]) / 10
except:
rate_mark = str(rate)
ranking_dict[rate_area] = rate_mark
##return ranking_dict
## review information
try:
reviewer_name = soup.find(
'div', {
'class': "prw_rup prw_reviews_member_info_resp_sur"
}).find('div', {
'class': "info_text"
}).find('div').get_text()
except:
reviewer_name = 'unknown'
try:
reviewer_contributions = int(
soup.find('div', {
'class': "prw_rup prw_reviews_member_info_resp_sur"
}).find('span', {
'class': 'badgetext'
}).get_text())
except:
reviewer_contributions = 'unknown'
try:
reviewer_location = soup.find(
'div', {
'class': "prw_rup prw_reviews_member_info_resp_sur"
}).find('div', {
'class': "userLoc"
}).get_text()
except:
reviewer_location = 'unknown'
#return reviewer_name, reviewer_contributions, reviewer_location
return review_date, title, content, overall_rating, stay_date, traveling_type, ranking_dict, reviewer_name, reviewer_contributions, reviewer_location
def rental_url_content(reviewer_url, review_code):
review_code = 'review_' + review_code
ntries = 10
for tries in range(ntries):
try:
response = urlopen(reviewer_url, timeout=20)
break
except error.URLError as err:
print('ReTry: %s' % reviewer_url)
time.sleep(30)
if tries == ntries - 1:
raise error.URLError('')
soup = BeautifulSoup(response)
# general information from json
json_data = json.loads(str(soup.find('script', {'type': 'application/ld+json'}).text))
title = json_data['name']
content = json_data['reviewBody']
overall_rating = json_data['reviewRating']['ratingValue']
###
### extract the table for the rest data
table = soup.find('div', {'id': str(review_code)})
###
# review date and stay date
date = table.find('span', {'class': 'ratingDate'}).get_text().replace('\n', '')
review_date_message = 'Reviewed '
stay_date_message = 'for a stay in '
try:
pointer = date.find(stay_date_message)
if pointer >= 1:
stay_date = date[pointer + len(stay_date_message):]
except:
stay_date = ''
try:
pointer = date.find(review_date_message)
if len(stay_date) == 0:
end_pointer = len(date)
else:
end_pointer = date.find(stay_date_message)
review_date = date[pointer + len(review_date_message):end_pointer]
except:
review_date = ''
###
# rank table
ranking_dict = dict()
try:
ranking_table = table.find('table', {'class': "vrReviewRatings"}).findAll('td')
for detail in ranking_table:
label = detail.find('span', {'class': 'vrReviewRatingLabel'}).get_text()
rank = ' '.join(detail.select('span[class*="ui_bubble"]')[0].get('class'))
rank = int((rank[rank.find('ui_bubble_rating bubble_') + len('ui_bubble_rating bubble_'):])) / 10
ranking_dict[label] = rank
except:
pass
###
# traveling type
try:
traveling_type = 'unknown'
all_div = table.select('div')
for div in all_div:
if div.find('span', {'class': 'vrReviewItem itemLabel'}) and (
div.find('span', {'class': 'vrReviewItem itemLabel'}).text) == "Traveling group:":
if div.div:
traveling_type = str(div.div.select('span[class="vrReviewItem"]')[0].text)
break
except:
traveling_type = 'unknown'
###
# reviewer data
try:
reviewer_name = soup.find('div', {'class': "username mo"}).text.replace('\n', '')
except:
reviewer_name = 'unknown'
try:
reviewer_location = soup.find('div', {'class': "location"}).text.replace('\n', '')
if reviewer_location == '':
reviewer_location = 'unknown'
except:
reviewer_location = 'unknown'
try:
reviewer_contributions = int(re.sub("[^0-9]", "", soup.find('div', {'class': "reviewerBadge badge"}).span.text))
except:
reviewer_contributions = 'unknown'
###
return title, content, overall_rating, review_date, stay_date, ranking_dict, traveling_type, reviewer_name, reviewer_contributions, reviewer_location
def airline_url_content(url):
ntries = 10
for tries in range(ntries):
try:
response = urlopen(url, timeout=20)
break
except error.URLError as err:
print('ReTry: %s' % url)
time.sleep(30)
if tries == ntries - 1:
raise error.URLError('')
soup = BeautifulSoup(response, "lxml")
review_id_pointer_start = url.find('-r') + len('-r')
review_id_pointer_end = url.find('-', review_id_pointer_start)
review_id = 'review_' + url[review_id_pointer_start:review_id_pointer_end]
## extract the general information
general_information = json.loads(
soup.find('script', type="application/ld+json").text)
# return general_information
try:
review_date = general_information['datePublished']
except:
try:
review_date = soup.find('div', {
'class':
"ui_column is-10-desktop is-12-tablet is-12-mobile"
}).find('span', {'class': "ratingDate"})['title']
except:
review_date = 'unknown'
try:
title = general_information['name']
except:
title = 'unknown'
try:
content = general_information['reviewBody']
except:
content = 'unknown'
try:
overall_rating = int(
general_information['reviewRating']['ratingValue'])
except:
overall_rating = 'unknown'
#return review_date, title, content, overall_rating
all_data = soup.find('div', {'id': review_id})
try:
stay_date = all_data.find('div', {
'class': "prw_rup prw_reviews_stay_date_hsx"
}).text
# remove the blank in the front
while True:
if 'Date of travel: ' in stay_date:
stay_date = stay_date.replace('Date of travel: ', '')
else:
break
while True:
if stay_date[0] == ' ':
stay_date = stay_date[1:]
else:
break
except:
stay_date = ''
#return review_date, title, content, overall_rating, stay_date
## ranking part
ranking_dict = dict()
try:
table = all_data.find_all('li', {'class': "recommend-answer"})
for rating in table:
rate = str(rating.select('div[class*="ui_bubble_rating bubble_"]'))
rate_pointer_start = rate.find(
'<div class="ui_bubble_rating bubble_') + len(
'<div class="ui_bubble_rating bubble_')
rate_pointer_end = rate.find('"></div>', rate_pointer_start)
rate = rate[rate_pointer_start:rate_pointer_end]
try:
rate_mark = int(str(rate)) / 10
except:
rate_mark = str(rate)
label = rating.find('div', {'class': "recommend-description"}).text
ranking_dict[label] = rate_mark
except:
pass
#return review_date, title, content, overall_rating, stay_date, ranking_dict
## review information
try:
reviewer_name = all_data.find('div', {'class': "username mo"}).text
except:
reviewer_name = 'unknown'
try:
reviewer_contributions = all_data.find('span', {
'class': "badgetext"
}).text
try:
reviewer_contributions = int(reviewer_contributions)
except:
pass
except:
reviewer_contributions = 'unknown'
try:
reviewer_location = soup.find('div', {'class': "location"}).text
except:
reviewer_location = 'unknown'
return review_date, title, content, overall_rating, stay_date, ranking_dict, reviewer_name, reviewer_contributions, reviewer_location
def errormessage(n):
if int(n) == 10:
return 100, 200
else:
raise error.URLError('')
