def get_data_from_url(url):
"""
Returns dict with data parsed from JSON code
On failure: returns None
:param url: source
:return: data dict
"""
try:
with urlopen(url) as url:
return json.loads(url.read().decode())
except URLError as e:
raise URLError("Can't connect to: \"" + str(url) + "\", " +
str(e.reason))
except ValueError as e:
raise ValueError("URL invalid: \"" + str(url) + "\", " + str(e))
def test_login_except(self, mocked_build_opener):
"""
Test that when logging in to SongSelect fails, the login method raises URLError
"""
# GIVEN: A bunch of mocked out stuff and an importer object
mocked_build_opener.open.side_effect = URLError('Fake URLError')
mock_callback = MagicMock()
importer = SongSelectImport(None)
# WHEN: The login method is called after being rigged to fail
result = importer.login('username', 'password', mock_callback)
# THEN: callback was called 1 time and False was returned
assert 1 == mock_callback.call_count, 'callback should have been called 1 times'
assert result is False, 'The login method should have returned False'
def GetMaxPage():
r = requests.get(CATELOG_URL, headers=GetHeader())
if r.status_code == 200:
doc = pq(r.text)
max_page = doc('.content__pg').attr('data-totalpage')
try:
max_page = int(max_page)
print('{} pages found'.format(max_page))
except ValueError:
print('Invalid max_page number fetched')
return max_page
else:
raise URLError('Connection status: {:s}'.format(r.status_code))
def test_mitoc_trips_api_down(self, verified_emails):
"""If the MITOC Trips API is down, the route still succeeds."""
email = ['*****@*****.**']
verified_emails.return_value = (email, [email])
self.configure_normal_update()
self.update_membership.side_effect = URLError("API is down!")
with mock.patch.object(extensions, 'sentry') as sentry:
response = self.client.post('/members/membership',
data=self.valid_payload)
sentry.captureException.assert_called_once()
self.assertTrue(response.is_json)
self.assertEqual(response.status_code, 201)
def test_call_with_wrong_jwks_host(
fetch_data_mock, route, client, valid_json, valid_jwt,
authorization_errors_expected_payload
):
mock_exception = MagicMock()
mock_exception.reason = 'test'
fetch_data_mock.side_effect = URLError(mock_exception)
response = client.post(
route, json=valid_json, headers=headers(valid_jwt())
)
assert response.status_code == HTTPStatus.OK
assert response.json == authorization_errors_expected_payload(
WRONG_JWKS_HOST
)
def run(self):
now_str = self._convert_datetime(self.now)
start_str = self._convert_datetime(self.start)
timespan = '{}/{}'.format(start_str, now_str)
url = 'https://geoservice.ist.supsi.ch/psos/sos?service=SOS&version=1.0.0&request=GetObservation&offering=temporary&procedure=Q_FAL_CHIA&observedProperty=urn:ogc:def:parameter:x-istsos:1.0:river:water:discharge&responseFormat=application/json&eventTime={}'.format(
timespan)
try:
with urllib.request.urlopen(url, timeout=15) as url:
data = json.loads(url.read().decode())
data = data['ObservationCollection']['member'][0]
self.data = data['result']['DataArray']['values']
except URLError as e:
raise URLError('OASI Problem: {}'.format(str(e))) from e
self._check_data()
self._store_data()
def base_test_failure_no_file_url_error(self):
resolve_bundle_mock = MagicMock(side_effect=URLError('reason', None))
stderr = MagicMock()
with patch('conductr_cli.resolver.resolve_bundle', resolve_bundle_mock):
logging_setup.configure_logging(MagicMock(**self.default_args), err_output=stderr)
result = conduct_load.load(MagicMock(**self.default_args))
self.assertFalse(result)
resolve_bundle_mock.assert_called_with(self.custom_settings, self.bundle_resolve_cache_dir,
self.bundle_file, self.offline_mode)
self.assertEqual(
as_error(strip_margin("""|Error: File not found: reason
|""")),
self.output(stderr))
def fake_urlopen(self, url):
"""Fake urlopen using test client"""
if 'example' in url:
response = StringIO('')
return addinfourl(response, {
'X-Pingback': '/xmlrpc.php',
'Content-Type': 'text/html'
}, url)
elif 'localhost' in url:
response = StringIO('<link rel="pingback" href="/xmlrpc/">')
return addinfourl(response, {'Content-Type': 'text/xhtml'}, url)
elif 'google' in url:
response = StringIO('PNG CONTENT')
return addinfourl(response, {'content-type': 'image/png'}, url)
elif 'error' in url:
raise URLError('Invalid ressource')
def get_polls(group, poll_id):
res = requests.get(f"https://www.meetup.com/{group}/polls/{poll_id}/")
soup = BeautifulSoup(res.text, "lxml")
votes = []
if not soup.select(".poll .line"):
raise URLError("Poll doesn't exists")
for entry in soup.select(".poll .line"):
count_text = entry.find(class_="count").text.strip()
match = votes_re.match(count_text) or {"n_votes": 0}
votes.append((entry.find("label").text.strip(), int(match["n_votes"])))
votes.sort(key=itemgetter(1), reverse=True)
votes = OrderedDict(votes)
return votes
def test_mitoc_trips_api_down_but_no_sentry(self, verified_emails):
"""If Sentry is not configured, the route still succeeds."""
email = ['*****@*****.**']
verified_emails.return_value = (email, [email])
self.configure_normal_update()
self.update_membership.side_effect = URLError("API is down!")
with mock.patch.object(views, 'extensions') as view_extensions:
view_extensions.sentry = None
response = self.client.post('/members/membership',
data=self.valid_payload)
self.assertTrue(response.is_json)
self.assertEqual(response.status_code, 201)
async def test_media_player_handle_URLerror(
hass: HomeAssistant, init_integration: MockConfigEntry, mock_remote: Mock
) -> None:
"""Test remote handle URLError as Unavailable."""
state_tv = hass.states.get("media_player.panasonic_viera_tv")
assert state_tv.state == STATE_ON
# simulate timeout error
mock_remote.get_mute = Mock(side_effect=URLError(None, None))
async_fire_time_changed(hass, utcnow() + timedelta(minutes=2))
await hass.async_block_till_done()
state_tv = hass.states.get("media_player.panasonic_viera_tv")
assert state_tv.state == STATE_UNAVAILABLE
def test_request_http_error(self):
test_servers = deque(TEST_SERVERS)
http_client = ElasticsearchRequestController(
test_servers, TEST_TIMEOUT, None, False, self._mocked_logger)
# expect HttpRetryError on URLError
with mock.patch.object(http_client, '_url_opener') as mock_url_opener:
mock_url_opener.open.side_effect = URLError('test error')
with self.assertRaises(HttpRetryError):
http_client._request_inner('/', data=None)
# expect real error on non-URLError
with mock.patch.object(http_client, '_url_opener') as mock_url_opener2:
mock_url_opener2.open.side_effect = KeyboardInterrupt('test error')
with self.assertRaises(KeyboardInterrupt):
http_client._request_inner('/', data=None)
def get_chart_csv_data(
chart_url: str,
auth_cookies: Optional[Dict[str, str]] = None) -> Optional[bytes]:
content = None
if auth_cookies:
opener = urllib.request.build_opener()
cookie_str = ";".join(
[f"{key}={val}" for key, val in auth_cookies.items()])
opener.addheaders.append(("Cookie", cookie_str))
response = opener.open(chart_url)
content = response.read()
if response.getcode() != 200:
raise URLError(response.getcode())
if content:
return content
return None
def _get_slice_data(schedule: SliceEmailSchedule) -> EmailContent:
slc = schedule.slice
slice_url = _get_url_path("Superset.explore_json",
csv="true",
form_data=json.dumps({"slice_id": slc.id}))
# URL to include in the email
url = _get_url_path("Superset.slice", slice_id=slc.id)
cookies = {}
for cookie in _get_auth_cookies():
cookies["session"] = cookie
opener = urllib.request.build_opener()
opener.addheaders.append(("Cookie", f"session={cookies['session']}"))
response = opener.open(slice_url)
if response.getcode() != 200:
raise URLError(response.getcode())
# TODO: Move to the csv module
content = response.read()
rows = [r.split(b",") for r in content.splitlines()]
if schedule.delivery_type == EmailDeliveryType.inline:
data = None
# Parse the csv file and generate HTML
columns = rows.pop(0)
with app.app_context(): # type: ignore
body = render_template(
"superset/reports/slice_data.html",
columns=columns,
rows=rows,
name=slc.slice_name,
link=url,
)
elif schedule.delivery_type == EmailDeliveryType.attachment:
data = {__("%(name)s.csv", name=slc.slice_name): content}
body = __(
'<b><a href="%(url)s">Explore in Superset</a></b><p></p>',
name=slc.slice_name,
url=url,
)
return EmailContent(body, data, None)
def retry_urlopen(url,
timeout,
retries,
delay,
logger=logging.getLogger(__name__)):
'''Open the URL url, which can be either a string or a Request object and
retry if an exception is raised up to retries times.
Parameters
----------
url : string
timeout : the optional timeout parameter specifies a timeout in seconds
for blocking operations like the connection attempt (if not specified,
the global default timeout setting will be used). This actually only
works for HTTP, HTTPS and FTP connections
retries : number of retries before an error is raised. The maximum number
of attempts is retries + 1
delay : the delay in seconds between subsequent retries
Returns
-------
'''
if retries < 0:
msg = 'retries={} must a non-negative integer'.format(retries)
raise ValueError(msg)
logger.info('opening URL')
for i in range(retries + 1):
try:
response = urlopen(url, timeout=timeout)
return response
except (URLError, socket.timeout) as err:
if i == 0 and retries == 0:
raise err
elif i == 0:
logger.warning(err)
else:
logger.warning('Retry {} got \'{}\''.format(i, err))
if i != retries:
sleep(delay)
raise URLError('urlopen failed after {} retries'.format(retries))
def transport_from_url(url):
""" Create a transport for the given URL.
"""
if '/' not in url and ':' in url and url.rsplit(':')[-1].isdigit():
url = 'scgi://' + url
url = urlparse.urlsplit(url, scheme="scgi", allow_fragments=False) # pylint: disable=redundant-keyword-arg
try:
transport = TRANSPORTS[url.scheme.lower()]
except KeyError:
if not any((url.netloc, url.query)) and url.path.isdigit():
# Support simplified "domain:port" URLs
return transport_from_url("scgi://%s:%s" % (url.scheme, url.path))
else:
raise URLError("Unsupported scheme in URL %r" % url.geturl())
else:
return transport(url)
def data(self) -> Optional[str]:
"""Reads the data from the File object.
First it checks if the File object has any data. If it doesn't, it retrieves
it and saves it to the File. Once the. It then reads it from the File and
returns it.
Returns:
Optional[str] -- The string data from the File object.
"""
if self.file.data is None:
try:
_, headers = urlretrieve(self.url, self.file.path)
except URLError:
# make sure that the right error string is returned
raise URLError('urlopen error: Name or service not known')
return self.file.data
def _get(url): try: return urlopen(url).read() except HTTPError: raise except URLError: # Fallback: Some users get "SSL: CERTIFICATE_VERIFY_FAILED" for urlopen. try: response = requests.get(url) except RequestException as e: raise URLError(e.__class__.__name__) if response.status_code != 200: raise HTTPError( url, response.status_code, response.reason, response.headers, None ) return response.content
def _get_swagger_spec(self, endpoint_name: str) -> Spec:
"""Get Swagger spec of specified service."""
logic_module = self._get_logic_module(endpoint_name)
schema_url = utils.get_swagger_url_by_logic_module(logic_module)
if schema_url not in self._specs:
try:
response = requests.get(schema_url)
spec_dict = response.json()
except URLError:
raise URLError(f'Make sure that {schema_url} is accessible.')
swagger_spec = Spec.from_dict(spec_dict,
config=self.SWAGGER_CONFIG)
self._specs[schema_url] = swagger_spec
return self._specs[schema_url]
def test_get_song_page_raises_exception(self, mocked_build_opener):
"""
Test that when BeautifulSoup gets a bad song page the get_song() method returns None
"""
# GIVEN: A bunch of mocked out stuff and an importer object
mocked_opener = MagicMock()
mocked_build_opener.return_value = mocked_opener
mocked_opener.open.read.side_effect = URLError('[Errno -2] Name or service not known')
mocked_callback = MagicMock()
importer = SongSelectImport(None)
# WHEN: get_song is called
result = importer.get_song({'link': 'link'}, callback=mocked_callback)
# THEN: The callback should have been called once and None should be returned
mocked_callback.assert_called_with()
assert result is None, 'The get_song() method should have returned None'
def test_calls_error_handler_on_exception(self):
on_error_mock = Mock()
error = URLError('reason')
sender = self._given_client(exception=error, on_error=on_error_mock)
success = sender.send_email({
'to': [self.recipient1, self.recipient2],
'from':
self.sender,
'subject':
self.test_calls_error_handler_on_error_response.__name__,
'message':
'email that gave 403 response'
})
self.assertFalse(success)
self.assertEqual(on_error_mock.call_count, 1)
def endpoint(temperatureScale):
'''
Determine which web service endpoint the user's input is, for conversion
to the other three temperature scales
@param temperatureScale: string representation of the temperature scale of
the user input.
@return: The web service endpoint that will perform the temperature conversion
'''
if temperatureScale.upper()[0] == 'K':
return '/convert/kelvin/'
elif temperatureScale.upper()[0] == 'F':
return '/convert/fahrenheit/'
elif temperatureScale.upper()[0] == 'R':
return '/convert/rankine/'
elif temperatureScale.upper()[0] == 'C':
return '/convert/celsius/'
raise URLError('Unknown conversion scale')
def get_history(self, epic, start_date=None, end_date=None):
"""Take a datetime object for start_date and/to end_date. If only
start_date is given then it uses datetime.datetime.now() as the
end_date"""
#check inputs
if end_date and not start_date:
raise ValueError("End date without start date")
if not start_date:
start_date = datetime.now() - relativedelta(months=3)
end_date = datetime.now()
if not isinstance(end_date, datetime) \
or not isinstance(start_date,datetime):
raise ValueError("Start date and End date must be datetime obects")
url = urljoin(self.base_url, "gateway/deal/prices/")
url = urljoin(url, epic)
self.headers['version'] = "3"
params = {'pageSize': int(1e6)}
params['from'] = "2017-03-20T12:30:00"
params['to'] = "2017-03-28T12:30:00"
# if start_date:
# str_start = start_date.strftime("%Y-%m-%dT%H:%M:%S")
# params['from'] = str_start
# if end_date:
# str_end = start_date.strftime("%Y-%m-%dT%H:%M:%S")
# params["to"] = str_end
r = requests.get(url, headers=self.headers, params=params)
return r
if r.ok:
data = json.loads(r.content.decode())['prices']
bid = {'time': [], 'open': [], 'close': [], 'high': [], 'low': []}
ask = {'time': [], 'open': [], 'close': [], 'high': [], 'low': []}
for price in data:
for sale, op in zip([bid, ask], ["bid", "ask"]):
sale['time'].append(
datetime.strptime(price['snapshotTimeUTC'],
"%Y-%m-%dT%H:%M:%S"))
sale['open'].append(price['openPrice'][op])
sale['close'].append(price['closePrice'][op])
sale['high'].append(price['highPrice'][op])
sale['low'].append(price['lowPrice'][op])
return bid, ask
else:
raise URLError(":Request failed with code {}".format(
r.status_code))
def test_network_loss(self):
""" Updates should be sent when the network is up again. """
filename = "networkless file.txt"
doc_id = self.remote.make_file("/",
filename,
content=b"Initial content.")
# Download file
local_path = f"/{doc_id}/{filename}"
with patch.object(self.manager_1,
"open_local_file",
new=open_local_file):
self.direct_edit._prepare_edit(pytest.nuxeo_url, doc_id)
self.wait_sync(timeout=2, fail_if_timeout=False)
# Simulate server error
self.engine_1.remote = RemoteTest(
pytest.nuxeo_url,
self.user_1,
"nxdrive-test-administrator-device",
pytest.version,
password=self.password_1,
)
error = URLError(
"[Errno 10051] A mocked socket operation was attempted to an unreachable network"
)
self.engine_1.remote.make_upload_raise(error)
try:
# Update file content
self.local.update_content(local_path, b"Updated")
time.sleep(5)
self.local.update_content(local_path, b"Updated twice")
time.sleep(5)
# The file should not be updated on the server
assert (self.remote.get_blob(
self.remote.get_info(doc_id)) == b"Initial content.")
finally:
self.engine_1.remote.make_upload_raise(None)
# Check the file is reuploaded when the network come again
# timout=30 to ensure the file is removed from the blacklist (which have a 30 sec delay)
self.wait_sync(timeout=30, fail_if_timeout=False)
assert (self.remote.get_blob(
self.remote.get_info(doc_id)) == b"Updated twice")
def test_mitoc_trips_api_down_but_no_sentry(self, update_membership):
"""If Sentry is not configured, the route still succeeds."""
update_membership.side_effect = URLError("API is down!")
all_emails = ['*****@*****.**']
with self._first_waiver('*****@*****.**', all_emails) as (db, verified_emails):
with mock.patch.object(views, 'extensions') as view_extensions:
view_extensions.sentry = None
resp = self.client.post('/members/waiver', data=self._waiver_data)
# This request goes through all the usual steps!
verified_emails.assert_called_once_with('*****@*****.**') # (from the XML)
db.add_person.assert_called_once_with('Tim', 'Beaver', '*****@*****.**')
db.add_waiver.assert_called_once_with(self.person_id, self.TIME_SIGNED)
db.update_affiliation.assert_called_once_with(self.person_id, 'Non-affiliate')
self.assertTrue(resp.is_json)
self.assertEqual(resp.status_code, 201)
def mailman_open(self, req):
list_manager = getUtility(IListManager)
# Parse urls of the form:
#
# mailman:///<fqdn_listname|list_id>/<language>/<template_name>
#
# where only the template name is required.
mlist = code = template = None
# Parse the full requested URL and be sure it's something we handle.
original_url = req.get_full_url()
parsed = urlparse(original_url)
assert parsed.scheme == 'mailman'
# The path can contain one, two, or three components. Since no empty
# path components are legal, filter them out.
parts = [p for p in parsed.path.split('/') if p]
if len(parts) == 0:
raise URLError('No template specified')
elif len(parts) == 1:
template = parts[0]
elif len(parts) == 2:
part0, template = parts
# Is part0 a language code or a mailing list? This is rather
# tricky because if it's a mailing list, it could be a list-id and
# that will contain dots, as could the language code.
language = getUtility(ILanguageManager).get(part0)
if language is None:
# part0 must be a fqdn-listname or list-id.
mlist = (list_manager.get(part0) if '@' in part0 else
list_manager.get_by_list_id(part0))
if mlist is None:
raise URLError('Bad language or list name')
else:
code = language.code
elif len(parts) == 3:
part0, code, template = parts
# part0 could be an fqdn-listname or a list-id.
mlist = (getUtility(IListManager).get(part0) if '@' in part0 else
getUtility(IListManager).get_by_list_id(part0))
if mlist is None:
raise URLError('Missing list')
language = getUtility(ILanguageManager).get(code)
if language is None:
raise URLError('No such language')
code = language.code
else:
raise URLError('No such file')
# Find the template, mutating any missing template exception.
try:
path, fp = find(template, mlist, code)
except TemplateNotFoundError:
raise URLError('No such file')
return addinfourl(fp, {}, original_url)
async def test_url_error_poll_report(
self, mock_get_selected_fields, mock_get_core_schema,
mock_write_schema, mock_get_bookmark, mock_sobject_to_dict,
mock_write_state, mock_write_bookmark, mock_metrics,
mock_write_records, mock_filter_selected_fields_many):
'''
Test that tap retry on the timeout error for 1 minute.
'''
mock_client = MockClient(
URLError('_ssl.c:1059: The handshake operation timed out'))
before_time = datetime.datetime.now()
try:
await tap_bing_ads.poll_report(mock_client, '', '', '', '', '')
except URLError:
after_time = datetime.datetime.now()
time_difference = (after_time - before_time).total_seconds()
# verify the code backed off for 60 seconds
# time_difference should be greater or equal 60 as some time elapsed while calculating `after_time`
self.assertGreaterEqual(time_difference, 60)
def __init__(self, url, data=None, headers={},
origin_req_host=None, unverifiable=False,
method=None):
url_match = _URL_REGEX.match(url)
if url_match is None:
raise URLError("Unvalid URL")
url_obj = parse_url(url)
url_addr = url_obj["url"]
url_auth = url_obj["auth"]
try:
super(Request, self).__init__(url_addr, data=data, headers=headers,
origin_req_host=origin_req_host, unverifiable=unverifiable,
method=method)
except TypeError:
super(Request, self).__init__(url_addr, data=data, headers=headers,
origin_req_host=origin_req_host, unverifiable=unverifiable)
self.get_method = lambda: method if method is not None else "GET"
if url_auth is not None:
self.add_header('Authorization', 'Basic %s' % url_auth)
def load_url(url, retry=2, retry_period=1, timeout=10):
try:
fh = urlopen(url, timeout=timeout)
except HTTPError as e:
if e.code == 503 and retry:
time.sleep(retry_period)
return load_url(
url, retry=retry - 1, retry_period=retry_period,
timeout=timeout)
e.msg += ' (%s)' % url
raise
except URLError as e:
if isinstance(e.reason, socket.timeout) and retry:
time.sleep(retry_period)
return load_url(
url, retry=retry - 1, retry_period=retry_period,
timeout=timeout)
raise URLError(str(e) + ' (%s)' % url)
return fh.read()
def download_handler(feed, placeholders):
import shlex
"""
Parse and execute the download handler
"""
value = feed.retrieve_config('downloadhandler', 'greg')
if value == 'greg':
while os.path.isfile(placeholders.fullpath):
placeholders.fullpath = placeholders.fullpath + '_'
placeholders.filename = placeholders.filename + '_'
urlretrieve(placeholders.link, placeholders.fullpath)
else:
value_list = shlex.split(value)
instruction_list = [substitute_placeholders(part, placeholders) for
part in value_list]
returncode = subprocess.call(instruction_list)
if returncode:
# Give a reason for the failure
raise URLError('Failed to download ' + placefolders.link)
