def parse_deck_page(url, mtgtop8=True):
if mtgtop8:
link = "http://mtgtop8.com/mtgo?{}".format(
mtgtop8compile.search(url).group())
else:
link = "https://www.mtggoldfish.com/deck/download{}".format(
mtggoldfishcompile.search(url).group())
global cached_files
if link in cached_files:
if exists(cached_files[link]):
return cached_files[link]
else:
del cached_files[link]
try:
resp = get(link)
except:
raise ConnectionError("Could not connect to website")
if resp.status_code != 200:
raise ConnectionError("Error code ".format(resp.status_code))
if resp.text == "Throttled":
print "Breaking for 30 seconds while downloading list."
sleep(30)
return parse_deck_page(url, mtgtop8)
with NamedTemporaryFile(delete=False, dir="./data/temp") as fp:
fp.write(resp.text)
name = fp.name
cached_files[link] = name
return name
def _wait_for_response(self, requests, to_json):
"""Issue a batch of requests and wait for the responses.
Args:
requests - A list of requests
to_json - A boolean, should the responses be returned as JSON blobs
Returns:
A list of dicts if to_json, a list of grequest.response otherwise
"""
all_responses = []
for retry in range(self._max_retry):
try:
responses = grequests.map(requests)
valid_responses = [
response for response in responses if response
]
failed_auth_responses = [
response for response in responses
if response.status_code == 403
]
if failed_auth_responses:
raise ConnectionError(
'Credentials not authorized to access URL')
if len(valid_responses) != len(requests):
continue
else:
break
except ConnectionError:
raise
except:
pass
if retry == self._max_retry:
raise ConnectionError(
'Unable to complete batch of requests within max_retry retries'
)
for request, response in zip(requests, responses):
if not response:
# should have caught this earlier, but if not ...
raise ConnectionError(
'Request to {0} had an empty response'.format(request.url))
if 200 != response.status_code:
write_error_message('url[{0}] status_code[{1}]'.format(
response.request.url, response.status_code))
if to_json:
all_responses.append(response.json())
else:
all_responses.append(response)
return all_responses
def test_display_routine(self, mock_get_config, mock_viewer, mock_joke,
mock_coin, mock_details, mock_logo):
mock_get_config.return_value = {
'ticker': {
'tell_jokes': True,
'crypto': 'btc',
'vs_currency': 'usd'
}
}
mock_joke.return_value = "FUNNY!"
mock_coin.return_value = {
'name': 'Bitcoin',
'id': 'btc',
'symbol': 'btc',
'logo': 'bitcoin_file.jpg'
}
mock_logo.return_value = 'bitcoin_file.jpg'
mock_details.return_value = '$31,000, +10.20%'
# Don't want to wait on sleep in tests
time.sleep = mock.MagicMock()
t = Ticker()
t.crypto = Crypto()
t.load_config()
print(t.config)
t.display_routine()
calls = [mock.call('BITCOIN', MessageType.FALLING, 'bitcoin_file.jpg')]
for j in range(0, 3):
calls.append(
mock.call('$31,000, +10.20%',
MessageType.BOUNCING,
'bitcoin_file.jpg',
delay=30))
calls.append(mock.call('JOKE TIME', MessageType.FALLING, delay=25))
calls.append(mock.call('FUNNY!', MessageType.SCROLLING))
t.viewer.display_message.assert_has_calls(calls)
mock_details.side_effect = ConnectionError()
t.display_routine()
t.viewer.display_message.assert_called_with(
"Error connecting to Price API", MessageType.SCROLLING)
mock_details.side_effect = None
mock_joke.side_effect = ConnectionError()
t.display_routine()
t.viewer.display_message.assert_called_with(
"Error connecting to Joke API", MessageType.SCROLLING)
mock_joke.side_effect = Exception()
t.display_routine()
t.viewer.display_message.assert_called_with(
"Encountered an Unknown Error", MessageType.SCROLLING)
def _post(self, endpoint, json):
addr = self.service_api + endpoint
r = self._sess.post(addr, json=json)
if r.status_code != 200:
raise ConnectionError("Connect to service mesh failed: {}".format(
r.status_code))
result = r.json()
if result.get('success') is not True:
raise ConnectionError("Connect to service mesh failed: {}".format(
result.get('errorMessage', "MISSING ERROR MESSAGE")))
return result
def test_error_update(self, mock_njs, mock_auth):
self._cleanup(self.jobid)
mlog = MockLogger()
os.environ['KB_AUTH_TOKEN'] = 'bogus'
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
jr.logger = mlog
jr.njs.check_job_canceled.return_value = {'finished': False}
jr.njs.update_job.side_effect = ConnectionError()
jr.njs.get_job_params.side_effect = ConnectionError()
with self.assertRaises(ConnectionError):
jr.run()
emsg = 'Failed to get job parameters. Exiting.'
self.assertEquals(mlog.errors[0], emsg)
def test_create_retry_handler_with_socket_errors(self):
handler = retryhandler.create_retry_handler(
self.retry_config, operation_name='OperationBar')
with self.assertRaises(ConnectionError):
handler(response=None, attempts=10,
caught_exception=ConnectionError())
# No connection error raised because attempts < max_attempts.
sleep_time = handler(response=None, attempts=1,
caught_exception=ConnectionError())
self.assertEqual(sleep_time, 1)
# But any other exception should be raised even if
# attempts < max_attempts.
with self.assertRaises(ValueError):
sleep_time = handler(response=None, attempts=1,
caught_exception=ValueError())
def get_articles(self, product_id, start=0, provided_oauth=None, **kwargs):
# https://api.cardmarket.com/ws/documentation/API_2.0:Articles
INCREMENT = 1000
url = '{}/articles/{}'.format(self.base_url, product_id)
mkm_oauth = self.__setup_service(url, provided_oauth)
logging.debug(">> Getting articles on product: " + str(product_id))
params = kwargs
if (start > 0):
params.update({'start': start, 'maxResults': INCREMENT})
r = mkm_oauth.get(url, params=params)
max_items = 0
if (r.status_code == requests.codes.partial_content):
max_items = self.__get_max_items_from_header(r)
logging.debug('> Content-Range header: ' +
r.headers['Content-Range'])
logging.debug('> # articles in response: ' +
str(len(r.json()['article'])))
if (start + INCREMENT >= max_items and self.__handle_response(r)):
return r.json()['article']
else:
return r.json()['article'] + self.get_articles(
product_id, start=start + INCREMENT, **kwargs)
elif (r.status_code == requests.codes.no_content):
raise NoResultsError('No products found in stock.')
elif (r.status_code == requests.codes.ok):
return r.json()['article']
else:
raise ConnectionError(r)
def test_okta_connection_error(
self,
mock_print_tty,
mock_makedirs,
mock_open,
mock_chmod
):
responses.add(
responses.POST,
'https://organization.okta.com/api/v1/authn',
body=ConnectionError()
)
with self.assertRaises(SystemExit):
Okta(
user_name="user_name",
user_pass="******",
organization="organization.okta.com"
)
print_tty_calls = [
call("Error: Connection Error")
]
mock_print_tty.assert_has_calls(print_tty_calls)
def test_get_bib_record_connection_issues(mock_get):
mock_get.side_effect = ConnectTimeout()
assert_equal(get_bib_record("placeholder_mmsid"),
{"error": "Alma Connection Error - try again later."})
mock_get.side_effect = ConnectionError()
assert_equal(get_bib_record("placeholder_mmsid"),
{"error": "Alma Connection Error - try again later."})
def login(self, login_id, login_pw):
"""
create requests session and log into ilias.uni-mannheim.de
:param args: login details (uni-id and password)
:type args: list
:raises TypeError:
"""
if type(login_id) is not str or type(login_pw) is not str:
raise TypeError("...")
data = {'username': login_id, 'password': login_pw}
head = {
'Accept-Encoding': 'gzip, deflate, sdch, br',
'Accept-Language': 'de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Connection': 'keep-alive',
}
self.session = session()
self.login_soup = BeautifulSoup(self.session.get("https://cas.uni-mannheim.de/cas/login").content, "lxml")
form_data = self.login_soup.select('form[action^="/cas/login"] input')
data.update({inp["name"]: inp["value"] for inp in form_data if inp["name"] not in data})
self.session.post('https://cas.uni-mannheim.de/cas/login', data=data, headers=head)
self.login_soup = BeautifulSoup(self.session.get(self.base_url).content, "lxml")
# Login successful? FIY
if not self.login_soup.find("a", {'id' : 'mm_desktop'}):
raise ConnectionError("Couldn't log into ILIAS. Make sure your provided uni-id and the password are correct.")
def wrapped(**kwargs):
response_log_path = _build_response_log_path(
test_func=test_function,
response_logs_dir=response_logs_dir,
**kwargs,
)
if response_log_path.exists() and enforce_online_test:
# Delete the file to enforce an online test
response_log_path.unlink()
if response_log_path.exists():
try:
LOGGER.info(
f"Running offline test based on file at {response_log_path}"
)
_run_offline_test(
response_log_path=response_log_path,
test_function=test_function,
**kwargs,
)
except ConnectionError as e:
msg = (
f"A required API call was missing from response logs file for this "
f"offline test ({response_log_path}). The response log file must be "
f"regenerated. Delete the existing file to automatically regenerate a "
f"new one. Caused by: {e}")
LOGGER.error(msg)
raise ConnectionError(e)
else:
_run_online_test(response_log_path=response_log_path,
test_function=test_function,
**kwargs)
def test_no_retry_on_connection_error(mock_session):
expected_call_count = 0
api_response = {'key': 'value'}
session_context = mock_session.return_value.__enter__.return_value
session_context.send.return_value.json.return_value = api_response
for verb in RETRY_VERBS:
expected_call_count += 1
request_info = dict(params={
'secondParameter': 'b',
'firstParameter': 'a'
},
json={},
url='https://api.civisanalytics.com/wobble/wubble',
method=verb)
request = Request(**request_info)
pre_request = session_context.prepare_request(request)
session_context.send.side_effect = ConnectionError()
try:
retry_request(verb, pre_request, session_context, 3)
except ConnectionError:
pass
assert session_context.send.call_count == expected_call_count
def _request(self,
method,
route,
payload=None): # noqa: C901 (lint ignore)
res = url = err = jsonObject = None
_payload = json.dumps(payload)
url = '{}{}'.format(self.endpoint, route)
try:
if method == 'get':
res = requests.get(url=url,
params=_payload,
headers=self.headers,
auth=(self.key, self.secret))
elif method == 'post':
res = requests.post(url=url,
data=_payload,
headers=self.headers,
auth=(self.key, self.secret))
elif method == 'patch':
res = requests.patch(url=url,
json=_payload,
headers=self.headers,
auth=(self.key, self.secret))
elif method == 'put':
res = requests.put(url=url,
data=_payload,
headers=self.headers,
auth=(self.key, self.secret))
elif method == 'delete':
res = requests.delete(url=url,
params=_payload,
headers=self.headers,
auth=(self.key, self.secret))
return bool(res.status_code == 204)
except requests.ConnectionError as ex:
raise ConnectionError(url, ex.message)
# If status_code is in range 200-209
if str(res.status_code)[0] == '2':
try:
if res.content:
jsonObject = res.json()
except Exception as ex:
err = {
'code': res.status_code,
'message': getattr(ex, 'message', ''),
'content': res.content
}
else:
err = {
'code': res.status_code,
'message': res.reason,
'content': res.content
}
if err:
raise Exception(err)
return jsonObject
def test_throws_when_piazza_is_unreachable(self, _):
with unittest.mock.patch('requests.post') as stub:
stub.side_effect = ConnectionError(request=unittest.mock.Mock())
with self.assertRaises(piazza.Unreachable):
piazza.deploy(data_id='test-data-id',
poll_interval=0,
max_poll_attempts=2)
def test_exists_on_pypi_with_exceptions(self, mock):
mock.side_effect = ConnectionError()
view = self.FakeView()
result = view.exists_on_pypi('http://pypi.python.org/pypi/example')
self.assertFalse(result)
def test_offline(self, mocked_client: JupyterHubClient):
msg = 'Failed to establish a new connection: [Errno 111] Connection refused'
mocked_client.agent.put.side_effect = ConnectionError(msg)
with pytest.raises(JupyterCommunicationError):
mocked_client.upload_notebook(self.notebook, self.path,
self.workspace)
def test_throws_if_geoserver_is_unreachable_when_setting_default_style(
self, m: requests_mock.Mocker):
m.post('/geoserver/rest/styles')
with patch('requests.put') as stub:
stub.side_effect = ConnectionError()
with self.assertRaises(geoserver.InstallError):
geoserver.install_style('test-style-id')
def _getJsonWithRetry(self, url):
"""Private method to wrap getting json data with retries. Only gets single page, nothing fancy. See getJson
:param url: url to get
:return: json data returned from server
"""
retry_count = 0
req = None
while retry_count < self._max_retries:
try:
req = requests.get(url, timeout=self._req_timeout)
if req.status_code == requests.codes.ok:
break
logging.warning('Failed to get %s, status %d, retry %d' %
(url, req.status_code, retry_count))
except requests.exceptions.RequestException:
pass # Just pass it, we will include it as a retry ahead
finally:
retry_count += 1
if retry_count >= self._max_retries:
logging.error("Exceeded max connection retries for %s" % url)
if req is not None:
logging.error("Request failure reason: %s" % req.reason)
raise ConnectionError(req.reason)
else:
logging.error("No request, no reason!")
raise ConnectionError
return req.json()
def _get(self, url, expected_status_code=200):
ret = self.session.get(url=url, headers=self.headers)
if (ret.status_code != expected_status_code):
raise ConnectionError(
'Status code {status} for url {url}\n{content}'.format(
status=ret.status_code, url=url, content=ret.text))
return ret
def test_recaptcha_server_connection_error(self, requests):
requests.post.side_effect = ConnectionError()
with self.assertRaises(ValidationError) as e:
self.field.validate('1234')
self.assertEqual('captcha-error', e.exception.code)
def post_request(data, endpoint):
"""
Sends a post request to the tower.
Args:
data (:obj:`dict`): a dictionary containing the data to be posted.
endpoint (:obj:`str`): the endpoint to send the post request.
Returns:
:obj:`dict`: a json-encoded dictionary with the server response if the data can be posted.
Raises:
:obj:`ConnectionError`: if the client cannot connect to the tower.
"""
try:
return requests.post(url=endpoint, json=data, timeout=5)
except Timeout:
message = "Cannot connect to the Eye of Satoshi's API. Connection timeout"
except ConnectionError:
message = "Cannot connect to the Eye of Satoshi's API. Server cannot be reached"
except (InvalidSchema, MissingSchema, InvalidURL):
message = "Invalid URL. No schema, or invalid schema, found ({})".format(
endpoint)
raise ConnectionError(message)
def get_articles(self, product_id, start=0, provided_oauth=None, **kwargs):
# https://api.cardmarket.com/ws/documentation/API_2.0:Articles
INCREMENT = 1000
url = "{}/articles/{}".format(self.base_url, product_id)
mkm_oauth = self.__setup_service(url, provided_oauth)
logging.debug(">> Getting articles on product: " + str(product_id))
params = kwargs
if start > 0:
params.update({"start": start, "maxResults": INCREMENT})
r = mkm_oauth.get(url, params=params)
max_items = 0
if r.status_code == requests.codes.partial_content:
max_items = self.__get_max_items_from_header(r)
logging.debug("> Content-Range header: " +
r.headers["Content-Range"])
logging.debug("> # articles in response: " +
str(len(r.json()["article"])))
if start + INCREMENT >= max_items and self.__handle_response(r):
return r.json()["article"]
else:
next_start = start + INCREMENT
params.update({"start": next_start, "maxResults": INCREMENT})
return r.json()["article"] + self.get_articles(
product_id, **kwargs)
elif r.status_code == requests.codes.no_content:
raise CardmarketError("No products found in stock.")
elif r.status_code == requests.codes.ok:
return r.json()["article"]
else:
raise ConnectionError(r)
def testFunction(domain_url):
status, briefing, message, api_TXT_record = apiLookup(domain_url)
if (status != 0):
status = 2
briefing = "This test was skipped because previous test "
briefing += "<strong>api_server_lookup</strong> has failed.<br/>"
new_message = briefing
new_message += "Reason:<br/>"
new_message += "<br/>" + message
return (status, briefing, new_message, None)
try:
req = Request(
'HEAD',
"%(protocol)s://%(domain)s:%(port)s%(path)s" % api_TXT_record)
r = req.prepare()
s = Session()
# s.mount('https://', SSLAdapter('TLSv1'))
if ((s.send(r, verify=False, timeout=1)).ok):
status = 0
briefing = "Connection successful to API server at "
briefing += "<strong>%(domain)s:%(port)s%(path)s</strong>" % api_TXT_record
message = briefing
return (status, briefing, message, None)
else:
raise ConnectionError()
except ConnectionError as e:
status = 1
error = "Could not connect to API server at "
error += "<strong>%(domain)s:%(port)s%(path)s</strong>." % api_TXT_record
briefing = error
message = briefing
if str(e) != "":
message += "<br/>Reason: %s" % str(e)
return (status, briefing, message, None)
except Exception as e:
status = 2
briefing = "Something odd happened while trying to connect to API server at "
briefing += "<strong>%(domain)s:%(port)s%(path)s</strong>!" % api_TXT_record
message = briefing
message += "<br/>This is the exception we got: {%s}" % str(e)
message += "<br/>It is probably a temporary issue with domain " + domain_url + "."
message += "<br/>But it could also be a bug in our Inspector."
message += " Let us know at <a href='https://github.com/buddycloud/buddycloud-tests-framework/issues'>our issue tracker</a> if you think so."
return (status, briefing, message, None)
def side_effect(_, events):
call_count[0] += 1
if call_count[0] == 1:
raise ConnectionError('Foo')
else:
events.append(event)
return event.id
def request(method, url, timeout=None):
self.assertEquals(method, 'GET', 'Incorrect HTTP method')
self.assertEquals(url, 'http://localhost:8080/swagger',
'Monitored URL is incorrect')
self.assertEquals(timeout, 0.010)
raise ConnectionError()
def __init__(self, api_key, encryption_password=None, proxy=None):
self.api_key = api_key
self._json_header = {"Content-Type": "application/json"}
self._session = requests.Session()
self._session.auth = (self.api_key, "")
self._session.headers.update(self._json_header)
if proxy:
if "https" not in [k.lower() for k in proxy.keys()]:
raise ConnectionError("You can only use HTTPS proxies!")
self._session.proxies.update(proxy)
self.refresh()
self._encryption_key = None
if encryption_password:
try:
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
except ImportError as e:
raise NoEncryptionModuleError(str(e))
kdf = PBKDF2HMAC(
algorithm=hashes.SHA256(),
length=32,
salt=self.user_info["iden"].encode("ASCII"),
iterations=30000,
backend=default_backend(),
)
self._encryption_key = kdf.derive(
encryption_password.encode("UTF-8"))
def test_recaptcha_pass_on_error_request_error(self, requests):
self.field.pass_on_error = True
requests.post.side_effect = ConnectionError()
self.field.validate('1234')
self.assertTrue(True)
def put(self, *args, **kwargs):
if any(ref in args[0] for ref in self.fail_on):
raise ConnectionError(
"Connection fails with lib2 and lib4 references!")
else:
return super(FailOnReferencesUploader,
self).put(*args, **kwargs)
def test_connection_error(requests):
"""Test connection error."""
requests.get.side_effect = ConnectionError()
c = get_client()
with pytest.raises(DataCiteHttpError):
c.doi_get("10.1234/foo.bar")
def __init__(self, hostname=None, username=None, password=None):
"""
:param hostname:
Checkmarx hostname
:param username:
Checkmarx username
:param password:
Checkmarx password
"""
self.logger = configure_logging(logging.getLogger(__name__))
self.hostname = hostname
self.username = username
self.password = password
self.resolver_url = "%s/cxwebinterface/cxwsresolver.asmx?wsdl" % self.hostname
session = Session()
session.verify = False
self.transport = Transport(session=session)
try:
self._resolver_client = Client(self.resolver_url,
transport=self.transport)
except Exception as error:
self.logger.error(
"Checkmarx connection failed: {error}".format(error=error))
raise ConnectionError(
f"Checkmarx connection failed. Wrong or inaccessible hostname: {hostname}"
) from None
self.session_id = None
self.clients = {}
