def _get_fc(data_dir=None, resume=True, verbose=1):
"""
Gets functional connections from MyConnectome parcelled time series data
Returns
-------
fc : (73, 198135) numpy.ndarray
Functional connections (lower triangle)
"""
# download time series data for all sessions
ts = []
for ses in SESSIONS:
if verbose > 0:
print('Fetching time series for session {}'.format(ses))
out = urlopen(TIMESERIES.format(ses))
if out.status == 200:
ts.append(np.loadtxt(out.readlines()))
else:
raise HTTPError('Failed to fetch time series data: session {}'
.format(ses))
# get upper triangle of correlation matrix for each session
fc = [np.corrcoef(ses.T)[np.tril_indices(len(ses.T), k=-1)] for ses in ts]
# return stacked sessions
return np.row_stack(fc)
def _get_panas(data_dir=None, resume=True, verbose=1):
"""
Gets PANAS subscales from MyConnectome behavioral data
Returns
-------
panas : dict
Where keys are PANAS subscales names and values are session-level
composite measures
"""
from numpy.lib.recfunctions import structured_to_unstructured as stu
# download behavioral data
out = urlopen(BEHAVIOR)
if out.status == 200:
data = out.readlines()
else:
raise HTTPError('Cannot fetch behavioral data')
# drop sessions with missing PANAS items
sessions = np.genfromtxt(data, delimiter='\t', usecols=0, dtype=object,
names=True, converters={0: lambda s: s.decode()})
keeprows = np.isin(sessions, ['ses-{}'.format(f) for f in SESSIONS])
panas = np.genfromtxt(data, delimiter='\t', names=True, dtype=float,
usecols=range(28, 91))[keeprows]
# create subscales from individual item scores
measures = {}
for subscale, items in PANAS.items():
measure = stu(panas[['panas{}'.format(f) for f in items]])
measures[subscale] = measure.sum(axis=-1)
return measures
def test_when_urlopen_throws_exception_the_http_error_is_used_to_instantiate_the_response(
self):
http_error = HTTPError('/', 500, '', None, None)
self.urlopen.side_effect = http_error
with mock.patch.object(self.sut_class, '_get_request'):
self.sut._get_response('GET', '/', {}, **{})
self.response.assert_called_once_with(http_error)
def redirect_request(self, req, fp, code, msg, headers, newurl):
# If does not request from this module, go to original method
if not hasattr(req, 'locations'):
if code == 308 and not hasattr(_HTTPRedirectHandler, 'http_error_308'):
return
return super().redirect_request(req, fp, code, msg, headers, newurl)
logger.debug('Redirect to URL: ' + newurl)
req.locations.append(newurl)
method = req.get_method()
if method not in self.rmethod:
raise HTTPError(req.full_url, code, msg, headers, fp)
data = req.data # is used by fixedly method redirections
newheaders = {k.lower(): v for k, v in req.headers.items()}
if code in self.rmethod:
for header in ('content-length', 'content-type', 'transfer-encoding'):
newheaders.pop(header, None)
data = None
if method != 'HEAD':
method = 'GET'
# Useless in our modules, memo for somebody may needs
#newurl = newurl.replace(' ', '%20')
newreq = _Request(newurl, data=data, headers=newheaders,
origin_req_host=req.origin_req_host,
unverifiable=True, method=method)
# Important attributes MUST be passed to new request
newreq.headget = req.headget
newreq.locations = req.locations
newreq.responses = req.responses
return newreq
def test_error(mocker):
from appconfig.__main__ import test_error
mocker.patch('appconfig.__main__.urlopen')
with pytest.raises(RuntimeError):
test_error('wals3')
mocker.patch('appconfig.__main__.urlopen',
mocker.Mock(side_effect=HTTPError('', 500, '', {}, None)))
test_error('wals3')
def test_add_hook_error(self):
request = self.factory.post('/')
request.user = self.user
resp = BytesIO(b'')
resp.status = 412
error = HTTPError('', 412, '', {}, resp)
with patch('urllib.request.urlopen', side_effect=error) as urlopen:
data = add_hook(request)
self.assertTrue(isinstance(data, HttpResponse))
self.assertTrue(data.status_code, 412)
def do_auth_capture(request):
req = create_authnet_checkout_request(request)
gcontext = ssl.SSLContext()
try:
response_xml = urlopen(req, context=gcontext)
print('ian')
# print(response_xml.read())
except HTTPError as err:
raise HTTPError(err)
except URLError as err:
raise URLError(err)
return response_xml
def validated_mpesa_access_token():
url = MpesaC2bCredential.API_URL
auth = base64.b64encode(
bytes(
'%s:%s' % (MpesaC2bCredential.consumer_key,
MpesaC2bCredential.consumer_secret), 'ascii'))
req = Request(url)
req.add_header("Authorization", "Basic %s" % auth.decode('utf-8'))
try:
result = urlopen(req).read()
except URLError as err:
raise URLError(err)
except HTTPError as err:
raise HTTPError(err)
r = result.decode(encoding='utf-8', errors='ignore')
mpesa_access_token = json.loads(r)
validated_mpesa_access_token1 = mpesa_access_token['access_token']
return validated_mpesa_access_token1
def _make_api_query(dtype,
includes=None,
criteria=None,
attributes=None,
suffix=None,
returns='msg',
verbose=False):
"""
"""
url = 'https://api.brain-map.org/api/v2/data/{}/query.json?'.format(dtype)
params = [includes, criteria, attributes]
for key, value in zip(['include', 'criteria', 'only'], params):
if value is not None:
if isinstance(value, list):
value = ','.join(value)
url += '{}={}&'.format(key, urllib.parse.quote_plus(value))
if suffix is not None:
url += suffix
if verbose:
print("Querying {}...".format(urllib.parse.unquote_plus(url)))
response = urlopen(url)
if response.status != 200:
raise HTTPError('Failed to query API with code {}: {}'.format(
response.status, response.reason))
info = json.loads(response.read().decode('utf-8'))
if not info['success']:
raise ValueError('Provided query {} is invalid. Please check '
'parameters and try again.'.format(
urllib.parse.unquote_plus(url)))
elif info['total_rows'] == 0:
raise ValueError('Provided query {} returned no results. Please '
'check parameters and try again.'.format(
urllib.parse.unquote_plus(url)))
if returns is not None:
info = info.get(returns, [])
return info
def test_wrong_exception_order(self):
tmp = self.mkdtemp()
path = os.path.join(tmp, 'xxx')
self.write_file(path)
dist_files = [('xxx', '2.6', path)]
self.write_file(self.rc, PYPIRC_LONG_PASSWORD)
pkg_dir, dist = self.create_dist(dist_files=dist_files)
tests = [(OSError('oserror'), 'oserror', OSError), (HTTPError('url',
400, 'httperror', {}, None), 'Upload failed (400): httperror',
DistutilsError)]
for exception, expected, raised_exception in tests:
with self.subTest(exception=type(exception).__name__):
with mock.patch('distutils.command.upload.urlopen', new=
mock.Mock(side_effect=exception)):
with self.assertRaises(raised_exception):
cmd = upload(dist)
cmd.ensure_finalized()
cmd.run()
results = self.get_logs(ERROR)
self.assertIn(expected, results[-1])
self.clear_logs()
def redirect_request(self, req, fp, code, msg, headers, newurl):
m = req.get_method()
if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
or code in (301, 302, 303) and m == "POST"):
# Strictly (according to RFC 2616), 301 or 302 in response
# to a POST MUST NOT cause a redirection without confirmation
# from the user (of urllib2, in this case). In practice,
# essentially all clients do redirect in this case, so we
# do the same.
# be conciliant with URIs containing a space
newurl = newurl.replace(' ', '%20')
newheaders = dict(
(k, v) for k, v in req.headers.items()
if k.lower() not in ("content-length", "content-type"))
return Request(newurl,
headers=newheaders,
origin_req_host=req.get_origin_req_host(),
unverifiable=True,
method=m)
else:
raise HTTPError(req.get_full_url(), code, msg, headers, fp)
def get_url_response(url):
"""
This function gets response from a url
:param url:
:return:
"""
try:
if validators.url(url) and urlGenerator.verify_host(url) == 0:
r = requests.get(url)
if r.status_code == 200:
return r
except HTTPError as err:
raise HTTPError("HTTP error with status", err.code)
except TypeError as err2:
raise TypeError("Type of arg ", url, " is ", type(url))
return None
except socket.gaierror:
pass
except urllib3.exceptions.NewConnectionError:
pass
except urllib3.exceptions.MaxRetryError:
pass
except ConnectionError as e:
pass
class TestUpload(moves.unittest.TestCase):
def setUp(self):
self.package = PackageDescription.from_string("""\
Name: foo
""")
self.cwd = tempfile.mkdtemp()
try:
self.old_cwd = os.getcwd()
os.chdir(self.cwd)
filename = op.join(self.cwd, "foo.bin")
fp = open(filename, "wb")
try:
fp.write(six.b("garbage"))
finally:
fp.close()
except:
shutil.rmtree(self.cwd)
raise
def tearDown(self):
os.chdir(self.old_cwd)
shutil.rmtree(self.cwd)
def test_upload_post_data(self):
post_data = build_upload_post_data("foo.bin", "bdist_dumb",
self.package)
self.assertEqual(post_data[":action"], "file_upload")
self.assertEqual(post_data["content"], ("foo.bin", six.b("garbage")))
def test_signing(self):
self.assertRaises(NotImplementedError, build_upload_post_data,
"foo.bin", "bdist_dumb", self.package, True)
def test_build_request(self):
repository = "http://localhost"
post_data = build_upload_post_data("foo.bin", "bdist_dumb",
self.package)
request = build_request(repository, post_data, "dummy_auth")
r_headers = {
"Content-type":
six.
b("multipart/form-data; boundary=--------------GHSKFJDLGDS7543FJKLFHRE75642756743254"
),
"Content-length":
"2238",
"Authorization":
"dummy_auth"
}
self.assertEqual(request.headers, r_headers)
@mock.patch("bento.pypi.upload_utils.urlopen",
lambda request: MockedResult(200, ""))
def test_upload(self):
config = PyPIConfig("john", "password", repository="http://localhost")
upload("foo.bin", "bdist_dumb", self.package, config)
@mock.patch("bento.pypi.upload_utils.urlopen",
my_urlopen_factory(
HTTPError("", 404, "url not found", {},
six.moves.StringIO())))
def test_upload_error_404(self):
config = PyPIConfig("john", "password", repository="http://localhost")
self.assertRaises(bento.errors.PyPIError, upload, "foo.bin",
"bdist_dumb", self.package, config)
@mock.patch("bento.pypi.upload_utils.urlopen",
my_urlopen_factory(URLError("dummy")))
def test_upload_error_no_host(self):
config = PyPIConfig("john", "password", repository="http://llocalhost")
self.assertRaises(URLError, upload, "foo.bin", "bdist_dumb",
self.package, config)
@mock.patch("bento.pypi.upload_utils.urlopen",
lambda request: MockedResult(200, ""))
def test_upload_auth(self):
config = PyPIConfig("john", "password", repository="http://localhost")
self.assertRaises(NotImplementedError, upload, "foo.bin", "bdist_dumb",
self.package, config, True)
def __init__(self, request, fp, code, message, headers):
error_message = '(%s) %s' % (code, message)
HTTPError.__init__(self, fp.geturl(), code, error_message, headers, fp)
self.request = request
self.message = error_message
def download_from_url(url, dest_file):
"""
Attempt to download file specified by url to 'dest_file'
Raises:
WrongFileTypeException
when content-type is not in the supported types or cannot
be derived from the URL
FileExceptionsException
If the filename (derived from the URL) already exists in
the destination directory.
HTTPError
...
"""
# Don't download files multiple times!
if type(dest_file) == str and pathexists(dest_file):
raise FileExistsException('URL [%s] already downloaded.' % url)
response = request(url)
info = response.info()
actual_url = response.url
if actual_url == 'http://i.imgur.com/removed.png':
raise HTTPError(actual_url, 404, "Imgur suggests the image was removed", None, None)
# Work out file type either from the response or the url.
if 'content-type' in info.keys():
filetype = info['content-type']
elif url.endswith('.jpg') or url.endswith('.jpeg'):
filetype = 'image/jpeg'
elif url.endswith('.png'):
filetype = 'image/png'
elif url.endswith('.gif'):
filetype = 'image/gif'
elif url.endswith('.mp4'):
filetype = 'video/mp4'
elif url.endswith('.webm'):
filetype = 'video/webm'
else:
filetype = 'unknown'
# Only try to download acceptable image types
if filetype not in ['image/jpeg', 'image/png', 'image/gif', 'video/webm', 'video/mp4']:
raise WrongFileTypeException('WRONG FILE TYPE: %s has type: %s!' % (url, filetype))
filedata = response.read()
if dest_file == '':
return
if type(dest_file) == str:
filehandle = open(dest_file, 'wb')
else:
filehandle = dest_file
filehandle.write(filedata)
if type(dest_file) == str:
filehandle.close()
with pytest.raises(ValidationError) as validation_error:
VerificationService.verify_airbnb(
'0x112234455C3a32FD11230C42E7Bccd4A84e02010', "12a34")
assert str(validation_error.value) == 'AirbnbUserId should be a number.'
# Verify attestation not stored
attestations = Attestation.query.all()
assert (len(attestations)) == 0
@mock.patch('logic.attestation_service.IPFSHelper')
@mock.patch('logic.attestation_service.urlopen',
side_effect=HTTPError(
'https://www.airbnb.com/users/show/99999999999999999', 404,
"User not found", {}, {}))
def test_verify_airbnb_verification_code_non_existing_user(
mock_urllib_request, mock_ipfs):
mock_ipfs.return_value.add_json.return_value = \
'QmYpVLAyQ2SV7NLATdN3xnHTewoQ3LYN85LAcvN1pr2k3z'
with pytest.raises(AirbnbVerificationError) as service_err:
VerificationService.verify_airbnb(
'0x112234455C3a32FD11230C42E7Bccd4A84e02010', "99999999999999999")
assert str(
service_err.value) == 'Airbnb user id: 99999999999999999 not found.'
# Verify attestation not stored
attestations = Attestation.query.all()
assert (len(attestations)) == 0
def __init__(self, request, *args):
HTTPError.__init__(*(self,)+args)
self.request = request
def test_http_500(self):
status = self.do_collect_jobs_error(
HTTPError(self.url, 500, None, None, None))
self.assertEqual(RequestStatus.ERROR,
status[("api.newrelic.com", "all")].request_status)
def test_http_404(self):
status = self.do_collect_jobs_error(
HTTPError(self.url, 404, None, None, None))
self.assertEqual(RequestStatus.NOT_FOUND,
status[("api.newrelic.com", "all")].request_status)
VerificationService.verify_airbnb(
'0x112234455C3a32FD11230C42E7Bccd4A84e02010',
"12a34"
)
assert str(validation_error.value) == 'AirbnbUserId should be a number.'
# Verify attestation not stored
attestations = Attestation.query.all()
assert(len(attestations)) == 0
@mock.patch('logic.attestation_service.urlopen', side_effect=HTTPError(
'https://www.airbnb.com/users/show/99999999999999999',
404,
"User not found",
{},
{}
))
def test_verify_airbnb_verification_code_non_existing_user(
mock_urllib_request):
with pytest.raises(AirbnbVerificationError) as service_err:
VerificationService.verify_airbnb(
'0x112234455C3a32FD11230C42E7Bccd4A84e02010',
"99999999999999999"
)
assert str(
service_err.value) == 'Airbnb user id: 99999999999999999 not found.'
# Verify attestation not stored
class TestRegisterUtils(unittest.TestCase):
def test_build_post_data(self):
r_content = six.b("""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="maintainer"\r\n\r\n\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="name"\r\n\r\n""" \
"""foo\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="license"\r\n\r\n\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="author"\r\n\r\n""" \
"""John Doe\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="url"\r\n\r\n\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name=":action"\r\n\r\n""" \
"""submit\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="download_url"\r\n\r\n\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="maintainer_email"\r\n\r\n\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="author_email"\r\n\r\n\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="version"\r\n\r\n""" \
"""1.0\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="long_description"\r\n\r\n\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \
"""Content-Disposition: form-data; name="description"\r\n\r\n\r\n""" \
"""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254--\r\n""" \
"""""")
bento_info = """\
Name: foo
Version: 1.0
Author: John Doe
"""
package = PackageDescription.from_string(bento_info)
post_data = build_post_data(package, "submit")
content_type, body = encode_multipart(list(post_data.items()), [])
self.assertEqual(r_content, body)
@mock.patch(_OPENER_DIRECTOR, mock.MagicMock())
def test_register_server(self):
package = PackageDescription(name="foo")
repository = 'http://testpypi.python.org/pypi'
realm = DEFAULT_REALM
config = PyPIConfig(username="******",
password="******",
repository=repository,
realm=realm)
auth = HTTPPasswordMgr()
host = urlparse(config.repository)[0]
auth.add_password(config.realm, host, config.username, config.password)
post_data = build_post_data(package, "submit")
code, msg = post_to_server(post_data, config, auth)
self.assertEqual(code, 200)
self.assertEqual(msg, "OK")
@mock.patch("%s.open" % _OPENER_DIRECTOR,
mock.MagicMock(side_effect=HTTPError("", 404, "", {}, None)))
def test_register_server_http_errors(self):
code, msg = self._test_register_server_errors()
self.assertEqual(code, 404)
self.assertEqual(msg, "")
@mock.patch("%s.open" % _OPENER_DIRECTOR,
mock.MagicMock(side_effect=URLError("")))
def test_register_server_url_errors(self):
code, msg = self._test_register_server_errors()
self.assertEqual(code, 500)
def _test_register_server_errors(self):
package = PackageDescription(name="foo")
config = PyPIConfig.from_string("""
[distutils]
index-servers = pypi
[pypi]
username = cdavid
password = yoyo
server = http://testpypi.python.org
""")
post_data = build_post_data(package, "submit")
return post_to_server(post_data, config)
def http_error_default(self, req, fp, code, msg, headers):
result = HTTPError(req.get_full_url(), code, msg, headers, fp)
result.status = code
return result
def __init__(self, request, *args):
HTTPError.__init__(*(self, ) + args)
self.request = request
def read(self):
if self.http_error:
raise HTTPError('http://example.com', 404, 'Not found', {}, None)
return self.read_data
