def _get_fc(data_dir=None, resume=True, verbose=1): """ Gets functional connections from MyConnectome parcelled time series data Returns ------- fc : (73, 198135) numpy.ndarray Functional connections (lower triangle) """ # download time series data for all sessions ts = [] for ses in SESSIONS: if verbose > 0: print('Fetching time series for session {}'.format(ses)) out = urlopen(TIMESERIES.format(ses)) if out.status == 200: ts.append(np.loadtxt(out.readlines())) else: raise HTTPError('Failed to fetch time series data: session {}' .format(ses)) # get upper triangle of correlation matrix for each session fc = [np.corrcoef(ses.T)[np.tril_indices(len(ses.T), k=-1)] for ses in ts] # return stacked sessions return np.row_stack(fc)
def _get_panas(data_dir=None, resume=True, verbose=1): """ Gets PANAS subscales from MyConnectome behavioral data Returns ------- panas : dict Where keys are PANAS subscales names and values are session-level composite measures """ from numpy.lib.recfunctions import structured_to_unstructured as stu # download behavioral data out = urlopen(BEHAVIOR) if out.status == 200: data = out.readlines() else: raise HTTPError('Cannot fetch behavioral data') # drop sessions with missing PANAS items sessions = np.genfromtxt(data, delimiter='\t', usecols=0, dtype=object, names=True, converters={0: lambda s: s.decode()}) keeprows = np.isin(sessions, ['ses-{}'.format(f) for f in SESSIONS]) panas = np.genfromtxt(data, delimiter='\t', names=True, dtype=float, usecols=range(28, 91))[keeprows] # create subscales from individual item scores measures = {} for subscale, items in PANAS.items(): measure = stu(panas[['panas{}'.format(f) for f in items]]) measures[subscale] = measure.sum(axis=-1) return measures
def test_when_urlopen_throws_exception_the_http_error_is_used_to_instantiate_the_response( self): http_error = HTTPError('/', 500, '', None, None) self.urlopen.side_effect = http_error with mock.patch.object(self.sut_class, '_get_request'): self.sut._get_response('GET', '/', {}, **{}) self.response.assert_called_once_with(http_error)
def redirect_request(self, req, fp, code, msg, headers, newurl): # If does not request from this module, go to original method if not hasattr(req, 'locations'): if code == 308 and not hasattr(_HTTPRedirectHandler, 'http_error_308'): return return super().redirect_request(req, fp, code, msg, headers, newurl) logger.debug('Redirect to URL: ' + newurl) req.locations.append(newurl) method = req.get_method() if method not in self.rmethod: raise HTTPError(req.full_url, code, msg, headers, fp) data = req.data # is used by fixedly method redirections newheaders = {k.lower(): v for k, v in req.headers.items()} if code in self.rmethod: for header in ('content-length', 'content-type', 'transfer-encoding'): newheaders.pop(header, None) data = None if method != 'HEAD': method = 'GET' # Useless in our modules, memo for somebody may needs #newurl = newurl.replace(' ', '%20') newreq = _Request(newurl, data=data, headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True, method=method) # Important attributes MUST be passed to new request newreq.headget = req.headget newreq.locations = req.locations newreq.responses = req.responses return newreq
def test_error(mocker): from appconfig.__main__ import test_error mocker.patch('appconfig.__main__.urlopen') with pytest.raises(RuntimeError): test_error('wals3') mocker.patch('appconfig.__main__.urlopen', mocker.Mock(side_effect=HTTPError('', 500, '', {}, None))) test_error('wals3')
def test_add_hook_error(self): request = self.factory.post('/') request.user = self.user resp = BytesIO(b'') resp.status = 412 error = HTTPError('', 412, '', {}, resp) with patch('urllib.request.urlopen', side_effect=error) as urlopen: data = add_hook(request) self.assertTrue(isinstance(data, HttpResponse)) self.assertTrue(data.status_code, 412)
def do_auth_capture(request): req = create_authnet_checkout_request(request) gcontext = ssl.SSLContext() try: response_xml = urlopen(req, context=gcontext) print('ian') # print(response_xml.read()) except HTTPError as err: raise HTTPError(err) except URLError as err: raise URLError(err) return response_xml
def validated_mpesa_access_token(): url = MpesaC2bCredential.API_URL auth = base64.b64encode( bytes( '%s:%s' % (MpesaC2bCredential.consumer_key, MpesaC2bCredential.consumer_secret), 'ascii')) req = Request(url) req.add_header("Authorization", "Basic %s" % auth.decode('utf-8')) try: result = urlopen(req).read() except URLError as err: raise URLError(err) except HTTPError as err: raise HTTPError(err) r = result.decode(encoding='utf-8', errors='ignore') mpesa_access_token = json.loads(r) validated_mpesa_access_token1 = mpesa_access_token['access_token'] return validated_mpesa_access_token1
def _make_api_query(dtype, includes=None, criteria=None, attributes=None, suffix=None, returns='msg', verbose=False): """ """ url = 'https://api.brain-map.org/api/v2/data/{}/query.json?'.format(dtype) params = [includes, criteria, attributes] for key, value in zip(['include', 'criteria', 'only'], params): if value is not None: if isinstance(value, list): value = ','.join(value) url += '{}={}&'.format(key, urllib.parse.quote_plus(value)) if suffix is not None: url += suffix if verbose: print("Querying {}...".format(urllib.parse.unquote_plus(url))) response = urlopen(url) if response.status != 200: raise HTTPError('Failed to query API with code {}: {}'.format( response.status, response.reason)) info = json.loads(response.read().decode('utf-8')) if not info['success']: raise ValueError('Provided query {} is invalid. Please check ' 'parameters and try again.'.format( urllib.parse.unquote_plus(url))) elif info['total_rows'] == 0: raise ValueError('Provided query {} returned no results. Please ' 'check parameters and try again.'.format( urllib.parse.unquote_plus(url))) if returns is not None: info = info.get(returns, []) return info
def test_wrong_exception_order(self): tmp = self.mkdtemp() path = os.path.join(tmp, 'xxx') self.write_file(path) dist_files = [('xxx', '2.6', path)] self.write_file(self.rc, PYPIRC_LONG_PASSWORD) pkg_dir, dist = self.create_dist(dist_files=dist_files) tests = [(OSError('oserror'), 'oserror', OSError), (HTTPError('url', 400, 'httperror', {}, None), 'Upload failed (400): httperror', DistutilsError)] for exception, expected, raised_exception in tests: with self.subTest(exception=type(exception).__name__): with mock.patch('distutils.command.upload.urlopen', new= mock.Mock(side_effect=exception)): with self.assertRaises(raised_exception): cmd = upload(dist) cmd.ensure_finalized() cmd.run() results = self.get_logs(ERROR) self.assertIn(expected, results[-1]) self.clear_logs()
def redirect_request(self, req, fp, code, msg, headers, newurl): m = req.get_method() if (code in (301, 302, 303, 307) and m in ("GET", "HEAD") or code in (301, 302, 303) and m == "POST"): # Strictly (according to RFC 2616), 301 or 302 in response # to a POST MUST NOT cause a redirection without confirmation # from the user (of urllib2, in this case). In practice, # essentially all clients do redirect in this case, so we # do the same. # be conciliant with URIs containing a space newurl = newurl.replace(' ', '%20') newheaders = dict( (k, v) for k, v in req.headers.items() if k.lower() not in ("content-length", "content-type")) return Request(newurl, headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True, method=m) else: raise HTTPError(req.get_full_url(), code, msg, headers, fp)
def get_url_response(url): """ This function gets response from a url :param url: :return: """ try: if validators.url(url) and urlGenerator.verify_host(url) == 0: r = requests.get(url) if r.status_code == 200: return r except HTTPError as err: raise HTTPError("HTTP error with status", err.code) except TypeError as err2: raise TypeError("Type of arg ", url, " is ", type(url)) return None except socket.gaierror: pass except urllib3.exceptions.NewConnectionError: pass except urllib3.exceptions.MaxRetryError: pass except ConnectionError as e: pass
class TestUpload(moves.unittest.TestCase): def setUp(self): self.package = PackageDescription.from_string("""\ Name: foo """) self.cwd = tempfile.mkdtemp() try: self.old_cwd = os.getcwd() os.chdir(self.cwd) filename = op.join(self.cwd, "foo.bin") fp = open(filename, "wb") try: fp.write(six.b("garbage")) finally: fp.close() except: shutil.rmtree(self.cwd) raise def tearDown(self): os.chdir(self.old_cwd) shutil.rmtree(self.cwd) def test_upload_post_data(self): post_data = build_upload_post_data("foo.bin", "bdist_dumb", self.package) self.assertEqual(post_data[":action"], "file_upload") self.assertEqual(post_data["content"], ("foo.bin", six.b("garbage"))) def test_signing(self): self.assertRaises(NotImplementedError, build_upload_post_data, "foo.bin", "bdist_dumb", self.package, True) def test_build_request(self): repository = "http://localhost" post_data = build_upload_post_data("foo.bin", "bdist_dumb", self.package) request = build_request(repository, post_data, "dummy_auth") r_headers = { "Content-type": six. b("multipart/form-data; boundary=--------------GHSKFJDLGDS7543FJKLFHRE75642756743254" ), "Content-length": "2238", "Authorization": "dummy_auth" } self.assertEqual(request.headers, r_headers) @mock.patch("bento.pypi.upload_utils.urlopen", lambda request: MockedResult(200, "")) def test_upload(self): config = PyPIConfig("john", "password", repository="http://localhost") upload("foo.bin", "bdist_dumb", self.package, config) @mock.patch("bento.pypi.upload_utils.urlopen", my_urlopen_factory( HTTPError("", 404, "url not found", {}, six.moves.StringIO()))) def test_upload_error_404(self): config = PyPIConfig("john", "password", repository="http://localhost") self.assertRaises(bento.errors.PyPIError, upload, "foo.bin", "bdist_dumb", self.package, config) @mock.patch("bento.pypi.upload_utils.urlopen", my_urlopen_factory(URLError("dummy"))) def test_upload_error_no_host(self): config = PyPIConfig("john", "password", repository="http://llocalhost") self.assertRaises(URLError, upload, "foo.bin", "bdist_dumb", self.package, config) @mock.patch("bento.pypi.upload_utils.urlopen", lambda request: MockedResult(200, "")) def test_upload_auth(self): config = PyPIConfig("john", "password", repository="http://localhost") self.assertRaises(NotImplementedError, upload, "foo.bin", "bdist_dumb", self.package, config, True)
def __init__(self, request, fp, code, message, headers): error_message = '(%s) %s' % (code, message) HTTPError.__init__(self, fp.geturl(), code, error_message, headers, fp) self.request = request self.message = error_message
def download_from_url(url, dest_file): """ Attempt to download file specified by url to 'dest_file' Raises: WrongFileTypeException when content-type is not in the supported types or cannot be derived from the URL FileExceptionsException If the filename (derived from the URL) already exists in the destination directory. HTTPError ... """ # Don't download files multiple times! if type(dest_file) == str and pathexists(dest_file): raise FileExistsException('URL [%s] already downloaded.' % url) response = request(url) info = response.info() actual_url = response.url if actual_url == 'http://i.imgur.com/removed.png': raise HTTPError(actual_url, 404, "Imgur suggests the image was removed", None, None) # Work out file type either from the response or the url. if 'content-type' in info.keys(): filetype = info['content-type'] elif url.endswith('.jpg') or url.endswith('.jpeg'): filetype = 'image/jpeg' elif url.endswith('.png'): filetype = 'image/png' elif url.endswith('.gif'): filetype = 'image/gif' elif url.endswith('.mp4'): filetype = 'video/mp4' elif url.endswith('.webm'): filetype = 'video/webm' else: filetype = 'unknown' # Only try to download acceptable image types if filetype not in ['image/jpeg', 'image/png', 'image/gif', 'video/webm', 'video/mp4']: raise WrongFileTypeException('WRONG FILE TYPE: %s has type: %s!' % (url, filetype)) filedata = response.read() if dest_file == '': return if type(dest_file) == str: filehandle = open(dest_file, 'wb') else: filehandle = dest_file filehandle.write(filedata) if type(dest_file) == str: filehandle.close()
with pytest.raises(ValidationError) as validation_error: VerificationService.verify_airbnb( '0x112234455C3a32FD11230C42E7Bccd4A84e02010', "12a34") assert str(validation_error.value) == 'AirbnbUserId should be a number.' # Verify attestation not stored attestations = Attestation.query.all() assert (len(attestations)) == 0 @mock.patch('logic.attestation_service.IPFSHelper') @mock.patch('logic.attestation_service.urlopen', side_effect=HTTPError( 'https://www.airbnb.com/users/show/99999999999999999', 404, "User not found", {}, {})) def test_verify_airbnb_verification_code_non_existing_user( mock_urllib_request, mock_ipfs): mock_ipfs.return_value.add_json.return_value = \ 'QmYpVLAyQ2SV7NLATdN3xnHTewoQ3LYN85LAcvN1pr2k3z' with pytest.raises(AirbnbVerificationError) as service_err: VerificationService.verify_airbnb( '0x112234455C3a32FD11230C42E7Bccd4A84e02010', "99999999999999999") assert str( service_err.value) == 'Airbnb user id: 99999999999999999 not found.' # Verify attestation not stored attestations = Attestation.query.all() assert (len(attestations)) == 0
def __init__(self, request, *args): HTTPError.__init__(*(self,)+args) self.request = request
def test_http_500(self): status = self.do_collect_jobs_error( HTTPError(self.url, 500, None, None, None)) self.assertEqual(RequestStatus.ERROR, status[("api.newrelic.com", "all")].request_status)
def test_http_404(self): status = self.do_collect_jobs_error( HTTPError(self.url, 404, None, None, None)) self.assertEqual(RequestStatus.NOT_FOUND, status[("api.newrelic.com", "all")].request_status)
VerificationService.verify_airbnb( '0x112234455C3a32FD11230C42E7Bccd4A84e02010', "12a34" ) assert str(validation_error.value) == 'AirbnbUserId should be a number.' # Verify attestation not stored attestations = Attestation.query.all() assert(len(attestations)) == 0 @mock.patch('logic.attestation_service.urlopen', side_effect=HTTPError( 'https://www.airbnb.com/users/show/99999999999999999', 404, "User not found", {}, {} )) def test_verify_airbnb_verification_code_non_existing_user( mock_urllib_request): with pytest.raises(AirbnbVerificationError) as service_err: VerificationService.verify_airbnb( '0x112234455C3a32FD11230C42E7Bccd4A84e02010', "99999999999999999" ) assert str( service_err.value) == 'Airbnb user id: 99999999999999999 not found.' # Verify attestation not stored
class TestRegisterUtils(unittest.TestCase): def test_build_post_data(self): r_content = six.b("""----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="maintainer"\r\n\r\n\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="name"\r\n\r\n""" \ """foo\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="license"\r\n\r\n\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="author"\r\n\r\n""" \ """John Doe\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="url"\r\n\r\n\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name=":action"\r\n\r\n""" \ """submit\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="download_url"\r\n\r\n\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="maintainer_email"\r\n\r\n\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="author_email"\r\n\r\n\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="version"\r\n\r\n""" \ """1.0\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="long_description"\r\n\r\n\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\r\n""" \ """Content-Disposition: form-data; name="description"\r\n\r\n\r\n""" \ """----------------GHSKFJDLGDS7543FJKLFHRE75642756743254--\r\n""" \ """""") bento_info = """\ Name: foo Version: 1.0 Author: John Doe """ package = PackageDescription.from_string(bento_info) post_data = build_post_data(package, "submit") content_type, body = encode_multipart(list(post_data.items()), []) self.assertEqual(r_content, body) @mock.patch(_OPENER_DIRECTOR, mock.MagicMock()) def test_register_server(self): package = PackageDescription(name="foo") repository = 'http://testpypi.python.org/pypi' realm = DEFAULT_REALM config = PyPIConfig(username="******", password="******", repository=repository, realm=realm) auth = HTTPPasswordMgr() host = urlparse(config.repository)[0] auth.add_password(config.realm, host, config.username, config.password) post_data = build_post_data(package, "submit") code, msg = post_to_server(post_data, config, auth) self.assertEqual(code, 200) self.assertEqual(msg, "OK") @mock.patch("%s.open" % _OPENER_DIRECTOR, mock.MagicMock(side_effect=HTTPError("", 404, "", {}, None))) def test_register_server_http_errors(self): code, msg = self._test_register_server_errors() self.assertEqual(code, 404) self.assertEqual(msg, "") @mock.patch("%s.open" % _OPENER_DIRECTOR, mock.MagicMock(side_effect=URLError(""))) def test_register_server_url_errors(self): code, msg = self._test_register_server_errors() self.assertEqual(code, 500) def _test_register_server_errors(self): package = PackageDescription(name="foo") config = PyPIConfig.from_string(""" [distutils] index-servers = pypi [pypi] username = cdavid password = yoyo server = http://testpypi.python.org """) post_data = build_post_data(package, "submit") return post_to_server(post_data, config)
def http_error_default(self, req, fp, code, msg, headers): result = HTTPError(req.get_full_url(), code, msg, headers, fp) result.status = code return result
def __init__(self, request, *args): HTTPError.__init__(*(self, ) + args) self.request = request
def read(self): if self.http_error: raise HTTPError('http://example.com', 404, 'Not found', {}, None) return self.read_data