def test_ensure_response_callback(self):
m = JsonModel()
request = HttpRequest(
None,
m.response,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body='{}',
headers={'content-type': 'application/json'})
h = HttpMockSequence([ ({'status': 200}, '{}')])
responses = []
def _on_response(resp, responses=responses):
responses.append(resp)
request.add_response_callback(_on_response)
request.execute(http=h)
self.assertEqual(1, len(responses))
def test_execute(self):
batch = BatchHttpRequest()
callbacks = Callbacks()
batch.add(self.request1, callback=callbacks.f)
batch.add(self.request2, callback=callbacks.f)
http = HttpMockSequence([
({'status': '200',
'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
BATCH_RESPONSE),
])
batch.execute(http=http)
self.assertEqual({'foo': 42}, callbacks.responses['1'])
self.assertEqual(None, callbacks.exceptions['1'])
self.assertEqual({'baz': 'qux'}, callbacks.responses['2'])
self.assertEqual(None, callbacks.exceptions['2'])
def test_failed_authentication(self, mock_credclass, mock_cache):
# http 401
# Disable the discovery cache, so that we can fully control the http requests
# with HttpMockSequence below
mock_cache.return_value = None
mock_creds = mock_credclass.from_json_keyfile_dict.return_value
mock_creds.authorize.return_value = HttpMockSequence([({
'status': '200'
}, self.valid_discovery), ({
'status': '401'
}, '')])
with self.assertRaises(HttpError):
analytics_upageviews([1068728, 1074760], self.start_date)
def test_submit_job_successfully(self):
http_mocked = HttpMockSequence([
({
'status': '200'
}, open('tests/fixtures/dataproc/first_request.json',
'rb').read()), ({
'status': '200'
}, 'echo_request_body'),
({
'status': '200'
}, open('tests/fixtures/dataproc/job_status_running.json',
'rb').read()),
({
'status': '200'
}, open('tests/fixtures/dataproc/job_status_running.json',
'rb').read()),
({
'status': '200'
}, open('tests/fixtures/dataproc/job_status_done.json',
'rb').read())
])
dataproc_test = dataproc.Dataproc(project='project',
region='region',
http=http_mocked)
result = dataproc_test.submit_job('CLUSTER', 'BUCKET',
['/path/to/jar/jarname.jar'], 'main',
['arg1', 'arg2'])
body_request_expected = {
'projectId': 'project',
'job': {
'placement': {
'clusterName': 'CLUSTER'
},
'sparkJob': {
'jarFileUris': ['/path/to/jar/jarname.jar'],
'mainClass': 'main',
'args': ['arg1', 'arg2']
},
'reference': {
'jobId': 'main_1994_04_27_12_00_01'
}
}
}
self.assertEqual(body_request_expected, result)
def test_userip_missing_is_not_added_to_discovery_uri(self):
# build() will raise an HttpError on a 400, use this to pick the request uri
# out of the raised exception.
try:
http = HttpMockSequence([
({
'status': '400'
}, open(datafile('zoo.json'), 'rb').read()),
])
zoo = build('zoo',
'v1',
http=http,
developerKey=None,
discoveryServiceUrl='http://example.com')
self.fail('Should have raised an exception.')
except HttpError as e:
self.assertEqual(e.uri, 'http://example.com')
def test_get_media(self):
http = HttpMock(datafile('zoo.json'), {'status': '200'})
zoo = build('zoo', 'v1', http=http)
request = zoo.animals().get_media(name='Lion')
parsed = urlparse(request.uri)
q = parse_qs(parsed[4])
self.assertEqual(q['alt'], ['media'])
self.assertEqual(request.headers['accept'], '*/*')
http = HttpMockSequence([
({
'status': '200'
}, 'standing in for media'),
])
response = request.execute(http=http)
self.assertEqual(b'standing in for media', response)
def test_credentials_good(self):
private_key = datafile('privatekey.%s' % self.format)
credentials = SignedJwtAssertionCredentials('*****@*****.**',
private_key,
scope='read+write',
sub='*****@*****.**')
http = HttpMockSequence([
({
'status': '200'
}, '{"access_token":"1/3w","expires_in":3600}'),
({
'status': '200'
}, 'echo_request_headers'),
])
http = credentials.authorize(http)
resp, content = http.request('http://example.org')
self.assertEqual('Bearer 1/3w', content['Authorization'])
def test_exchange_code_for_token_fail(self):
http = HttpMockSequence([
({
'status': '400'
}, '{"error":"invalid_request"}'),
])
try:
credentials = credentials_from_code(self.client_id,
self.client_secret,
self.scope,
self.code,
redirect_uri=self.redirect_uri,
http=http)
self.fail('should raise exception if exchange doesn\'t get 200')
except FlowExchangeError:
pass
def test_execute_batch_http_error(self):
callbacks = Callbacks()
batch = BatchHttpRequest(callback=callbacks.f)
batch.add(self.request1)
batch.add(self.request2)
http = HttpMockSequence([
({'status': '200',
'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
BATCH_ERROR_RESPONSE),
])
batch.execute(http=http)
self.assertEqual({'foo': 42}, callbacks.responses['1'])
expected = ('<HttpError 403 when requesting '
'https://www.googleapis.com/someapi/v1/collection/?foo=bar returned '
'"Access Not Configured">')
self.assertEqual(expected, str(callbacks.exceptions['2']))
def __enter__(self):
http = HttpMockSequence(self._responses)
native_request_method = http.request
# Collecting requests to validate at __exit__.
def _request_wrapper(*args, **kwargs):
self._actual_requests.append(args + (kwargs.get('body', ''), ))
return native_request_method(*args, **kwargs)
http.request = _request_wrapper
discovery = requests.get(
'https://www.googleapis.com/discovery/v1/apis/ml/v1/rest')
service_mock = build_from_document(discovery.json(), http=http)
with mock.patch.object(hook.MLEngineHook,
'get_conn',
return_value=service_mock):
return hook.MLEngineHook()
def test_user_does_not_have_analytics_account(self, mock_credclass,
mock_cache):
# http 403
# Disable the discovery cache, so that we can fully control the http requests
# with HttpMockSequence below
mock_cache.return_value = None
mock_creds = mock_credclass.from_json_keyfile_dict.return_value
mock_creds.authorize.return_value = HttpMockSequence([({
"status": "200"
}, self.valid_discovery), ({
"status": "403"
}, "")])
with self.assertRaises(HttpError):
analytics_upageviews([1068728, 1074760], self.start_date)
def _credentials_refresh(self, credentials):
http = HttpMockSequence([
({
'status': '200'
}, '{"access_token":"1/3w","expires_in":3600}'),
({
'status': '401'
}, ''),
({
'status': '200'
}, '{"access_token":"3/3w","expires_in":3600}'),
({
'status': '200'
}, 'echo_request_headers'),
])
http = credentials.authorize(http)
resp, content = http.request('http://example.org')
return content
def test_successful_query(self, mock_credclass, mock_cache):
# Disable the discovery cache, so that we can fully control the http requests
# with HttpMockSequence below
mock_cache.return_value = None
mock_creds = mock_credclass.from_json_keyfile_dict.return_value
mock_creds.authorize.return_value = HttpMockSequence([
({
"status": "200"
}, self.valid_discovery),
({
"status": "200"
}, self.valid_response),
])
results = analytics_upageviews([1068728, 1074760], self.start_date)
self.assertEqual(results, {1068728: 18775, 1074760: 753})
def mock_google_sheets_responses(fixture_files=None):
"""
Function to mock one or multiple requests to sheets.
:param fixture_files: Fixture file name (must be located in the fixture folder).
:return: An HttpMockSequence object.
"""
mocks = [({'status': '200'}, open_fixture('discovery.json'))]
# If input is a string, transform it into list of one item
if isinstance(fixture_files, str):
fixture_files = [fixture_files]
# Add each fixture as a request mock if any
if fixture_files:
for file in fixture_files:
mocks.append(({'status': '200'}, open_fixture(file)))
http_mocks = HttpMockSequence(mocks)
return http_mocks
def test_token_refresh_failure(self):
for status_code in REFRESH_STATUS_CODES:
http = HttpMockSequence([
({
'status': status_code
}, ''),
({
'status': '400'
}, '{"error":"access_denied"}'),
])
http = self.credentials.authorize(http)
try:
http.request('http://example.com')
self.fail('should raise AccessTokenRefreshError exception')
except AccessTokenRefreshError:
pass
self.assertTrue(self.credentials.access_token_expired)
self.assertEqual(None, self.credentials.token_response)
def func(*args):
arg_files = [data_file(x) for x in list(args)]
sequence = [({
"status": "200"
}, x) for x in [data_file("tagmanager_v2_discovery.json"), *arg_files]]
http = HttpMockSequence(sequence)
return (
build(SERVICE_NAME,
SERVICE_VERSION,
http=http,
cache_discovery=False),
[
json.loads(x) if x not in HTTP_MOCK_KEYWORDS else x
for x in arg_files
],
)
def test_create_file_success(self, mock_from_service_account_file): # pylint: disable=unused-argument
"""
Test normal case for uploading a file.
"""
fake_file_id = 'fake-file-id'
http_mock_sequence = HttpMockSequence([
# First, a request is made to the discovery API to construct a client object for Drive.
({'status': '200'}, self.mock_discovery_response_content),
# Then, a request is made to upload the file.
({'status': '200'}, '{{"id": "{}"}}'.format(fake_file_id)),
])
test_client = DriveApi('non-existent-secrets.json', http=http_mock_sequence)
response = test_client.create_file_in_folder(
'fake-folder-id',
'Fake Filename',
BytesIO('fake file contents'.encode('ascii')),
'text/plain',
)
assert response == fake_file_id
def test_gdrive(self):
image_folder = "imagehosting"
api_key = "your_api_key"
folder_id = "givne_folder_id"
image_file_name = "image_0.png"
image_ids = [f"given_image_id_{i}" for i in range(10)]
article_title = "test_article"
article_folder_id = "test-article-id"
http = HttpMockSequence(
[
({"status": "200"}, open("tests/data/drive.json", "rb").read()),
({"status": "200"}, json.dumps({"files": [{"id": folder_id}]})),
({"status": "200"}, json.dumps({"files": [{"id": article_folder_id}]})),
# (
# {"status": "200"},
# json.dumps(
# {
# "files": [
# {"id": image_ids[0], "md5Checksum": "asdf0"},
# {"id": image_ids[1], "md5Checksum": "asdf1"},
# ]
# }
# ),
# ),
(
{"status": "200", "location": "location"},
json.dumps({"id": image_ids[0]}),
),
({"status": "200"}, json.dumps({"id": image_ids[0]})),
({"status": "200"}, "{}"),
]
)
mock_gdrive_config = MagicMock()
mock_gdrive_config.article_title = article_title
mock_gdrive_config.blog_folder = image_folder
mock_gdrive_config.credit = api_key
service = build("drive", "v3", http=http, developerKey=api_key)
gdrive_service = GdriveService(config=mock_gdrive_config, api_service=service)
result = gdrive_service.upload_img(
image_file_name, f"tests/data/{image_file_name}"
)
assert result == f"https://drive.google.com/uc?export=view&id={image_ids[0]}"
def test_delete_cluster(self):
http_mocked = HttpMockSequence([
({
'status': '200'
}, open('tests/fixtures/dataproc/first_request.json',
'rb').read()),
({
'status': '200'
}, 'echo_request_headers_as_json'),
({
'status': '200'
}, open('tests/fixtures/dataproc/cluster_deleting.json',
'rb').read())
])
dataproc_test = dataproc.Dataproc(project="project",
region="region",
http=http_mocked)
result = dataproc_test.delete_cluster("NAME")
self.assertEqual(result['content-length'], '0')
def test_exchange_failure_with_json_error(self):
# Some providers have 'error' attribute as a JSON object
# in place of regular string.
# This test makes sure no strange object-to-string coversion
# exceptions are being raised instead of FlowExchangeError.
http = HttpMockSequence([
({
'status': '400'
}, """ {"error": {
"type": "OAuthException",
"message": "Error validating verification code."} }"""),
])
try:
credentials = self.flow.step2_exchange('some random code',
http=http)
self.fail('should raise exception if exchange doesn\'t get 200')
except FlowExchangeError, e:
pass
def test_token_refresh_success(self):
for status_code in REFRESH_STATUS_CODES:
token_response = {'access_token': '1/3w', 'expires_in': 3600}
http = HttpMockSequence([
({
'status': status_code
}, ''),
({
'status': '200'
}, simplejson.dumps(token_response)),
({
'status': '200'
}, 'echo_request_headers'),
])
http = self.credentials.authorize(http)
resp, content = http.request('http://example.com')
self.assertEqual('Bearer 1/3w', content['Authorization'])
self.assertFalse(self.credentials.access_token_expired)
self.assertEqual(token_response, self.credentials.token_response)
def test_media_io_base_download_handle_redirects(self):
self.request.http = HttpMockSequence([
({
'status': '200',
'content-location': 'https://secure.example.net/lion'
}, b''),
({
'status': '200',
'content-range': '0-2/5'
}, b'abc'),
])
download = MediaIoBaseDownload(fd=self.fd,
request=self.request,
chunksize=3)
status, done = download.next_chunk()
self.assertEqual('https://secure.example.net/lion', download._uri)
def test_list_with_iterate_pages(self):
http_mocked = HttpMockSequence([
({
'status': '200'
}, open('tests/fixtures/dataproc/first_request.json',
'rb').read()),
({
'status': '200'
}, open('tests/fixtures/dataproc/list_clusters_page_1.json',
'rb').read()),
({
'status': '200'
}, open('tests/fixtures/dataproc/list_clusters_page_2.json',
'rb').read())
])
dataproc_test = dataproc.Dataproc(project="",
region="",
http=http_mocked)
result = dataproc_test.list_clusters()
self.assertEqual(len(result), 2)
def test_get_latest(self):
deployments = [
Deployment("kf-vfoo-00", "2019-04-01T23:59:59+00:00"),
Deployment("kf-vfoo-01", "2019-04-02T23:59:59+00:00"),
Deployment("kf-vfoo-02", "2019-04-03T23:59:59+00:00"),
]
list_resp = {
"deployments": create_mock_list_resp(deployments),
}
http = HttpMockSequence([
({'status': '200'}, self.dm_api),
({'status': '200'}, json.dumps(list_resp)),
({"status": "200"}, json.dumps(create_mock_resource_resp(deployments[0]))),
({"status": "200"}, json.dumps(create_mock_resource_resp(deployments[1]))),
({"status": "200"}, json.dumps(create_mock_resource_resp(deployments[2]))),
])
self.assertEqual(get_kf_testing_cluster.get_latest(
project=TEST_PROJECT, base_name="kf-vfoo-??", http=http),
get_kf_testing_cluster.get_deployment_endpoint(TEST_PROJECT,
"kf-vfoo-02"))
def test_found(self):
video = YoutubeVideo(url="https://www.youtube.com/watch?v=03H1qSot9_s",
n_watch=128,
n_like=64)
response = read_file(
"tests/api_responses/youtube/videos/list/no_mention.json")
http = HttpMockSequence([({
"status": 200
}, api_discovery), ({
"status": 200
}, response)])
youtube = YouTube(secret=PLACE_HOLDER, http=http)
out = youtube.get_video_detail(video)
self.assertIsInstance(out, YoutubeVideo)
self.assertEqual(out.url, video.url)
self.assertEqual(out.n_watch, video.n_watch)
self.assertEqual(out.n_like, video.n_like)
self.assertEqual(out.published_at, "2018-02-16T04:40:17.000Z")
def test_resumable_media_fail_unknown_response_code_first_request(self):
"""Not a multipart upload."""
self.http = HttpMock(datafile('zoo.json'), {'status': '200'})
zoo = build('zoo', 'v1', http=self.http)
media_upload = MediaFileUpload(datafile('small.png'), resumable=True)
request = zoo.animals().insert(media_body=media_upload, body=None)
http = HttpMockSequence([
({
'status': '400',
'location': 'http://upload.example.com'
}, ''),
])
try:
request.execute(http=http)
self.fail('Should have raised ResumableUploadError.')
except ResumableUploadError as e:
self.assertEqual(400, e.resp.status)
def test_media_io_base_next_chunk_retries(self):
try:
import io
except ImportError:
return
f = open(datafile('small.png'), 'r')
fd = io.BytesIO(f.read())
upload = MediaIoBaseUpload(
fd=fd, mimetype='image/png', chunksize=500, resumable=True)
# Simulate 5XXs for both the request that creates the resumable upload and
# the upload itself.
http = HttpMockSequence([
({'status': '500'}, ''),
({'status': '500'}, ''),
({'status': '503'}, ''),
({'status': '200', 'location': 'location'}, ''),
({'status': '500'}, ''),
({'status': '500'}, ''),
({'status': '503'}, ''),
({'status': '200'}, '{}'),
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
headers={},
resumable=upload)
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=3)
self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
def test_list_deployments_multi_pages(self):
deployments = [
Deployment("kf-vfoo-n00", "2019-04-01T23:59:59+00:00"),
Deployment("kf-vfoo-n01", "2019-04-02T23:59:59+00:00"),
Deployment("kf-vfoo-n02", "2019-04-03T23:59:59+00:00"),
]
list_resp1 = {
"deployments": create_mock_list_resp(deployments[:1]),
"nextPageToken": "bar",
}
list_resp2 = {
"deployments": create_mock_list_resp(deployments[1:]),
}
http = HttpMockSequence([
({
'status': '200'
}, self.dm_api),
({
'status': '200'
}, json.dumps(list_resp1)),
({
"status": "200"
}, json.dumps(create_mock_resource_resp(deployments[0]))),
({
"status": "200"
}, json.dumps(list_resp2)),
({
"status": "200"
}, json.dumps(create_mock_resource_resp(deployments[1]))),
({
"status": "200"
}, json.dumps(create_mock_resource_resp(deployments[2]))),
])
actual = get_kf_testing_cluster.list_deployments(TEST_PROJECT,
"kf-vfoo",
TEST_LABEL,
http=http)
expected = sorted(create_expected_list_resp(deployments),
key=lambda entry: entry["insertTime"],
reverse=True)
self.assertListEqual(actual, expected)
def test_media_io_base_download_unknown_media_size(self):
self.request.http = HttpMockSequence([
({'status': '200'}, b'123')
])
download = MediaIoBaseDownload(
fd=self.fd, request=self.request, chunksize=3)
self.assertEqual(self.fd, download._fd)
self.assertEqual(0, download._progress)
self.assertEqual(None, download._total_size)
self.assertEqual(False, download._done)
self.assertEqual(self.request.uri, download._uri)
status, done = download.next_chunk()
self.assertEqual(self.fd.getvalue(), b'123')
self.assertEqual(True, done)
self.assertEqual(3, download._progress)
self.assertEqual(None, download._total_size)
self.assertEqual(0, status.progress())
def test_media_io_base_download_empty_file(self):
self.request.http = HttpMockSequence([
({'status': '200',
'content-range': '0-0/0'}, b''),
])
download = MediaIoBaseDownload(
fd=self.fd, request=self.request, chunksize=3)
self.assertEqual(self.fd, download._fd)
self.assertEqual(0, download._progress)
self.assertEqual(None, download._total_size)
self.assertEqual(False, download._done)
self.assertEqual(self.request.uri, download._uri)
status, done = download.next_chunk()
self.assertEqual(True, done)
self.assertEqual(0, download._progress)
self.assertEqual(0, download._total_size)
self.assertEqual(0, status.progress())
