def test_download_to_filename(self):
import os
import time
from six.moves.http_client import OK
from six.moves.http_client import PARTIAL_CONTENT
from gcloud._testing import _NamedTemporaryFile
BLOB_NAME = 'blob-name'
chunk1_response = {'status': PARTIAL_CONTENT,
'content-range': 'bytes 0-2/6'}
chunk2_response = {'status': OK,
'content-range': 'bytes 3-5/6'}
connection = _Connection(
(chunk1_response, b'abc'),
(chunk2_response, b'def'),
)
client = _Client(connection)
bucket = _Bucket(client)
MEDIA_LINK = 'http://example.com/media/'
properties = {'mediaLink': MEDIA_LINK,
'updated': '2014-12-06T13:13:50.690Z'}
blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 3
with _NamedTemporaryFile() as temp:
blob.download_to_filename(temp.name)
with open(temp.name, 'rb') as file_obj:
wrote = file_obj.read()
mtime = os.path.getmtime(temp.name)
updatedTime = time.mktime(blob.updated.timetuple())
self.assertEqual(wrote, b'abcdef')
self.assertEqual(mtime, updatedTime)
def test_download_to_filename(self):
import os
import time
from six.moves.http_client import OK
from six.moves.http_client import PARTIAL_CONTENT
from gcloud._testing import _NamedTemporaryFile
BLOB_NAME = 'blob-name'
chunk1_response = {'status': PARTIAL_CONTENT,
'content-range': 'bytes 0-2/6'}
chunk2_response = {'status': OK,
'content-range': 'bytes 3-5/6'}
connection = _Connection(
(chunk1_response, b'abc'),
(chunk2_response, b'def'),
)
client = _Client(connection)
bucket = _Bucket(client)
MEDIA_LINK = 'http://example.com/media/'
properties = {'mediaLink': MEDIA_LINK,
'updated': '2014-12-06T13:13:50.690Z'}
blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 3
with _NamedTemporaryFile() as temp:
blob.download_to_filename(temp.name)
with open(temp.name, 'rb') as file_obj:
wrote = file_obj.read()
mtime = os.path.getmtime(temp.name)
updatedTime = time.mktime(blob.updated.timetuple())
self.assertEqual(wrote, b'abcdef')
self.assertEqual(mtime, updatedTime)
def test_upload_from_file_resumable(self):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _Monkey
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import http_wrapper
from gcloud.streaming import transfer
BLOB_NAME = "blob-name"
UPLOAD_URL = "http://example.com/upload/name/key"
DATA = b"ABCDEF"
loc_response = {"status": OK, "location": UPLOAD_URL}
chunk1_response = {"status": http_wrapper.RESUME_INCOMPLETE, "range": "bytes 0-4"}
chunk2_response = {"status": OK}
# Need valid JSON on last response, since resumable.
connection = _Connection((loc_response, b""), (chunk1_response, b""), (chunk2_response, b"{}"))
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
# Set the threshhold low enough that we force a resumable uploada.
with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
with _NamedTemporaryFile() as temp:
with open(temp.name, "wb") as file_obj:
file_obj.write(DATA)
with open(temp.name, "rb") as file_obj:
blob.upload_from_file(file_obj, rewind=True)
rq = connection.http._requested
self.assertEqual(len(rq), 3)
# Requested[0]
headers = dict([(x.title(), str(y)) for x, y in rq[0].pop("headers").items()])
self.assertEqual(headers["X-Upload-Content-Length"], "6")
self.assertEqual(headers["X-Upload-Content-Type"], "application/octet-stream")
uri = rq[0].pop("uri")
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, "http")
self.assertEqual(netloc, "example.com")
self.assertEqual(path, "/b/name/o")
self.assertEqual(dict(parse_qsl(qs)), {"uploadType": "resumable", "name": BLOB_NAME})
self.assertEqual(rq[0], {"method": "POST", "body": "", "connection_type": None, "redirections": 5})
# Requested[1]
headers = dict([(x.title(), str(y)) for x, y in rq[1].pop("headers").items()])
self.assertEqual(headers["Content-Range"], "bytes 0-4/6")
self.assertEqual(
rq[1], {"method": "PUT", "uri": UPLOAD_URL, "body": DATA[:5], "connection_type": None, "redirections": 5}
)
# Requested[2]
headers = dict([(x.title(), str(y)) for x, y in rq[2].pop("headers").items()])
self.assertEqual(headers["Content-Range"], "bytes 5-5/6")
self.assertEqual(
rq[2], {"method": "PUT", "uri": UPLOAD_URL, "body": DATA[5:], "connection_type": None, "redirections": 5}
)
def test_download_to_filename(self):
import os
import time
from six.moves.http_client import OK
from six.moves.http_client import PARTIAL_CONTENT
from gcloud._testing import _NamedTemporaryFile
BLOB_NAME = "blob-name"
chunk1_response = {"status": PARTIAL_CONTENT, "content-range": "bytes 0-2/6"}
chunk2_response = {"status": OK, "content-range": "bytes 3-5/6"}
connection = _Connection((chunk1_response, b"abc"), (chunk2_response, b"def"))
client = _Client(connection)
bucket = _Bucket(client)
MEDIA_LINK = "http://example.com/media/"
properties = {"mediaLink": MEDIA_LINK, "updated": "2014-12-06T13:13:50.690Z"}
blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 3
with _NamedTemporaryFile() as temp:
blob.download_to_filename(temp.name)
with open(temp.name, "rb") as file_obj:
wrote = file_obj.read()
mtime = os.path.getmtime(temp.name)
updatedTime = time.mktime(blob.updated.timetuple())
self.assertEqual(wrote, b"abcdef")
self.assertEqual(mtime, updatedTime)
def test_success(self):
from gcloud._testing import _NamedTemporaryFile
with _NamedTemporaryFile() as temp:
with open(temp.name, mode='w') as creds_file:
creds_file.write('{"project_id": "test-project-id"}')
creds_file.seek(0)
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = creds_file.name
self.assertEqual('test-project-id', self._callFUT())
def test_upload_from_file_resumable_w_error(self):
from six.moves.http_client import NOT_FOUND
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _Monkey
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import transfer
from gcloud.streaming.exceptions import HttpError
BLOB_NAME = 'blob-name'
DATA = b'ABCDEF'
loc_response = {'status': NOT_FOUND}
connection = _Connection(
(loc_response, b'{"error": "no such bucket"}'), )
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
# Set the threshhold low enough that we force a resumable uploada.
with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
with open(temp.name, 'rb') as file_obj:
with self.assertRaises(HttpError):
blob.upload_from_file(file_obj, rewind=True)
rq = connection.http._requested
self.assertEqual(len(rq), 1)
# Requested[0]
headers = dict([(x.title(), str(y))
for x, y in rq[0].pop('headers').items()])
self.assertEqual(headers['X-Upload-Content-Length'], '6')
self.assertEqual(headers['X-Upload-Content-Type'],
'application/octet-stream')
uri = rq[0].pop('uri')
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)), {
'uploadType': 'resumable',
'name': BLOB_NAME
})
self.assertEqual(
rq[0], {
'method': 'POST',
'body': '',
'connection_type': None,
'redirections': 5,
})
def test_success(self):
import os
from gcloud._testing import _NamedTemporaryFile
with _NamedTemporaryFile() as temp:
with open(temp.name, mode='w') as creds_file:
creds_file.write('{"project_id": "test-project-id"}')
creds_file.seek(0)
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = creds_file.name
self.assertEqual('test-project-id', self._callFUT())
def test_upload_from_file_w_slash_in_name(self):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import http_wrapper
BLOB_NAME = 'parent/child'
UPLOAD_URL = 'http://example.com/upload/name/parent%2Fchild'
DATA = b'ABCDEF'
loc_response = {'status': OK, 'location': UPLOAD_URL}
chunk1_response = {
'status': http_wrapper.RESUME_INCOMPLETE,
'range': 'bytes 0-4'
}
chunk2_response = {'status': OK}
connection = _Connection(
(loc_response, '{}'),
(chunk1_response, ''),
(chunk2_response, ''),
)
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
with open(temp.name, 'rb') as file_obj:
blob.upload_from_file(file_obj, rewind=True)
self.assertEqual(file_obj.tell(), len(DATA))
rq = connection.http._requested
self.assertEqual(len(rq), 1)
self.assertEqual(rq[0]['redirections'], 5)
self.assertEqual(rq[0]['body'], DATA)
self.assertEqual(rq[0]['connection_type'], None)
self.assertEqual(rq[0]['method'], 'POST')
uri = rq[0]['uri']
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)), {
'uploadType': 'media',
'name': 'parent/child'
})
headers = dict([(x.title(), str(y))
for x, y in rq[0]['headers'].items()])
self.assertEqual(headers['Content-Length'], '6')
self.assertEqual(headers['Content-Type'], 'application/octet-stream')
def test_upload_from_file_resumable_w_error(self):
from six.moves.http_client import NOT_FOUND
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _Monkey
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import transfer
from gcloud.streaming.exceptions import HttpError
BLOB_NAME = 'blob-name'
DATA = b'ABCDEF'
loc_response = {'status': NOT_FOUND}
connection = _Connection(
(loc_response, b'{"error": "no such bucket"}'),
)
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
# Set the threshhold low enough that we force a resumable uploada.
with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
with open(temp.name, 'rb') as file_obj:
with self.assertRaises(HttpError):
blob.upload_from_file(file_obj, rewind=True)
rq = connection.http._requested
self.assertEqual(len(rq), 1)
# Requested[0]
headers = dict(
[(x.title(), str(y)) for x, y in rq[0].pop('headers').items()])
self.assertEqual(headers['X-Upload-Content-Length'], '6')
self.assertEqual(headers['X-Upload-Content-Type'],
'application/octet-stream')
uri = rq[0].pop('uri')
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)),
{'uploadType': 'resumable', 'name': BLOB_NAME})
self.assertEqual(rq[0], {
'method': 'POST',
'body': '',
'connection_type': None,
'redirections': 5,
})
def _upload_from_filename_test_helper(self,
properties=None,
content_type_arg=None,
expected_content_type=None):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import http_wrapper
BLOB_NAME = 'blob-name'
UPLOAD_URL = 'http://example.com/upload/name/key'
DATA = b'ABCDEF'
loc_response = {'status': OK, 'location': UPLOAD_URL}
chunk1_response = {
'status': http_wrapper.RESUME_INCOMPLETE,
'range': 'bytes 0-4'
}
chunk2_response = {'status': OK}
connection = _Connection(
(loc_response, '{}'),
(chunk1_response, ''),
(chunk2_response, ''),
)
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
with _NamedTemporaryFile(suffix='.jpeg') as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
blob.upload_from_filename(temp.name, content_type=content_type_arg)
rq = connection.http._requested
self.assertEqual(len(rq), 1)
self.assertEqual(rq[0]['method'], 'POST')
uri = rq[0]['uri']
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)), {
'uploadType': 'media',
'name': BLOB_NAME
})
headers = dict([(x.title(), str(y))
for x, y in rq[0]['headers'].items()])
self.assertEqual(headers['Content-Length'], '6')
self.assertEqual(headers['Content-Type'], expected_content_type)
def test_upload_from_file_w_slash_in_name(self):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import http_wrapper
BLOB_NAME = 'parent/child'
UPLOAD_URL = 'http://example.com/upload/name/parent%2Fchild'
DATA = b'ABCDEF'
loc_response = {'status': OK, 'location': UPLOAD_URL}
chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
'range': 'bytes 0-4'}
chunk2_response = {'status': OK}
connection = _Connection(
(loc_response, '{}'),
(chunk1_response, ''),
(chunk2_response, ''),
)
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
with open(temp.name, 'rb') as file_obj:
blob.upload_from_file(file_obj, rewind=True)
self.assertEqual(file_obj.tell(), len(DATA))
rq = connection.http._requested
self.assertEqual(len(rq), 1)
self.assertEqual(rq[0]['redirections'], 5)
self.assertEqual(rq[0]['body'], DATA)
self.assertEqual(rq[0]['connection_type'], None)
self.assertEqual(rq[0]['method'], 'POST')
uri = rq[0]['uri']
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)),
{'uploadType': 'media', 'name': 'parent/child'})
headers = dict(
[(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
self.assertEqual(headers['Content-Length'], '6')
self.assertEqual(headers['Content-Type'], 'application/octet-stream')
def _upload_from_filename_test_helper(self, properties=None,
content_type_arg=None,
expected_content_type=None):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import http_wrapper
BLOB_NAME = 'blob-name'
UPLOAD_URL = 'http://example.com/upload/name/key'
DATA = b'ABCDEF'
loc_response = {'status': OK, 'location': UPLOAD_URL}
chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
'range': 'bytes 0-4'}
chunk2_response = {'status': OK}
connection = _Connection(
(loc_response, '{}'),
(chunk1_response, ''),
(chunk2_response, ''),
)
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket,
properties=properties)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
with _NamedTemporaryFile(suffix='.jpeg') as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
blob.upload_from_filename(temp.name,
content_type=content_type_arg)
rq = connection.http._requested
self.assertEqual(len(rq), 1)
self.assertEqual(rq[0]['method'], 'POST')
uri = rq[0]['uri']
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)),
{'uploadType': 'media', 'name': BLOB_NAME})
headers = dict(
[(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
self.assertEqual(headers['Content-Length'], '6')
self.assertEqual(headers['Content-Type'], expected_content_type)
def _upload_from_file_simple_test_helper(self,
properties=None,
content_type_arg=None,
expected_content_type=None,
chunk_size=5,
status=None):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _NamedTemporaryFile
BLOB_NAME = 'blob-name'
DATA = b'ABCDEF'
if status is None:
status = OK
response = {'status': status}
connection = _Connection((response, b'{}'), )
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = chunk_size
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
with open(temp.name, 'rb') as file_obj:
blob.upload_from_file(file_obj,
rewind=True,
content_type=content_type_arg)
rq = connection.http._requested
self.assertEqual(len(rq), 1)
self.assertEqual(rq[0]['method'], 'POST')
uri = rq[0]['uri']
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)), {
'uploadType': 'media',
'name': BLOB_NAME
})
headers = dict([(x.title(), str(y))
for x, y in rq[0]['headers'].items()])
self.assertEqual(headers['Content-Length'], '6')
self.assertEqual(headers['Content-Type'], expected_content_type)
def _upload_from_file_simple_test_helper(self, properties=None,
content_type_arg=None,
expected_content_type=None,
chunk_size=5,
status=None):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _NamedTemporaryFile
BLOB_NAME = 'blob-name'
DATA = b'ABCDEF'
if status is None:
status = OK
response = {'status': status}
connection = _Connection(
(response, b'{}'),
)
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = chunk_size
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
with open(temp.name, 'rb') as file_obj:
blob.upload_from_file(file_obj, rewind=True,
content_type=content_type_arg)
rq = connection.http._requested
self.assertEqual(len(rq), 1)
self.assertEqual(rq[0]['method'], 'POST')
uri = rq[0]['uri']
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)),
{'uploadType': 'media', 'name': BLOB_NAME})
headers = dict(
[(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
self.assertEqual(headers['Content-Length'], '6')
self.assertEqual(headers['Content-Type'], expected_content_type)
def test_upload_from_file_w_slash_in_name(self):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import http_wrapper
BLOB_NAME = "parent/child"
UPLOAD_URL = "http://example.com/upload/name/parent%2Fchild"
DATA = b"ABCDEF"
loc_response = {"status": OK, "location": UPLOAD_URL}
chunk1_response = {"status": http_wrapper.RESUME_INCOMPLETE, "range": "bytes 0-4"}
chunk2_response = {"status": OK}
connection = _Connection((loc_response, "{}"), (chunk1_response, ""), (chunk2_response, ""))
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
with _NamedTemporaryFile() as temp:
with open(temp.name, "wb") as file_obj:
file_obj.write(DATA)
with open(temp.name, "rb") as file_obj:
blob.upload_from_file(file_obj, rewind=True)
self.assertEqual(file_obj.tell(), len(DATA))
rq = connection.http._requested
self.assertEqual(len(rq), 1)
self.assertEqual(rq[0]["redirections"], 5)
self.assertEqual(rq[0]["body"], DATA)
self.assertEqual(rq[0]["connection_type"], None)
self.assertEqual(rq[0]["method"], "POST")
uri = rq[0]["uri"]
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, "http")
self.assertEqual(netloc, "example.com")
self.assertEqual(path, "/b/name/o")
self.assertEqual(dict(parse_qsl(qs)), {"uploadType": "media", "name": "parent/child"})
headers = dict([(x.title(), str(y)) for x, y in rq[0]["headers"].items()])
self.assertEqual(headers["Content-Length"], "6")
self.assertEqual(headers["Content-Type"], "application/octet-stream")
def test_it(self):
from gcloud import credentials as MUT
from gcloud._testing import _Monkey
from gcloud._testing import _NamedTemporaryFile
CLIENT_EMAIL = '*****@*****.**'
PRIVATE_KEY = b'SEEkR1t'
client = _Client()
with _Monkey(MUT, client=client):
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(PRIVATE_KEY)
found = self._callFUT(CLIENT_EMAIL, temp.name)
self.assertTrue(found is client._signed)
expected_called_with = {
'service_account_name': CLIENT_EMAIL,
'private_key': PRIVATE_KEY,
'scope': None,
}
self.assertEqual(client._called_with, expected_called_with)
def _upload_from_filename_test_helper(self, properties=None, content_type_arg=None, expected_content_type=None):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import http_wrapper
BLOB_NAME = "blob-name"
UPLOAD_URL = "http://example.com/upload/name/key"
DATA = b"ABCDEF"
loc_response = {"status": OK, "location": UPLOAD_URL}
chunk1_response = {"status": http_wrapper.RESUME_INCOMPLETE, "range": "bytes 0-4"}
chunk2_response = {"status": OK}
connection = _Connection((loc_response, "{}"), (chunk1_response, ""), (chunk2_response, ""))
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
with _NamedTemporaryFile(suffix=".jpeg") as temp:
with open(temp.name, "wb") as file_obj:
file_obj.write(DATA)
blob.upload_from_filename(temp.name, content_type=content_type_arg)
rq = connection.http._requested
self.assertEqual(len(rq), 1)
self.assertEqual(rq[0]["method"], "POST")
uri = rq[0]["uri"]
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, "http")
self.assertEqual(netloc, "example.com")
self.assertEqual(path, "/b/name/o")
self.assertEqual(dict(parse_qsl(qs)), {"uploadType": "media", "name": BLOB_NAME})
headers = dict([(x.title(), str(y)) for x, y in rq[0]["headers"].items()])
self.assertEqual(headers["Content-Length"], "6")
self.assertEqual(headers["Content-Type"], expected_content_type)
def _upload_from_file_simple_test_helper(
self, properties=None, content_type_arg=None, expected_content_type=None, chunk_size=5
):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _NamedTemporaryFile
BLOB_NAME = "blob-name"
DATA = b"ABCDEF"
response = {"status": OK}
connection = _Connection((response, b"{}"))
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = chunk_size
with _NamedTemporaryFile() as temp:
with open(temp.name, "wb") as file_obj:
file_obj.write(DATA)
with open(temp.name, "rb") as file_obj:
blob.upload_from_file(file_obj, rewind=True, content_type=content_type_arg)
rq = connection.http._requested
self.assertEqual(len(rq), 1)
self.assertEqual(rq[0]["method"], "POST")
uri = rq[0]["uri"]
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, "http")
self.assertEqual(netloc, "example.com")
self.assertEqual(path, "/b/name/o")
self.assertEqual(dict(parse_qsl(qs)), {"uploadType": "media", "name": BLOB_NAME})
headers = dict([(x.title(), str(y)) for x, y in rq[0]["headers"].items()])
self.assertEqual(headers["Content-Length"], "6")
self.assertEqual(headers["Content-Type"], expected_content_type)
def test_upload_from_file_resumable(self):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _Monkey
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import http_wrapper
from gcloud.streaming import transfer
BLOB_NAME = 'blob-name'
UPLOAD_URL = 'http://example.com/upload/name/key'
DATA = b'ABCDEF'
loc_response = {'status': OK, 'location': UPLOAD_URL}
chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
'range': 'bytes 0-4'}
chunk2_response = {'status': OK}
# Need valid JSON on last response, since resumable.
connection = _Connection(
(loc_response, b''),
(chunk1_response, b''),
(chunk2_response, b'{}'),
)
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
# Set the threshhold low enough that we force a resumable uploada.
with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
with open(temp.name, 'rb') as file_obj:
blob.upload_from_file(file_obj, rewind=True)
rq = connection.http._requested
self.assertEqual(len(rq), 3)
# Requested[0]
headers = dict(
[(x.title(), str(y)) for x, y in rq[0].pop('headers').items()])
self.assertEqual(headers['X-Upload-Content-Length'], '6')
self.assertEqual(headers['X-Upload-Content-Type'],
'application/octet-stream')
uri = rq[0].pop('uri')
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)),
{'uploadType': 'resumable', 'name': BLOB_NAME})
self.assertEqual(rq[0], {
'method': 'POST',
'body': '',
'connection_type': None,
'redirections': 5,
})
# Requested[1]
headers = dict(
[(x.title(), str(y)) for x, y in rq[1].pop('headers').items()])
self.assertEqual(headers['Content-Range'], 'bytes 0-4/6')
self.assertEqual(rq[1], {
'method': 'PUT',
'uri': UPLOAD_URL,
'body': DATA[:5],
'connection_type': None,
'redirections': 5,
})
# Requested[2]
headers = dict(
[(x.title(), str(y)) for x, y in rq[2].pop('headers').items()])
self.assertEqual(headers['Content-Range'], 'bytes 5-5/6')
self.assertEqual(rq[2], {
'method': 'PUT',
'uri': UPLOAD_URL,
'body': DATA[5:],
'connection_type': None,
'redirections': 5,
})
def test_upload_from_file_resumable(self):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _Monkey
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import http_wrapper
from gcloud.streaming import transfer
BLOB_NAME = 'blob-name'
UPLOAD_URL = 'http://example.com/upload/name/key'
DATA = b'ABCDEF'
loc_response = {'status': OK, 'location': UPLOAD_URL}
chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
'range': 'bytes 0-4'}
chunk2_response = {'status': OK}
# Need valid JSON on last response, since resumable.
connection = _Connection(
(loc_response, b''),
(chunk1_response, b''),
(chunk2_response, b'{}'),
)
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
# Set the threshhold low enough that we force a resumable uploada.
with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
with open(temp.name, 'rb') as file_obj:
blob.upload_from_file(file_obj, rewind=True)
rq = connection.http._requested
self.assertEqual(len(rq), 3)
# Requested[0]
headers = dict(
[(x.title(), str(y)) for x, y in rq[0].pop('headers').items()])
self.assertEqual(headers['X-Upload-Content-Length'], '6')
self.assertEqual(headers['X-Upload-Content-Type'],
'application/octet-stream')
uri = rq[0].pop('uri')
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)),
{'uploadType': 'resumable', 'name': BLOB_NAME})
self.assertEqual(rq[0], {
'method': 'POST',
'body': '',
'connection_type': None,
'redirections': 5,
})
# Requested[1]
headers = dict(
[(x.title(), str(y)) for x, y in rq[1].pop('headers').items()])
self.assertEqual(headers['Content-Range'], 'bytes 0-4/6')
self.assertEqual(rq[1], {
'method': 'PUT',
'uri': UPLOAD_URL,
'body': DATA[:5],
'connection_type': None,
'redirections': 5,
})
# Requested[2]
headers = dict(
[(x.title(), str(y)) for x, y in rq[2].pop('headers').items()])
self.assertEqual(headers['Content-Range'], 'bytes 5-5/6')
self.assertEqual(rq[2], {
'method': 'PUT',
'uri': UPLOAD_URL,
'body': DATA[5:],
'connection_type': None,
'redirections': 5,
})
