def test_download_to_filename(self):
        import os
        import time
        from six.moves.http_client import OK
        from six.moves.http_client import PARTIAL_CONTENT
        from gcloud._testing import _NamedTemporaryFile

        BLOB_NAME = 'blob-name'
        chunk1_response = {'status': PARTIAL_CONTENT,
                           'content-range': 'bytes 0-2/6'}
        chunk2_response = {'status': OK,
                           'content-range': 'bytes 3-5/6'}
        connection = _Connection(
            (chunk1_response, b'abc'),
            (chunk2_response, b'def'),
        )
        client = _Client(connection)
        bucket = _Bucket(client)
        MEDIA_LINK = 'http://example.com/media/'
        properties = {'mediaLink': MEDIA_LINK,
                      'updated': '2014-12-06T13:13:50.690Z'}
        blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 3

        with _NamedTemporaryFile() as temp:
            blob.download_to_filename(temp.name)
            with open(temp.name, 'rb') as file_obj:
                wrote = file_obj.read()
                mtime = os.path.getmtime(temp.name)
                updatedTime = time.mktime(blob.updated.timetuple())

        self.assertEqual(wrote, b'abcdef')
        self.assertEqual(mtime, updatedTime)
Exemple #2
0
    def test_download_to_filename(self):
        import os
        import time
        from six.moves.http_client import OK
        from six.moves.http_client import PARTIAL_CONTENT
        from gcloud._testing import _NamedTemporaryFile

        BLOB_NAME = 'blob-name'
        chunk1_response = {'status': PARTIAL_CONTENT,
                           'content-range': 'bytes 0-2/6'}
        chunk2_response = {'status': OK,
                           'content-range': 'bytes 3-5/6'}
        connection = _Connection(
            (chunk1_response, b'abc'),
            (chunk2_response, b'def'),
        )
        client = _Client(connection)
        bucket = _Bucket(client)
        MEDIA_LINK = 'http://example.com/media/'
        properties = {'mediaLink': MEDIA_LINK,
                      'updated': '2014-12-06T13:13:50.690Z'}
        blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 3

        with _NamedTemporaryFile() as temp:
            blob.download_to_filename(temp.name)
            with open(temp.name, 'rb') as file_obj:
                wrote = file_obj.read()
                mtime = os.path.getmtime(temp.name)
                updatedTime = time.mktime(blob.updated.timetuple())

        self.assertEqual(wrote, b'abcdef')
        self.assertEqual(mtime, updatedTime)
    def test_upload_from_file_resumable(self):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _Monkey
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import http_wrapper
        from gcloud.streaming import transfer

        BLOB_NAME = "blob-name"
        UPLOAD_URL = "http://example.com/upload/name/key"
        DATA = b"ABCDEF"
        loc_response = {"status": OK, "location": UPLOAD_URL}
        chunk1_response = {"status": http_wrapper.RESUME_INCOMPLETE, "range": "bytes 0-4"}
        chunk2_response = {"status": OK}
        # Need valid JSON on last response, since resumable.
        connection = _Connection((loc_response, b""), (chunk1_response, b""), (chunk2_response, b"{}"))
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        # Set the threshhold low enough that we force a resumable uploada.
        with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
            with _NamedTemporaryFile() as temp:
                with open(temp.name, "wb") as file_obj:
                    file_obj.write(DATA)
                with open(temp.name, "rb") as file_obj:
                    blob.upload_from_file(file_obj, rewind=True)

        rq = connection.http._requested
        self.assertEqual(len(rq), 3)

        # Requested[0]
        headers = dict([(x.title(), str(y)) for x, y in rq[0].pop("headers").items()])
        self.assertEqual(headers["X-Upload-Content-Length"], "6")
        self.assertEqual(headers["X-Upload-Content-Type"], "application/octet-stream")

        uri = rq[0].pop("uri")
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, "http")
        self.assertEqual(netloc, "example.com")
        self.assertEqual(path, "/b/name/o")
        self.assertEqual(dict(parse_qsl(qs)), {"uploadType": "resumable", "name": BLOB_NAME})
        self.assertEqual(rq[0], {"method": "POST", "body": "", "connection_type": None, "redirections": 5})

        # Requested[1]
        headers = dict([(x.title(), str(y)) for x, y in rq[1].pop("headers").items()])
        self.assertEqual(headers["Content-Range"], "bytes 0-4/6")
        self.assertEqual(
            rq[1], {"method": "PUT", "uri": UPLOAD_URL, "body": DATA[:5], "connection_type": None, "redirections": 5}
        )

        # Requested[2]
        headers = dict([(x.title(), str(y)) for x, y in rq[2].pop("headers").items()])
        self.assertEqual(headers["Content-Range"], "bytes 5-5/6")
        self.assertEqual(
            rq[2], {"method": "PUT", "uri": UPLOAD_URL, "body": DATA[5:], "connection_type": None, "redirections": 5}
        )
    def test_download_to_filename(self):
        import os
        import time
        from six.moves.http_client import OK
        from six.moves.http_client import PARTIAL_CONTENT
        from gcloud._testing import _NamedTemporaryFile

        BLOB_NAME = "blob-name"
        chunk1_response = {"status": PARTIAL_CONTENT, "content-range": "bytes 0-2/6"}
        chunk2_response = {"status": OK, "content-range": "bytes 3-5/6"}
        connection = _Connection((chunk1_response, b"abc"), (chunk2_response, b"def"))
        client = _Client(connection)
        bucket = _Bucket(client)
        MEDIA_LINK = "http://example.com/media/"
        properties = {"mediaLink": MEDIA_LINK, "updated": "2014-12-06T13:13:50.690Z"}
        blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 3

        with _NamedTemporaryFile() as temp:
            blob.download_to_filename(temp.name)
            with open(temp.name, "rb") as file_obj:
                wrote = file_obj.read()
                mtime = os.path.getmtime(temp.name)
                updatedTime = time.mktime(blob.updated.timetuple())

        self.assertEqual(wrote, b"abcdef")
        self.assertEqual(mtime, updatedTime)
    def test_success(self):
        from gcloud._testing import _NamedTemporaryFile

        with _NamedTemporaryFile() as temp:
            with open(temp.name, mode='w') as creds_file:
                creds_file.write('{"project_id": "test-project-id"}')
                creds_file.seek(0)
                os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = creds_file.name

                self.assertEqual('test-project-id', self._callFUT())
Exemple #6
0
    def test_upload_from_file_resumable_w_error(self):
        from six.moves.http_client import NOT_FOUND
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _Monkey
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import transfer
        from gcloud.streaming.exceptions import HttpError

        BLOB_NAME = 'blob-name'
        DATA = b'ABCDEF'
        loc_response = {'status': NOT_FOUND}
        connection = _Connection(
            (loc_response, b'{"error": "no such bucket"}'), )
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        # Set the threshhold low enough that we force a resumable uploada.
        with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
            with _NamedTemporaryFile() as temp:
                with open(temp.name, 'wb') as file_obj:
                    file_obj.write(DATA)
                with open(temp.name, 'rb') as file_obj:
                    with self.assertRaises(HttpError):
                        blob.upload_from_file(file_obj, rewind=True)

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)

        # Requested[0]
        headers = dict([(x.title(), str(y))
                        for x, y in rq[0].pop('headers').items()])
        self.assertEqual(headers['X-Upload-Content-Length'], '6')
        self.assertEqual(headers['X-Upload-Content-Type'],
                         'application/octet-stream')

        uri = rq[0].pop('uri')
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'example.com')
        self.assertEqual(path, '/b/name/o')
        self.assertEqual(dict(parse_qsl(qs)), {
            'uploadType': 'resumable',
            'name': BLOB_NAME
        })
        self.assertEqual(
            rq[0], {
                'method': 'POST',
                'body': '',
                'connection_type': None,
                'redirections': 5,
            })
    def test_success(self):
        import os
        from gcloud._testing import _NamedTemporaryFile

        with _NamedTemporaryFile() as temp:
            with open(temp.name, mode='w') as creds_file:
                creds_file.write('{"project_id": "test-project-id"}')
                creds_file.seek(0)
                os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = creds_file.name

                self.assertEqual('test-project-id', self._callFUT())
Exemple #8
0
    def test_upload_from_file_w_slash_in_name(self):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import http_wrapper

        BLOB_NAME = 'parent/child'
        UPLOAD_URL = 'http://example.com/upload/name/parent%2Fchild'
        DATA = b'ABCDEF'
        loc_response = {'status': OK, 'location': UPLOAD_URL}
        chunk1_response = {
            'status': http_wrapper.RESUME_INCOMPLETE,
            'range': 'bytes 0-4'
        }
        chunk2_response = {'status': OK}
        connection = _Connection(
            (loc_response, '{}'),
            (chunk1_response, ''),
            (chunk2_response, ''),
        )
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        with _NamedTemporaryFile() as temp:
            with open(temp.name, 'wb') as file_obj:
                file_obj.write(DATA)
            with open(temp.name, 'rb') as file_obj:
                blob.upload_from_file(file_obj, rewind=True)
                self.assertEqual(file_obj.tell(), len(DATA))

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)
        self.assertEqual(rq[0]['redirections'], 5)
        self.assertEqual(rq[0]['body'], DATA)
        self.assertEqual(rq[0]['connection_type'], None)
        self.assertEqual(rq[0]['method'], 'POST')
        uri = rq[0]['uri']
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'example.com')
        self.assertEqual(path, '/b/name/o')
        self.assertEqual(dict(parse_qsl(qs)), {
            'uploadType': 'media',
            'name': 'parent/child'
        })
        headers = dict([(x.title(), str(y))
                        for x, y in rq[0]['headers'].items()])
        self.assertEqual(headers['Content-Length'], '6')
        self.assertEqual(headers['Content-Type'], 'application/octet-stream')
Exemple #9
0
    def test_upload_from_file_resumable_w_error(self):
        from six.moves.http_client import NOT_FOUND
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _Monkey
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import transfer
        from gcloud.streaming.exceptions import HttpError

        BLOB_NAME = 'blob-name'
        DATA = b'ABCDEF'
        loc_response = {'status': NOT_FOUND}
        connection = _Connection(
            (loc_response, b'{"error": "no such bucket"}'),
        )
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        # Set the threshhold low enough that we force a resumable uploada.
        with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
            with _NamedTemporaryFile() as temp:
                with open(temp.name, 'wb') as file_obj:
                    file_obj.write(DATA)
                with open(temp.name, 'rb') as file_obj:
                    with self.assertRaises(HttpError):
                        blob.upload_from_file(file_obj, rewind=True)

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)

        # Requested[0]
        headers = dict(
            [(x.title(), str(y)) for x, y in rq[0].pop('headers').items()])
        self.assertEqual(headers['X-Upload-Content-Length'], '6')
        self.assertEqual(headers['X-Upload-Content-Type'],
                         'application/octet-stream')

        uri = rq[0].pop('uri')
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'example.com')
        self.assertEqual(path, '/b/name/o')
        self.assertEqual(dict(parse_qsl(qs)),
                         {'uploadType': 'resumable', 'name': BLOB_NAME})
        self.assertEqual(rq[0], {
            'method': 'POST',
            'body': '',
            'connection_type': None,
            'redirections': 5,
        })
Exemple #10
0
    def _upload_from_filename_test_helper(self,
                                          properties=None,
                                          content_type_arg=None,
                                          expected_content_type=None):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import http_wrapper

        BLOB_NAME = 'blob-name'
        UPLOAD_URL = 'http://example.com/upload/name/key'
        DATA = b'ABCDEF'
        loc_response = {'status': OK, 'location': UPLOAD_URL}
        chunk1_response = {
            'status': http_wrapper.RESUME_INCOMPLETE,
            'range': 'bytes 0-4'
        }
        chunk2_response = {'status': OK}
        connection = _Connection(
            (loc_response, '{}'),
            (chunk1_response, ''),
            (chunk2_response, ''),
        )
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        with _NamedTemporaryFile(suffix='.jpeg') as temp:
            with open(temp.name, 'wb') as file_obj:
                file_obj.write(DATA)
            blob.upload_from_filename(temp.name, content_type=content_type_arg)

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)
        self.assertEqual(rq[0]['method'], 'POST')
        uri = rq[0]['uri']
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'example.com')
        self.assertEqual(path, '/b/name/o')
        self.assertEqual(dict(parse_qsl(qs)), {
            'uploadType': 'media',
            'name': BLOB_NAME
        })
        headers = dict([(x.title(), str(y))
                        for x, y in rq[0]['headers'].items()])
        self.assertEqual(headers['Content-Length'], '6')
        self.assertEqual(headers['Content-Type'], expected_content_type)
Exemple #11
0
    def test_upload_from_file_w_slash_in_name(self):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import http_wrapper

        BLOB_NAME = 'parent/child'
        UPLOAD_URL = 'http://example.com/upload/name/parent%2Fchild'
        DATA = b'ABCDEF'
        loc_response = {'status': OK, 'location': UPLOAD_URL}
        chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
                           'range': 'bytes 0-4'}
        chunk2_response = {'status': OK}
        connection = _Connection(
            (loc_response, '{}'),
            (chunk1_response, ''),
            (chunk2_response, ''),
        )
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        with _NamedTemporaryFile() as temp:
            with open(temp.name, 'wb') as file_obj:
                file_obj.write(DATA)
            with open(temp.name, 'rb') as file_obj:
                blob.upload_from_file(file_obj, rewind=True)
                self.assertEqual(file_obj.tell(), len(DATA))

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)
        self.assertEqual(rq[0]['redirections'], 5)
        self.assertEqual(rq[0]['body'], DATA)
        self.assertEqual(rq[0]['connection_type'], None)
        self.assertEqual(rq[0]['method'], 'POST')
        uri = rq[0]['uri']
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'example.com')
        self.assertEqual(path, '/b/name/o')
        self.assertEqual(dict(parse_qsl(qs)),
                         {'uploadType': 'media', 'name': 'parent/child'})
        headers = dict(
            [(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
        self.assertEqual(headers['Content-Length'], '6')
        self.assertEqual(headers['Content-Type'], 'application/octet-stream')
Exemple #12
0
    def _upload_from_filename_test_helper(self, properties=None,
                                          content_type_arg=None,
                                          expected_content_type=None):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import http_wrapper

        BLOB_NAME = 'blob-name'
        UPLOAD_URL = 'http://example.com/upload/name/key'
        DATA = b'ABCDEF'
        loc_response = {'status': OK, 'location': UPLOAD_URL}
        chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
                           'range': 'bytes 0-4'}
        chunk2_response = {'status': OK}
        connection = _Connection(
            (loc_response, '{}'),
            (chunk1_response, ''),
            (chunk2_response, ''),
        )
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket,
                             properties=properties)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        with _NamedTemporaryFile(suffix='.jpeg') as temp:
            with open(temp.name, 'wb') as file_obj:
                file_obj.write(DATA)
            blob.upload_from_filename(temp.name,
                                      content_type=content_type_arg)

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)
        self.assertEqual(rq[0]['method'], 'POST')
        uri = rq[0]['uri']
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'example.com')
        self.assertEqual(path, '/b/name/o')
        self.assertEqual(dict(parse_qsl(qs)),
                         {'uploadType': 'media', 'name': BLOB_NAME})
        headers = dict(
            [(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
        self.assertEqual(headers['Content-Length'], '6')
        self.assertEqual(headers['Content-Type'], expected_content_type)
Exemple #13
0
    def _upload_from_file_simple_test_helper(self,
                                             properties=None,
                                             content_type_arg=None,
                                             expected_content_type=None,
                                             chunk_size=5,
                                             status=None):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _NamedTemporaryFile

        BLOB_NAME = 'blob-name'
        DATA = b'ABCDEF'
        if status is None:
            status = OK
        response = {'status': status}
        connection = _Connection((response, b'{}'), )
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = chunk_size

        with _NamedTemporaryFile() as temp:
            with open(temp.name, 'wb') as file_obj:
                file_obj.write(DATA)

            with open(temp.name, 'rb') as file_obj:
                blob.upload_from_file(file_obj,
                                      rewind=True,
                                      content_type=content_type_arg)

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)
        self.assertEqual(rq[0]['method'], 'POST')
        uri = rq[0]['uri']
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'example.com')
        self.assertEqual(path, '/b/name/o')
        self.assertEqual(dict(parse_qsl(qs)), {
            'uploadType': 'media',
            'name': BLOB_NAME
        })
        headers = dict([(x.title(), str(y))
                        for x, y in rq[0]['headers'].items()])
        self.assertEqual(headers['Content-Length'], '6')
        self.assertEqual(headers['Content-Type'], expected_content_type)
Exemple #14
0
    def _upload_from_file_simple_test_helper(self, properties=None,
                                             content_type_arg=None,
                                             expected_content_type=None,
                                             chunk_size=5,
                                             status=None):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _NamedTemporaryFile

        BLOB_NAME = 'blob-name'
        DATA = b'ABCDEF'
        if status is None:
            status = OK
        response = {'status': status}
        connection = _Connection(
            (response, b'{}'),
        )
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = chunk_size

        with _NamedTemporaryFile() as temp:
            with open(temp.name, 'wb') as file_obj:
                file_obj.write(DATA)

            with open(temp.name, 'rb') as file_obj:
                blob.upload_from_file(file_obj, rewind=True,
                                      content_type=content_type_arg)

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)
        self.assertEqual(rq[0]['method'], 'POST')
        uri = rq[0]['uri']
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'example.com')
        self.assertEqual(path, '/b/name/o')
        self.assertEqual(dict(parse_qsl(qs)),
                         {'uploadType': 'media', 'name': BLOB_NAME})
        headers = dict(
            [(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
        self.assertEqual(headers['Content-Length'], '6')
        self.assertEqual(headers['Content-Type'], expected_content_type)
Exemple #15
0
    def test_upload_from_file_w_slash_in_name(self):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import http_wrapper

        BLOB_NAME = "parent/child"
        UPLOAD_URL = "http://example.com/upload/name/parent%2Fchild"
        DATA = b"ABCDEF"
        loc_response = {"status": OK, "location": UPLOAD_URL}
        chunk1_response = {"status": http_wrapper.RESUME_INCOMPLETE, "range": "bytes 0-4"}
        chunk2_response = {"status": OK}
        connection = _Connection((loc_response, "{}"), (chunk1_response, ""), (chunk2_response, ""))
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        with _NamedTemporaryFile() as temp:
            with open(temp.name, "wb") as file_obj:
                file_obj.write(DATA)
            with open(temp.name, "rb") as file_obj:
                blob.upload_from_file(file_obj, rewind=True)
                self.assertEqual(file_obj.tell(), len(DATA))

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)
        self.assertEqual(rq[0]["redirections"], 5)
        self.assertEqual(rq[0]["body"], DATA)
        self.assertEqual(rq[0]["connection_type"], None)
        self.assertEqual(rq[0]["method"], "POST")
        uri = rq[0]["uri"]
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, "http")
        self.assertEqual(netloc, "example.com")
        self.assertEqual(path, "/b/name/o")
        self.assertEqual(dict(parse_qsl(qs)), {"uploadType": "media", "name": "parent/child"})
        headers = dict([(x.title(), str(y)) for x, y in rq[0]["headers"].items()])
        self.assertEqual(headers["Content-Length"], "6")
        self.assertEqual(headers["Content-Type"], "application/octet-stream")
    def test_it(self):
        from gcloud import credentials as MUT
        from gcloud._testing import _Monkey
        from gcloud._testing import _NamedTemporaryFile

        CLIENT_EMAIL = '*****@*****.**'
        PRIVATE_KEY = b'SEEkR1t'
        client = _Client()
        with _Monkey(MUT, client=client):
            with _NamedTemporaryFile() as temp:
                with open(temp.name, 'wb') as file_obj:
                    file_obj.write(PRIVATE_KEY)
                found = self._callFUT(CLIENT_EMAIL, temp.name)

        self.assertTrue(found is client._signed)
        expected_called_with = {
            'service_account_name': CLIENT_EMAIL,
            'private_key': PRIVATE_KEY,
            'scope': None,
        }
        self.assertEqual(client._called_with, expected_called_with)
Exemple #17
0
    def _upload_from_filename_test_helper(self, properties=None, content_type_arg=None, expected_content_type=None):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import http_wrapper

        BLOB_NAME = "blob-name"
        UPLOAD_URL = "http://example.com/upload/name/key"
        DATA = b"ABCDEF"
        loc_response = {"status": OK, "location": UPLOAD_URL}
        chunk1_response = {"status": http_wrapper.RESUME_INCOMPLETE, "range": "bytes 0-4"}
        chunk2_response = {"status": OK}
        connection = _Connection((loc_response, "{}"), (chunk1_response, ""), (chunk2_response, ""))
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        with _NamedTemporaryFile(suffix=".jpeg") as temp:
            with open(temp.name, "wb") as file_obj:
                file_obj.write(DATA)
            blob.upload_from_filename(temp.name, content_type=content_type_arg)

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)
        self.assertEqual(rq[0]["method"], "POST")
        uri = rq[0]["uri"]
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, "http")
        self.assertEqual(netloc, "example.com")
        self.assertEqual(path, "/b/name/o")
        self.assertEqual(dict(parse_qsl(qs)), {"uploadType": "media", "name": BLOB_NAME})
        headers = dict([(x.title(), str(y)) for x, y in rq[0]["headers"].items()])
        self.assertEqual(headers["Content-Length"], "6")
        self.assertEqual(headers["Content-Type"], expected_content_type)
Exemple #18
0
    def _upload_from_file_simple_test_helper(
        self, properties=None, content_type_arg=None, expected_content_type=None, chunk_size=5
    ):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _NamedTemporaryFile

        BLOB_NAME = "blob-name"
        DATA = b"ABCDEF"
        response = {"status": OK}
        connection = _Connection((response, b"{}"))
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = chunk_size

        with _NamedTemporaryFile() as temp:
            with open(temp.name, "wb") as file_obj:
                file_obj.write(DATA)

            with open(temp.name, "rb") as file_obj:
                blob.upload_from_file(file_obj, rewind=True, content_type=content_type_arg)

        rq = connection.http._requested
        self.assertEqual(len(rq), 1)
        self.assertEqual(rq[0]["method"], "POST")
        uri = rq[0]["uri"]
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, "http")
        self.assertEqual(netloc, "example.com")
        self.assertEqual(path, "/b/name/o")
        self.assertEqual(dict(parse_qsl(qs)), {"uploadType": "media", "name": BLOB_NAME})
        headers = dict([(x.title(), str(y)) for x, y in rq[0]["headers"].items()])
        self.assertEqual(headers["Content-Length"], "6")
        self.assertEqual(headers["Content-Type"], expected_content_type)
Exemple #19
0
    def test_upload_from_file_resumable(self):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _Monkey
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import http_wrapper
        from gcloud.streaming import transfer

        BLOB_NAME = 'blob-name'
        UPLOAD_URL = 'http://example.com/upload/name/key'
        DATA = b'ABCDEF'
        loc_response = {'status': OK, 'location': UPLOAD_URL}
        chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
                           'range': 'bytes 0-4'}
        chunk2_response = {'status': OK}
        # Need valid JSON on last response, since resumable.
        connection = _Connection(
            (loc_response, b''),
            (chunk1_response, b''),
            (chunk2_response, b'{}'),
        )
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        # Set the threshhold low enough that we force a resumable uploada.
        with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
            with _NamedTemporaryFile() as temp:
                with open(temp.name, 'wb') as file_obj:
                    file_obj.write(DATA)
                with open(temp.name, 'rb') as file_obj:
                    blob.upload_from_file(file_obj, rewind=True)

        rq = connection.http._requested
        self.assertEqual(len(rq), 3)

        # Requested[0]
        headers = dict(
            [(x.title(), str(y)) for x, y in rq[0].pop('headers').items()])
        self.assertEqual(headers['X-Upload-Content-Length'], '6')
        self.assertEqual(headers['X-Upload-Content-Type'],
                         'application/octet-stream')

        uri = rq[0].pop('uri')
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'example.com')
        self.assertEqual(path, '/b/name/o')
        self.assertEqual(dict(parse_qsl(qs)),
                         {'uploadType': 'resumable', 'name': BLOB_NAME})
        self.assertEqual(rq[0], {
            'method': 'POST',
            'body': '',
            'connection_type': None,
            'redirections': 5,
        })

        # Requested[1]
        headers = dict(
            [(x.title(), str(y)) for x, y in rq[1].pop('headers').items()])
        self.assertEqual(headers['Content-Range'], 'bytes 0-4/6')
        self.assertEqual(rq[1], {
            'method': 'PUT',
            'uri': UPLOAD_URL,
            'body': DATA[:5],
            'connection_type': None,
            'redirections': 5,
        })

        # Requested[2]
        headers = dict(
            [(x.title(), str(y)) for x, y in rq[2].pop('headers').items()])
        self.assertEqual(headers['Content-Range'], 'bytes 5-5/6')
        self.assertEqual(rq[2], {
            'method': 'PUT',
            'uri': UPLOAD_URL,
            'body': DATA[5:],
            'connection_type': None,
            'redirections': 5,
        })
Exemple #20
0
    def test_upload_from_file_resumable(self):
        from six.moves.http_client import OK
        from six.moves.urllib.parse import parse_qsl
        from six.moves.urllib.parse import urlsplit
        from gcloud._testing import _Monkey
        from gcloud._testing import _NamedTemporaryFile
        from gcloud.streaming import http_wrapper
        from gcloud.streaming import transfer

        BLOB_NAME = 'blob-name'
        UPLOAD_URL = 'http://example.com/upload/name/key'
        DATA = b'ABCDEF'
        loc_response = {'status': OK, 'location': UPLOAD_URL}
        chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
                           'range': 'bytes 0-4'}
        chunk2_response = {'status': OK}
        # Need valid JSON on last response, since resumable.
        connection = _Connection(
            (loc_response, b''),
            (chunk1_response, b''),
            (chunk2_response, b'{}'),
        )
        client = _Client(connection)
        bucket = _Bucket(client)
        blob = self._makeOne(BLOB_NAME, bucket=bucket)
        blob._CHUNK_SIZE_MULTIPLE = 1
        blob.chunk_size = 5

        # Set the threshhold low enough that we force a resumable uploada.
        with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
            with _NamedTemporaryFile() as temp:
                with open(temp.name, 'wb') as file_obj:
                    file_obj.write(DATA)
                with open(temp.name, 'rb') as file_obj:
                    blob.upload_from_file(file_obj, rewind=True)

        rq = connection.http._requested
        self.assertEqual(len(rq), 3)

        # Requested[0]
        headers = dict(
            [(x.title(), str(y)) for x, y in rq[0].pop('headers').items()])
        self.assertEqual(headers['X-Upload-Content-Length'], '6')
        self.assertEqual(headers['X-Upload-Content-Type'],
                         'application/octet-stream')

        uri = rq[0].pop('uri')
        scheme, netloc, path, qs, _ = urlsplit(uri)
        self.assertEqual(scheme, 'http')
        self.assertEqual(netloc, 'example.com')
        self.assertEqual(path, '/b/name/o')
        self.assertEqual(dict(parse_qsl(qs)),
                         {'uploadType': 'resumable', 'name': BLOB_NAME})
        self.assertEqual(rq[0], {
            'method': 'POST',
            'body': '',
            'connection_type': None,
            'redirections': 5,
        })

        # Requested[1]
        headers = dict(
            [(x.title(), str(y)) for x, y in rq[1].pop('headers').items()])
        self.assertEqual(headers['Content-Range'], 'bytes 0-4/6')
        self.assertEqual(rq[1], {
            'method': 'PUT',
            'uri': UPLOAD_URL,
            'body': DATA[:5],
            'connection_type': None,
            'redirections': 5,
        })

        # Requested[2]
        headers = dict(
            [(x.title(), str(y)) for x, y in rq[2].pop('headers').items()])
        self.assertEqual(headers['Content-Range'], 'bytes 5-5/6')
        self.assertEqual(rq[2], {
            'method': 'PUT',
            'uri': UPLOAD_URL,
            'body': DATA[5:],
            'connection_type': None,
            'redirections': 5,
        })