示例#1
0
 def test_persist(self):
     assert_gcs_environ()
     uri = os.environ.get('GCS_TEST_FILE_URI')
     if not uri:
         raise unittest.SkipTest("No GCS URI available for testing")
     data = b"TestGCSFilesStore: \xe2\x98\x83"
     buf = BytesIO(data)
     meta = {'foo': 'bar'}
     path = 'full/filename'
     store = GCSFilesStore(uri)
     store.POLICY = 'authenticatedRead'
     expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'}
     yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
     s = yield store.stat_file(path, info=None)
     self.assertIn('last_modified', s)
     self.assertIn('checksum', s)
     self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
     u = urlparse(uri)
     content, acl, blob = get_gcs_content_and_delete(
         u.hostname, u.path[1:] + path)
     self.assertEqual(content, data)
     self.assertEqual(blob.metadata, {'foo': 'bar'})
     self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
     self.assertEqual(blob.content_type, 'application/octet-stream')
     self.assertIn(expected_policy, acl)
示例#2
0
 def test_blob_path_consistency(self):
     """Test to make sure that paths used to store files is the same as the one used to get
     already uploaded files.
     """
     assert_gcs_environ()
     try:
         import google.cloud.storage  # noqa
     except ModuleNotFoundError:
         raise unittest.SkipTest("google-cloud-storage is not installed")
     else:
         with mock.patch('google.cloud.storage') as _:
             with mock.patch('scrapy.pipelines.files.time') as _:
                 uri = 'gs://my_bucket/my_prefix/'
                 store = GCSFilesStore(uri)
                 store.bucket = mock.Mock()
                 path = 'full/my_data.txt'
                 yield store.persist_file(path,
                                          mock.Mock(),
                                          info=None,
                                          meta=None,
                                          headers=None)
                 yield store.stat_file(path, info=None)
                 expected_blob_path = store.prefix + path
                 store.bucket.blob.assert_called_with(expected_blob_path)
                 store.bucket.get_blob.assert_called_with(
                     expected_blob_path)
 def test_persist(self):
     assert_gcs_environ()
     uri = os.environ.get('GCS_TEST_FILE_URI')
     if not uri:
         raise unittest.SkipTest("No GCS URI available for testing")
     data = b"TestGCSFilesStore: \xe2\x98\x83"
     buf = BytesIO(data)
     meta = {'foo': 'bar'}
     path = 'full/filename'
     store = GCSFilesStore(uri)
     yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
     s = yield store.stat_file(path, info=None)
     self.assertIn('last_modified', s)
     self.assertIn('checksum', s)
     self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
     u = urlparse(uri)
     content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path)
     self.assertEqual(content, data)
     self.assertEqual(blob.metadata, {'foo': 'bar'})
     self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
     self.assertEqual(blob.content_type, 'application/octet-stream')