def storer():
storer = GCSStorer()
yield storer
import gzip
import json
import base64
import asyncio
import requests
import google.auth
import google.oauth2.id_token
import google.auth.transport.requests
from google.cloud import pubsub_v1
from google.cloud import firestore
from xialib_gcp import PubsubPublisher, PubsubSubscriber, FirestoreDepositor, GCSStorer, GCSListArchiver
from pyinsight import Insight, Cleaner
project_id = google.auth.default()[1]
firestore_db = firestore.Client()
gcs_storer = GCSStorer()
cleaner_url = 'https://insight-cleaner-zmfr66omva-ew.a.run.app'
def get_auth_header(target_url: str) -> dict:
credentials, project_id = google.auth.default(
scopes=["https://www.googleapis.com/auth/cloud-platform"])
auth_req = google.auth.transport.requests.Request()
credentials.refresh(auth_req)
target_audience = target_url
id_token = google.oauth2.id_token.fetch_id_token(auth_req, target_audience)
return {'Authorization': ' '.join(['Bearer', id_token])}
def get_envelope(header: dict, data: list) -> dict:
return {
def test_simple_flow(storer: GCSStorer):
storer.mkdir(storer.join("gs://xialib-gcp-test", "gcs-storer"))
file_path = storer.join("gs://xialib-gcp-test", "gcs-storer", "schema.json")
dest_file = storer.join("gs://xialib-gcp-test", "gcs-storer", "schema1.json")
assert storer.exists(file_path)
data_copy1 = storer.read(file_path)
for data_io in storer.get_io_stream(file_path):
new_file = storer.write(data_io, dest_file)
assert new_file == dest_file
for data_io in storer.get_io_stream(file_path):
for write_io in storer.get_io_wb_stream(dest_file):
write_io.write(data_io.read())
for data_io in storer.get_io_stream(dest_file):
data_copy2 = data_io.read()
assert data_copy1 == data_copy2
assert storer.remove(dest_file)
assert not storer.remove(dest_file)
storer.write(data_copy1, dest_file)
storer.remove(dest_file)