def storer(): storer = GCSStorer() yield storer
import gzip import json import base64 import asyncio import requests import google.auth import google.oauth2.id_token import google.auth.transport.requests from google.cloud import pubsub_v1 from google.cloud import firestore from xialib_gcp import PubsubPublisher, PubsubSubscriber, FirestoreDepositor, GCSStorer, GCSListArchiver from pyinsight import Insight, Cleaner project_id = google.auth.default()[1] firestore_db = firestore.Client() gcs_storer = GCSStorer() cleaner_url = 'https://insight-cleaner-zmfr66omva-ew.a.run.app' def get_auth_header(target_url: str) -> dict: credentials, project_id = google.auth.default( scopes=["https://www.googleapis.com/auth/cloud-platform"]) auth_req = google.auth.transport.requests.Request() credentials.refresh(auth_req) target_audience = target_url id_token = google.oauth2.id_token.fetch_id_token(auth_req, target_audience) return {'Authorization': ' '.join(['Bearer', id_token])} def get_envelope(header: dict, data: list) -> dict: return {
def test_simple_flow(storer: GCSStorer): storer.mkdir(storer.join("gs://xialib-gcp-test", "gcs-storer")) file_path = storer.join("gs://xialib-gcp-test", "gcs-storer", "schema.json") dest_file = storer.join("gs://xialib-gcp-test", "gcs-storer", "schema1.json") assert storer.exists(file_path) data_copy1 = storer.read(file_path) for data_io in storer.get_io_stream(file_path): new_file = storer.write(data_io, dest_file) assert new_file == dest_file for data_io in storer.get_io_stream(file_path): for write_io in storer.get_io_wb_stream(dest_file): write_io.write(data_io.read()) for data_io in storer.get_io_stream(dest_file): data_copy2 = data_io.read() assert data_copy1 == data_copy2 assert storer.remove(dest_file) assert not storer.remove(dest_file) storer.write(data_copy1, dest_file) storer.remove(dest_file)