def initialize_db(): global db get_env_creds() if not db: firebase_admin.initialize_app() db = firestore.client() return db
def gcs_file_upload(bucket_name, file_type, file_path): """Uploads a blob to the bucket.""" # file_path = "local/path/to/file" # GCS_BUCKET_NAME = "bucketname" # https://cloud.google.com/storage/docs/uploading-objects#storage-upload-object-python try: # source and destination file name are kept same get_env_creds() storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) file_name = file_from_path(file_path) blob = bucket.blob(file_type + "/" + file_name) blob.upload_from_filename(file_path) print("File {} has been uploaded".format(file_name)) except NameError: print("Please set the environment variable {}".format(GCS_BUCKET_NAME))
def deploy(model_type, model_location, model_path, **kwargs): ''' model_path is path inside cloud bucket, not local path ''' try: get_env_creds() file_name = get_filename_noext(model_path) model_name = filter_alpha(file_name) date = get_date_for_id() model_id = "{}-{}".format(model_name, date) region = kwargs.get(GCP_REGION, os.environ.get( GCP_REGION, GCP_DEFAULT_REGION)) auth = kwargs.get(CLOUD_RUN_AUTH, os.environ.get( CLOUD_RUN_AUTH, CLOUD_RUN_DEFAULT_AUTH)) local_model_path = gcs_download_file( model_path) if model_location == CLOUD else model_path if model_location == LOCAL: upload(MODELS, local_model_path) if model_type == SKLEARN: get_container_files(model_type) add_env_to_dockerfile() gcp_auth() gcp_setproject() upload_model_metadata(local_model_path, model_id, model_type) build_container(model_id) deploy_container(model_id, region, auth) elif model_type == TF: get_container_files(model_type) # adding .pb to ensure .pb file is always picked add_modelfilename_to_dockerfile( file_name+".pb", model_type) gcp_auth() gcp_setproject() upload_model_metadata(local_model_path, model_id, model_type) # cloud build yaml file has build + deploy build_container(model_id) deploy_container(model_id, region, auth) cleanup_postdeploy() except: raise Exception(DEPLOY_ERROR)
def gcs_list_files(bucket_name, prefix, delimiter=None): """Lists all the blobs in the bucket that begin with the prefix. This can be used to list all blobs in a "folder", e.g. "public/". The delimiter argument can be used to restrict the results to only the "files" in the given "folder". Without the delimiter, the entire tree under the prefix is returned. For example, given these blobs: a/1.txt a/b/2.txt If you just specify prefix = 'a', you'll get back: a/1.txt a/b/2.txt However, if you specify prefix='a' and delimiter='/', you'll get back: a/1.txt Additionally, the same request will return blobs.prefixes populated with: a/b/ """ # Note: Client.list_blobs requires at least package version 1.17.0. # https://cloud.google.com/storage/docs/listing-objects get_env_creds() storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) files_list = storage_client.list_blobs(bucket, prefix=prefix, delimiter=delimiter) files_list = files_list.prefixes if delimiter else files_list directory = prefix directory += delimiter if delimiter else "" print("Files in {}:".format(directory)) for file_name in files_list: print(" -> {}".format(file_name.name))
def submit_pyspark_job(spark_job_path, **kwargs): get_env_creds() project_id = kwargs.get(PROJECT_ID, os.environ[PROJECT_ID]) region = kwargs.get(GCP_REGION, os.environ.get(GCP_REGION, GCP_DEFAULT_REGION)) cluster_name = kwargs.get(GCP_DATAPROC_CLUSTER, os.environ[GCP_DATAPROC_CLUSTER]) bucket_name = kwargs.get(GCS_BUCKET_NAME, os.environ[GCS_BUCKET_NAME]) _, dataproc_job_client = set_cluster_clients() spark_bucket_output_path = kwargs.get( OUTPUT_PATH_ARG, "gs://{}/{}".format(bucket_name, DATA)) mod_job_path = add_outputpath_to_job(spark_job_path, spark_bucket_output_path) gcs_file_upload(bucket_name, DATAPROC, mod_job_path) submit_dataproc_pyspark_job(dataproc_job_client, mod_job_path, spark_bucket_output_path, project_id, region, cluster_name, bucket_name) os.remove(mod_job_path)
def gcs_download_file(source_file_path): """Downloads a blob from the bucket.""" # GCS_BUCKET_NAME = "your-bucket-name" # source_file_path = "models/storage-object-name" # destination_file_name = "local/path/to/file" try: get_env_creds() storage_client = storage.Client() bucket = storage_client.bucket(os.environ[GCS_BUCKET_NAME]) blob = bucket.blob(source_file_path) model_file_name = source_file_path.rsplit('/', 1)[1] # currently will only download to current directory set_cwd() destination_file_name = os.getcwd() + "/" + model_file_name blob.download_to_filename(destination_file_name) print("Blob {} downloaded to {}.".format(source_file_path, destination_file_name)) return destination_file_name except FileNotFoundError: print( "The file either doesn't exist in the bucket or hasn't been specified" )
def list_files(list_type, **kwargs): get_env_creds() bucket_name = kwargs.get(GCS_BUCKET_NAME, os.environ[GCS_BUCKET_NAME]) gcs_list_files(bucket_name, list_type)
def upload(file_type, file_path, **kwargs): get_env_creds() bucket_name = kwargs.get(GCS_BUCKET_NAME, os.environ[GCS_BUCKET_NAME]) gcs_file_upload(bucket_name, file_type, file_path)