class GoogleCloudStorage(FileStoreBase): def __init__(self, *args, **kwargs): self.client = Client(**kwargs) def list_files(self, bucket_name, prefix, **kwargs): try: basic_request = {'prefix': prefix} list_files_request = {**basic_request, **kwargs} bucket = self.client.get_bucket(bucket_name) response_elements = bucket.list_blobs(**list_files_request) return [x.name for x in response_elements] except Exception as e: logging.exception( 'Exception in [GoogleCloudStorage.list_files] with bucket_name {} and prefix {} and kwargs {}' .format(bucket_name, prefix, kwargs)) raise e def delete_object(self, bucket_name, key, **kwargs): try: bucket = self.client.get_bucket(bucket_name) bucket.delete_blob(key) except Exception as e: logging.exception( 'Exception in [GoogleCloudStorage.delete_object] with bucket_name {} and key {}' .format(bucket_name, key)) raise e def put_object(self, bucket_name, key, obj, **kwargs): try: bucket = self.client.get_bucket(bucket_name) blob = Blob(key, bucket) blob.upload_from_string(obj) except Exception as e: logging.exception( 'Exception in [GoogleCloudStorage.put_object] with bucket_name {} and key {}' .format(bucket_name, key)) raise e def delete_bucket(self, bucket_name): try: bucket = self.client.get_bucket(bucket_name) bucket.delete() except Exception as e: logging.exception( 'Exception in [GoogleCloudStorage.delete_bucket] with bucket_name {}' .format(bucket_name)) raise e def create_bucket(self, bucket_name, **kwargs): try: basic_request = {'bucket_name': bucket_name} create_bucket_request = {**basic_request, **kwargs} self.client.create_bucket(**create_bucket_request) except Exception as e: logging.exception( 'Exception in [GoogleCloudStorage.create_bucket] with bucket_name {} and kwargs {}' .format(bucket_name, kwargs)) raise e
class GoogleStorage(object): def __init__(self, config=None): self.config = config if config else newhive.config # initialize s3 connection if self.config.buckets: try: from google.cloud.storage.client import Client self.con = Client() self.buckets = { k: self.con.get_bucket(name) for k, name in self.config.buckets.items() } except: print('google.cloud.storage.client failure') def upload_file(self, file, bucket_name, path, name, mimetype, md5=None): bucket = self.buckets[bucket_name] remote = bucket.blob(path) if mimetype: remote.content_type = mimetype remote.cache_control = 'max-age=' + str(86400 * 3650) if md5: remote.md5_hash = b64encode(b16decode(md5.upper())) if isinstance(file, basestring): remote.upload_from_filename(file) else: file.seek(0) remote.upload_from_file(file, num_retries=3) return self.url(bucket_name, path) def delete_file(self, bucket, path): bucket = self.buckets[bucket] remote = bucket.blob(path) if remote.exists(): remote.delete() return True return False def file_exists(self, bucket, path): bucket = self.buckets[bucket] remote = bucket.blob(path) return remote.exists() def bucket_url(self, bucket='media'): return '//' + self.config.buckets[bucket] + '/' def url(self, bucket='media', key='', bucket_name=None, http=False, secure=False): url = self.bucket_url(bucket) + key if http: url = 'http' + url if secure: url = 'https' + url return url