class GoogleCloudStorage(FileStoreBase):
def __init__(self, *args, **kwargs):
self.client = Client(**kwargs)
def list_files(self, bucket_name, prefix, **kwargs):
try:
basic_request = {'prefix': prefix}
list_files_request = {**basic_request, **kwargs}
bucket = self.client.get_bucket(bucket_name)
response_elements = bucket.list_blobs(**list_files_request)
return [x.name for x in response_elements]
except Exception as e:
logging.exception(
'Exception in [GoogleCloudStorage.list_files] with bucket_name {} and prefix {} and kwargs {}'
.format(bucket_name, prefix, kwargs))
raise e
def delete_object(self, bucket_name, key, **kwargs):
try:
bucket = self.client.get_bucket(bucket_name)
bucket.delete_blob(key)
except Exception as e:
logging.exception(
'Exception in [GoogleCloudStorage.delete_object] with bucket_name {} and key {}'
.format(bucket_name, key))
raise e
def put_object(self, bucket_name, key, obj, **kwargs):
try:
bucket = self.client.get_bucket(bucket_name)
blob = Blob(key, bucket)
blob.upload_from_string(obj)
except Exception as e:
logging.exception(
'Exception in [GoogleCloudStorage.put_object] with bucket_name {} and key {}'
.format(bucket_name, key))
raise e
def delete_bucket(self, bucket_name):
try:
bucket = self.client.get_bucket(bucket_name)
bucket.delete()
except Exception as e:
logging.exception(
'Exception in [GoogleCloudStorage.delete_bucket] with bucket_name {}'
.format(bucket_name))
raise e
def create_bucket(self, bucket_name, **kwargs):
try:
basic_request = {'bucket_name': bucket_name}
create_bucket_request = {**basic_request, **kwargs}
self.client.create_bucket(**create_bucket_request)
except Exception as e:
logging.exception(
'Exception in [GoogleCloudStorage.create_bucket] with bucket_name {} and kwargs {}'
.format(bucket_name, kwargs))
raise e
class GoogleStorage(object):
def __init__(self, config=None):
self.config = config if config else newhive.config
# initialize s3 connection
if self.config.buckets:
try:
from google.cloud.storage.client import Client
self.con = Client()
self.buckets = {
k: self.con.get_bucket(name)
for k, name in self.config.buckets.items()
}
except:
print('google.cloud.storage.client failure')
def upload_file(self, file, bucket_name, path, name, mimetype, md5=None):
bucket = self.buckets[bucket_name]
remote = bucket.blob(path)
if mimetype:
remote.content_type = mimetype
remote.cache_control = 'max-age=' + str(86400 * 3650)
if md5:
remote.md5_hash = b64encode(b16decode(md5.upper()))
if isinstance(file, basestring):
remote.upload_from_filename(file)
else:
file.seek(0)
remote.upload_from_file(file, num_retries=3)
return self.url(bucket_name, path)
def delete_file(self, bucket, path):
bucket = self.buckets[bucket]
remote = bucket.blob(path)
if remote.exists():
remote.delete()
return True
return False
def file_exists(self, bucket, path):
bucket = self.buckets[bucket]
remote = bucket.blob(path)
return remote.exists()
def bucket_url(self, bucket='media'):
return '//' + self.config.buckets[bucket] + '/'
def url(self,
bucket='media',
key='',
bucket_name=None,
http=False,
secure=False):
url = self.bucket_url(bucket) + key
if http: url = 'http' + url
if secure: url = 'https' + url
return url
