def check_bucket(self, bucket_name):
"""
Checks if a buckete exists.
Args:
bucket_name: `str`. Name of the bucket
"""
try:
self.client.head_bucket(Bucket=bucket_name)
return True
except ClientError as e:
logger.info(e.response["Error"]["Message"])
return False
def check_blob(self, blob, bucket_name=None):
"""
Checks for the existence of a file in Google Cloud Storage.
Args:
blob: `str`. the path to the object to check in the Google cloud storage bucket.
bucket_name: `str`. Name of the bucket in which the file is stored
"""
try:
return bool(self.get_blob(blob=blob, bucket_name=bucket_name))
except Exception as e:
logger.info('Block does not exist %s', e)
return False
def check_key(self, key, bucket_name=None):
"""
Checks if a key exists in a bucket
Args:
key: `str`. S3 key that will point to the file
bucket_name: `str`. Name of the bucket in which the file is stored
"""
if not bucket_name:
(bucket_name, key) = self.parse_s3_url(key)
try:
self.client.head_object(Bucket=bucket_name, Key=key)
return True
except ClientError as e:
logger.info(e.response["Error"]["Message"])
return False
def get_gc_credentials(key_path=None, keyfile_dict=None, scopes=None):
"""
Returns the Credentials object for Google API
"""
key_path = key_path or get_key_path()
keyfile_dict = keyfile_dict or get_keyfile_dict()
scopes = scopes or get_scopes()
if scopes is not None:
scopes = [s.strip() for s in scopes.split(',')]
else:
scopes = DEFAULT_SCOPES
if not key_path and not keyfile_dict:
logger.info('Getting connection using `google.auth.default()` '
'since no key file is defined for hook.')
credentials, _ = google.auth.default(scopes=scopes)
elif key_path:
# Get credentials from a JSON file.
if key_path.endswith('.json'):
logger.info('Getting connection using a JSON key file.')
credentials = Credentials.from_service_account_file(
os.path.abspath(key_path), scopes=scopes)
else:
raise PolyaxonStoresException(
'Unrecognised extension for key file.')
else:
# Get credentials from JSON data.
try:
if not isinstance(keyfile_dict, Mapping):
keyfile_dict = json.loads(keyfile_dict)
# Convert escaped newlines to actual newlines if any.
keyfile_dict['private_key'] = keyfile_dict['private_key'].replace(
'\\n', '\n')
credentials = Credentials.from_service_account_info(keyfile_dict,
scopes=scopes)
except ValueError: # json.decoder.JSONDecodeError does not exist on py2
raise PolyaxonStoresException('Invalid key JSON.')
return credentials
