def check_bucket(self, bucket_name): """ Checks if a buckete exists. Args: bucket_name: `str`. Name of the bucket """ try: self.client.head_bucket(Bucket=bucket_name) return True except ClientError as e: logger.info(e.response["Error"]["Message"]) return False
def check_blob(self, blob, bucket_name=None): """ Checks for the existence of a file in Google Cloud Storage. Args: blob: `str`. the path to the object to check in the Google cloud storage bucket. bucket_name: `str`. Name of the bucket in which the file is stored """ try: return bool(self.get_blob(blob=blob, bucket_name=bucket_name)) except Exception as e: logger.info('Block does not exist %s', e) return False
def check_key(self, key, bucket_name=None): """ Checks if a key exists in a bucket Args: key: `str`. S3 key that will point to the file bucket_name: `str`. Name of the bucket in which the file is stored """ if not bucket_name: (bucket_name, key) = self.parse_s3_url(key) try: self.client.head_object(Bucket=bucket_name, Key=key) return True except ClientError as e: logger.info(e.response["Error"]["Message"]) return False
def get_gc_credentials(key_path=None, keyfile_dict=None, scopes=None): """ Returns the Credentials object for Google API """ key_path = key_path or get_key_path() keyfile_dict = keyfile_dict or get_keyfile_dict() scopes = scopes or get_scopes() if scopes is not None: scopes = [s.strip() for s in scopes.split(',')] else: scopes = DEFAULT_SCOPES if not key_path and not keyfile_dict: logger.info('Getting connection using `google.auth.default()` ' 'since no key file is defined for hook.') credentials, _ = google.auth.default(scopes=scopes) elif key_path: # Get credentials from a JSON file. if key_path.endswith('.json'): logger.info('Getting connection using a JSON key file.') credentials = Credentials.from_service_account_file( os.path.abspath(key_path), scopes=scopes) else: raise PolyaxonStoresException( 'Unrecognised extension for key file.') else: # Get credentials from JSON data. try: if not isinstance(keyfile_dict, Mapping): keyfile_dict = json.loads(keyfile_dict) # Convert escaped newlines to actual newlines if any. keyfile_dict['private_key'] = keyfile_dict['private_key'].replace( '\\n', '\n') credentials = Credentials.from_service_account_info(keyfile_dict, scopes=scopes) except ValueError: # json.decoder.JSONDecodeError does not exist on py2 raise PolyaxonStoresException('Invalid key JSON.') return credentials