class TestBaseGcpSystem(unittest.TestCase, LoggingMixin):
def __init__(self, method_name, gcp_key, project_extra=None):
super().__init__(methodName=method_name)
self.gcp_authenticator = GcpAuthenticator(gcp_key=gcp_key,
project_extra=project_extra)
self.setup_called = False
@staticmethod
def skip_check(key_name):
return GcpAuthenticator(key_name).full_key_path is None
@staticmethod
def skip_long(key_name):
if os.environ.get('GCP_ENABLE_LONG_TESTS') == 'True':
return GcpAuthenticator(key_name).full_key_path is None
return True
def setUp(self):
self.gcp_authenticator.gcp_store_authentication()
self.gcp_authenticator.gcp_authenticate()
# We checked that authentication works. Ne we revoke it to make
# sure we are not relying on the default authentication
self.gcp_authenticator.gcp_revoke_authentication()
self.setup_called = True
# noinspection PyPep8Naming
def tearDown(self):
self.gcp_authenticator.gcp_restore_authentication()
def skip_gcp_system(service_key: str,
long_lasting: bool = False,
require_local_executor: bool = False):
"""
Decorator for skipping GCP system tests.
:param service_key: name of the service key that will be used to provide credentials
:type service_key: str
:param long_lasting: set True if a test take relatively long time
:type long_lasting: bool
:param require_local_executor: set True if test config must use local executor
:type require_local_executor: bool
"""
if GcpAuthenticator(service_key).full_key_path is None:
return unittest.skip(SKIP_TEST_WARNING)
if long_lasting and os.environ.get("GCP_ENABLE_LONG_TESTS") == "True":
return unittest.skip(SKIP_LONG_TEST_WARNING)
if require_local_executor and POSTGRES_LOCAL_EXECUTOR != os.environ.get(
"AIRFLOW_CONFIG"):
return unittest.skip(
LOCAL_EXECUTOR_WARNING.format(POSTGRES_LOCAL_EXECUTOR))
return lambda cls: cls
def delete_bucket(self):
self.execute_cmd(['gsutil', 'rm', '-r', "gs://%s/" % GCP_BUCKET_NAME])
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Create and remove a bucket for system tests.')
parser.add_argument(
'--action',
dest='action',
required=True,
choices=('create-bucket', 'delete-bucket', 'before-tests', 'after-tests'),
)
action = parser.parse_args().action
helper = GCPVisionTestHelper()
gcp_authenticator = GcpAuthenticator(GCP_AI_KEY)
helper.log.info('Starting action: {}'.format(action))
gcp_authenticator.gcp_store_authentication()
try:
gcp_authenticator.gcp_authenticate()
if action == 'before-tests':
pass
elif action == 'after-tests':
pass
elif action == 'create-bucket':
helper.create_bucket()
elif action == 'delete-bucket':
helper.delete_bucket()
else:
raise Exception("Unknown action: {}".format(action))
'--action',
dest='action',
required=True,
choices=(
'create-s3-bucket',
'delete-s3-bucket',
'create-gcs-buckets',
'delete-gcs-buckets',
'before-tests',
'after-tests',
),
)
action = parser.parse_args().action
helper = GCPTransferTestHelper()
gcp_authenticator = GcpAuthenticator(GCP_GCS_TRANSFER_KEY)
helper.log.info('Starting action: {}'.format(action))
gcp_authenticator.gcp_store_authentication()
try:
gcp_authenticator.gcp_authenticate()
if action == 'before-tests':
pass
elif action == 'after-tests':
pass
elif action == 'create-s3-bucket':
helper.create_s3_bucket()
elif action == 'delete-s3-bucket':
helper.delete_s3_bucket()
elif action == 'create-gcs-buckets':
helper.create_gcs_buckets()
def skip_long(key_name):
if os.environ.get('GCP_ENABLE_LONG_TESTS') == 'True':
return GcpAuthenticator(key_name).full_key_path is None
return True
def skip_check(key_name):
return GcpAuthenticator(key_name).full_key_path is None
def __init__(self, method_name, gcp_key, project_extra=None):
super().__init__(methodName=method_name)
self.gcp_authenticator = GcpAuthenticator(gcp_key=gcp_key,
project_extra=project_extra)
self.setup_called = False
from tests.gcp.utils.gcp_authenticator import GCP_CLOUDSQL_KEY, GcpAuthenticator
QUERY_SUFFIX = "_QUERY"
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Create or delete Cloud SQL instances for system tests.')
parser.add_argument('--action',
required=True,
choices=('create', 'delete', 'setup-instances',
'before-tests', 'after-tests'))
action = parser.parse_args().action
helper = CloudSqlQueryTestHelper()
gcp_authenticator = GcpAuthenticator(gcp_key=GCP_CLOUDSQL_KEY)
helper.log.info('Starting action: {}'.format(action))
gcp_authenticator.gcp_store_authentication()
try:
gcp_authenticator.gcp_authenticate()
if action == 'before-tests':
helper.create_instances(instance_suffix=QUERY_SUFFIX)
helper.setup_instances(instance_suffix=QUERY_SUFFIX)
elif action == 'after-tests':
helper.delete_instances(instance_suffix=QUERY_SUFFIX)
elif action == 'create':
helper.create_instances(instance_suffix=QUERY_SUFFIX)
elif action == 'delete':
helper.delete_instances(instance_suffix=QUERY_SUFFIX)
elif action == 'setup-instances':
helper.setup_instances(instance_suffix=QUERY_SUFFIX)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Create and delete bucket for system tests.")
parser.add_argument(
"--action",
dest="action",
required=True,
choices=("create-bucket", "delete-bucket", "before-tests",
"after-tests"),
)
action = parser.parse_args().action
helper = GCPBigQueryTestHelper()
gcp_authenticator = GcpAuthenticator(GCP_BIGQUERY_KEY)
helper.log.info("Starting action: %s", action)
gcp_authenticator.gcp_store_authentication()
try:
gcp_authenticator.gcp_authenticate()
if action == "before-tests":
pass
elif action == "after-tests":
pass
elif action == "create-bucket":
helper.create_repository_and_bucket()
elif action == "delete-bucket":
helper.delete_bucket()
else:
raise Exception("Unknown action: {}".format(action))
dest="action",
required=True,
choices=(
"create-repo-and-bucket",
"delete-repo",
"delete-bucket",
"delete-docker-images",
"delete-repo-and-bucket-and-docker-images",
"before-tests",
"after-tests",
),
)
action = parser.parse_args().action
helper = GCPCloudBuildTestHelper()
gcp_authenticator = GcpAuthenticator(GCP_CLOUD_BUILD_KEY)
helper.log.info("Starting action: %s", action)
gcp_authenticator.gcp_store_authentication()
try:
gcp_authenticator.gcp_authenticate()
if action == "before-tests":
pass
elif action == "after-tests":
pass
elif action == "create-repo-and-bucket":
helper.create_repository_and_bucket()
elif action == "delete-repo-and-bucket-and-docker-images":
helper.delete_repo()
helper.delete_bucket()
helper.delete_docker_images()
