def AcquireFromGCE(account=None, use_google_auth=False):
"""Get credentials from a GCE metadata server.
Args:
account: str, The account name to use. If none, the default is used.
use_google_auth: bool, True to load credentials of google-auth if it is
supported in the current authentication scenario. False to load
credentials of oauth2client.
Returns:
oauth2client.client.Credentials or google.auth.credentials.Credentials based
on use_google_auth and whether google-auth is supported in the current
authentication sceanrio.
Raises:
c_gce.CannotConnectToMetadataServerException: If the metadata server cannot
be reached.
TokenRefreshError: If the credentials fail to refresh.
TokenRefreshReauthError: If the credentials fail to refresh due to reauth.
"""
if use_google_auth:
email = account or 'default'
credentials = google_auth_gce.Credentials(service_account_email=email)
else:
credentials = oauth2client_gce.AppAssertionCredentials(email=account)
Refresh(credentials)
return credentials
def initializeGDS(self):
global credentials
global client
print("Setup Database Connection")
credentials = compute_engine.Credentials()
# service account
client = datastore.Client.from_service_account_json('sa.json')
def __init__(self, service_account_json):
from google.auth import compute_engine
credentials = compute_engine.Credentials()
#credentials = ServiceAccountCredentials.from_json_keyfile_name(
# service_account_json, API_SCOPES)
if not credentials:
sys.exit('Could not load service account credential '
'from {}'.format(service_account_json))
discovery_url = '{}?version={}'.format(DISCOVERY_API, API_VERSION)
self._service = discovery.build(
SERVICE_NAME,
API_VERSION,
discoveryServiceUrl=discovery_url,
credentials=credentials,
cache_discovery=False)
self._bigquery_client = bigquery.Client()
dataset_id = 'IoT'
table_id = 'humidity'
# Set the destination table
self._table = self._bigquery_client.get_table(
self._bigquery_client.dataset(dataset_id).table(table_id))
# Used to serialize the calls to the
# modifyCloudToDeviceConfig REST method. This is needed
# because the google-api-python-client library is built on top
# of the httplib2 library, which is not thread-safe. For more
# details, see: https://developers.google.com/
# api-client-library/python/guide/thread_safety
self._update_config_mutex = Lock()
def test_gke(data, context):
project_id = "[PROJECT-ID]"
zone = "[ZONE]"
cluster_id = "[CLUSTER-NAME]"
credentials = compute_engine.Credentials()
cluster_manager_client = ClusterManagerClient(credentials=credentials)
cluster = cluster_manager_client.get_cluster(project_id, zone, cluster_id)
configuration = client.Configuration()
configuration.host = f"https://{cluster.endpoint}:443"
configuration.verify_ssl = False
configuration.api_key = {"authorization": "Bearer " + credentials.token}
client.Configuration.set_default(configuration)
v1 = client.AppsV1Api()
#Try to list all the deployments.
try:
deployments = v1.list_deployment_for_all_namespaces()
except ApiException as e:
print("Exception when calling AppsV1Api->list_deployment_for_all_namespaces: %s\n" % e)
#Loop through the deployments to find the specfic hello-server deployment and update accordingly.
for deployment in deployments.items:
if 'hello' in deployment.metadata.name:
print("%s\t%s" % (deployment.metadata.namespace, deployment.metadata.name))
update_deployment(v1,deployment)
return("Ok")
def analyze_sentiment(text):
"""
Analyze sentiment in a text
Args:
text -> The content to be analyzed
"""
try:
client = language_v1.LanguageServiceClient().from_service_account_json(
"./service_account_key.json")
except FileNotFoundError:
credentials = compute_engine.Credentials()
client = language_v1.LanguageServiceClient(credentials=credentials)
type_ = enums.Document.Type.PLAIN_TEXT
language = "en"
encoding_type = enums.EncodingType.UTF8
document = {"content": text, "type": type_, "language": language}
response = client.analyze_sentiment(document, encoding_type=encoding_type)
# Get overall sentiment
document_sentiment = response.document_sentiment
score = document_sentiment.score
sentiment = assign_sentiment(score)
return sentiment
def get_values_from_sheet():
"""
Use the Google Sheets API to fetch and process rows of data.
Returns:
a list of dicts mapping headers to data
"""
if 'API_KEY' in os.environ:
service = build('sheets', 'v4', developerKey=os.environ['API_KEY'])
else:
credentials = compute_engine.Credentials()
service = build('sheets', 'v4', credentials=credentials)
sheet = service.spreadsheets()
# NB: if sheet headers change the capture range may have to change
sheet_range = f"{os.environ['TAB_ID']}!A{os.environ['START_ROW']}:H"
result = sheet.values().get(spreadsheetId=os.environ['SHEET_ID'],
range=sheet_range).execute()
# NB: sheet headers may change!
headers = [
'name', 'country', 'city', 'state', 'type', 'link', 'mixcloud',
'genre', 'notes'
]
return [
row for value in result.get('values', [])
if (row := process_row(value, headers)) is not None
]
def _get_gce_credentials(request=None):
"""Gets credentials and project ID from the GCE Metadata Service."""
# Ping requires a transport, but we want application default credentials
# to require no arguments. So, we'll use the _http_client transport which
# uses http.client. This is only acceptable because the metadata server
# doesn't do SSL and never requires proxies.
# While this library is normally bundled with compute_engine, there are
# some cases where it's not available, so we tolerate ImportError.
try:
from google.auth import compute_engine
from google.auth.compute_engine import _metadata
except ImportError:
_LOGGER.warning("Import of Compute Engine auth library failed.")
return None, None
if request is None:
request = google.auth.transport._http_client.Request()
if _metadata.ping(request=request):
# Get the project ID.
try:
project_id = _metadata.get_project_id(request=request)
except exceptions.TransportError:
project_id = None
return compute_engine.Credentials(), project_id
else:
_LOGGER.warning(
"Authentication failed using Compute Engine authentication due to unavailable metadata server."
)
return None, None
def _get_gce_credentials(request=None):
"""Gets credentials and project ID from the GCE Metadata Service."""
# Ping requires a transport, but we want application default credentials
# to require no arguments. So, we'll use the _http_client transport which
# uses http.client. This is only acceptable because the metadata server
# doesn't do SSL and never requires proxies.
from google.auth import compute_engine
from google.auth.compute_engine import _metadata
if request is None:
request = google.auth.transport._http_client.Request()
if _metadata.ping(request=request):
# Get the project ID.
try:
project_id = _metadata.get_project_id(request=request)
except exceptions.TransportError:
_LOGGER.warning(
'No project ID could be determined from the Compute Engine '
'metadata service. Consider setting the %s environment '
'variable.', environment_vars.PROJECT)
project_id = None
return compute_engine.Credentials(), project_id
else:
return None, None
def upload_blob(bucket_name, dst_blob_name, src_file_name):
storage_client = storage.Client(credentials=compute_engine.Credentials(),
project='voxsrc-2020-dev-1')
bucket = storage_client.bucket(bucket_name)
blob = bucket.blob(dst_blob_name)
blob.upload_from_filename(src_file_name)
class UtilsTests(sdk_test_base.SdkBase, parameterized.TestCase):
@parameterized.parameters((True, {}, 'fake_token_host'), (True, {
'token_uri': 'another_token_host'
}, 'fake_token_host'), (False, {}, properties.VALUES.auth.DEFAULT_TOKEN_HOST),
(False, {
'token_uri': 'another_token_host'
}, 'another_token_host'))
def testGetEffectiveTokenUri(self, explicitly_set, cred_json, expected_value):
if explicitly_set:
properties.VALUES.auth.token_host.Set('fake_token_host')
self.assertEqual(expected_value, creds.GetEffectiveTokenUri(cred_json))
@parameterized.parameters(
(google_auth_credentials.UserCredWithReauth('access_token',
'refresh_token'), True, True),
(google_auth_credentials.UserCredWithReauth(
'access_token', 'refresh_token'), False, True),
(google_auth_gce.Credentials(), True, True),
(google_auth_gce.Credentials(), False, False),
(UnKnownCredentials(), True, False),
(UnKnownCredentials(), False, False),
)
def testIsUserAccountCredentialsGoogleAuth(self, credentials, is_devshell,
expected_result):
self.StartObjectPatch(
devshell, 'IsDevshellEnvironment', return_value=is_devshell)
self.assertEqual(
creds.IsUserAccountCredentials(credentials), expected_result)
@parameterized.parameters(
(client.OAuth2Credentials('token', 'client_id', 'client_secret',
'refresh_token', None, None, None), True, True),
(client.OAuth2Credentials('token', 'client_id', 'client_secret',
'refresh_token', None, None,
None), False, True),
(gce.AppAssertionCredentials(), True, True),
(gce.AppAssertionCredentials(), False, False),
(UnKnownCredentials(), True, False),
(UnKnownCredentials(), False, False),
)
def testIsUserAccountCredentialsOauth2client(self, credentials, is_devshell,
expected_result):
self.StartObjectPatch(
devshell, 'IsDevshellEnvironment', return_value=is_devshell)
self.assertEqual(
creds.IsUserAccountCredentials(credentials), expected_result)
def getData(self, project, project_name, folder_name, file_name):
credentials = compute_engine.Credentials()
storage_client = storage.Client(credentials=credentials,
project=project)
bucket = storage_client.get_bucket(project_name)
blob = bucket.blob(folder_name + '/' + file_name)
content = blob.download_as_string()
return content
def get_client(self):
import requests
from google.cloud import datastore
from google.auth import compute_engine
if os.environ.get('DATASTORE_EMULATOR_HOST'):
return datastore.Client(_http=requests.Session,
project='virustotal-avs-control')
return datastore.Client(credentials=compute_engine.Credentials())
def __init__(self):
"""Initializes the Connection"""
from google.cloud import datastore
from google.auth import compute_engine
global credentials
global client
credentials = compute_engine.Credentials()
self.client = datastore.Client.from_service_account_json('sa.json')
def explicit_compute_engine(project):
# Explicitly use Compute Engine credentials. These credentials are
# available on Compute Engine, App Engine Flexible, and Container Engine.
credentials = compute_engine.Credentials()
# Create the client using the credentials and specifying a project ID.
storage_client = storage.Client(credentials=credentials, project=project)
return storage_client
def list_workers():
creds = compute_engine.Credentials()
compute = googleapiclient.discovery.build(credentials=creds, serviceName = "compute", version = "v1")
result = compute.instances().list(project=config["DEFAULT"]["project_id"], zone=config["DEFAULT"]["zone"]).execute()
instance_names = []
for i in range(len(result["items"])):
if result["items"][i]["name"].startswith("worker"):
instance_names.append(result["items"][i]["name"])
return instance_names
def download_blob(bucket_name, src_blob_name, dst_file_name):
"""Downloads a blob from the bucket."""
storage_client = storage.Client(credentials=compute_engine.Credentials(),
project='voxsrc-2020-dev-1')
bucket = storage_client.bucket(bucket_name)
blob = bucket.blob(src_blob_name)
blob.download_to_filename(dst_file_name)
print(f"Blob {src_blob_name} downloaded to {dst_file_name}.")
def instances_list(request, service, version):
"""
API endpoint that lists the versions of a service
"""
credentials = compute_engine.Credentials()
authed_session = AuthorizedSession(credentials)
res = authed_session.get('https://appengine.googleapis.com/v1/apps/{}/services/{}/versions/{}/instances'.format(os.environ.get("GOOGLE_CLOUD_PROJECT"), service, version))
return Response(res.json())
def service_details(request, service):
"""
API endpoint that lists the services for the current application
"""
credentials = compute_engine.Credentials()
authed_session = AuthorizedSession(credentials)
res = authed_session.get('https://appengine.googleapis.com/v1/apps/{}/services/{}'.format(os.environ.get("GOOGLE_CLOUD_PROJECT"), service))
return Response(res.json())
def application_details(request):
"""
API endpoint that gets information about the running application
"""
credentials = compute_engine.Credentials()
authed_session = AuthorizedSession(credentials)
res = authed_session.get('https://appengine.googleapis.com/v1/apps/{}'.format(os.environ.get("GOOGLE_CLOUD_PROJECT")))
return Response(res.json())
def GetComputeEngineCredentials():
"""Get the compute engine credentials.
Returns:
An instance of compute_engine.Credentials.
"""
credentials = compute_engine.Credentials()
request = requests.Request()
credentials.refresh(request)
return credentials
def gcp_credentials(service_account_file):
if service_account_file:
with io.open(service_account_file, 'r', encoding='utf-8') as json_fi:
credentials_info = json.load(json_fi)
credentials = service_account.Credentials.from_service_account_info(
credentials_info)
else:
# Explicitly use Compute Engine credentials. These credentials are
# available on Compute Engine, App Engine Flexible, and Container Engine.
credentials = compute_engine.Credentials()
return credentials
def add_instances(node_chunk):
node_list = node_chunk['nodes']
pg_name = None
if 'pg' in node_chunk:
pg_name = node_chunk['pg']
log.debug(f"node_list:{node_list} pg:{pg_name}")
auth_http = None
if not cfg.google_app_cred_path:
http = set_user_agent(httplib2.Http(),
"Slurm_GCP_Scripts/1.2 (GPN:SchedMD)")
creds = compute_engine.Credentials()
auth_http = google_auth_httplib2.AuthorizedHttp(creds, http=http)
compute = googleapiclient.discovery.build('compute',
'v1',
http=auth_http,
cache_discovery=False)
pid = util.get_pid(node_list[0])
instance_def = cfg.instance_defs[pid]
try:
operation = create_instance(compute, instance_def, node_list, pg_name)
except googleapiclient.errors.HttpError as e:
log.error(
f"failed to add {node_list[0]}*{len(node_list)} to slurm, {e}")
if instance_def.exclusive:
os._exit(1)
down_nodes(node_list, e)
return
result = util.wait_for_operation(compute, cfg.project, operation)
if not result or 'error' in result:
grp_err_msg = result['error']['errors'][0]['message']
log.error(f"group operation failed: {grp_err_msg}")
if instance_def.exclusive:
os._exit(1)
group_ops = util.get_group_operations(compute, cfg.project, result)
failed_nodes = {}
for op in group_ops['items']:
if op['operationType'] != 'insert':
continue
if 'error' in op:
err_msg = op['error']['errors'][0]['message']
failed_node = op['targetLink'].split('/')[-1]
if err_msg not in failed_nodes:
failed_nodes[err_msg] = [failed_node]
else:
failed_nodes[err_msg].append(failed_node)
if failed_nodes:
log.error(f"insert requests failed: {failed_nodes}")
for msg, nodes in failed_nodes.items():
down_nodes(nodes, msg)
def _upload_blob(bucket_name, source_file_name, destination_blob_name):
# Should be extracted to its own method
credentials = compute_engine.Credentials()
storage_client = storage.Client(project=PROJECT_ID)
bucket = storage_client.get_bucket(bucket_name)
blob = bucket.blob(destination_blob_name)
blob.upload_from_filename(source_file_name)
print('File {} uploaded to {}.'.format(source_file_name,
destination_blob_name))
def test_refresh(http_request, token_info):
credentials = compute_engine.Credentials()
credentials.refresh(http_request)
assert credentials.token is not None
assert credentials.service_account_email is not None
info = token_info(credentials.token)
info_scopes = _helpers.string_to_scopes(info["scope"])
assert set(info_scopes) == set(credentials.scopes)
def get_storage_client():
# grab storage client, with credentials from compute engine unless
# the google sdk credentials env var is set (like what we do in
# local dev)
if "GOOGLE_APPLICATION_CREDENTIALS" in os.environ:
# use configured user credentials
return storage.Client(project='voxsrc-2020-dev-1')
else:
# use embedded compute engine (GCP) credentials
return storage.Client(credentials=compute_engine.Credentials(),
project='voxsrc-2020-dev-1')
def delete_path_from_storage(bucket_name,
path,
use_cloud_engine_credentials=False):
credentials = None
if use_cloud_engine_credentials:
credentials = compute_engine.Credentials()
storage_client = storage.Client(credentials=credentials)
bucket = storage_client.get_bucket(bucket_name)
blobs = bucket.list_blobs(prefix=path)
for blob in blobs:
blob.delete()
def download_from_cloud(source_blob_name):
source_blob_name = source_blob_name.replace("\\", "/")
# Should be extracted to its own method
credentials = compute_engine.Credentials()
storage_client = storage.Client(project=PROJECT_ID)
bucket = storage_client.get_bucket(BUCKET_NAME)
print("Soure blob name: ", source_blob_name)
blob = bucket.get_blob(source_blob_name[18:])
destination_file_name = source_blob_name[18:]
blob.download_to_filename(destination_file_name)
print('Blob {} downloaded to {}.'.format(source_blob_name,
destination_file_name))
def upload_file_to_storage(project_id,
bucket,
file_path: str,
gs_path,
use_cloud_engine_credentials=False):
gs_path = gs_path.replace(f'gs://{bucket}/', "")
credentials = None
if use_cloud_engine_credentials:
credentials = compute_engine.Credentials()
client = storage.Client(project=project_id, credentials=credentials)
bucket = client.get_bucket(bucket)
blob = bucket.blob('{}/{}'.format(gs_path, file_path.split('/')[-1]))
blob.upload_from_filename(file_path)
def build_credentials(self) -> Credentials:
try:
if not self._credentials or not self._credentials.valid:
logger.debug(
'Fetch compute_engine.Credentials and token refresh.')
self._credentials = compute_engine.Credentials()
self._credentials.refresh(request=self._build_request())
return self._credentials
except ServiceAccountConfigurationError:
raise
except RuntimeError as e:
raise ServiceAccountConfigurationError(e)
def explicit_compute_engine(project):
from google.auth import compute_engine
from google.cloud import storage
# Explicitly use Compute Engine credentials. These credentials are
# available on Compute Engine, App Engine Flexible, and Container Engine.
credentials = compute_engine.Credentials()
# Create the client using the credentials and specifying a project ID.
storage_client = storage.Client(credentials=credentials, project=project)
# Make an authenticated API request
buckets = list(storage_client.list_buckets())
print(buckets)
