def __init__(self, project, service_file, token=None, session=None):
self.service_file = service_file
self.session = session
self.token = token or Token(project, self.service_file,
session=self.session,
scopes=[READ_WRITE_SCOPE])
def __init__(self,
project: str,
service_file: str,
token=None,
session=None):
"""Initialize a new Instance.
You should interact inside a async context for interaction with their
APIs.
:param str project:
google project name
:param str service_file:
path to service account json. you should give permissions to this service
account for Machine Learning API (Owner)
:param gcloud.aio.auth.Token token:
(optional) a pre-initialized Google Cloud token
:param aiohttp.ClientSession session:
(optional) a pre-initialized aiohttp client session
"""
self.project = project
self.service_file = service_file
self.session = session or aiohttp.ClientSession()
self.token = token or Token(self.project,
self.service_file,
session=self.session,
scopes=[MLENGINE_SCOPE])
def __init__(self, *,
service_file: Optional[Union[str, IO[AnyStr]]] = None,
token: Optional[Token] = None,
session: Optional[Session] = None) -> None:
self.session = AioSession(session, verify_ssl=VERIFY_SSL)
self.token = token or Token(service_file=service_file, scopes=SCOPES,
session=self.session.session)
async def test_token_does_not_require_session(creds, project):
scopes = ['https://www.googleapis.com/auth/taskqueue']
token = Token(project, creds, scopes=scopes)
result = await token.get()
assert result is not None
def __init__(self,
*,
service_file: Optional[str] = None,
token: Optional[Token] = None,
session: Optional[aiohttp.ClientSession] = None) -> None:
self.session = session
self.token = token or Token(
service_file=service_file, session=self.session, scopes=SCOPES)
async def test_token_is_created(creds, project):
scopes = ['https://www.googleapis.com/auth/taskqueue']
async with aiohttp.ClientSession() as session:
token = Token(project, creds, session=session, scopes=scopes)
result = await token.get()
assert result is not None
def __init__(self, project, service_file, taskqueue, location=LOCATION,
session=None, token=None):
self.api_root = (f'{API_ROOT}/projects/{project}/'
f'locations/{location}/queues/{taskqueue}')
self.session = session
self.token = token or Token(project, service_file, scopes=SCOPES,
session=self.session)
def __init__(self,
project: Optional[str] = None,
service_file: Optional[Union[str, io.IOBase]] = None,
session: Optional[Session] = None,
token: Optional[Token] = None) -> None:
self._project = project
self.session = AioSession(session)
self.token = token or Token(service_file=service_file, scopes=SCOPES,
session=self.session.session)
async def test_token_is_created():
project = os.environ['GCLOUD_PROJECT']
creds = os.environ['GOOGLE_APPLICATION_CREDENTIALS']
scopes = ['https://www.googleapis.com/auth/taskqueue']
async with aiohttp.ClientSession() as session:
token = Token(project, creds, session=session, scopes=scopes)
result = await token.get()
assert result is not None
def __init__(self, project, service_file, keyproject, keyring, keyname,
location=LOCATION, session=None, token=None):
# pylint: disable=too-many-arguments
self.api_root = (f'{API_ROOT}/projects/{keyproject}/'
f'locations/{location}/keyRings/{keyring}/'
f'cryptoKeys/{keyname}')
self.session = session
self.token = token or Token(project, service_file, scopes=SCOPES,
session=self.session)
async def test_token_does_not_require_creds() -> None:
scopes = ['https://www.googleapis.com/auth/taskqueue']
token = Token(scopes=scopes)
result = await token.get()
assert result
assert token.access_token is not None
assert token.access_token_duration != 0
assert token.access_token_acquired_at != datetime.datetime(1970, 1, 1)
def __init__(self, keyproject: str, keyring: str, keyname: str,
service_file: Optional[str] = None, location: str = LOCATION,
session: Optional[aiohttp.ClientSession] = None,
token: Optional[Token] = None) -> None:
self.api_root = (f'{API_ROOT}/projects/{keyproject}/'
f'locations/{location}/keyRings/{keyring}/'
f'cryptoKeys/{keyname}')
self.session = session
self.token = token or Token(service_file=service_file, scopes=SCOPES,
session=self.session)
async def test_token_is_created(creds: str) -> None:
scopes = ['https://www.googleapis.com/auth/taskqueue']
async with Session() as session:
token = Token(service_file=creds, session=session, scopes=scopes)
result = await token.get()
assert result
assert token.access_token is not None
assert token.access_token_duration != 0
assert token.access_token_acquired_at != datetime.datetime(1970, 1, 1)
def make_download(project,
service_file,
bucket_name,
session=None,
token=None):
token = token or Token(project, service_file, scopes=[READ_ONLY_SCOPE])
storage = Storage(project, service_file, session=session, token=token)
bucket = storage.get_bucket(bucket_name)
return functools.partial(download, bucket)
def __init__(self, project: str, taskqueue: str,
service_file: Optional[Union[str, io.IOBase]] = None,
location: str = LOCATION,
session: Optional[aiohttp.ClientSession] = None,
token: Optional[Token] = None) -> None:
self.base_api_root = f'{API_ROOT}/v2beta3'
self.api_root = (f'{self.base_api_root}/projects/{project}/'
f'locations/{location}/queues/{taskqueue}')
self.session = session
self.token = token or Token(service_file=service_file, scopes=SCOPES,
session=self.session)
def __init__(self,
project: str,
service_file: str,
namespace: str = '',
session: aiohttp.ClientSession = None,
token: Token = None) -> None:
self.project = project
self.namespace = namespace
self.session = session
self.token = token or Token(
project, service_file, session=session, scopes=SCOPES)
def __init__(self,
project: str,
service_file: str,
*,
token: Token = None,
session: aiohttp.ClientSession = None) -> None:
self.service_file = service_file
self.session = session
self.token = token or Token(project,
self.service_file,
session=self.session,
scopes=[READ_WRITE_SCOPE])
def __init__(self,
dataset_name: str,
table_name: str,
project: Optional[str] = None,
service_file: Optional[Union[str, io.IOBase]] = None,
session: Optional[aiohttp.ClientSession] = None,
token: Optional[Token] = None) -> None:
self._project = project
self.dataset_name = dataset_name
self.table_name = table_name
self.session = session
self.token = token or Token(
service_file=service_file, session=session, scopes=SCOPES)
def __init__(self,
project,
service_file,
dataset_name,
table_name,
session=None,
token=None):
self.project = project
self.table_name = table_name
self.dataset_name = dataset_name
self.session = session
self.token = token or Token(
project, service_file, session=session, scopes=SCOPES)
def __init__(self,
project,
service_file,
dataset_name,
table_name,
session=None,
token=None):
# pylint: disable=too-many-arguments
self.project = project
self.table_name = table_name
self.dataset_name = dataset_name
self.session = session
self.token = token or Token(
project, service_file, session=session, scopes=SCOPES)
def __init__(self, project: Optional[str] = None,
service_file: Optional[str] = None, namespace: str = '',
session: Optional[aiohttp.ClientSession] = None,
token: Optional[Token] = None) -> None:
self.namespace = namespace
self.session = session
if IS_DEV:
self._project = os.environ.get('DATASTORE_PROJECT_ID', 'dev')
# Tokens are not needed when using dev emulator
self.token = None
else:
self._project = project
self.token = token or Token(service_file=service_file,
session=session, scopes=SCOPES)
def create_token(self, credentials: Optional[JSON]) -> Optional[Token]:
if credentials is None:
return None
tmpfile = tempfile.NamedTemporaryFile(mode="w+", delete=False)
json.dump(credentials, tmpfile)
tmpfile.close()
try:
token = Token(
service_file=tmpfile.name,
session=self.http_client,
scopes=["https://www.googleapis.com/auth/cloud-platform"],
)
finally:
os.remove(tmpfile.name)
return token
def __init__(self,
project,
service_file,
taskqueue,
location=LOCATION,
session=None,
token=None):
# pylint: disable=too-many-arguments
self.session = session or aiohttp.ClientSession(conn_timeout=10,
read_timeout=10)
self.api_root = (f'{API_ROOT}/projects/{project}/'
f'locations/{location}/queues/{taskqueue}')
self.token = token or Token(
project, service_file, scopes=SCOPES, session=self.session)
async def start(self, num_retries=15):
self.session = self.session or aiohttp.ClientSession()
self.auth_client = Token(session=self.session)
endpoint = ("https://container.googleapis.com/v1/"
f"projects/{self.project_id}/"
f"zones/{self.zone}/"
f"clusters")
network = f"projects/{self.project_id}/global/networks/{NETWORK_NAME}"
request = {
"cluster": {
"name":
self.cluster_id,
"masterAuth": {
"clientCertificateConfig": {}
},
"network":
network,
"addonsConfig": {
"httpLoadBalancing": {},
"horizontalPodAutoscaling": {},
"kubernetesDashboard": {
"disabled": True
},
"dnsCacheConfig": {},
},
"nodePools": [{
"name": NODE_POOL_NAME,
"config": {
"machineType":
self.machine_type,
"diskSizeGb":
DISK_SIZE_GB,
"oauthScopes": [
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/service.management.readonly", # noqa
"https://www.googleapis.com/auth/trace.append",
],
"metadata": {
"disable-legacy-endpoints": "true"
},
"preemptible":
USE_PREEMPTIBLES,
"diskType":
DISK_TYPE,
"shieldedInstanceConfig": {
"enableIntegrityMonitoring": True
},
},
"initialNodeCount": self.num_nodes,
"autoscaling": {},
"management": {
"autoUpgrade": True,
"autoRepair": True
},
"version": GKE_VERSION,
"upgradeSettings": {
"maxSurge": 1
},
}],
}
}
headers = {"Authorization": f"Bearer {await self.auth_client.get()}"}
for i in range(num_retries):
async with self.session.post(endpoint,
json=request,
headers=headers) as response:
if response.status == 400 and (i + 1) < num_retries: # retry
await asyncio.sleep(POLL_INTERVAL)
continue
if response.status != 200:
print(response)
print(await response.text())
assert response.status == 200, response.status
break
self.started = True
# Poll for cluster endpoint
poll_endpoint = f"{endpoint}/{self.cluster_id}"
while not self.cluster_endpoint:
await asyncio.sleep(POLL_INTERVAL)
async with self.session.get(poll_endpoint,
headers=headers) as response:
if response.status != 200:
print(response)
print(await response.text())
assert response.status == 200, response.status
response_json = await response.json()
if response_json.get("status") == "RUNNING":
self.cluster_endpoint = response_json.get("endpoint")
print(
f"Started Kubernetes cluster {self.cluster_id} at {self.cluster_endpoint}"
)
def __init__(self, project, service_file, session=None, token=None):
self.project = project
self.session = session
self.token = token or Token(
project, service_file, session=session, scopes=SCOPES)
