def _GetAuthorizedHttp(credentials=None):
"""Get the authorized http from the stored credentials.
The client library will validate and refresh credentials as needed.
Args:
credentials: Optional credentials to use instead of any in the datastore.
Returns:
authorized http, a "httplib2.Http" instance, with the proper authentication
header, access token, and credential.
Raises:
SetupNeeded: An exception that there are no credentails in the datastore.
"""
if not credentials:
credential_storage = appengine.StorageByKeyName(
appengine.CredentialsModel, datastore.CURRENT_DOMAIN,
'credentials')
credentials = credential_storage.get()
if credentials:
logging.debug('Successfully got credentials from storage.')
else:
if config.SERVICE_ACCOUNT:
credentials = setup.LoadCredentialsFromPem()
else:
raise SetupNeeded('Credentials not in storage')
return credentials.authorize(httplib2.Http())
def post(self):
"""Start instances using the gce_appengine helper class."""
gce_project_id = data_handler.stored_user_data[
user_data.GCE_PROJECT_ID]
gce_zone_name = data_handler.stored_user_data[user_data.GCE_ZONE_NAME]
user_id = users.get_current_user().user_id()
credentials = oauth2client.StorageByKeyName(
oauth2client.CredentialsModel, user_id, 'credentials').get()
gce_project = gce.GceProject(credentials,
project_id=gce_project_id,
zone_name=gce_zone_name)
num_instances = int(self.request.get('num_instances'))
instances = [
gce.Instance('%s-%d' % (DEMO_NAME, i))
for i in range(num_instances)
]
response = gce_appengine.GceAppEngine().run_gce_request(
self,
gce_project.bulk_insert,
'Error inserting instances: ',
resources=instances)
# Record objective in datastore so we can recover work in progress.
updateObjective(gce_project_id, num_instances)
if response:
self.response.headers['Content-Type'] = 'text/plain'
self.response.out.write('starting cluster')
def _StoreCredentials(credentials):
if datastore.HOSTED:
domain = users.get_current_user().email().split('@')[1]
else: # In non-hosted, the user will not be logged in on /report/ requests.
domain = config.DOMAIN.split(',')[0]
credential_storage = appengine.StorageByKeyName(appengine.CredentialsModel,
domain, 'credentials')
credential_storage.put(credentials)
def get_predict_service():
#TODO(lambert): we need to cache this somehow, if we use this, since it appears to not even use memcache for credentials.
import httplib2
from apiclient.discovery import build
from oauth2client import appengine
credentials = appengine.StorageByKeyName(appengine.CredentialsModel, MAGIC_USER_ID, 'credentials').get()
http = credentials.authorize(httplib2.Http())
service = build("prediction", "v1.5", http=http)
return service
def post(self):
"""Stop instances with names containing the tag."""
user = users.get_current_user()
credentials = oauth2client.StorageByKeyName(
oauth2client.CredentialsModel, user.user_id(),
'credentials').get()
gce_project_id = data_handler.stored_user_data[
user_data.GCE_PROJECT_ID]
gce_project = gce.GceProject(credentials, project_id=gce_project_id)
gce_appengine.GceAppEngine().delete_demo_instances(
self, gce_project, DEMO_NAME)
def post(self):
"""Stop instances using the gce_appengine helper class."""
gce_project_id = data_handler.stored_user_data[
user_data.GCE_PROJECT_ID]
gce_zone_name = data_handler.stored_user_data[user_data.GCE_ZONE_NAME]
user_id = users.get_current_user().user_id()
credentials = oauth2client.StorageByKeyName(
oauth2client.CredentialsModel, user_id, 'credentials').get()
gce_project = gce.GceProject(credentials,
project_id=gce_project_id,
zone_name=gce_zone_name)
gce_appengine.GceAppEngine().delete_demo_instances(
self, gce_project, DEMO_NAME)
# Record reset objective in datastore so we can recover work in progress.
updateObjective(gce_project_id, 0)
def post(self):
"""Remove all cloud storage contents from the given bucket and dir."""
user_id = users.get_current_user().user_id()
credentials = oauth2client.StorageByKeyName(
oauth2client.CredentialsModel, user_id, 'credentials').get()
gcs_project_id = data_handler.stored_user_data[
user_data.GCS_PROJECT_ID]
gcs_bucket = data_handler.stored_user_data[user_data.GCS_BUCKET]
gcs_directory = data_handler.stored_user_data.get(
user_data.GCS_DIRECTORY, None)
gcs_helper = gcs_appengine.GcsAppEngineHelper(credentials,
gcs_project_id)
file_regex = None
if gcs_directory:
file_regex = r'^%s/%s.*' % (gcs_directory, DEMO_NAME)
else:
file_regex = r'^%s.*' % DEMO_NAME
gcs_helper.delete_bucket_contents(gcs_bucket, gcs_directory,
file_regex)
self.response.headers['Content-Type'] = 'text/plain'
self.response.out.write('cleaning cloud storage bucket')
def post(self):
"""Insert instances with a startup script, metadata, and scopes.
Startup script is randomly chosen to either rotate images left or right.
Metadata includes the image to rotate, the demo name tag, and the machine
number. Service account scopes include Compute and storage.
"""
user = users.get_current_user()
credentials = oauth2client.StorageByKeyName(
oauth2client.CredentialsModel, user.user_id(),
'credentials').get()
gce_project_id = data_handler.stored_user_data[
user_data.GCE_PROJECT_ID]
gce_project = gce.GceProject(credentials, project_id=gce_project_id)
# Get the bucket info for the instance metadata.
gcs_bucket = data_handler.stored_user_data[user_data.GCS_BUCKET]
gcs_directory = data_handler.stored_user_data.get(
user_data.GCS_DIRECTORY, None)
gcs_path = None
if gcs_directory:
gcs_path = '%s/%s' % (gcs_bucket, gcs_directory)
else:
gcs_path = gcs_bucket
# Figure out the image. Use custom image if it exists.
(image_project, image_name) = self._get_image_name(gce_project)
# Create a list of instances to insert.
instances = []
num_instances = int(self.request.get('num_instances'))
for i in range(num_instances):
startup_script = os.path.join(os.path.dirname(__file__),
'startup.sh')
instances.append(
gce.Instance(name='%s-%d' % (DEMO_NAME, i),
image_project_id=image_project,
image_name=image_name,
service_accounts=gce_project.
settings['cloud_service_account'],
metadata=[{
'key':
'startup-script',
'value':
open(startup_script, 'r').read()
}, {
'key': 'image',
'value': random.choice(IMAGES)
}, {
'key': 'seq',
'value': random.choice(SEQUENCES)
}, {
'key': 'machine-num',
'value': i
}, {
'key': 'tag',
'value': DEMO_NAME
}, {
'key': 'gcs-path',
'value': gcs_path
}]))
response = gce_appengine.GceAppEngine().run_gce_request(
self,
gce_project.bulk_insert,
'Error inserting instances: ',
resources=instances)
if response:
self.response.headers['Content-Type'] = 'text/plain'
self.response.out.write('starting cluster')
def post(self):
"""Start instances using the gce_appengine helper class."""
user_info = getUserDemoInfo(users.get_current_user())
gce_project_id = data_handler.stored_user_data[
user_data.GCE_PROJECT_ID]
gce_zone_name = data_handler.stored_user_data[user_data.GCE_ZONE_NAME]
user_id = users.get_current_user().user_id()
credentials = oauth2client.StorageByKeyName(
oauth2client.CredentialsModel, user_id, 'credentials').get()
gce_project = gce.GceProject(credentials,
project_id=gce_project_id,
zone_name=gce_zone_name)
# Create a user specific route. We will apply this route to all
# instances without an IP address so their requests are routed
# through the first instance acting as a proxy.
# gce_project.list_routes()
proxy_instance = gce.Instance(name='%s-0' % user_info['demo_id'],
zone_name=gce_zone_name)
proxy_instance.gce_project = gce_project
route_name = '%s-0' % user_info['demo_id']
gce_route = gce.Route(name=route_name,
network_name='default',
destination_range='0.0.0.0/0',
next_hop_instance=proxy_instance,
priority=200,
tags=['qs-%s' % user_info['ldap']])
response = gce_appengine.GceAppEngine().run_gce_request(
self,
gce_project.insert,
'Error inserting route: ',
resource=gce_route)
# Define a network interfaces list here that requests an ephemeral
# external IP address. We will apply this configuration to the first
# VM started by quick start. All other VMs will take the default
# network configuration, which requests no external IP address.
network = gce.Network('default')
network.gce_project = gce_project
ext_net = [{
'network':
network.url,
'accessConfigs': [{
'name': 'External IP access config',
'type': 'ONE_TO_ONE_NAT'
}]
}]
num_instances = int(self.request.get('num_instances'))
instances = [
gce.Instance(
'%s-%d' % (user_info['demo_id'], i),
zone_name=gce_zone_name,
network_interfaces=(ext_net if i == 0 else None),
metadata=([{
'key': 'startup-script',
'value': user_data.STARTUP_SCRIPT % 'false'
}] if i == 0 else [{
'key': 'startup-script',
'value': user_data.STARTUP_SCRIPT % 'true'
}]),
service_accounts=[{
'email':
'default',
'scopes': ['https://www.googleapis.com/auth/compute']
}],
disk_mounts=[
gce.DiskMount(init_disk_name='%s-%d' %
(user_info['demo_id'], i),
boot=True)
],
can_ip_forward=(True if i == 0 else False),
tags=(['qs-proxy']
if i == 0 else ['qs-%s' % user_info['ldap']]))
for i in range(num_instances)
]
response = gce_appengine.GceAppEngine().run_gce_request(
self,
gce_project.bulk_insert,
'Error inserting instances: ',
resources=instances)
# Record objective in datastore so we can recover work in progress.
updateObjective(user_info['project_id'], num_instances)
if response:
self.response.headers['Content-Type'] = 'text/plain'
self.response.out.write('starting cluster')