def get_big_query_service():
global _bigquery_service
if _bigquery_service is None:
credentials = AppAssertionCredentials(scope='https://www.googleapis.com/auth/bigquery')
http = credentials.authorize(httplib2.Http(memcache))
_bigquery_service = build('bigquery', 'v2', http=http)
return _bigquery_service
def _get_content(self, path, method='POST', body=None):
scope = [
'https://www.googleapis.com/auth/genomics'
]
# The API Key is required when deployed to app engine
api_key = os.environ['API_KEY']
credentials = AppAssertionCredentials(scope=scope)
http = httplib2.Http(cache=memcache)
http = credentials.authorize(http)
try:
response, content = http.request(
uri="https://www.googleapis.com/genomics/v1beta/%s?key=%s"
% (path, api_key),
method=method, body=json.dumps(body) if body else None,
headers={'Content-Type': 'application/json; charset=UTF-8'})
except DeadlineExceededError:
raise ApiException('API fetch timed out')
try:
content = json.loads(content)
except ValueError:
logging.error("non-json api content %s" % content)
raise ApiException('The API returned invalid JSON')
if response.status >= 300:
logging.error("error api response %s" % response)
logging.error("error api content %s" % content)
if 'error' in content:
raise ApiException(content['error']['message'])
else:
raise ApiException('Something went wrong with the API call!')
return content
def __init__ (self, table, date = None):
if date is not None:
self.today = date
else:
self.today = datetime.today()
logger.info("date:" + str(self.today))
urlfetch.set_default_fetch_deadline(60) # Increase url fetch deadline for slow Google Analytics API calls
# Fetch the API key if we haven't pulled it from the keyfile already
global api_key
if api_key == "":
with open ("key.txt", "r") as keyfile:
api_key=keyfile.read().replace('\n', '')
# Set Query Range
self.startdate = self.today - query_range
self.extended_startdate = self.today - extended_query_range;
self.expdate = self.today - cache_time
# Setup analytics service authentication
credentials = AppAssertionCredentials(scope='https://www.googleapis.com/auth/analytics.readonly')
http_auth = credentials.authorize(Http(memcache))
self.service = build('analytics', 'v3', http=http_auth, developerKey=api_key)
self.table_id = table
def get_service():
global _service
if not _service:
credentials = AppAssertionCredentials(scope=_BIGQUERY_OAUTH_SCOPE)
http = credentials.authorize(httplib2.Http())
_service = discovery.build('bigquery', 'v2', http=http)
return _service
def __init__(self):
self._is_gae = os.getenv(
'SERVER_SOFTWARE', '').startswith('Google App Engine')
if self._is_gae:
credentials = AppAssertionCredentials(scope=self._SCOPE)
http = credentials.authorize(httplib2.Http())
else:
# parser = argparse.ArgumentParser(description=__doc__,
# formatter_class=argparse.RawDescriptionHelpFormatter,
# parents=[tools.argparser])
# flags = parser.parse_args([])
# self._FLOW = client.flow_from_clientsecrets(self._CLIENT_SECRETS,
# scope=[
# 'https://www.googleapis.com/auth/analytics.readonly'
# ],
# redirect_uri=
# message=tools.message_if_missing(self._CLIENT_SECRETS))
# storage = file.Storage('analytics.dat')
# credentials = storage.get()
# if credentials is None or credentials.invalid:
# credentials = tools.run_flow(self._FLOW, storage, flags)
# http = httplib2.Http()
http = auth_google_api.get_http_service()
self._service = build('analytics', 'v3', http=http)
def createDriveService(credential_path=None):
"""
Create drive service based on service account credentialed from api_key in secret_files/config.json. Utilizes 'https://www.googleapis.com/auth/drive' scope.
"""
# Determine if running in local appengine SDK dev_server environment.
# AppAssertionCredentials does not load in this context.
# Therefore, service must be mimicked locally using an installed client oauth flow.
if os.environ.get('SERVER_SOFTWARE') is None:
os.environ['SERVER_SOFTWARE'] = 'Development'
if 'Development' in os.environ.get('SERVER_SOFTWARE'):
# Acquire credentials stored from running `python secret.py` from the commandline.
if credential_path is None:
credential_path = secret_cred.CREDENTIALS
storage = Storage(credential_path)
credentials = storage.get()
http = httplib2.Http()
http = credentials.authorize(http)
return build('drive', 'v2', http=http)
else:
service_account_email = secret_cred.SERVICE_ACCOUNT
api_key = secret_cred.API_KEY
credentials = AppAssertionCredentials(scope=OAUTH_SCOPE)
http = httplib2.Http()
http = credentials.authorize(http)
return build('drive', 'v2', http=http, developerKey=api_key)
def setUp(self):
app_identity_stub = self.AppIdentityStubImpl()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service",
app_identity_stub)
self.scope = "http://www.googleapis.com/scope"
self.credentials = AppAssertionCredentials(self.scope)
def DeleteCalendar_func(calendarId):
credentials = AppAssertionCredentials(
'https://www.googleapis.com/auth/calendar')
http_auth = credentials.authorize(Http())
calendar = build('calendar', 'v3', http=http_auth)
service = discovery.build('calendar', 'v3', http=http_auth)
service.calendars().delete(calendarId=calendarId).execute()
def __init__(self):
self._is_gae = os.getenv(
'SERVER_SOFTWARE', '').startswith('Google App Engine')
if self._is_gae:
credentials = AppAssertionCredentials(scope=self._SCOPE)
http = credentials.authorize(httplib2.Http())
else:
http = auth_google_api.get_http_service()
self._service = build('bigquery', 'v2', http=http)
def test_get_access_token(self):
app_identity_stub = self.AppIdentityStubImpl()
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service", app_identity_stub)
apiproxy_stub_map.apiproxy.RegisterStub("memcache", memcache_stub.MemcacheServiceStub())
credentials = AppAssertionCredentials(["dummy_scope"])
token = credentials.get_access_token()
self.assertEqual("a_token_123", token.access_token)
self.assertEqual(None, token.expires_in)
def test_get_access_token_on_refresh(self):
app_identity_stub = self.AppIdentityStubImpl()
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service",
app_identity_stub)
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache', memcache_stub.MemcacheServiceStub())
scope = [
"http://www.googleapis.com/scope",
"http://www.googleapis.com/scope2"]
credentials = AppAssertionCredentials(scope)
http = httplib2.Http()
credentials.refresh(http)
self.assertEqual('a_token_123', credentials.access_token)
json = credentials.to_json()
credentials = Credentials.new_from_json(json)
self.assertEqual(
'http://www.googleapis.com/scope http://www.googleapis.com/scope2',
credentials.scope)
scope = "http://www.googleapis.com/scope http://www.googleapis.com/scope2"
credentials = AppAssertionCredentials(scope)
http = httplib2.Http()
credentials.refresh(http)
self.assertEqual('a_token_123', credentials.access_token)
self.assertEqual(
'http://www.googleapis.com/scope http://www.googleapis.com/scope2',
credentials.scope)
def TestQuery():
"""Runs a test query against the measurement-lab BigQuery database.
Returns:
(string) The query results formatted as an HTML page.
"""
# Certify BigQuery access credentials.
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/bigquery')
http = credentials.authorize(httplib2.Http(memcache))
service = build('bigquery', 'v2', http=http)
job_runner = service.jobs()
# Run a query against the BigQuery database.
logging.debug('Query: %s' % TEST_QUERY)
jobdata = {'configuration': {'query': {'query': TEST_QUERY}}}
insert = job_runner.insert(projectId=PROJECT_ID,
body=jobdata).execute()
logging.debug('Response: %s' % insert)
currentRow = 0
queryReply = job_runner.getQueryResults(
projectId=PROJECT_ID,
jobId=insert['jobReference']['jobId'],
startIndex=currentRow).execute()
results = queryReply
while 'rows' in queryReply and currentRow < queryReply['totalRows'] :
currentRow += len(queryReply['rows'])
queryReply = job_runner.getQueryResults(
projectId=PROJECT_ID,
jobId=queryReply['jobReference']['jobId'],
startIndex=currentRow).execute()
if 'schema' not in results or 'fields' not in results['schema']:
if 'schema' in queryReply and 'fields' in queryReply['schema']:
results['schema'] = queryReply['schema']
if 'rows' in queryReply:
results['rows'].extend(queryReply['rows'])
# Format the results as an HTML page.
body = '<h2>The Query</h2><pre>%s</pre>\n<hr>\n' % TEST_QUERY
tablerows = '<tr>'
for field in results['schema']['fields']:
tablerows += '<th>%s</th>' % field['name']
for row in results['rows']:
tablerows += '</tr><tr>'
for value in row['f']:
tablerows += '<td>%s</td>' % value['v']
tablerows += '</tr>'
body += '<table border=1>\n%s\n</table>\n' % tablerows
return '<!DOCTYPE html><html><body>%s</body></html>' % body
def initService():
api_key = 'YOUR GOOGLE API KEY'
credentials = AppAssertionCredentials(scope='https://www.googleapis.com/auth/calendar')
http = httplib2.Http(memcache)
http = credentials.authorize(http)
service = build("calendar", "v3", http=http, developerKey=api_key)
return service
def index(request):
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/bigquery')
http = credentials.authorize(httplib2.Http())
bigquery_service = build('bigquery', 'v2', http=http)
query_request = bigquery_service.jobs()
query_data = {'query':'SELECT * FROM [maveriks_assessment_sprint_1.test_new]'}
query_response = query_request.query(projectId=PROJECT_NUMBER,
body=query_data).execute()
return render_to_response('AssessingPie_toBeremoved/testmeveriks.html',{'result':query_response['rows']},context_instance = RequestContext(request))
def createDriveService():
"""Builds and returns a Drive service object authorized with the
application's service account.
Returns:
Drive service object.
"""
credentials = AppAssertionCredentials(scope='https://www.googleapis.com/auth/drive')
http = httplib2.Http()
http = credentials.authorize(http)
return build('drive', 'v1', http=http, developerKey='AIzaSyA9j1GWqNWUjpBA6DhRQzAQYeJQalfJSWs')
def test_get_access_token(self):
app_identity_stub = self.AppIdentityStubImpl()
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service",
app_identity_stub)
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache', memcache_stub.MemcacheServiceStub())
credentials = AppAssertionCredentials(['dummy_scope'])
token = credentials.get_access_token()
self.assertEqual('a_token_123', token)
def build_bq_client():
from googleapiclient.discovery import build
from oauth2client.appengine import AppAssertionCredentials
import httplib2
SCOPE = 'https://www.googleapis.com/auth/bigquery'
credentials = AppAssertionCredentials(scope=SCOPE)
http = credentials.authorize(httplib2.Http())
bigquery_service = build('bigquery', 'v2', http=http)
return bigquery_service
def build_bq_client():
from googleapiclient.discovery import build
from oauth2client.appengine import AppAssertionCredentials
import httplib2
SCOPE = 'https://www.googleapis.com/auth/bigquery'
credentials = AppAssertionCredentials(scope=SCOPE)
http = credentials.authorize(httplib2.Http())
bigquery_service = build('bigquery', 'v2', http=http)
return bigquery_service
def _decorated(self, *args, **kwargs):
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/devstorage.full_control')
http = credentials.authorize(httplib2.Http(memcache))
self.gcs_service = build('storage',
'v1',
http=http,
developerKey=settings.DEVELOPER_KEY)
self.gcs_service.BUCKET = settings.BUCKET
return function(self, *args, **kwargs)
def test_get_access_token(self):
app_identity_stub = self.AppIdentityStubImpl()
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service",
app_identity_stub)
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache', memcache_stub.MemcacheServiceStub())
credentials = AppAssertionCredentials(['dummy_scope'])
token = credentials.get_access_token()
self.assertEqual('a_token_123', token.access_token)
self.assertEqual(None, token.expires_in)
def post(self):
self.response.headers['Content-Type'] = 'text/plain'
jobid = self.request.get('jobid')
job = job_info.JobInfo.get_by_id(jobid)
if not job:
return
payload = urllib.urlencode({'q': 'MAX_TRACE_HANDLES=10'})
query_url = '%s/query?%s' % (_PERFORMANCE_INSIGHTS_URL, payload)
result = urlfetch.fetch(url=query_url,
payload=payload,
method=urlfetch.GET,
follow_redirects=False,
deadline=10)
logging.info(result.content)
taskid = str(uuid.uuid4())
traces = json.loads(result.content)
default_retry_params = gcs.RetryParams(initial_delay=0.2,
max_delay=5.0,
backoff_factor=2,
max_retry_period=15)
gcs_file = gcs.open(_DEFAULT_BUCKET.format(name=taskid),
'w',
content_type='text/plain',
options={},
retry_params=default_retry_params)
gcs_file.write(json.dumps(traces))
gcs_file.close()
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/compute')
http = credentials.authorize(httplib2.Http(memcache))
compute = build("compute", "v1", http=http)
startup_script = _STARTUP_SCRIPT.format(
revision=job.revision)
result = self._CreateGCEInstace(
compute, 'mr-%s' % jobid, startup_script)
logging.info('Call to instances().insert response:\n')
for k, v in sorted(result.iteritems()):
logging.info(' %s: %s' % (k, v))
job.status = 'COMPLETE'
job.put()
response = {'success': False}
self.response.out.write(json.dumps(response))
def search_command(self, message=None):
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/urlshortener')
http = credentials.authorize(httplib2.Http(memcache))
service = build("urlshortener", "v1", http=http)
credentials.refresh(http)
long_url = message.arg
url = "http://gymkhana.iitb.ac.in/~ugacademics/wiki/index.php?search=" + long_url + "&go=Go&title=Special%3ASearch"
shortened = service.url().insert(body={"longUrl": url}).execute()
shortened1 = service.url().list().execute()
message.reply(str(shortened1["items"][0]['id']))
def run(self, csv_output):
credentials = AppAssertionCredentials(scope=SCOPE)
http = credentials.authorize(httplib2.Http())
bigquery_service = build("bigquery", "v2", http=http)
jobs = bigquery_service.jobs()
table_name = 'datastore_data_%s' % datetime.datetime.utcnow().strftime(
'%m%d%Y_%H%M%S')
files = [str(f.replace('/gs/', 'gs://')) for f in csv_output]
result = jobs.insert(projectId=PROJECT_ID,
body=build_job_data(table_name, files))
result.execute()
def test_get_access_token_on_refresh(self):
app_identity_stub = self.AppIdentityStubImpl()
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service",
app_identity_stub)
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache', memcache_stub.MemcacheServiceStub())
scope = "http://www.googleapis.com/scope"
credentials = AppAssertionCredentials(scope)
http = httplib2.Http()
credentials.refresh(http)
self.assertEqual('a_token_123', credentials.access_token)
def createDriveService1():
"""Builds and returns a Drive service object authorized with the
application's service account.
Returns:
Drive service object.
"""
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/drive')
http = httplib2.Http()
http = credentials.authorize(http)
return build('drive', 'v2', http=http, developerKey=API_KEY)
def run(self, csv_output):
credentials = AppAssertionCredentials(scope=SCOPE)
http = credentials.authorize(httplib2.Http())
bigquery_service = build("bigquery", "v2", http=http)
jobs = bigquery_service.jobs()
table_name = 'datastore_data_%s' % datetime.datetime.utcnow().strftime(
'%m%d%Y_%H%M%S')
files = [str(f.replace('/gs/', 'gs://')) for f in csv_output]
result = jobs.insert(projectId=PROJECT_ID,
body=build_job_data(table_name,files))
result.execute()
def test_custom_service_account(self):
scope = "http://www.googleapis.com/scope"
account_id = "*****@*****.**"
with mock.patch.object(
app_identity, "get_access_token", return_value=("a_token_456", None), autospec=True
) as get_access_token:
credentials = AppAssertionCredentials(scope, service_account_id=account_id)
http = httplib2.Http()
credentials.refresh(http)
self.assertEqual("a_token_456", credentials.access_token)
self.assertEqual(scope, credentials.scope)
get_access_token.assert_called_once_with([scope], service_account_id=account_id)
def sendToBQ(project_id, dataset_id, table_id, csv_name, schema):
source_csv="gs://" + csv_name
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/bigquery')
http = credentials.authorize(httplib2.Http(memcache))
service = build("bigquery", "v2", http=http)
loadTable(service, project_id, dataset_id, table_id, source_csv, schema)
###############################################################################
def search_command(self,message=None):
credentials = AppAssertionCredentials(scope='https://www.googleapis.com/auth/urlshortener')
http = credentials.authorize(httplib2.Http(memcache))
service = build("urlshortener", "v1", http=http)
credentials.refresh(http)
long_url = message.arg
url="http://gymkhana.iitb.ac.in/~ugacademics/wiki/index.php?search="+long_url+"&go=Go&title=Special%3ASearch"
shortened = service.url().insert(body={"longUrl": url}).execute()
shortened1 = service.url().list().execute()
message.reply( str(shortened1["items"][0]['id']))
def test_raise_correct_type_of_exception(self):
app_identity_stub = self.ErroringAppIdentityStubImpl()
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service", app_identity_stub)
apiproxy_stub_map.apiproxy.RegisterStub("memcache", memcache_stub.MemcacheServiceStub())
scope = "http://www.googleapis.com/scope"
try:
credentials = AppAssertionCredentials(scope)
http = httplib2.Http()
credentials.refresh(http)
self.fail("Should have raised an AccessTokenRefreshError")
except AccessTokenRefreshError:
pass
def createDriveService():
"""
Builds and returns a Drive service object authorized with the
application's service account.
Returns:
Drive service object.
"""
from oauth2client.appengine import AppAssertionCredentials
from apiclient.discovery import build
credentials = AppAssertionCredentials(scope="https://www.googleapis.com/auth/drive")
http = httplib2.Http()
http = credentials.authorize(http)
return build("drive", "v2", http=http, developerKey=API_KEY)
def sendToBQ(project_id, dataset_id, table_id, csv_name, schema):
source_csv = "gs://" + csv_name
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/bigquery')
http = credentials.authorize(httplib2.Http(memcache))
service = build("bigquery", "v2", http=http)
loadTable(service, project_id, dataset_id, table_id, source_csv, schema)
###############################################################################
def index(request):
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/bigquery')
http = credentials.authorize(httplib2.Http())
bigquery_service = build('bigquery', 'v2', http=http)
query_request = bigquery_service.jobs()
query_data = {
'query': 'SELECT * FROM [maveriks_assessment_sprint_1.test_new]'
}
query_response = query_request.query(projectId=PROJECT_NUMBER,
body=query_data).execute()
return render_to_response('AssessingPie_toBeremoved/testmeveriks.html',
{'result': query_response['rows']},
context_instance=RequestContext(request))
def createDriveService():
"""Builds and returns a Drive service object authorized with the
application's service account.
Returns:
Drive service object.
"""
credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/drive')
http = httplib2.Http()
http = credentials.authorize(http)
return build('drive',
'v1',
http=http,
developerKey='AIzaSyA9j1GWqNWUjpBA6DhRQzAQYeJQalfJSWs')
def test_custom_service_account(self):
scope = "http://www.googleapis.com/scope"
account_id = "*****@*****.**"
m = mox.Mox()
m.StubOutWithMock(app_identity, "get_access_token")
app_identity.get_access_token([scope], service_account_id=account_id).AndReturn(("a_token_456", None))
m.ReplayAll()
credentials = AppAssertionCredentials(scope, service_account_id=account_id)
http = httplib2.Http()
credentials.refresh(http)
m.VerifyAll()
m.UnsetStubs()
self.assertEqual("a_token_456", credentials.access_token)
self.assertEqual(scope, credentials.scope)
def test_custom_service_account(self):
scope = "http://www.googleapis.com/scope"
account_id = "*****@*****.**"
with mock.patch.object(app_identity, 'get_access_token',
return_value=('a_token_456', None),
autospec=True) as get_access_token:
credentials = AppAssertionCredentials(
scope, service_account_id=account_id)
http = httplib2.Http()
credentials.refresh(http)
self.assertEqual('a_token_456', credentials.access_token)
self.assertEqual(scope, credentials.scope)
get_access_token.assert_called_once_with(
[scope], service_account_id=account_id)
def build_service(service_name, api_version, scopes):
"""Get an authorized service account http connection"""
if DEBUG:
if config.SERVICE_EMAIL and config.SERVICE_KEY_PATH and os.path.exists(
config.SERVICE_KEY_PATH):
from oauth2client.client import SignedJwtAssertionCredentials
# must extract key first since pycrypto doesn't support p12 files
# openssl pkcs12 -passin pass:notasecret -in privatekey.p12 -nocerts -passout pass:notasecret -out key.pem
# openssl pkcs8 -nocrypt -in key.pem -passin pass:notasecret -topk8 -out privatekey.pem
# rm key.pem
key_str = open(config.SERVICE_KEY_PATH).read()
credentials = SignedJwtAssertionCredentials(
config.SERVICE_EMAIL, key_str, scopes)
http = credentials.authorize(httplib2.Http(api.memcache))
return build(service_name, api_version, http=http)
else:
logging.warn(
"Please create a service account and download your key add to appengine_config.py."
)
raise AppError(
"Service '{}' not availble from localhost without a service account set up and added to appengine_config.py."
.format(service_name))
credentials = AppAssertionCredentials(scope=scopes)
http = credentials.authorize(httplib2.Http(api.memcache))
return build(service_name, api_version, http=http)
def setUp(self):
app_identity_stub = self.AppIdentityStubImpl()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service",
app_identity_stub)
self.scope = "http://www.googleapis.com/scope"
self.credentials = AppAssertionCredentials(self.scope)
def test_raise_correct_type_of_exception(self):
app_identity_stub = self.ErroringAppIdentityStubImpl()
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
apiproxy_stub_map.apiproxy.RegisterStub('app_identity_service',
app_identity_stub)
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache', memcache_stub.MemcacheServiceStub())
scope = 'http://www.googleapis.com/scope'
try:
credentials = AppAssertionCredentials(scope)
http = httplib2.Http()
credentials.refresh(http)
self.fail('Should have raised an AccessTokenRefreshError')
except AccessTokenRefreshError:
pass
def test_custom_service_account(self):
scope = "http://www.googleapis.com/scope"
account_id = "*****@*****.**"
m = mox.Mox()
m.StubOutWithMock(app_identity, 'get_access_token')
app_identity.get_access_token(
[scope], service_account_id=account_id).AndReturn(('a_token_456', None))
m.ReplayAll()
credentials = AppAssertionCredentials(scope, service_account_id=account_id)
http = httplib2.Http()
credentials.refresh(http)
m.VerifyAll()
m.UnsetStubs()
self.assertEqual('a_token_456', credentials.access_token)
self.assertEqual(scope, credentials.scope)
def reallyinitEarthEngineService():
util.positional_parameters_enforcement = util.POSITIONAL_IGNORE # avoid the WARNING [util.py:129] new_request() takes at most 1 positional argument (4 given)
OAuthInfo.SCOPE += ' https://www.googleapis.com/auth/fusiontables.readonly'
print OAuthInfo.SCOPE
try:
if os.environ['SERVER_SOFTWARE'].startswith('Development'):
logging.info("Initialising Earth Engine authenticated connection from devserver")
try:
acct = os.environ['MY_SERVICE_ACCOUNT']
key = os.environ['MY_PRIVATE_KEY_FILE']
except KeyError:
logging.warning("Environment variable MY_SERVICE_ACCOUNT and MY_PRIVATE_KEY_FILE not set. Using settings.py")
logging.debug (os.environ)
acct = settings.MY_LOCAL_SERVICE_ACCOUNT
key = settings.MY_LOCAL_PRIVATE_KEY_FILE
EE_CREDENTIALS = ee.ServiceAccountCredentials(acct, key)
else:
logging.info("Initialising Earth Engine authenticated connection from App Engine")
EE_CREDENTIALS = AppAssertionCredentials(OAuthInfo.SCOPE)
ee.Initialize(EE_CREDENTIALS)
#print EE_CREDENTIALS
# permit long wait for response from earth engine
urlfetch.set_default_fetch_deadline(60)
return EE_CREDENTIALS
except Exception, e:
logging.error("Failed to connect to Earth Engine Google API. Exception: %s", e)
return False
def __init__(self):
is_running_locally = os.environ.get('APPLICATION_ID', '').startswith('dev')
self.bigquery_table = constants.BIGQUERY_TABLE
if is_running_locally:
self.bigquery_dataset = constants.BIGQUERY_DATASET_LOCAL
else:
self.bigquery_dataset = constants.BIGQUERY_DATASET_PROD
# Attempt to initialize a connection to BigQuery.
self.bigquery = None
if is_running_locally:
# Local instances require a 'secrets.json' file.
secrets_path = os.path.join(os.path.dirname(__file__), 'secrets.json')
if (os.path.exists(secrets_path)):
with open(secrets_path) as f:
auth = json.load(f)
credentials = oauth2client.client.SignedJwtAssertionCredentials(
auth['client_email'], auth['private_key'],
constants.BIGQUERY_URL)
self.bigquery = self._build_bigquery_object(credentials)
else:
logging.warning(
'No credentials provided for BigQuery. Logging disabled.')
else:
# Use the GAE service credentials.
credentials = AppAssertionCredentials(
scope=constants.BIGQUERY_URL)
self.bigquery = self._build_bigquery_object(credentials)
def post(self):
credentials = AppAssertionCredentials(scope='https://www.googleapis.com/auth/devstorage.full_control')
http = credentials.authorize(httplib2.Http(memcache))
gcs_service = build('storage', 'v1', http=http, developerKey=settings.DEVELOPER_KEY)
object_name = self.request.get('object_name')
bucket_name = self.request.get('bucket_name')
logging.info('delete tasks qeueu: delete gcs: %s/%s' % (bucket_name, object_name))
try:
req = gcs_service.objects().delete(bucket=bucket_name, object=object_name.encode('utf8'))
resp = req.execute()
except HttpError, error:
logging.info(error)
def get(self):
if not super(CalendarStatus, self).get():
return
credentials = AppAssertionCredentials(
'https://www.googleapis.com/auth/calendar')
http_auth = credentials.authorize(Http())
calendar = build('calendar', 'v3', http=http_auth)
service = discovery.build('calendar', 'v3', http=http_auth)
template = JINJA_ENVIRONMENT.\
get_template('templates/calendar_status.html')
calendars = service.calendarList().list().execute()
self.render_data['calendars'] = []
item = {}
for calendar in calendars['items']:
self.render_data['calendars'].append(calendar)
self.response.write(template.render(self.render_data))
def CreateEvent_func(calendar_id, summary, start_time, end_time, description):
credentials = AppAssertionCredentials(
'https://www.googleapis.com/auth/calendar')
http_auth = credentials.authorize(Http())
calendar = build('calendar', 'v3', http=http_auth)
service = discovery.build('calendar', 'v3', http=http_auth) #это часть для auth, нужна везде
event = {
'summary': summary,
'start': {
'dateTime': start_time + ':00+03:00' #html возвращает в формате немного другом
},
'end': {
'dateTime': end_time + ':00+03:00'
},
'description': description
}
event = service.events().insert(calendarId=calendar_id, body=event).execute()
return event['id']
def google_api_access(scope):
credentials = AppAssertionCredentials(scope=scope)
http = credentials.authorize(httplib2.Http(memcache, 60 * 9))
def token_refresh_decorator(func):
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except client.AccessTokenRefreshError:
logging.warn(
"The credentials have been revoked or expired, refreshing the token",
exc_info=True)
credentials.authorize(http)
return func(*args, **kwargs)
return wrapper
return token_refresh_decorator, http
def _get_http_auth():
"""Get an authorized `httplib2.Http` object.
Detect if we are running on appengine or locally and authorize
accordingly. (If locally, a json file with service account credentials is
searched for in the project root directory.) The service has read/write
access.
"""
if _IS_APPENGINE:
credentials = AppAssertionCredentials(scope=_SCOPES)
else:
# Look for credentials in a file named "gcs_credentials.json"
# in the project's root directory.
cred_dir = os.path.abspath(os.path.join(*(_LOCATION, os.path.pardir)))
with open(os.path.join(cred_dir, 'gcs_credentials.json')) as _infile:
creds = json.load(_infile)
credentials = SignedJwtAssertionCredentials(creds['client_email'],
creds['private_key'],
scope=_SCOPES)
return credentials.authorize(httplib2.Http())
def _get_content(self, path, method='POST', body=None):
scope = [
'https://www.googleapis.com/auth/genomics',
'https://www.googleapis.com/auth/devstorage.read_write'
]
api_key = os.environ['API_KEY']
credentials = AppAssertionCredentials(scope=scope)
http = httplib2.Http()
http = credentials.authorize(http)
try:
response, content = http.request(
uri="https://www.googleapis.com/genomics/v1beta/%s?key=%s" %
(path, api_key),
method=method,
body=json.dumps(body) if body else None,
headers={'Content-Type': 'application/json; charset=UTF-8'})
except DeadlineExceededError:
raise ApiException('API fetch timed out')
# Log results to debug
logging.debug("Response:")
logging.debug(response)
logging.debug("Content:")
logging.debug(content)
# Parse the content as json.
content = json.loads(content)
if response.status == 404:
raise ApiException('API not found')
elif response.status == 400:
raise ApiException('API request malformed')
elif response.status != 200:
if 'error' in content:
logging.error(
"Error Code: %s Message: %s" %
(content['error']['code'], content['error']['message']))
raise ApiException("Something went wrong with the API call. "
"Please check the logs for more details.")
return content
def _get_content(self, path, method='POST', body=None):
# Genomics requires both the genomics scope and devstorage
scope = [
'https://www.googleapis.com/auth/genomics',
'https://www.googleapis.com/auth/devstorage.read_write'
]
# The API Key may or may not be required.
api_key = os.environ['API_KEY']
credentials = AppAssertionCredentials(scope=scope)
http = httplib2.Http(cache=memcache)
http = credentials.authorize(http)
try:
response, content = http.request(
uri="https://www.googleapis.com/genomics/v1beta/%s?key=%s"
% (path, api_key),
method=method, body=json.dumps(body) if body else None,
headers={'Content-Type': 'application/json; charset=UTF-8'})
except DeadlineExceededError:
raise ApiException('API fetch timed out')
# Log results to debug
logging.debug("Response:")
logging.debug(response)
logging.debug("Content:")
logging.debug(content)
# Parse the content as json.
content = json.loads(content)
if response.status == 404:
raise ApiException('API not found')
elif response.status == 400:
raise ApiException('API request malformed')
elif response.status != 200:
if 'error' in content:
logging.error("Error Code: %s Message: %s",
content['error']['code'], content['error']['message'])
raise ApiException("Something went wrong with the API call. "
"Please check the logs for more details.")
return content
def CreateCalendar_func(group):
credentials = AppAssertionCredentials(
'https://www.googleapis.com/auth/calendar')
http_auth = credentials.authorize(Http())
calendar = build('calendar', 'v3', http=http_auth)
service = discovery.build('calendar', 'v3', http=http_auth)
calendar = {
'summary': group,
'timeZone': 'Europe/Moscow'
}
created_calendar = service.calendars().insert(body=calendar).execute()
cal_id = created_calendar['id']
rule = {
'scope': {
'type': 'default'
},
'role': 'reader'
}
created_rule = service.acl().insert(calendarId=cal_id, body=rule).execute()
return cal_id
def get_worksheet(fname, sheet):
credentials = AppAssertionCredentials(scope=SCOPE)
gc = gspread.authorize(credentials)
try:
wks = gc.open(fname).worksheet(sheet)
except Exception as err:
logging.error(
"[get_datasehet] oops, failure getting fname=%s, sheet=%s" %
(fname, sheet))
raise
return wks
class AppAssertionCredentialsBQClient(_BigQueryClient):
"""BigQuery client implemented with App Assertion Credentials.
Use this BigQuery client if the application credentials should be used for
BigQuery transactions.
"""
def _Connect(self):
# Certify BigQuery access credentials.
self._credentials = AppAssertionCredentials(
scope='https://www.googleapis.com/auth/bigquery')
self._http = self._credentials.authorize(httplib2.Http(memcache))
self._service = build('bigquery', 'v2', http=self._http)
def read_edm_file(self, edm_object_name):
credentials = AppAssertionCredentials(scope='https://www.googleapis.com/auth/devstorage.full_control')
http = credentials.authorize(httplib2.Http(memcache))
gcs_service = build('storage', 'v1', http=http, developerKey=settings.DEVELOPER_KEY)
data = memcache.get(edm_object_name)
if data is not None:
return data
else:
fh = io.BytesIO()
request = gcs_service.objects().get_media(bucket=settings.BUCKET, object=edm_object_name.encode('utf8'))
downloader = MediaIoBaseDownload(fh, request, chunksize=settings.CHUNKSIZE)
done = False
while not done:
status, done = downloader.next_chunk()
if status:
logging.info('Download %d%%.' % int(status.progress() * 100))
logging.info('Download %s Complete!' % edm_object_name)
data = fh.getvalue()
memcache.add(edm_object_name, data, settings.EDM_CONTENT_MEMCACHE_TIME)
return data
class TestAppAssertionCredentials(unittest.TestCase):
account_name = "*****@*****.**"
signature = "signature"
class AppIdentityStubImpl(apiproxy_stub.APIProxyStub):
def __init__(self):
super(TestAppAssertionCredentials.AppIdentityStubImpl, self).__init__(
'app_identity_service')
def _Dynamic_GetServiceAccountName(self, request, response):
return response.set_service_account_name(
TestAppAssertionCredentials.account_name)
def _Dynamic_SignForApp(self, request, response):
return response.set_signature_bytes(
TestAppAssertionCredentials.signature)
def setUp(self):
app_identity_stub = self.AppIdentityStubImpl()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service",
app_identity_stub)
self.scope = "http://www.googleapis.com/scope"
self.credentials = AppAssertionCredentials(self.scope)
def test_assertion(self):
assertion = self.credentials._generate_assertion()
parts = assertion.split(".")
self.assertTrue(len(parts) == 3)
header, body, signature = [base64.b64decode(part) for part in parts]
header_dict = simplejson.loads(header)
self.assertEqual(header_dict['typ'], 'JWT')
self.assertEqual(header_dict['alg'], 'RS256')
body_dict = simplejson.loads(body)
self.assertEqual(body_dict['aud'],
'https://accounts.google.com/o/oauth2/token')
self.assertEqual(body_dict['scope'], self.scope)
self.assertEqual(body_dict['iss'], self.account_name)
issuedAt = body_dict['iat']
self.assertTrue(issuedAt > 0)
self.assertEqual(body_dict['exp'], issuedAt + 3600)
self.assertEqual(signature, self.signature)
class TestAppAssertionCredentials(unittest.TestCase):
account_name = "*****@*****.**"
signature = "signature"
class AppIdentityStubImpl(apiproxy_stub.APIProxyStub):
def __init__(self):
super(TestAppAssertionCredentials.AppIdentityStubImpl, self).__init__(
'app_identity_service')
def _Dynamic_GetServiceAccountName(self, request, response):
return response.set_service_account_name(
TestAppAssertionCredentials.account_name)
def _Dynamic_SignForApp(self, request, response):
return response.set_signature_bytes(
TestAppAssertionCredentials.signature)
def setUp(self):
app_identity_stub = self.AppIdentityStubImpl()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service",
app_identity_stub)
self.scope = "http://www.googleapis.com/scope"
self.credentials = AppAssertionCredentials(self.scope)
def test_assertion(self):
assertion = self.credentials._generate_assertion()
parts = assertion.split(".")
self.assertTrue(len(parts) == 3)
header, body, signature = [base64.b64decode(part) for part in parts]
header_dict = simplejson.loads(header)
self.assertEqual(header_dict['typ'], 'JWT')
self.assertEqual(header_dict['alg'], 'RS256')
body_dict = simplejson.loads(body)
self.assertEqual(body_dict['aud'],
'https://accounts.google.com/o/oauth2/token')
self.assertEqual(body_dict['scope'], self.scope)
self.assertEqual(body_dict['iss'], self.account_name)
issuedAt = body_dict['iat']
self.assertTrue(issuedAt > 0)
self.assertEqual(body_dict['exp'], issuedAt + 3600)
self.assertEqual(signature, self.signature)
def images():
if not request.args or not request.args['key'] or request.args['key'] != IOS_API_KEY:
abort(401)
if not request.args or not request.args['name'] or not request.args['length']:
logging.error("name: %s", request.args.get('name'))
logging.error("length: %s", request.args.get('length'))
abort(401)
logging.info(request.args)
name = request.args['name']
length = request.args['length']
if request.method == 'GET':
credentials = AppAssertionCredentials(scope='https://www.googleapis.com/auth/devstorage.read_write')
http_auth = credentials.authorize(httplib2.Http())
logging.info("successfully created http_auth object")
try:
url = "https://www.googleapis.com/upload/storage/v1/b/" + _BUCKET_NAME + "/o?uploadType=resumable&name=" + name
resp, content = http_auth.request(url, method="POST", headers={'Content-Length':'0',
'Content-Type':'application/json; charset=UTF-8',
'X-Upload-Content-Type':'image/jpeg',
'X-Upload-Content-Length':length})
return jsonify(resp)
except HttpError, e:
logging.error("response status: %s", e.resp.status)
logging.error("response content: %s", e.content)
def test_get_access_token_on_refresh(self):
app_identity_stub = self.AppIdentityStubImpl()
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
apiproxy_stub_map.apiproxy.RegisterStub("app_identity_service", app_identity_stub)
apiproxy_stub_map.apiproxy.RegisterStub("memcache", memcache_stub.MemcacheServiceStub())
scope = ["http://www.googleapis.com/scope", "http://www.googleapis.com/scope2"]
credentials = AppAssertionCredentials(scope)
http = httplib2.Http()
credentials.refresh(http)
self.assertEqual("a_token_123", credentials.access_token)
json = credentials.to_json()
credentials = Credentials.new_from_json(json)
self.assertEqual("http://www.googleapis.com/scope http://www.googleapis.com/scope2", credentials.scope)
scope = "http://www.googleapis.com/scope http://www.googleapis.com/scope2"
credentials = AppAssertionCredentials(scope)
http = httplib2.Http()
credentials.refresh(http)
self.assertEqual("a_token_123", credentials.access_token)
self.assertEqual("http://www.googleapis.com/scope http://www.googleapis.com/scope2", credentials.scope)
from mapreduce import base_handler
from oauth2client.appengine import AppAssertionCredentials
from apiclient.http import BatchHttpRequest
import mapreduce.third_party.pipeline as pipeline
import mapreduce.third_party.pipeline.common as pipeline_common
import logging
logger = logging.getLogger('pipeline')
credentials = AppAssertionCredentials(
scope=[
'https://www.googleapis.com/auth/prediction',
'https://www.googleapis.com/auth/devstorage.full_control'
]
)
http = credentials.authorize(httplib2.Http(memcache))
service = build('bigquery', 'v2', http=http)
batch = BatchHttpRequest()
class Check(base_handler.PipelineBase):
def run(self, projectId, modelId, delays=10):
result = service.trainedmodels().get(
project=projectId,
id=modelId
).execute()
if result['trainingStatus'] == "RUNNING":
delay = yield pipeline_common.Delay(seconds=delays)
with pipeline.After(delay):
yield PredictCheck(projectId, modelId, delays)
