def get_service(api='gmail',
version='v1',
auth='service',
scopes=None,
uri_file=None):
global DISCOVERY_CACHE
key = api + version + auth + str(threading.current_thread().ident)
if key not in DISCOVERY_CACHE:
credentials = get_credentials(auth)
if uri_file:
uri_file = uri_file.strip()
if uri_file.startswith('{'):
DISCOVERY_CACHE[key] = discovery.build_from_document(
uri_file, credentials=credentials)
else:
with open(uri_file, 'r') as cache_file:
DISCOVERY_CACHE[key] = discovery.build_from_document(
cache_file.read(), credentials=credentials)
else:
DISCOVERY_CACHE[key] = discovery.build(api,
version,
credentials=credentials)
return DISCOVERY_CACHE[key]
def get_service(config,
api='gmail',
version='v1',
auth='service',
scopes=None,
headers=None,
key=None,
uri_file=None):
global DISCOVERY_CACHE
class HttpRequestCustom(HttpRequest):
def __init__(self, *args, **kwargs):
if headers:
kwargs['headers'].update(headers)
super(HttpRequestCustom, self).__init__(*args, **kwargs)
if not key:
key = config.recipe['setup'].get(key, '')
cache_key = api + version + auth + str(key) + str(
threading.current_thread().ident)
if cache_key not in DISCOVERY_CACHE:
credentials = get_credentials(config, auth)
if uri_file:
uri_file = uri_file.strip()
if uri_file.startswith('{'):
DISCOVERY_CACHE[cache_key] = discovery.build_from_document(
uri_file,
credentials=credentials,
developerKey=key,
requestBuilder=HttpRequestCustom)
else:
with open(uri_file, 'r') as cache_file:
DISCOVERY_CACHE[cache_key] = discovery.build_from_document(
cache_file.read(),
credentials=credentials,
developerKey=key,
requestBuilder=HttpRequestCustom)
else:
try:
DISCOVERY_CACHE[cache_key] = discovery.build(
api,
version,
credentials=credentials,
developerKey=key,
requestBuilder=HttpRequestCustom,
static_discovery=False)
# PATCH: static_discovery not present in google-api-python-client < 2, default version in colab
# ALTERNATE WORKAROUND: pip install update google-api-python-client==2.3 --no-deps --force-reinstall
except TypeError:
DISCOVERY_CACHE[cache_key] = discovery.build(
api,
version,
credentials=credentials,
developerKey=key,
requestBuilder=HttpRequestCustom)
return DISCOVERY_CACHE[cache_key]
def __init__(self, project_id, language, flags):
self.flags = flags
client_secrets = os.path.join(os.path.dirname(__file__),
CLIENT_SECRETS_FILE)
flow = client.flow_from_clientsecrets(
client_secrets,
SCOPES,
message=tools.message_if_missing(client_secrets))
storage = file.Storage(CREDENTIALS_STORAGE)
credentials = storage.get()
if credentials is None or credentials.invalid:
credential_flags = argparse.Namespace(
noauth_local_webserver=True,
logging_level=logging.getLevelName(
logging.getLogger().getEffectiveLevel()))
credentials = tools.run_flow(flow, storage, flags=credential_flags)
http = credentials.authorize(http=build_http())
self.gmb_services = {}
for service_name in FEDERATED_SERVICES:
with open(f"{service_name}{DISCOVERY_FILE_SUFFIX}"
) as discovery_file:
self.gmb_services[
service_name] = discovery.build_from_document(
discovery_file.read(),
base="https://www.googleapis.com/",
http=http)
with open(GMB_DISCOVERY_FILE) as gmb_discovery_file:
self.gmb_service = discovery.build_from_document(
gmb_discovery_file.read(),
base="https://www.googleapis.com/",
http=http)
self.project_id = project_id
self.dataset_exists = False
self.existing_tables = {}
self.language = language
with open(SCHEMAS_FILE) as schemas_file:
self.schemas = json.load(schemas_file)
self.bq_service = discovery.build("bigquery", "v2", http=http)
self.nlp_service = discovery.build("language", "v1", http=http)
if flags["topic_clustering"]:
self.topic_clustering = TopicClustering()
def get_service(api='gmail',
version='v1',
auth='service',
scopes=None,
headers=None,
key=None,
uri_file=None):
global DISCOVERY_CACHE
class HttpRequestCustom(HttpRequest):
def __init__(self, *args, **kwargs):
if headers:
kwargs['headers'].update(headers)
super(HttpRequestCustom, self).__init__(*args, **kwargs)
if not key:
key = project.recipe['setup'].get(key, '')
cache_key = api + version + auth + key + str(threading.current_thread().ident)
if cache_key not in DISCOVERY_CACHE:
credentials = get_credentials(auth)
if uri_file:
uri_file = uri_file.strip()
if uri_file.startswith('{'):
DISCOVERY_CACHE[cache_key] = discovery.build_from_document(
uri_file,
credentials=credentials,
developerKey=key,
requestBuilder=HttpRequestCustom
)
else:
with open(uri_file, 'r') as cache_file:
DISCOVERY_CACHE[cache_key] = discovery.build_from_document(
cache_file.read(),
credentials=credentials,
developerKey=key,
requestBuilder=HttpRequestCustom
)
else:
DISCOVERY_CACHE[cache_key] = discovery.build(
api,
version,
credentials=credentials,
developerKey=key,
requestBuilder=HttpRequestCustom,
static_discovery=False
)
return DISCOVERY_CACHE[cache_key]
def __init__(self, oauth, http_client=None):
self.oauth = oauth
http_client = http_client or httplib2.Http(cache="/tmp/.pygsheets_cache", timeout=10)
http = self.oauth.authorize(http_client)
data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
with open(os.path.join(data_path, "sheets_discovery.json")) as jd:
self.service = discovery.build_from_document(jload(jd), http=http)
with open(os.path.join(data_path, "drive_discovery.json")) as jd:
self.driveService = discovery.build_from_document(jload(jd), http=http)
self._spreadsheeets = []
self.batch_requests = dict()
self.retries = 1
self._fetch_sheets()
def run(self):
"""
1. run label detection API
2. write the result into the :py:meth:`*.label.json` target on Storage
"""
credentials = GoogleCredentials.get_application_default()
with open(API_DISCOVERY_FILE, 'r') as f:
doc = f.read()
video_service = discovery.build_from_document(doc,
credentials=credentials)
with open(OPERATIONS_DISCOVERY_FILE, 'r') as f:
op_doc = f.read()
op_service = discovery.build_from_document(op_doc,
credentials=credentials)
video_service_request = video_service.videos().annotate(
body={
'inputUri': self.gs_path_video,
'features': ['LABEL_DETECTION']
})
response = video_service_request.execute()
name = response['name']
op_service_request = op_service.operations().get(name=name)
response = op_service_request.execute()
op_start_time = str(
response['metadata']['annotationProgress'][0]['startTime'])
print('Operation {} started: {}'.format(name, op_start_time))
while True:
response = op_service_request.execute()
time.sleep(30)
if 'done' in response and response['done'] is True:
break
else:
print('Operation processing ...')
print('The video has been successfully processed.')
lblData = response['response']['annotationResults'][0][
'labelAnnotations']
# output data
f = self.output().open('w')
json.dump(lblData, f)
f.close()
def test_can_build_from_local_deserialized_document(self):
discovery = open(datafile('plus.json')).read()
discovery = json.loads(discovery)
plus = build_from_document(discovery,
base="https://www.googleapis.com/")
self.assertTrue(plus is not None)
self.assertTrue(hasattr(plus, 'activities'))
def main(photo_file):
'''Run a label request on a single image'''
credentials = GoogleCredentials.get_application_default()
with open(API_DISCOVERY_FILE, 'r') as f:
doc = f.read()
service = discovery.build_from_document(doc,
credentials=credentials,
http=httplib2.Http())
with open(photo_file, 'rb') as image:
image_content = base64.b64encode(image.read())
service_request = service.images().annotate(
body={
'requests': [{
'image': {
'content': image_content
},
'features': [{
'type': 'LABEL_DETECTION',
'maxResults': 5,
}]
}]
})
response = service_request.execute()
# print(response)
labels = map(lambda x: x['description'],
response['responses'][0]['labelAnnotations'])
sys.stdout.write("There are " + ", ".join(labels))
def service(self):
"""Setup the service object.
"""
self._check_access_token()
if self._service is None:
build_content = _get_build_content(
self._api_name,
self._api_version,
discoveryServiceUrl=self._url,
cache_discovery=True,
cache=DiscoveryCache(),
proxy_info=self.proxy_info,
)
self._service = build_from_document(build_content,
credentials=self.creds)
if self.no_verify:
self._service._http.http.disable_ssl_certificate_validation = True
if self.proxy_info:
self._service._http.http.proxy_info = httplib2.ProxyInfo(
httplib2.socks.PROXY_TYPE_HTTP_NO_TUNNEL,
self.proxy_info["host"],
self.proxy_info["port"],
proxy_user=self.proxy_info["user"],
proxy_pass=self.proxy_info["password"],
)
return self._service
def __init__(self, api_key, image_path):
with open(API_DISCOVERY_FILE, "r") as f:
doc = f.read()
self.service = discovery.build_from_document(doc, developerKey=api_key, http=httplib2.Http())
self.types = ["FACE_DETECTION", "LABEL_DETECTION", "LANDMARK_DETECTION", "LOGO_DETECTION", "TEXT_DETECTION"]
print("use image %s" % image_path)
self.image = image_path # cv2.imread(image_path)
def main():
photo_file = 'target.jpg'
with picamera.PiCamera() as camera:
camera.resolution = (640, 480)
camera.start_preview()
time.sleep(2) # Adjust condition
camera.capture(photo_file)
camera.stop_preview()
credentials = GoogleCredentials.get_application_default()
with open(API_DISCOVERY_FILE, 'r') as f:
doc = f.read()
service = discovery.build_from_document(
doc, credentials=credentials, http=httplib2.Http())
with open(photo_file, 'rb') as image:
image_content = base64.b64encode(image.read())
service_request = service.images().annotate(
body={
'requests': [{
'image': {
'content': image_content
},
'features': [{
'type': 'LABEL_DETECTION',
'maxResults': 2,
}]
}]
})
response = service_request.execute()
# print(response)
labels = map(lambda x: x['description'], response[
'responses'][0]['labelAnnotations'])
sys.stdout.write("There are " + ", ".join(labels))
def __init__(self, account, project_id, region):
self.account = account
self.project_id = project_id
self.region = region
self.client_id = f'client{"".join([str(random.choice([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])) for i in range(0, 8)])}'
self.api = discovery.build_from_document(
service=_read_api_document(self.project_id))
def __init__(self,
http,
data_path,
seconds_per_quota=100,
retries=1,
logger=logging.getLogger(__name__)):
"""A wrapper class for the Google Sheets API v4.
All calls to the the API are made in this class. This ensures that the quota is never hit.
The default quota for the API is 100 requests per 100 seconds. Each request is made immediately and counted.
When 100 seconds have passed the counter is reset. Should the counter reach 101 the request is delayed until seconds_per_quota
seconds since the first request pass.
:param http: The http object used to execute the requests.
:param data_path: Where the discovery json file is stored.
:param seconds_per_quota: Default value is 100 seconds
:param retries: How often the requests will be repeated if the connection times out. (Default 1)
:param logger:
"""
self.logger = logger
with open(os.path.join(data_path, "sheets_discovery.json")) as jd:
self.service = discovery.build_from_document(json.load(jd),
http=http)
self.retries = retries
self.seconds_per_quota = seconds_per_quota
def google_service(argv, name, version, doc, filename, parents, discovery_filename=None):
# Define the auth scopes to request.
scope = ['https://www.googleapis.com/auth/webmasters.readonly'] # Read and Analyze Data
parent_parsers = [tools.argparser]
parent_parsers.extend(parents)
parser = argparse.ArgumentParser(description=doc,
formatter_class=argparse.RawDescriptionHelpFormatter,
parents=parent_parsers)
flags = parser.parse_args(argv[1:])
client_secrets = os.path.join(os.path.dirname(filename), 'client_secrets.json')
flow = client.flow_from_clientsecrets(client_secrets, scope=scope, message=tools.message_if_missing(client_secrets))
storage = file.Storage(name + '.dat')
credentials = storage.get()
if credentials is None or credentials.invalid:
credentials = tools.run_flow(flow, storage, flags)
http = credentials.authorize(http=httplib2.Http())
if discovery_filename is None:
# Construct a service object via the discovery service.
service = discovery.build(name, version, http=http)
else:
# Construct a service object using a local discovery document file.
with open(discovery_filename) as discovery_file:
service = discovery.build_from_document(discovery_file.read(),
base='https://www.googleapis.com/',
http=http)
return service, flags
def main(photo_file):
'''Run a label request on a single image'''
credentials = GoogleCredentials.get_application_default()
with open(API_DISCOVERY_FILE, 'r') as f:
doc = f.read()
service = discovery.build_from_document(
doc, credentials=credentials, http=httplib2.Http())
with open(photo_file, 'rb') as image:
image_content = base64.b64encode(image.read())
service_request = service.images().annotate(
body={
'requests': [{
'image': {
'content': image_content
},
'features': [{
'type': 'LABEL_DETECTION',
'maxResults': 5,
}]
}]
})
response = service_request.execute()
# print(response)
labels = map(lambda x: x['description'], response[
'responses'][0]['labelAnnotations'])
sys.stdout.write("There are " + ", ".join(labels))
def main():
photo_file = 'target.jpg'
with picamera.PiCamera() as camera:
camera.resolution = (640, 480)
camera.start_preview()
time.sleep(2) # Adjust condition
camera.capture(photo_file)
camera.stop_preview()
credentials = GoogleCredentials.get_application_default()
with open(API_DISCOVERY_FILE, 'r') as f:
doc = f.read()
service = discovery.build_from_document(doc,
credentials=credentials,
http=httplib2.Http())
with open(photo_file, 'rb') as image:
image_content = base64.b64encode(image.read())
service_request = service.images().annotate(
body={
'requests': [{
'image': {
'content': image_content
},
'features': [{
'type': 'LABEL_DETECTION',
'maxResults': 2,
}]
}]
})
response = service_request.execute()
# print(response)
labels = map(lambda x: x['description'],
response['responses'][0]['labelAnnotations'])
sys.stdout.write("There are " + ", ".join(labels))
def get(name, version, doc, filename, discovery_filename, scope):
# Name of a file containing the OAuth 2.0 information details
client_secrets = os.path.join(os.path.dirname(filename),
'client_secrets_adsense.json')
# Set up a Flow object to be used if we need to authenticate.
flow = client.flow_from_clientsecrets(
client_secrets,
scope=scope,
message=tools.message_if_missing(client_secrets))
# Prepare credentials, and authorize HTTP object with them.
# If the credentials don't exist or are invalid run through the native client
# flow. The Storage object will ensure that if successful the good
# credentials will get written back to a file.
storage = file.Storage(name + '.dat')
credentials = storage.get()
if credentials is None or credentials.invalid:
flags = None
credentials = tools.run_flow(flow, storage, flags)
http = credentials.authorize(http=httplib2.Http())
if discovery_filename is None:
# Construct a service object via the discovery service.
service = discovery.build(name, version, http=http)
else:
# Construct a service object using a local discovery document file.
with open(discovery_filename) as discovery_file:
service = discovery.build_from_document(
discovery_file.read(),
base='https://www.googleapis.com/',
http=http)
return service
def initCredentialsAndService(self, name, version, scope = None, discovery_filename = None):
if scope is None:
scope = 'https://www.googleapis.com/auth/' + name
client_secrets = os.path.join(self._directory, 'client_secrets.json')
flow = client.flow_from_clientsecrets(client_secrets,
scope=scope,
message=tools.message_if_missing(client_secrets))
storage = file.Storage(name + '.dat')
credentials = storage.get()
if credentials is None or credentials.invalid:
flags = Expando()
flags.nonoauth_local_webserver = True
credentials = tools.run_flow(flow, storage, flags)
http = credentials.authorize(http=httplib2.Http())
if discovery_filename is None:
service = discovery.build(name, version, http=http)
else:
with open(discovery_filename) as discovery_file:
service = discovery.build_from_document(
discovery_file.read(),
base='https://www.googleapis.com/',
http=http)
return (credentials, service)
def __init__(self, oauth_credentials=None, descriptor='', http_=None):
authenticate_kwargs = gcp.get_authenticate_kwargs(oauth_credentials, http_)
if descriptor:
self.client = discovery.build_from_document(descriptor, **authenticate_kwargs)
else:
self.client = discovery.build('bigquery', 'v2', **authenticate_kwargs)
def __init__(self, oauth_credentials=None, descriptor='', http_=None):
authenticate_kwargs = gcp.get_authenticate_kwargs(oauth_credentials, http_)
if descriptor:
self.client = discovery.build_from_document(descriptor, **authenticate_kwargs)
else:
self.client = discovery.build('bigquery', 'v2', cache_discovery=False, **authenticate_kwargs)
def get_vision_api():
credentials = get_api_credentials(
'https://www.googleapis.com/auth/cloud-platform')
with open(API_DISCOVERY_FILE, 'r') as f:
doc = f.read()
return discovery.build_from_document(doc,
credentials=credentials,
http=httplib2.Http())
def get_speech_service():
# [START authenticating]
credentials = GoogleCredentials.get_application_default().create_scoped(
['https://www.googleapis.com/auth/cloud-platform'])
with open(API_DISCOVERY_FILE, 'r') as f:
doc = f.read()
return discovery.build_from_document(
doc, credentials=credentials, http=httplib2.Http())
def __init__(self, oauth_credentials=None, descriptor='', http_=None):
if not oauth_credentials:
oauth_credentials = oauth2client.client.GoogleCredentials.get_application_default()
if descriptor:
self.client = discovery.build_from_document(descriptor, credentials=oauth_credentials, http=http_)
else:
self.client = discovery.build('bigquery', 'v2', credentials=oauth_credentials, http=http_)
def __init__(self, credentials=None):
# project_id = Customer's Project ID
if credentials is None:
credentials, project_id = google.auth.default(scopes=BQ_DTS_OAUTH_SCOPES)
with open(BQ_DTS_PARTNER_API_DISCOVERY_DOC_PATH) as fp:
bq_dts_discover_doc = fp.read()
self._rest_client = discovery.build_from_document(service=bq_dts_discover_doc, credentials=credentials)
def __init__(self, oauth_credentials=None, descriptor='', http_=None,
chunksize=CHUNKSIZE):
self.chunksize = chunksize
authenticate_kwargs = gcp.get_authenticate_kwargs(oauth_credentials, http_)
if descriptor:
self.client = discovery.build_from_document(descriptor, **authenticate_kwargs)
else:
self.client = discovery.build('storage', 'v1', **authenticate_kwargs)
def get_speech_service():
# [START authenticating]
credentials = GoogleCredentials.get_application_default().create_scoped(
['https://www.googleapis.com/auth/cloud-platform'])
with open(API_DISCOVERY_FILE, 'r') as f:
doc = f.read()
return discovery.build_from_document(
doc, credentials=credentials, http=httplib2.Http())
def create_caip_optimizer_client(project_id):
from google.cloud import storage
_OPTIMIZER_API_DOCUMENT_BUCKET = 'caip-optimizer-public'
_OPTIMIZER_API_DOCUMENT_FILE = 'api/ml_public_google_rest_v1.json'
client = storage.Client(project_id)
bucket = client.get_bucket(_OPTIMIZER_API_DOCUMENT_BUCKET)
blob = bucket.get_blob(_OPTIMIZER_API_DOCUMENT_FILE)
discovery_document = blob.download_as_bytes()
return discovery.build_from_document(service=discovery_document)
def __init__(self, oauth_credentials=None, descriptor="", http_=None):
http_ = http_ or httplib2.Http()
if not oauth_credentials:
oauth_credentials = oauth2client.client.GoogleCredentials.get_application_default()
if descriptor:
self.client = discovery.build_from_document(descriptor, credentials=oauth_credentials, http=http_)
else:
self.client = discovery.build("bigquery", "v2", credentials=oauth_credentials, http=http_)
def get_youtube_url(query):
with open("./rest.json") as f:
service = json.load(f)
yt = discovery.build_from_document(service, developerKey=YOUTUBE_API_KEY)
req = yt.search().list(q=query, part='snippet', maxResults=1, type='video')
res = req.execute()
return ('https://youtu.be/' + res['items'][0]['id']['videoId']), unescape(
res['items'][0]['snippet']['title'])
def get_authenticated_service():
args = argparser.parse_args()
flow = flow_from_clientsecrets(CLIENT_SECRETS_FILE, scope=YOUTUBE_READ_WRITE_SSL_SCOPE,
message=MISSING_CLIENT_SECRETS_MESSAGE)
storage = Storage("%s-oauth2.json" % sys.argv[0])
credentials = storage.get()
if credentials is None or credentials.invalid:
credentials = run_flow(flow, storage, args)
with open("youtube-v3-discoverydocument.json", "r") as f:
doc = f.read()
return build_from_document(doc, http=credentials.authorize(httplib2.Http()))
def __init__(self, oauth, http_client=None, retries=1, no_cache=False):
if no_cache:
cache = None
else:
cache = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
self.oauth = oauth
http_client = http_client or httplib2.Http(cache=cache, timeout=20)
http = self.oauth.authorize(http_client)
data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
with open(os.path.join(data_path, "sheets_discovery.json")) as jd:
self.service = discovery.build_from_document(jload(jd), http=http)
with open(os.path.join(data_path, "drive_discovery.json")) as jd:
self.driveService = discovery.build_from_document(jload(jd), http=http)
self._spreadsheeets = []
self.batch_requests = dict()
self.retries = retries
self.enableTeamDriveSupport = False # if teamdrive files should be included
self.teamDriveId = None # teamdrive to search for spreadsheet
self._fetch_sheets()
def __init__(self, oauth_credentials=None, descriptor='', http_=None,
chunksize=CHUNKSIZE):
self.chunksize = chunksize
if not oauth_credentials:
oauth_credentials = oauth2client.client.GoogleCredentials.get_application_default()
if descriptor:
self.client = discovery.build_from_document(descriptor, credentials=oauth_credentials, http=http_)
else:
self.client = discovery.build('storage', 'v1', credentials=oauth_credentials, http=http_)
def document_api(name, version, uri, doc_destination_dir):
"""Document the given API.
Args:
name (str): Name of the API.
version (str): Version of the API.
uri (str): URI of the API's discovery document
doc_destination_dir (str): relative path where the reference
documentation should be saved.
"""
http = build_http()
resp, content = http.request(
uri or uritemplate.expand(
FLAGS.discovery_uri_template, {"api": name, "apiVersion": version}
)
)
if resp.status == 200:
discovery = json.loads(content)
service = build_from_document(discovery)
version = safe_version(version)
doc_name = "{}.{}.json".format(name, version.replace("_", ""))
discovery_file_path = DISCOVERY_DOC_DIR / doc_name
revision = None
pathlib.Path(discovery_file_path).touch(exist_ok=True)
# Write discovery artifact to disk if revision equal or newer
with open(discovery_file_path, "r+") as f:
try:
json_data = json.load(f)
revision = json_data['revision']
except json.JSONDecodeError:
revision = None
if revision is None or discovery['revision'] >= revision:
# Reset position to the beginning
f.seek(0)
# Write the changes to disk
json.dump(discovery, f, indent=2, sort_keys=True)
# Truncate anything left as it's not needed
f.truncate()
elif resp.status == 404:
print("Warning: {} {} not found. HTTP Code: {}".format(name, version, resp.status))
return
else:
print("Warning: {} {} could not be built. HTTP Code: {}".format(name, version, resp.status))
return
document_collection_recursive(
service, "{}_{}.".format(name, version), discovery, discovery, doc_destination_dir
)
def __initialise_client(self):
authenticate_kwargs = gcp.get_authenticate_kwargs(
self.oauth_credentials, self.http_)
if self.descriptor:
self.client = discovery.build_from_document(
self.descriptor, **authenticate_kwargs)
else:
self.client = discovery.build('bigquery',
'v2',
cache_discovery=False,
**authenticate_kwargs)
def __init__(self, http, data_path, retries=3, logger=logging.getLogger(__name__)):
try:
with open(os.path.join(data_path, "drive_discovery.json")) as jd:
self.service = discovery.build_from_document(json.load(jd), http=http)
except:
self.service = discovery.build('drive', 'v3', http=http)
self.team_drive_id = None
self.include_team_drive_items = True
"""Include files from TeamDrive when executing requests."""
self.logger = logger
self._spreadsheet_mime_type_query = "mimeType='application/vnd.google-apps.spreadsheet'"
self.retries = retries
def __init__(self,
api_definition,
project,
zone,
tpu_names,
job_name='tpu_worker',
credentials='default',
service=None):
"""Creates a new TPUClusterResolver object.
The ClusterResolver will then use the parameters to query the Cloud TPU APIs
for the IP addresses and ports of each Cloud TPU listed.
Args:
api_definition: (Alpha only) A copy of the JSON API definitions for
Cloud TPUs. This will be removed once Cloud TPU enters beta.
project: Name of the GCP project containing Cloud TPUs
zone: Zone where the TPUs are located
tpu_names: A list of names of the target Cloud TPUs.
job_name: Name of the TensorFlow job the TPUs belong to.
credentials: GCE Credentials. If None, then we use default credentials
from the oauth2client
service: The GCE API object returned by the googleapiclient.discovery
function. If you specify a custom service object, then the credentials
parameter will be ignored.
Raises:
ImportError: If the googleapiclient is not installed.
"""
self._project = project
self._zone = zone
self._tpu_names = tpu_names
self._job_name = job_name
self._credentials = credentials
if credentials == 'default':
if _GOOGLE_API_CLIENT_INSTALLED:
self._credentials = GoogleCredentials.get_application_default()
if service is None:
if not _GOOGLE_API_CLIENT_INSTALLED:
raise ImportError(
'googleapiclient must be installed before using the '
'TPU cluster resolver')
# TODO(frankchn): Remove once Cloud TPU API Definitions are public and
# replace with discovery.build('tpu', 'v1')
self._service = discovery.build_from_document(
api_definition, credentials=self._credentials)
else:
self._service = service
def __init__(self,
api_definition,
project,
zone,
tpu_names,
job_name='tpu_worker',
credentials='default',
service=None):
"""Creates a new TPUClusterResolver object.
The ClusterResolver will then use the parameters to query the Cloud TPU APIs
for the IP addresses and ports of each Cloud TPU listed.
Args:
api_definition: (Alpha only) A copy of the JSON API definitions for
Cloud TPUs. This will be removed once Cloud TPU enters beta.
project: Name of the GCP project containing Cloud TPUs
zone: Zone where the TPUs are located
tpu_names: A list of names of the target Cloud TPUs.
job_name: Name of the TensorFlow job the TPUs belong to.
credentials: GCE Credentials. If None, then we use default credentials
from the oauth2client
service: The GCE API object returned by the googleapiclient.discovery
function. If you specify a custom service object, then the credentials
parameter will be ignored.
Raises:
ImportError: If the googleapiclient is not installed.
"""
self._project = project
self._zone = zone
self._tpu_names = tpu_names
self._job_name = job_name
self._credentials = credentials
if credentials == 'default':
if _GOOGLE_API_CLIENT_INSTALLED:
self._credentials = GoogleCredentials.get_application_default()
if service is None:
if not _GOOGLE_API_CLIENT_INSTALLED:
raise ImportError('googleapiclient must be installed before using the '
'TPU cluster resolver')
# TODO(frankchn): Remove once Cloud TPU API Definitions are public and
# replace with discovery.build('tpu', 'v1')
self._service = discovery.build_from_document(
api_definition,
credentials=self._credentials)
else:
self._service = service
def __init__(self, oauth_credentials=None, descriptor='', http_=None,
chunksize=CHUNKSIZE, **discovery_build_kwargs):
self.chunksize = chunksize
authenticate_kwargs = gcp.get_authenticate_kwargs(oauth_credentials, http_)
build_kwargs = authenticate_kwargs.copy()
build_kwargs.update(discovery_build_kwargs)
if descriptor:
self.client = discovery.build_from_document(descriptor, **build_kwargs)
else:
build_kwargs.setdefault('cache_discovery', False)
self.client = discovery.build('storage', 'v1', **build_kwargs)
def __init__(self, oauth_credentials=None, descriptor='', http_=None,
chunksize=CHUNKSIZE, **discovery_build_kwargs):
self.chunksize = chunksize
authenticate_kwargs = gcp.get_authenticate_kwargs(oauth_credentials, http_)
build_kwargs = authenticate_kwargs.copy()
build_kwargs.update(discovery_build_kwargs)
if descriptor:
self.client = discovery.build_from_document(descriptor, **build_kwargs)
else:
build_kwargs.setdefault('cache_discovery', False)
self.client = discovery.build('storage', 'v1', **build_kwargs)
def get_authenticated_service(args):
flow = flow_from_clientsecrets(CLIENT_SECRETS_FILE, scope=YOUTUBE_READ_WRITE_SSL_SCOPE,
message=MISSING_CLIENT_SECRETS_MESSAGE)
storage = Storage("%s-oauth2.json" % sys.argv[0])
credentials = storage.get()
if credentials is None or credentials.invalid:
credentials = run_flow(flow, storage, args)
# Trusted testers can download this discovery document from the developers page
# and it should be in the same directory with the code.
with open("youtube-v3-api-captions.json", "r") as f:
doc = f.read()
return build_from_document(doc, http=credentials.authorize(httplib2.Http()))
def __enter__(self):
http = HttpMockSequence(self._responses)
native_request_method = http.request
# Collecting requests to validate at __exit__.
def _request_wrapper(*args, **kwargs):
self._actual_requests.append(args + (kwargs.get('body', ''),))
return native_request_method(*args, **kwargs)
http.request = _request_wrapper
discovery = requests.get(
'https://www.googleapis.com/discovery/v1/apis/ml/v1/rest')
service_mock = build_from_document(discovery.json(), http=http)
with mock.patch.object(
hook.MLEngineHook, 'get_conn', return_value=service_mock):
return hook.MLEngineHook()
def document_api_from_discovery_document(uri):
"""Document the given API.
Args:
uri: string, URI of discovery document.
"""
http = httplib2.Http()
response, content = http.request(FLAGS.discovery_uri)
discovery = simplejson.loads(content)
service = build_from_document(discovery)
name = discovery['version']
version = safe_version(discovery['version'])
document_collection_recursive(
service, '%s_%s.' % (name, version), discovery, discovery)
def build_and_update(res_path, api, version):
from oauth2client.client import GoogleCredentials
from googleapiclient.discovery import build_from_document
path = os.path.join(res_path, '{}.{}'.format(api, version))
try:
age = time.time() - os.path.getmtime(path)
if age > MAX_AGE:
_update_discovery_doc(api, version, path)
except os.error:
_update_discovery_doc(api, version, path)
with open(path, 'r') as discovery_doc:
return build_from_document(discovery_doc.read(),
http=httplib2.Http(),
credentials=GoogleCredentials
.get_application_default())
def build_and_update(api, version):
from oauth2client.client import GoogleCredentials
from googleapiclient.discovery import build_from_document
path = os.path.join(RESOURCE_PATH, '{}.{}'.format(api, version))
try:
age = time.time() - os.path.getmtime(path)
if age > MAX_AGE:
_update_discovery_doc(api, version, path)
except os.error:
_update_discovery_doc(api, version, path)
credentials = GoogleCredentials.get_application_default()
if credentials.create_scoped_required():
credentials = credentials.create_scoped(BIGQUERY_SCOPES)
with open(path, 'r') as discovery_doc:
return build_from_document(discovery_doc.read(),
http=httplib2.Http(),
credentials=credentials)
def GetServiceFromFile(discovery_file):
"""Builds a service using the specified discovery document.
Args:
discovery_file: a str path to the JSON discovery file for the service to be
created.
Returns:
A googleapiclient.discovery.Resource instance used to interact with the
service specified by the discovery_file.
"""
credentials = _GetCredentials()
with open(discovery_file, 'r') as handler:
discovery_doc = handler.read()
service = build_from_document(service=discovery_doc, credentials=credentials)
return service
def upload_audio(self, speech, sample_rate):
credentials = GoogleCredentials.get_application_default().create_scoped([SERVICE_URL])
with open(self.discovery_file, 'r') as f:
doc = f.read()
speech_content = base64.b64encode(speech)
service = discovery.build_from_document(doc, credentials=credentials, http=httplib2.Http())
service_request = service.speech().recognize(
body={
'initialRequest': {
'encoding': 'LINEAR16',
'sampleRate': sample_rate
},
'audioRequest': {
'content': speech_content.decode('UTF-8')
}
})
response = service_request.execute()
return json.dumps(response)
def __init__(self, http, data_path, seconds_per_quota=100, retries=1, logger=logging.getLogger(__name__)):
"""A wrapper class for the Google Sheets API v4.
All calls to the the API are made in this class. This ensures that the quota is never hit.
The default quota for the API is 100 requests per 100 seconds. Each request is made immediately and counted.
When 100 seconds have passed the counter is reset. Should the counter reach 101 the request is delayed until seconds_per_quota
seconds since the first request pass.
:param http: The http object used to execute the requests.
:param data_path: Where the discovery json file is stored.
:param seconds_per_quota: Default value is 100 seconds
:param retries: How often the requests will be repeated if the connection times out. (Default 1)
:param logger:
"""
self.logger = logger
with open(os.path.join(data_path, "sheets_discovery.json")) as jd:
self.service = discovery.build_from_document(json.load(jd), http=http)
self.retries = retries
self.seconds_per_quota = seconds_per_quota
def main(photo_file):
credentials = GoogleCredentials.get_application_default()
with open(API_DISCOVERY_FILE, 'r') as f:
doc = f.read()
service = discovery.build_from_document(doc, credentials=credentials, http=httplib2.Http())
with open(photo_file, 'rb') as image:
image_content = base64.b64encode(image.read())
service_request = service.images().annotate(
body={
'requests': [{
'image': {
'content': image_content
},
'features': [{
'type': 'LABEL_DETECTION',
'maxResults': 1,
}]
}]
})
response = service_request.execute()
label = response['responses'][0]['labelAnnotations'][0]['description']
print('Found label: %s for %s' % (label, photo_file))
return 0
def _create_cloudtasks_client_testing(self, http):
with open(
os.path.join(os.path.dirname(__file__),
'../../artman/conductors/cloudtasks.json'), 'r') as f:
return build_from_document(f.read(), http=http)
def init(argv, name, version, doc, filename, scope=None, parents=[], discovery_filename=None):
"""A common initialization routine for samples.
Many of the sample applications do the same initialization, which has now
been consolidated into this function. This function uses common idioms found
in almost all the samples, i.e. for an API with name 'apiname', the
credentials are stored in a file named apiname.dat, and the
client_secrets.json file is stored in the same directory as the application
main file.
Args:
argv: list of string, the command-line parameters of the application.
name: string, name of the API.
version: string, version of the API.
doc: string, description of the application. Usually set to __doc__.
file: string, filename of the application. Usually set to __file__.
parents: list of argparse.ArgumentParser, additional command-line flags.
scope: string, The OAuth scope used.
discovery_filename: string, name of local discovery file (JSON). Use when discovery doc not available via URL.
Returns:
A tuple of (service, flags), where service is the service object and flags
is the parsed command-line flags.
"""
if scope is None:
scope = 'https://www.googleapis.com/auth/' + name
# Parser command-line arguments.
parent_parsers = [tools.argparser]
parent_parsers.extend(parents)
parser = argparse.ArgumentParser(
description=doc,
formatter_class=argparse.RawDescriptionHelpFormatter,
parents=parent_parsers)
flags = parser.parse_args(argv[1:])
# Name of a file containing the OAuth 2.0 information for this
# application, including client_id and client_secret, which are found
# on the API Access tab on the Google APIs
# Console <http://code.google.com/apis/console>.
client_secrets = os.path.join(os.path.dirname(filename),
'client_secrets.json')
# Set up a Flow object to be used if we need to authenticate.
flow = client.flow_from_clientsecrets(client_secrets,
scope=scope,
message=tools.message_if_missing(client_secrets))
# Prepare credentials, and authorize HTTP object with them.
# If the credentials don't exist or are invalid run through the native client
# flow. The Storage object will ensure that if successful the good
# credentials will get written back to a file.
storage = file.Storage(name + '.dat')
credentials = storage.get()
if credentials is None or credentials.invalid:
credentials = tools.run_flow(flow, storage, flags)
http = credentials.authorize(http = httplib2.Http())
if discovery_filename is None:
# Construct a service object via the discovery service.
service = discovery.build(name, version, http=http)
else:
# Construct a service object using a local discovery document file.
with open(discovery_filename) as discovery_file:
service = discovery.build_from_document(
discovery_file.read(),
base='https://www.googleapis.com/',
http=http)
return (service, flags)
def test_building_with_explicit_http(self):
http = HttpMock()
discovery = open(datafile('plus.json')).read()
plus = build_from_document(
discovery, base="https://www.googleapis.com/", http=http)
self.assertEquals(plus._http, http)
def test_building_with_optional_http(self):
discovery = open(datafile('plus.json')).read()
plus = build_from_document(discovery, base="https://www.googleapis.com/")
self.assertTrue(isinstance(plus._http, httplib2.Http))
def test_building_with_base_remembers_base(self):
discovery = open(datafile('plus.json')).read()
base = "https://www.example.com/"
plus = build_from_document(discovery, base=base)
self.assertEquals("https://www.googleapis.com/plus/v1/", plus._baseUrl)
def test_can_build_from_local_deserialized_document(self):
discovery = open(datafile('plus.json')).read()
discovery = json.loads(discovery)
plus = build_from_document(discovery, base="https://www.googleapis.com/")
self.assertTrue(plus is not None)
self.assertTrue(hasattr(plus, 'activities'))
def get_vision_api(api_key):
#credentials = get_api_credentials('https://www.googleapis.com/auth/cloud-platform')
#credentials = gce.AppAssertionCredentials(scope='https://www.googleapis.com/auth/devstorage.read_write')
with open(API_DISCOVERY_FILE, 'r') as f:
doc = f.read()
return discovery.build_from_document(doc, developerKey=api_key, http=httplib2.Http())
def _create_tasks_client():
credentials = GoogleCredentials.get_application_default()
with open(
os.path.join(os.path.dirname(__file__), 'cloudtasks.json'), 'r') as f:
return build_from_document(f.read(), credentials=credentials)
def patched_build(serviceName, version, http=None, **kwargs):
doc = _get_discovery_document(serviceName, version, http=http)
return discovery.build_from_document(doc, http=http, **kwargs)
