def get_http(): http = httplib2.Http() GoogleCredentials.get_application_default().create_scoped([ 'https://www.googleapis.com/auth/firebase.database', 'https://www.googleapis.com/auth/userinfo.email' ]).authorize(http) return http
def get_storage_resource(): try: credentials = GoogleCredentials.get_application_default() except: # todo: replace filename with variable credentials = GoogleCredentials.from_stream("privatekey.json").create_scoped(STORAGE_SCOPES) return discovery.build('storage', 'v1', credentials=credentials)
def get_service(): from googleapiclient.discovery import build from oauth2client.client import GoogleCredentials credentials = GoogleCredentials.get_application_default() if credentials.create_scoped_required(): credentials = credentials.create_scoped('https://www.googleapis.com/auth/bigquery') return build('bigquery','v2', credentials=GoogleCredentials.get_application_default())
def hello(): credentials = AppAssertionCredentials([]) client = datastore.Client(project = 'mq-cloud-prototyping-3', credentials = credentials) sys.stdout.write(credentials.to_json()) query = client.query(kind='Person') res = query.fetch() all = dict(res) sys.stdout.write(str(all)) return credentials.to_json() try : token = '' #ouath O_AUTH_EMAIL_SCOPE = 'https://www.googleapis.com/auth/userinfo.email' credentials = GoogleCredentials.get_application_default() if credentials.create_scoped_required(): credentials = credentials.create_scoped(PUBSUB_SCOPES) http = httplib2.Http() credentials.authorize(http) cl = discovery.build('pubsub', 'v1', http=http) return credentials.to_json() credentials = GoogleCredentials.get_application_default() credentials = credentials.create_scoped([O_AUTH_EMAIL_SCOPE]) http = httplib2.Http() credentials.authorize(http) return credentials.to_json() #if not http: # http = httplib2.Http() #credentials.authorize(http) #temp hardcoded token #token = 'ya29.CjjlAlrvqUwXrujCnJuqa08HTtmNilyP7K1GGrHQ40Gt489H6NGT9WQAxEL92OSQ6anGYeFPRcvI4g' tokenBearer = 'Bearer %s' % token url = 'https://admin-dot-mq-vouchers.appspot.com/api/communities/mtv1/campaigns?page=0&size=1000&sorting=campaignName,ASC' req = urllib2.Request(url, headers = {'Content-Type': 'application/json', 'Authorization' : tokenBearer}) f = urllib2.urlopen(req) response = f.read() sys.stdout.write(str(response)) respjson = json.loads(response) f.close() #respjson = '3333' #sys.stdout.write(str(all)) return str(response) except urllib2.HTTPError, error: return ('get failed %s' % error)
def __init__(self, project_id, bucket_name, credential_file=None): BaseTransfer.__init__(self) self.project_id = project_id if credential_file: creds = GoogleCredentials.from_stream(credential_file) else: creds = GoogleCredentials.get_application_default() gs = build("storage", "v1", credentials=creds) self.gs_buckets = gs.buckets() # pylint: disable=no-member self.gs_objects = gs.objects() # pylint: disable=no-member self.bucket_name = self.get_or_create_bucket(bucket_name) self.log.debug("GoogleTransfer initialized")
def test_get_application_default_environment_not_set_up(self): # It is normal for this test to fail if run inside # a Google Compute Engine VM or after 'gcloud auth login' command # has been executed on a non Windows machine. os.environ['SERVER_SOFTWARE'] = '' os.environ[GOOGLE_APPLICATION_CREDENTIALS] = '' os.environ['APPDATA'] = '' # we can't use self.assertRaisesRegexp() because it is only in Python 2.7+ try: GoogleCredentials.get_application_default() self.fail('An exception was expected!') except ApplicationDefaultCredentialsError as error: self.assertEqual(ADC_HELP_MSG, str(error))
def test_get_default_from_environment_variable_malformed_file(self): os.environ['SERVER_SOFTWARE'] = '' environment_variable_file = datafile( os.path.join('gcloud', 'credentials_default_malformed_3.json')) os.environ[GOOGLE_CREDENTIALS_DEFAULT] = environment_variable_file # we can't use self.assertRaisesRegexp() because it is only in Python 2.7+ try: GoogleCredentials.get_default() self.fail('An exception was expected!') except DefaultCredentialsError as error: self.assertTrue(str(error).startswith( 'An error was encountered while reading json file: ' + environment_variable_file + ' (pointed to by ' + GOOGLE_CREDENTIALS_DEFAULT + ' environment variable):'))
def main(photo_file): '''Run a label request on a single image''' API_DISCOVERY_FILE = 'https://vision.googleapis.com/$discovery/rest?version=v1' http = httplib2.Http() credentials = GoogleCredentials.get_application_default().create_scoped( ['https://www.googleapis.com/auth/cloud-platform']) credentials.authorize(http) service = build('vision', 'v1', http, discoveryServiceUrl=API_DISCOVERY_FILE) with open(photo_file, 'rb') as image: image_content = base64.b64encode(image.read()) service_request = service.images().annotate( body={ 'requests': [{ 'image': { 'content': image_content }, 'features': [{ 'type': 'LABEL_DETECTION', 'maxResults': 1, }] }] }) response = service_request.execute() label = response['responses'][0]['labelAnnotations'][0]['description'] print('Found label: %s for %s' % (label, photo_file)) return 0
def run(project, zone, instance_name): credentials = GoogleCredentials.get_application_default() compute = build('compute', 'v1', credentials=credentials) print('Creating instance.') operation = create_instance(compute, project, zone, instance_name) wait_for_operation(compute, project, zone, operation['name']) instances = list_instances(compute, project, zone) print('Instances in project %s and zone %s:' % (project, zone)) for instance in instances: print(' - ' + instance['name']) print(""" Instance created. It will take a minute or two for the instance to complete work. Check this URL: http://storage.googleapis.com/%s/output.png Once the image is uploaded press enter to delete the instance. """ % project) input() print('Deleting instance.') operation = delete_instance(compute, project, zone, instance_name) wait_for_operation(compute, project, zone, operation['name'])
def get_service(): credentials = GoogleCredentials.get_application_default() scoped_credentials = credentials.create_scoped( ['https://www.googleapis.com/auth/cloud-platform']) http = httplib2.Http() scoped_credentials.authorize(http) return discovery.build('language', 'v1beta1', http=http)
def main(bucket, sources): # Get the application default credentials. When running locally, these are # available after running `gcloud init`. When running on compute # engine, these are available from the environment. credentials = GoogleCredentials.get_application_default() # Construct the service object for the interacting with the Cloud Storage # API. service = discovery.build('storage', 'v1', credentials=credentials) print "\n-----------\n" print "sources===", sources print "\n-----------\n" # Upload the source files. for filename in sources: req = service.objects().insert( media_body=filename, name=filename, bucket=bucket) resp = req.execute() print('> Uploaded source file {}'.format(filename)) print(json.dumps(resp, indent=2)) """
def get_service(): """Build a client to the Google Cloud Natural Language API.""" credentials = GoogleCredentials.get_application_default() return discovery.build('language', 'v1beta1', credentials=credentials)
def run(project, zone, instance_name, package): credentials = GoogleCredentials.get_application_default() compute = build('compute', 'v1', credentials=credentials) print 'Creating instance.' #creating instance operation = create_instance(compute, project, zone, instance_name, package) #execute operations wait_for_operation(compute, project, zone, operation['name']) #listing instances instances = list_instances(compute, project, zone) print 'Instances in project %s and zone %s:' % (project, zone) for instance in instances: print ' - ' + instance['name'] print """ Instance created """ raw_input() print 'Deleting instance' operation = delete_instance(compute, project, zone, instance_name) wait_for_operation(compute, project, zone, operation['name'])
def get_service_credentials(): """For internal use only; no backwards-compatibility guarantees. Get credentials to access Google services.""" user_agent = 'beam-python-sdk/1.0' if is_running_in_gce: # We are currently running as a GCE taskrunner worker. # # TODO(ccy): It's not entirely clear if these credentials are thread-safe. # If so, we can cache these credentials to save the overhead of creating # them again. return _GCEMetadataCredentials(user_agent=user_agent) else: client_scopes = [ 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/devstorage.full_control', 'https://www.googleapis.com/auth/userinfo.email', 'https://www.googleapis.com/auth/datastore' ] try: credentials = GoogleCredentials.get_application_default() credentials = credentials.create_scoped(client_scopes) logging.debug('Connecting using Google Application Default ' 'Credentials.') return credentials except Exception as e: logging.warning( 'Unable to find default credentials to use: %s\n' 'Connecting anonymously.', e) return None
def _tpuService(self): """Creates a new Cloud TPU API object. This works around an issue where the underlying HTTP connection sometimes times out when the script has been running for too long. Other methods in this object calls this method to get a new API object whenever they need to communicate with the Cloud API. Returns: A Google Cloud TPU API object. """ if self._service: return self._service credentials = self._credentials if credentials is None or credentials == 'default': credentials = GoogleCredentials.get_application_default() if self._discovery_url: return discovery.build( 'tpu', 'v1alpha1', credentials=credentials, discoveryServiceUrl=self._discovery_url) else: return discovery.build( 'tpu', 'v1alpha1', credentials=credentials)
def _connect_google_monitoring(): c = getattr(GoogleMonitoringV3.threadlocal, 'cm_conn', None) if c is None: creds = GoogleCredentials.get_application_default() GoogleMonitoringV3.threadlocal.cm_conn = c = build('monitoring', 'v3', credentials=creds).projects() return c
def main(): conn = None device_opt = ["port", "no_password", "zone", "project"] atexit.register(atexit_handler) define_new_opts() all_opt["power_timeout"]["default"] = "60" options = check_input(device_opt, process_input(device_opt)) docs = {} docs["shortdesc"] = "Fence agent for GCE (Google Cloud Engine)" docs["longdesc"] = "fence_gce is an I/O Fencing agent for GCE (Google Cloud " \ "Engine). It uses the googleapiclient library to connect to GCE.\n" \ "googleapiclient can be configured with Google SDK CLI or by " \ "executing 'gcloud auth application-default login'.\n" \ "For instructions see: https://cloud.google.com/compute/docs/tutorials/python-guide" docs["vendorurl"] = "http://cloud.google.com" show_docs(options, docs) run_delay(options) try: credentials = GoogleCredentials.get_application_default() conn = discovery.build('compute', 'v1', credentials=credentials) except: fail_usage("Failed: Unable to connect to GCE. Check your configuration.") # Operate the fencing device result = fence_action(conn, options, set_power_status, get_power_status, get_nodes_list) sys.exit(result)
def main(): credentials = GoogleCredentials.get_application_default() http = credentials.authorize(httplib2.Http()) projectId = raw_input('Enter the project ID: ') datasetId = raw_input('Enter a dataset ID: ') tableId = raw_input('Enter a table name to load the data to: ') schema_path = raw_input( 'Enter the path to the schema file for the table: ') with open(schema_path, 'r') as schema_file: schema = schema_file.read() data_path = raw_input('Enter the path to the data file: ') with open(data_path, 'r') as data_file: data = data_file.read() resp, content = make_post(http, schema, data, projectId, datasetId, tableId) if resp.status == 200: job_resource = json.loads(content) service = get_service(credentials) poll_job(service, **job_resource['jobReference']) print("Success!") else: print("Http error code: {}".format(resp.status))
def main( description, project_id, day, month, year, hours, minutes, source_bucket, access_key, secret_access_key, sink_bucket ): """Create a one-off transfer from Amazon S3 to Google Cloud Storage.""" credentials = GoogleCredentials.get_application_default() storagetransfer = discovery.build("storagetransfer", "v1", credentials=credentials) # Edit this template with desired parameters. # Specify times below using US Pacific Time Zone. transfer_job = { "description": description, "status": "ENABLED", "projectId": project_id, "schedule": { "scheduleStartDate": {"day": day, "month": month, "year": year}, "scheduleEndDate": {"day": day, "month": month, "year": year}, "startTimeOfDay": {"hours": hours, "minutes": minutes}, }, "transferSpec": { "awsS3DataSource": { "bucketName": source_bucket, "awsAccessKey": {"accessKeyId": access_key, "secretAccessKey": secret_access_key}, }, "gcsDataSink": {"bucketName": sink_bucket}, }, } result = storagetransfer.transferJobs().create(body=transfer_job).execute() print("Returned transferJob: {}".format(json.dumps(result, indent=4)))
def handle(self, *args, **options): if len(args) < 2: raise CommandError("Not enough arguments") org = args[0] client_secret_path = args[1] # Fail early if the university does not exist university = University.objects.get(code=org) # is the file a client-secret file or a credentials file? with open(client_secret_path) as f: json_data = f.read() file_data = json.loads(json_data) if "scopes" in file_data: self.stdout.write("Importing access token...") credentials = GoogleCredentials.from_json(json_data) else: self.stdout.write("Start flow from client secrets...") credentials = flow_from_client_secrets(client_secret_path) # Save credentials to database YoutubeAuth.objects.create( university=university, client_id=credentials.client_id, client_secret=credentials.client_secret, access_token=credentials.access_token, refresh_token=credentials.refresh_token, token_expiry=credentials.token_expiry, ) self.stdout.write("Youtube auth token created")
def main(photo_file): """Run a label request on a single image""" # [START authenticate] credentials = GoogleCredentials.get_application_default() service = discovery.build('vision', 'v1', credentials=credentials, discoveryServiceUrl=DISCOVERY_URL) # [END authenticate] # [START construct_request] with open(photo_file, 'rb') as image: image_content = base64.b64encode(image.read()) service_request = service.images().annotate(body={ 'requests': [{ 'image': { 'content': image_content.decode('UTF-8') }, 'features': [{ 'type': 'LABEL_DETECTION', 'maxResults': 1 }] }] }) # [END construct_request] # [START parse_response] response = service_request.execute() label = response['responses'][0]['labelAnnotations'][0]['description'] print('Found label: %s for %s' % (label, photo_file)) return 0
def __init__(self, project, session_id, logger): super(GCEService, self).__init__(project, session_id, logger) self.credentials = GoogleCredentials.get_application_default() self.compute = discovery.build('compute', 'v1', credentials=self.credentials) self.storage = discovery.build('storage', 'v1', credentials=self.credentials)
def __init__(self, config, logger): """See README.md for the config format.""" self._project_name = config['project_name'] self._taskqueue_tag = config['taskqueue_tag'] self._src_path = config['src_path'] self._credentials = GoogleCredentials.get_application_default() self._logger = logger # Separate the cloud storage path into the bucket and the base path under # the bucket. storage_path_components = config['cloud_storage_path'].split('/') self._bucket_name = storage_path_components[0] self._base_path_in_bucket = '' if len(storage_path_components) > 1: self._base_path_in_bucket = '/'.join(storage_path_components[1:]) if not self._base_path_in_bucket.endswith('/'): self._base_path_in_bucket += '/' self._google_storage_accessor = GoogleStorageAccessor( credentials=self._credentials, project_name=self._project_name, bucket_name=self._bucket_name) self._traces_dir = os.path.join(self._base_path_in_bucket, 'traces') self._trace_database_path = os.path.join( self._traces_dir, config.get('trace_database_filename', 'trace_database.json')) # Recover any existing trace database in case the worker died. self._DownloadTraceDatabase() # Initialize the global options that will be used during trace generation. options.OPTIONS.ParseArgs([]) options.OPTIONS.local_binary = config['chrome_path']
def anaylze_content(text): '''Run a sentiment analysis request on text. This function is a modified version of the tutorial at this link on 7/24/16 https://cloud.google.com/natural-language/docs/sentiment-tutorial ''' http = httplib2.Http() credentials = GoogleCredentials.get_application_default().create_scoped( ['https://www.googleapis.com/auth/cloud-platform']) http=httplib2.Http() credentials.authorize(http) service = discovery.build('language', 'v1beta1', http=http, discoveryServiceUrl=DISCOVERY_URL) service_request = service.documents().annotateText( body={ "document":{ "type":"PLAIN_TEXT", "content": text }, "features":{ "extractDocumentSentiment":True }, "encodingType":"UTF8" }) try: response = service_request.execute() return response except: # Normally you don't want to catch all errors but what is a side project # without tech debt. return {"error_msg": str(sys.exc_info()[1])}
def __init__(self, config=None): # Initialize Handler Handler.__init__(self, config) if discovery is None: logging.error("Failed to load apiclient.discovery") return elif GoogleCredentials is None: logging.error("Failed to load " "oauth2client.client.GoogleCredentials") return # Initialize options self.topic = self.config['topic'] self.scopes = self.config['scopes'] self.retries = int(self.config['retries']) self.batch = self.config['batch'] self.batch_size = int(self.config['batch_size']) self.metrics = [] tags_items = self.config['tags'] self.tags = {} for item in tags_items: k, v = item.split(':') self.tags[k] = v # Initialize client credentials = GoogleCredentials.get_application_default() if credentials.create_scoped_required(): credentials = credentials.create_scoped(self.scopes) self.client = discovery.build('pubsub', 'v1', credentials=credentials)
def main(project_id, dataset_id, table_name, schema_path, data_path): credentials = GoogleCredentials.get_application_default() http = credentials.authorize(httplib2.Http()) bigquery = discovery.build('bigquery', 'v2', credentials=credentials) with open(schema_path, 'r') as schema_file: schema = schema_file.read() with open(data_path, 'r') as data_file: data = data_file.read() resp, content = make_post( http, schema, data, project_id, dataset_id, table_name) if resp.status == 200: job = json.loads(content) poll_job(bigquery, job) print("Success!") else: print("Http error code: {}".format(resp.status))
def main(project_id): credentials = GoogleCredentials.get_application_default() # Construct the service object for interacting with the BigQuery API. bigquery = discovery.build('bigquery', 'v2', credentials=credentials) list_datasets(bigquery, project_id) list_projects(bigquery)
def main(bucket): # Get the application default credentials. When running locally, these are # available after running `gcloud init`. When running on compute # engine, these are available from the environment. credentials = GoogleCredentials.get_application_default() # Construct the service object for interacting with the Cloud Storage API - # the 'storage' service, at version 'v1'. # You can browse other available api services and versions here: # https://developers.google.com/api-client-library/python/apis/ service = discovery.build('storage', 'v1', credentials=credentials) # Make a request to buckets.get to retrieve a list of objects in the # specified bucket. req = service.buckets().get(bucket=bucket) resp = req.execute() print(json.dumps(resp, indent=2)) # Create a request to objects.list to retrieve a list of objects. fields_to_return = \ 'nextPageToken,items(name,size,contentType,metadata(my-key))' req = service.objects().list(bucket=bucket, fields=fields_to_return) # If you have too many items to list in one request, list_next() will # automatically handle paging with the pageToken. while req: resp = req.execute() print(json.dumps(resp, indent=2)) req = service.objects().list_next(req, resp)
def main(project_id): # [START build_service] # Grab the application's default credentials from the environment. credentials = GoogleCredentials.get_application_default() # Construct the service object for interacting with the BigQuery API. bigquery_service = build('bigquery', 'v2', credentials=credentials) # [END build_service] try: # [START run_query] query_request = bigquery_service.jobs() query_data = { 'query': ( 'SELECT TOP(corpus, 10) as title, ' 'COUNT(*) as unique_words ' 'FROM [publicdata:samples.shakespeare];') } query_response = query_request.query( projectId=project_id, body=query_data).execute() # [END run_query] # [START print_results] print('Query Results:') for row in query_response['rows']: print('\t'.join(field['v'] for field in row['f'])) # [END print_results] except HttpError as err: print('Error: {}'.format(err.content)) raise err
def OCR(photo_file): """Run a label request on a single image""" #print photo_file credentials = GoogleCredentials.get_application_default() service = discovery.build('vision', 'v1', credentials=credentials, discoveryServiceUrl=DISCOVERY_URL) with open(photo_file, 'rb') as image: image_content = base64.b64encode(image.read()) service_request = service.images().annotate(body={ 'requests': [{ 'image': { 'content': image_content.decode('UTF-8') }, 'features': [{ 'type': 'TEXT_DETECTION', 'maxResults': 1 }] }] }) response = service_request.execute() line = response['responses'][0]['textAnnotations'][0]['description'] string =str(line) proc = string.splitlines() #print proc card = BusinessCard(proc) #print response #label = response['responses'][0]['labelAnnotations'][0]['description'] #print('Found label: %s for %s' % (label, photo_file)) return card.cleanup()
def clean_up_resource(args, deployments): """Clean up deployment / app config from previous test Args: args: The args from ArgParse. deployments set(string): which contains all deployment names in current test round. Returns: bool: True if cleanup is done """ logging.info("Clean up project resource (backend service and deployment)") # Will reuse source repo for continuous tests # Within 7 days after repo deleted, source repo won't allow recreation with same name # Delete deployment credentials = GoogleCredentials.get_application_default() service = discovery.build('deploymentmanager', 'v2', credentials=credentials) delete_done = False for deployment in deployments: try: request = service.deployments().delete(project=args.project, deployment=deployment) request.execute() except Exception as e: logging.info("Deployment doesn't exist, continue") # wait up to 10 minutes till delete finish. end_time = datetime.datetime.now() + datetime.timedelta(minutes=10) while datetime.datetime.now() < end_time: sleep(10) try: request = service.deployments().list(project=args.project) response = request.execute() if ('deployments' not in response) or (len(deployments & set( d['name'] for d in response['deployments'])) == 0): delete_done = True break except Exception: logging.info( "Failed listing current deployments, retry in 10 seconds") # Delete forwarding-rules delete_gcloud_resource(args, 'forwarding-rules', dlt_params=['--global']) # Delete target-http-proxies delete_gcloud_resource(args, 'target-http-proxies') # Delete target-http-proxies delete_gcloud_resource(args, 'target-https-proxies') # Delete url-maps delete_gcloud_resource(args, 'url-maps') # Delete backend-services delete_gcloud_resource(args, 'backend-services', dlt_params=['--global']) # Delete instance-groups for zone in LOADTEST_ZONE: delete_gcloud_resource(args, 'instance-groups unmanaged', filter=' --filter=INSTANCES:0', dlt_params=['--zone=' + zone]) # Delete ssl-certificates delete_gcloud_resource(args, 'ssl-certificates') # Delete health-checks delete_gcloud_resource(args, 'health-checks') if not delete_done: logging.error( "failed to clean up resources for project %s deployments %s", args.project, deployments) return delete_done
def main ( sqlFilename, billingProjID, tmpDestTable, delTableFlag, maxCost, gcsPath, localDir, outFile, verboseFlag ): ## let's read in the SQL query first ( queryString, BQtableList ) = getBQueryFromFile ( sqlFilename, verboseFlag ) ## first we need to get credentials and build the service if ( verboseFlag ): print " " print " getting credentials to access BigQuery ... " credentials = GoogleCredentials.get_application_default() BQsvc = discovery.build('bigquery', 'v2', credentials=credentials) ( destProjID, destDatasetID, destTableID ) = parseTableName ( tmpDestTable, billingProjID ) if ( verboseFlag ): print " " print " " ## submit the SQL query as a dry-run to validate and verify cost estCost = submitDryRunQueryJob ( BQsvc, queryString, billingProjID, \ destProjID, destDatasetID, destTableID ) if ( estCost > maxCost ): print " " print " The estimated cost exceeds the maximum specified cost. " print " If you want this to run, rerun with a higher maxCost parameter. " print " " sys.exit(-1) ## now submit the SQL query for real jobInfo = submitRealQueryJob ( BQsvc, queryString, billingProjID, \ destProjID, destDatasetID, destTableID ) ## and poll for job completion pollJob ( BQsvc, jobInfo ) ## the 'destination' table from the previous query is now the 'source' table srcProjID = destProjID srcDatasetID = destDatasetID srcTableID = destTableID ## once we've saved the results of the query, the next step is to ## export the contents of the table to GCS -- depending on the ## size of the table, there may be multiple output files ("shards") jobInfo = exportBQtable2GCS ( BQsvc, gcsPath, billingProjID, srcProjID, srcDatasetID, srcTableID ) ## and poll this job too ... pollJob ( BQsvc, jobInfo ) ## at this point we can delete that table if this has been requested if ( delTableFlag ): deleteBQtable ( BQsvc, srcProjID, srcDatasetID, srcTableID ) ## now starts the the second part of this ... we need to download the files/shards ## that have been written to GCS by the 'export' request, and then parse them ## and 'un-melt' the input 'tidy-formatted' data ... readShards_write2dTSV ( gcsPath, localDir, outFile, verboseFlag ) print " " print " DONE! " print " "
def __init__(self, api_discovery_file='vision_api.json'): self.credentials = GoogleCredentials.get_application_default() self.service = discovery.build('vision', 'v1', credentials=self.credentials, discoveryServiceUrl=DISCOVERY_URL)
def connect(self): credentials = GoogleCredentials.get_application_default() self.conn = discovery.build('storage', 'v1', credentials=credentials)
def get_job_id(pipeline_options: Dict[str, str]) -> str: """Captures the job_id of the pipeline job specified by the given options. For local jobs, generates a job_id using the given job_timestamp. For jobs running on Dataflow, finds the currently running job with the same job name as the current pipeline. Note: this works because there can only be one job with the same job name running on Dataflow at a time. Args: pipeline_options: Dictionary containing details about the pipeline. Returns: The job_id string of the current pipeline job. """ runner = pipeline_options.get('runner') if runner == 'DataflowRunner': # Job is running on Dataflow. Get job_id. project = pipeline_options.get('project') region = pipeline_options.get('region') job_name = pipeline_options.get('job_name') if not project: raise ValueError("No project provided in pipeline options: " f"{pipeline_options}") if not region: raise ValueError("No region provided in pipeline options: " f"{pipeline_options}") if not job_name: raise ValueError("No job_name provided in pipeline options: " f"{pipeline_options}") try: logging.info("Looking for job_id on Dataflow.") service_name = 'dataflow' dataflow_api_version = 'v1b3' credentials = GoogleCredentials.get_application_default() dataflow = build(serviceName=service_name, version=dataflow_api_version, credentials=credentials) result = dataflow.projects().locations().jobs().list( projectId=project, location=region, ).execute() pipeline_job_id = 'none' for job in result['jobs']: if job['name'] == job_name: if job['currentState'] == 'JOB_STATE_RUNNING': pipeline_job_id = job['id'] break if pipeline_job_id == 'none': msg = "Could not find currently running job with the " \ f"name: {job_name}." logging.error(msg) raise LookupError(msg) except Exception as e: logging.error("Error retrieving Job ID") raise LookupError(e) else: # Job is running locally. Generate id from the timestamp. pipeline_job_id = '_local_job' job_timestamp = pipeline_options.get('job_timestamp') if not job_timestamp: raise ValueError("Must provide a job_timestamp for local jobs.") pipeline_job_id = job_timestamp + pipeline_job_id return pipeline_job_id
def teardown_minikube(args): """Delete the VM used for minikube.""" credentials = GoogleCredentials.get_application_default() gce = discovery.build("compute", "v1", credentials=credentials) instances = gce.instances() request = instances.delete(project=args.project, zone=args.zone, instance=args.vm_name) response = request.execute() op_id = response.get("name") final_op = vm_util.wait_for_operation(gce, args.project, args.zone, op_id) logging.info("Final result for delete operation: %s", final_op) if final_op.get("status") != "DONE": raise ValueError("Delete operation has status %s", final_op.get("status")) if final_op.get("error"): message = "Delete operation resulted in error %s".format( final_op.get("error")) logging.error(message) raise ValueError(message) # Ensure the disk is deleted. The disk should be auto-deleted with # the VM but just in case we issue a delete request anyway. disks = gce.disks() request = disks.delete(project=args.project, zone=args.zone, disk=args.vm_name) response = None try: response = request.execute() except errors.HttpError as e: if not e.content: raise content = json.loads(e.content) if content.get("error", {}).get("code") == requests.codes.NOT_FOUND: logging.info("Disk %s in zone %s in project %s already deleted.", args.vm_name, args.zone, args.project) else: raise if response: logging.info("Waiting for disk to be deleted.") op_id = response.get("name") final_op = vm_util.wait_for_operation(gce, args.project, args.zone, op_id) logging.info("Final result for disk delete operation: %s", final_op) if final_op.get("status") != "DONE": raise ValueError("Disk delete operation has status %s", final_op.get("status")) if final_op.get("error"): message = "Delete disk operation resulted in error %s".format( final_op.get("error")) logging.error(message) raise ValueError(message)
def settings(request): account = request.POST.get('account', '') if account == '' and request.GET.get('code', False) == False: profile = Profile.objects.filter(user_id=request.user.id).first() # GET REFERRAL OBJECT OF THIS USER url = '%s://%s/i/%s' % ( request.scheme, request.META['HTTP_HOST'], profile.profilelink.short ) if profile and profile.profilelink and profile.profilelink.short else '' profilerank = profile.profilerank if profile else None return render(request, 'settings/settings.html', { 'profile': profile, 'profilerank': profilerank, 'url': url }) elif account == '': if request.GET.get('code', False): flow = client.flow_from_clientsecrets( 'client_secret_sb_webapp_GA.json', scope='https://www.googleapis.com/auth/analytics', redirect_uri="%s://%s%s" % (request.scheme, request.META['HTTP_HOST'], '/settings/'), prompt='select_account') google_authcode = request.GET['code'] credentials = flow.step2_exchange(google_authcode) access_token1 = credentials.access_token refresh_token1 = credentials.refresh_token client_id = credentials.client_id client_secret = credentials.client_secret token_expiry = credentials.token_expiry token_uri = credentials.token_uri user_agent = credentials.user_agent revoke_uri = credentials.revoke_uri Profile.objects.filter(user_id=request.user.id).update( google_authcode=google_authcode, access_token=access_token1, refresh_token=refresh_token1, client_id=client_id, client_secret=client_secret, token_expiry=token_expiry, token_uri=token_uri, user_agent=user_agent, revoke_uri=revoke_uri) http_auth = credentials.authorize(httplib2.Http()) service = build('analytics', 'v3', http_auth, cache_discovery=False) accounts = service.management().accounts().list().execute() if accounts.get('items'): account1 = [] account2 = [] for account in accounts.get('items'): account1 = account1 + [account.get('name')] account2 = account2 + [account.get('id')] zipped_data = zip(account1, account2) return render(request, 'settings/settings.html', {'accounts': zipped_data}) else: profile = Profile.objects.filter( user_id=request.user.id).first() # GET REFERRAL OBJECT OF THIS USER url = '%s://%s/i/%s' % ( request.scheme, request.META['HTTP_HOST'], profile.profilelink.short ) if profile and profile.profilelink and profile.profilelink.short else '' profilerank = profile.profilerank if profile else None message = "You don't have analytics account " return render( request, 'settings/settings.html', { 'profile': profile, 'profilerank': profilerank, 'url': url, 'message': message }) elif account: profile = Profile.objects.filter(user_id=request.user.id).first() access_token = profile.access_token refresh_token = profile.refresh_token client_id = profile.client_id client_secret = profile.client_secret token_expiry = profile.token_expiry token_uri = profile.token_uri user_agent = profile.user_agent revoke_uri = profile.revoke_uri credentials = GoogleCredentials(access_token, client_id, client_secret, refresh_token, token_expiry, token_uri, 'my-user-agent/1.0', revoke_uri) http_auth = credentials.authorize(httplib2.Http()) service = build('analytics', 'v3', http_auth, cache_discovery=False) # Get a list of all the properties for the first account. profile1 = None properties = service.management().webproperties().list( accountId=account).execute() if properties.get('items'): # Get the first property id. property = properties.get('items')[0].get('id') # Get a list of all views (profiles) for the first property. profiles = service.management().profiles().list( accountId=account, webPropertyId=property).execute() if profiles.get('items'): # return the first view (profile) id. profile1 = profiles.get('items')[0].get('id') if profile1 == None: profile = Profile.objects.filter(user_id=request.user.id).first() # GET REFERRAL OBJECT OF THIS USER url = '%s://%s/i/%s' % ( request.scheme, request.META['HTTP_HOST'], profile.profilelink.short ) if profile and profile.profilelink and profile.profilelink.short else '' profilerank = profile.profilerank if profile else None message = "Your account don't have sufficient permission " return render( request, 'settings/settings.html', { 'profile': profile, 'profilerank': profilerank, 'url': url, 'message': message }) service = build('analytics', 'v4', http_auth, cache_discovery=False) message = 'Not have permission of' try: conversion_rate = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '100daysAgo', 'endDate': 'today' }], 'metrics': [{ 'expression': 'ga:goalConversionRateAll' }] }] }).execute() for ccr in conversion_rate['reports']: for rate in ccr['data']['totals']: conversion_vals = rate['values'] # Profile._meta.get_field('current_conversion_rate').default = float(vals[0]) # Profile._meta.get_field('optimal_conversion_rate').default = float(vals[0]) except: # raise Exception('Permission Denied') conversion_vals = [0] message = message + ' Conversion Rate ' pass # return sb_traceback(request) try: bounce_rate = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '100daysAgo', 'endDate': 'today' }], 'metrics': [{ 'expression': 'ga:bounceRate' }], }] }).execute() for cbr in bounce_rate['reports']: for rate in cbr['data']['totals']: bounce_vals = rate['values'] # Profile._meta.get_field('current_bounce_rate').default = float(vals[0]) # Profile._meta.get_field('optimal_bounce_rate').default = float(vals[0]) except: # print traceback.print_exc() # print e bounce_vals = [0] message = message + ' Bounce Rate ' pass # return sb_traceback(request) # raise Exception('Permission Denied') try: metric_expression = { 'expression': 'ga:transactionRevenue/ga:transactions', 'formattingType': 'FLOAT' } avg_order_value = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '100daysAgo', 'endDate': 'today' }], 'metrics': [metric_expression], }] }).execute() for caov in avg_order_value['reports']: for order in caov['data']['totals']: order_value = order['values'] # Profile._meta.get_field('current_average_order_value').default = float(value[0]) # Profile._meta.get_field('optimal_average_order_value').default = float(value[0]) except: order_value = [0] message = message + ' Average Order Value ' pass # return sb_traceback(request) # raise Exception('Permission Denied') try: cart_abandonment_rate = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '100daysAgo', 'endDate': 'today' }], 'metrics': [{ 'expression': 'ga:goalAbandonRateAll' }], }] }).execute() for ccar in cart_abandonment_rate['reports']: for cart_rate in ccar['data']['totals']: cart_vals = cart_rate['values'] # Profile._meta.get_field('current_shopping_cart_abandonment_rate').default = float(cart_vals[0]) # Profile._meta.get_field('optimal_shopping_cart_abandonment_rate').default = float(cart_vals[0]) except: cart_vals = [0] message = message + ' Cart Abandonment Rate ' pass # return sb_traceback(request) # raise Exception('Permission Denied') try: revenue_per_user = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '100daysAgo', 'endDate': 'today' }], 'metrics': [{ 'expression': 'ga:revenuePerUser' }], # "dimensions":[{"name":"ga:transactionId"}], }] }).execute() for crpu in revenue_per_user['reports']: for revenue in crpu['data']['totals']: revenue_vals = revenue['values'] # Profile._meta.get_field('current_revenue_per_user').default = float(vals[0]) # Profile._meta.get_field('optimal_revenue_per_user').default = float(vals[0]) except: revenue_vals = [0] message = message + ' Revenue Per User ' pass # return sb_traceback(request) # raise Exception('Permission Denied') try: traffic = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '100daysAgo', 'endDate': 'today' }], 'metrics': [{ 'expression': 'ga:organicSearches' }], "dimensions": [{ "name": "ga:month" }], }] }).execute() for current_montn_traffic in traffic['reports']: for dimension in current_montn_traffic['data']['rows']: pass a = max(int(d) for d in dimension['dimensions']) for traffic in current_montn_traffic['data']['totals']: traffic_vals = cart_rate['values'] # Profile._meta.get_field('optimal_traffic').default = float(traffic_vals[0]) # Analytics matrics of Traffic This Month traffic_this_month = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '100daysAgo', 'endDate': 'today' }], 'metrics': [{ 'expression': 'ga:organicSearches' }], "dimensions": [{ "name": "ga:month" }], "dimensionFilterClauses": [{ "filters": [{ "dimensionName": "ga:month", "operator": "EXACT", "expressions": ["0%s" % a] }] }] }] }).execute() for cttm in traffic_this_month['reports']: for traffic in cttm['data']['totals']: traffic_this_vals = traffic['values'] # Profile._meta.get_field('traffic_this_month').default = float(vals[0]) except: traffic_this_vals = [0] message = message + ' Traffic This Month ' pass # return sb_traceback(request) # raise Exception('Permission Denied') try: # Analytics matrics of Traffic Last Month traffic_last_month = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '30daysAgo', 'endDate': 'today' }], 'metrics': [{ 'expression': 'ga:organicSearches' }], # "dimensions":[{"name":"ga:month"}], }] }).execute() for ctlm in traffic_last_month['reports']: for traffic_last in ctlm['data']['totals']: traffic_last_vals = traffic_last['values'] # Profile._meta.get_field('traffic_last_month').default = float(last_vals[0]) except: traffic_last_vals = [0] message = message + ' Traffic Last Month ' pass # return sb_traceback(request) # raise Exception('Permission Denied') try: # Analytics matrics of Revenue This Month revenue_this_month = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '30daysAgo', 'endDate': 'today' }], 'metrics': [{ 'expression': 'ga:transactionRevenue' }], "dimensions": [{ "name": "ga:month" }], }] }).execute() for crtm in revenue_this_month['reports']: for revenue in crtm['data']['totals']: revenue_this_vals = revenue['values'] # Profile._meta.get_field('revenue_this_month').default = float(vals[0]) except: revenue_this_vals = [0] message = message + ' Revenue This Month ' pass # return sb_traceback(request) # raise Exception('Permission Denied') try: # Analytics matrics of Revenue Last Month revenue_last_month = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '60daysAgo', 'endDate': '30daysago' }], 'metrics': [{ 'expression': 'ga:transactionRevenue' }], # "dimensions":[{"name":"ga:month"}], }] }).execute() for crlm in revenue_last_month['reports']: for revenue in crlm['data']['totals']: revenue_last_vals = revenue['values'] # Profile._meta.get_field('revenue_last_month').default = float(vals[0]) except: revenue_last_vals = [0] message = message + ' Revenue Last Month ' pass # return sb_traceback(request) # raise Exception('Permission Denied') try: optimal_revenue = service.reports().batchGet( body={ 'reportRequests': [{ 'viewId': profile1, 'dateRanges': [{ 'startDate': '30daysAgo', 'endDate': 'today' }], 'metrics': [{ 'expression': 'ga:transactionRevenue' }], # "dimensions":[{"name":"ga:month"}], }] }).execute() for optrev in optimal_revenue['reports']: for revenue in optrev['data']['totals']: optimal_revenue_vals = revenue['values'] # Profile._meta.get_field('optimal_revenue').default = float(vals[0]) except: optimal_revenue_vals = [0] message = message + ' Optimal Revenue ' pass # return sb_traceback(request) # raise Exception('Permission Denied') profile = Profile.objects.filter(user_id=request.user.id).first() # if not UserTaskHistory: # guide = Guide.objects.get(pk=8) # steps = Step.objects.filter(guide=guide.id).first() # tasks = Task.objects.filter(step=steps).first() # else: # pass user = User.objects.get(id=request.user.id) if not profile: pr = Profile.objects.create( current_conversion_rate=float(conversion_vals[0]), optimal_conversion_rate=float(conversion_vals[0]), current_bounce_rate=float(bounce_vals[0]), optimal_bounce_rate=float(bounce_vals[0]), current_average_order_value=float(order_value[0]), optimal_average_order_value=float(order_value[0]), current_shopping_cart_abandonment_rate=float(cart_vals[0]), optimal_shopping_cart_abandonment_rate=float(cart_vals[0]), traffic_last_month=float(traffic_last_vals[0]), traffic_this_month=float(traffic_this_vals[0]), optimal_traffic=float(revenue_this_vals[0]), revenue_last_month=float(revenue_last_vals[0]), revenue_this_month=float(conversion_vals[0]), optimal_revenue=float(optimal_revenue_vals[0]), current_revenue_per_user=float(revenue_vals[0]), optimal_revenue_per_user=float(revenue_vals[0]), current_user_name=request.user.username, user_id=user, connected=True, accountid=account, ) pr.save() else: Profile.objects.filter(user_id=request.user.id).update( current_conversion_rate=float(conversion_vals[0]), optimal_conversion_rate=float(conversion_vals[0]), current_bounce_rate=float(bounce_vals[0]), optimal_bounce_rate=float(bounce_vals[0]), current_average_order_value=float(order_value[0]), optimal_average_order_value=float(order_value[0]), current_shopping_cart_abandonment_rate=float(cart_vals[0]), optimal_shopping_cart_abandonment_rate=float(cart_vals[0]), traffic_last_month=float(traffic_last_vals[0]), traffic_this_month=float(traffic_this_vals[0]), optimal_traffic=float(revenue_this_vals[0]), revenue_last_month=float(revenue_last_vals[0]), revenue_this_month=float(conversion_vals[0]), optimal_revenue=float(optimal_revenue_vals[0]), current_revenue_per_user=float(revenue_vals[0]), optimal_revenue_per_user=float(revenue_vals[0]), current_user_name=request.user.username, connected=True, accountid=account, ) profile = Profile.objects.filter(user_id=request.user.id).first() # GET REFERRAL OBJECT OF THIS USER url = '%s://%s/i/%s' % ( request.scheme, request.META['HTTP_HOST'], profile.profilelink.short ) if profile and profile.profilelink and profile.profilelink.short else '' profilerank = profile.profilerank if profile else None message = '' if message == 'Not have permission of' else message return render( request, 'settings/settings.html', { 'profile': profile, 'profilerank': profilerank, 'url': url, 'message': message })
https://cloud.google.com/sdk and run `gcloud beta auth application-default login`. For more information, see https://developers.google.com/identity/protocols/application-default-credentials 3. Install the Python client library for Google APIs by running `pip install --upgrade google-api-python-client` """ from pprint import pprint from googleapiclient import discovery from oauth2client.client import GoogleCredentials from time import sleep credentials = GoogleCredentials.get_application_default() service = discovery.build('sqladmin', 'v1beta4', credentials=credentials) # Project ID of the project that contains the instance. project = 'able-cogency-234306' # TODO: Update placeholder value. # Cloud SQL instance ID. This does not include the project ID. instance = 'testddd' # TODO: Update placeholder value. instances_import_request_body = { "importContext": { "database": "recommendation_spark", "fileType": "CSV", "uri": "gs://able-cogency-234306/tmp/accommodation.csv", "csvImportOptions": {
def provide_gce_cluster(nodes_num, algorithm): # 1. spin machines [configure daemons will be run by the startup script on every node] credentials = GoogleCredentials.get_application_default() gce = discovery.build("compute", "v1", credentials=credentials) config_dir = "config/" config_file_template = "provision_node_{}_config.json" zone_operations, vm_ids, cluster = [], [], [] for i in range(1, nodes_num + 1): vm_ids.append("vm-node-{}-{}".format(i, get_random_string(6))) metadata = {"bucket": GCS_BUCKET, "clusterConfig": config_dir + config_file_template.format(vm_ids[-1]), "myid": vm_ids[-1]} print("Starting a new virtual machine with name {}".format(vm_ids[-1])) zone_operations.append(create_instance(gce, vm_ids[-1], metadata, "gce-startup-script.sh")) # vm creation is run asynchronously: check that the operation is really completed for (i, zone) in [(i, zone_op["name"]) for (i, zone_op) in enumerate(zone_operations)]: print("Checking if virtual machine {} is ready...".format(vm_ids[i])) while True: result = gce.zoneOperations().get(project=GCP_PROJECT_ID, zone=GCE_ZONE_ID, operation=zone).execute() if result["status"] == "DONE": print("Virtual machine {} is ready.".format(vm_ids[i])) result = gce.instances().get(project=GCP_PROJECT_ID, zone=GCE_ZONE_ID, instance=vm_ids[i]).execute() new_node = { "id": i + 1, "address": result["networkInterfaces"][0]["accessConfigs"][0]["natIP"], # ephemeral external IP "port": CONSENSUS_ALGORITHM_PORT, "rpcPort": NETWORK_MANAGER_PORT, "interface": "eth0", "vmID": vm_ids[i], "internalIP": result["networkInterfaces"][0]["networkIP"] # internal IP } print("Adding node to the configuration: {}".format(new_node)) cluster.append(new_node) break sleep(2) # ✓ 1. spin machines # 1.1 allow network traffic on VMs if needed try: gce.firewalls().get(project=GCP_PROJECT_ID, firewall=GCE_FIREWALL_RULE_NAME).execute() print("Firewall rule to allow traffic already exists.") except: # rule does not exist: create it print("Creating firewall rule to allow traffic...") firewall_rule = { "description": "Allow traffic on every TCP/UDP port", "allowed": [{"IPProtocol": "tcp", "ports": ["1-65535"]}, {"IPProtocol": "udp", "ports": ["1-65535"]}, {"IPProtocol": "icmp"}], "name": GCE_FIREWALL_RULE_NAME, } gce.firewalls().insert(project=GCP_PROJECT_ID, body=firewall_rule).execute() print("Firewall rule to allow traffic created: {}.".format(firewall_rule)) # ✓ 1.1 allow network traffic on VMs # 1.2 wait for startup scripts print("Waiting for startup scripts to be completed on VMs...") gcs = discovery.build('storage', 'v1', credentials=credentials) for vm_id_file in vm_ids: while not gcs_file_exists(gcs, vm_id_file): # acknowledge file created by startup script at the end sleep(2) gcs.objects().delete(bucket=GCS_BUCKET, object=vm_id_file).execute() # clean up # ✓ 1.2 wait for startup scripts # 1.3 provide node-specific configuration files [via configure daemons] configure_daemons = [rpcClient('http://{}:{}'.format(node["address"], CONFIGURE_DAEMON_PORT)) for node in cluster] for node in cluster: node_config_file = "/tmp/" + config_file_template.format(node["vmID"]) with open(node_config_file, "w") as out_f: if algorithm == "rethinkdb": node_conf = get_node_config(cluster, node, "gce", additional_ports=[GCE_RETHINKDB_PORTS["cluster_port"]]) else: node_conf = get_node_config(cluster, node, "gce") json.dump(node_conf, out_f, indent=4) upload_object(gcs, node_config_file, config_dir + config_file_template.format(node["vmID"])) # send file to VM for configure_daemon in configure_daemons: configure_daemon.download_node_config() # ✓ 1.3 provide node-specific configuration files # 2. run algorithm [via configure daemons] test_daemon_endpoint = cluster[0]["address"] + ":" + str(TEST_DAEMON_PORT) # arbitrarily run the test daemon on first node print("Going to run algorithm {} on cluster...".format(algorithm)) if algorithm == "pso": configure_pso_gce(cluster, configure_daemons, test_daemon_endpoint) elif algorithm == "paxos": configure_paxos_gce(cluster, configure_daemons, test_daemon_endpoint) elif algorithm == "rethinkdb": configure_rethinkdb_gce(cluster, configure_daemons) elif algorithm == "datastore": pass # ✓ 2. run algorithm [via configure daemons] # 3. run network managers [via configure daemons] print("Running network manager on every node...") for node in cluster: configure_daemons[node["id"] - 1].run_network_manager() # this rpc will download the node-specific configuration file from gs and run the network manager # each node can discover its configuration file by querying its own metadata print("Network manager active on {}:{}".format(node["address"], str(node["rpcPort"]))) # ✓ 3. run network managers # 4. run test daemon [via configure daemon] print("Running the test daemon on {}...".format(test_daemon_endpoint)) configure_daemons[0].run_test_daemon(algorithm, cluster[0]["port"]) print("Test daemon active on {}".format(test_daemon_endpoint)) # ✓ 4. run test daemon print("Provisioning completed: cluster is ready to execute tests on {} at {}".format(algorithm, test_daemon_endpoint)) cluster_config_file = "masterConfig.json" # this file will be used to tear down the cluster with open(cluster_config_file, "w") as out_f: json.dump({ "mode": "gce", "algorithm": algorithm, "testDaemon": test_daemon_endpoint, "nodes": cluster }, out_f, indent=4) print("Cluster configuration saved in file {}.\n".format(cluster_config_file)) print("To perform a test run: test_executor.py {} test_file_name".format(cluster_config_file)) print("To stop the cluster run: tear_down.py {}".format(cluster_config_file))
def deploy_minikube(args): """Create a VM and setup minikube.""" credentials = GoogleCredentials.get_application_default() gce = discovery.build( "compute", "v1", credentials=credentials, cache_discovery=False) instances = gce.instances() body = { "name": args.vm_name, "machineType": "zones/{0}/machineTypes/n1-standard-16".format(args.zone), "disks": [ { "boot": True, "initializeParams": { "sourceImage": "projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts", "diskSizeGb": 100, "autoDelete": True, }, }, ], "networkInterfaces": [ { "accessConfigs": [ { "name": "external-nat", "type": "ONE_TO_ONE_NAT", }, ], "network": "global/networks/default", }, ], } request = instances.insert(project=args.project, zone=args.zone, body=body) response = None try: response = request.execute() print("done") except errors.HttpError as e: if not e.content: raise content = json.loads(e.content) if content.get("error", {}).get("code") == requests.codes.CONFLICT: # We don't want to keep going so we reraise the error after logging # a helpful error message. logging.error( "Either the VM or the disk %s already exists in zone " "%s in project %s ", args.vm_name, args.zone, args.project) raise else: raise op_id = response.get("name") final_op = vm_util.wait_for_operation(gce, args.project, args.zone, op_id) logging.info("Final result for insert operation: %s", final_op) if final_op.get("status") != "DONE": raise ValueError("Insert operation has status %s", final_op.get("status")) if final_op.get("error"): message = "Insert operation resulted in error %s".format( final_op.get("error")) logging.error(message) raise ValueError(message) # Locate the install minikube script. install_script = os.path.join( os.path.dirname(__file__), "install_minikube.sh") if not os.path.exists(install_script): logging.error("Could not find minikube install script: %s", install_script) vm_util.wait_for_vm(args.project, args.zone, args.vm_name) vm_util.execute_script(args.project, args.zone, args.vm_name, install_script) # Copy the .kube and .minikube files to test_dir target = "~/.kube" full_target = "{0}:{1}".format(args.vm_name, target) logging.info("Copying %s to %s", target, args.test_dir) util.run([ "gcloud", "compute", "--project=" + args.project, "scp", "--recurse", full_target, args.test_dir, "--zone=" + args.zone ]) # The .minikube directory contains some really large ISO and other files that we don't need; so we # only copy the files we need. minikube_dir = os.path.join(args.test_dir, ".minikube") try: os.makedirs(minikube_dir) except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir(minikube_dir): pass else: raise for target in ["~/.minikube/*.crt", "~/.minikube/client.key"]: full_target = "{0}:{1}".format(args.vm_name, target) logging.info("Copying %s to %s", target, minikube_dir) util.run([ "gcloud", "compute", "--project=" + args.project, "scp", "--recurse", full_target, minikube_dir, "--zone=" + args.zone ]) config_path = os.path.join(args.test_dir, ".kube", "config") modify_minikube_config(config_path, minikube_dir)
def getSpreadsheetAuth(): auth.authenticate_user() spreadsheetAuth = gspread.authorize( GoogleCredentials.get_application_default()) return spreadsheetAuth
def get_credentials(): """Get the Google credentials needed to access our services.""" credentials = GoogleCredentials.get_application_default() if credentials.create_scoped_required(): credentials = credentials.create_scoped(SCOPES) return credentials
def google_compute(): credentials = GoogleCredentials.get_application_default() return discovery.build('compute', 'v1', credentials=credentials)
def get_client(): """Creates Pub/Sub client and returns it.""" if not hasattr(client_store, 'client'): client_store.client = get_client_from_credentials( GoogleCredentials.get_application_default()) return client_store.client
def deploy(): # See: https://cloud.google.com/ai-platform/prediction/docs/regional-endpoints#python json_data = request.get_json() region = 'us-central1' runtime_version = '2.2' python_version = '3.7' deployment_uri = None model_name = 'babyweight' version_name = None for key in json_data.keys(): if key == 'region': region = json_data[key] elif key == 'runtimeVersion': runtime_version = json_data[key] elif key == 'pythonVersion': python_version = json_data[key] elif key == 'deploymentUri': deployment_uri = json_data[key] elif key == 'modelName': model_name = json_data[key] elif key == 'versionName': version_name = json_data[key] if deployment_uri is None or version_name is None: resp = { 'message': 'Option deploymentUri or versionName is not specified.' } return resp, 500 if region == 'us-central1': endpoint = 'https://ml.googleapis.com' else: endpoint = 'https://{}-ml.googleapis.com'.format(region) client_options = ClientOptions(api_endpoint=endpoint) credentials = GoogleCredentials.get_application_default() api = discovery.build('ml', 'v1', credentials=credentials, cache_discovery=False, client_options=client_options) api_request = api.projects().models().get( name='projects/{}/models/{}'.format(PROJECT_ID, model_name)) try: resp = api_request.execute() except googleapiclient.errors.HttpError as err: # Create model request_body = {'name': model_name} api_request = api.projects().models().create( parent='projects/{}'.format(PROJECT_ID), body=request_body) api_request.execute() request_body = { 'name': version_name, 'deploymentUri': deployment_uri, 'runtimeVersion': runtime_version, 'pythonVersion': python_version } api_request = api.projects().models().versions().create( parent='projects/{}/models/{}'.format(PROJECT_ID, model_name), body=request_body) try: resp = api_request.execute() except googleapiclient.errors.HttpError as err: resp = {'message': err._get_reason()} return resp, 500 return resp, 200
def train(): json_data = request.get_json() scale_tier = 'BASIC_GPU' region = 'us-central1' runtime_version = '2.2' python_version = '3.7' data_dir = None job_dir = None num_train_examples = '60000000' num_eval_examples = '50000' num_evals = '100' learning_rate = '0.0001' for key in json_data.keys(): if key == 'scaleTier': scale_tier = json_data[key] elif key == 'region': region = json_data[key] elif key == 'runtimeVersion': runtime_version = json_data[key] elif key == 'pythonVersion': python_version = json_data[key] elif key == 'dataDir': data_dir = json_data[key] elif key == 'jobDir': job_dir = json_data[key] elif key == 'numTrainExamples': num_train_examples = str(json_data[key]) elif key == 'numEvalExamples': num_eval_examples = str(json_data[key]) elif key == 'numEvals': num_evals = str(json_data[key]) elif key == 'learningRate': learning_rate = str(json_data[key]) if data_dir is None or job_dir is None: resp = {'message': 'Option dataDir or jobDir is not specified.'} return resp, 500 with tempfile.TemporaryDirectory() as tmpdir: id_string = str(uuid.uuid4()) job_id = 'train-babyweight-{}'.format(id_string).replace('-', '_') job_dir = os.path.join(job_dir, id_string) clone_cmd = 'cd {}; git init; git config core.sparsecheckout true;'\ 'git remote add origin {};'\ 'echo {} > .git/info/sparse-checkout;'\ 'git pull origin {}' clone_cmd = clone_cmd.format(tmpdir, GIT_REPO, MODEL_PATH, BRANCH) subprocess.run(clone_cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) train_dir = os.path.join(tmpdir, MODEL_PATH) subprocess.run('cd {};python3 setup.py sdist'.format(train_dir), shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) package_file = os.path.join(train_dir, 'dist', 'trainer-0.0.0.tar.gz') package = '{}/trainer-0.0.0.tar.gz'.format(job_dir) subprocess.run('gsutil cp {} {}'.format(package_file, package), shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) training_inputs = { 'scaleTier': scale_tier, 'packageUris': [package], 'pythonModule': 'trainer.task', 'region': region, 'jobDir': job_dir, 'runtimeVersion': runtime_version, 'pythonVersion': python_version, 'args': [ '--data-dir', data_dir, '--num-train-examples', num_train_examples, '--num-eval-examples', num_eval_examples, '--num-evals', num_evals, '--learning-rate', learning_rate ] } job_spec = {'jobId': job_id, 'trainingInput': training_inputs} credentials = GoogleCredentials.get_application_default() api = discovery.build('ml', 'v1', credentials=credentials, cache_discovery=False) api_request = api.projects().jobs().create( body=job_spec, parent='projects/{}'.format(PROJECT_ID)) resp = None try: resp = api_request.execute() except googleapiclient.errors.HttpError as err: resp = {'message': err._get_reason()} return resp, 500 return resp, 200
def task_manager_scope(session): credentials = GoogleCredentials.get_application_default() ml = discovery.build('ml', 'v1', credentials=credentials) projectID = 'projects/{}'.format(settings.GOOGLE_PROJECT_NAME) project = get_current_project(session=session) version = get_current_version(session=session) machine_learning_settings = get_ml_settings(session=session, version=version) Images_db = session.query(Image).filter_by(version_id=version.id, is_test_image=True) REGION = "us-central1" RUNTIME_VERSION = "1.2" modelName = "a_" + str(project.id) versionName = "a_" + str(version.id) + "_" + str( machine_learning_settings.ml_compute_engine_id) versionName += "_" + str(machine_learning_settings.re_train_id) modelVersionName = '{}/models/{}/versions/{}'.format( projectID, modelName, versionName) gcs = storage.Client() bucket = gcs.get_bucket(settings.CLOUD_STORAGE_BUCKET) filenames = [] root_dir = str(project.id) + "/" + str(version.id) + "/" for image in Images_db: #print(image.is_test_image, file=sys.stderr) if image.soft_delete != True: filenames.append(root_dir + "images/" + str(image.id)) break Rows = [] Images = [] print("len(filenames):", len(filenames), file=sys.stderr) for file in filenames: blob = bucket.blob(file) image = blob.download_as_string() # Resize image = scipy.misc.imread(BytesIO(image)) if image is None: raise IOError("Could not open") # TODO BETTER WAY #image = scipy.misc.imresize(image, (640, 960)) temp = tempfile.mkdtemp() new_temp_filename = temp + "/resized.jpg" scipy.misc.imsave(new_temp_filename, image) # Otherwise have strange byte issues blob = bucket.blob(file + "_test_resized") blob.upload_from_filename(new_temp_filename, content_type="image/jpg") image = blob.download_as_string() encoded_contents = base64.b64encode(image).decode('UTF-8') row = {'b64': encoded_contents} Rows.append(row) Images.append(image) output = {'instances': Rows} ml_request = ml.projects().predict(name=modelVersionName, body=output) PATH_TO_LABELS = root_dir + "ml/" + str( machine_learning_settings.ml_compute_engine_id ) + "/label_map.pbtext" label_map_blob = bucket.blob(PATH_TO_LABELS) label_map_data = label_map_blob.download_as_string() category_index = categoryMap(session=session) try: time0 = time.time() response = ml_request.execute() time1 = time.time() print("Time in seconds", (time1 - time0), file=sys.stderr) print(response, file=sys.stderr) for i in range(len(Images)): response = response['predictions'][i] # First one boxes = response['detection_boxes'] scores = response['detection_scores'] classes = response['detection_classes'] boxes = np.array(boxes) scores = np.array(scores) classes = np.array(classes, dtype=int) print(classes, file=sys.stderr) image_np = scipy.misc.imread(BytesIO(Images[i])) # Handle gray scale if len(image_np.shape) == 2: image_np = np.stack((image_np, ) * 3, axis=2) print(image_np.shape) visualization_utils.visualize_boxes_and_labels_on_image_array( image_np, boxes, classes, scores, category_index, use_normalized_coordinates=True, min_score_thresh=.3, line_thickness=2) blob = bucket.blob(root_dir + "test_inference_out/" + str(i) + "_.jpg") temp = tempfile.mkdtemp() new_temp_filename = temp + "/inference_" + str(i) + "_.jpg" scipy.misc.imsave(new_temp_filename, image_np) blob.upload_from_filename(new_temp_filename, content_type="image/jpg") min_score_thresh = .05 for i in range(len(boxes)): if scores[i] > min_score_thresh: class_name = category_index[classes[i]]['name'] print(class_name, scores[i], file=sys.stderr) # TODO add pub sub messaging out = 'success' except errors.HttpError as EOFError: print('There was an error. Check the details:', file=sys.stderr) print(EOFError._get_reason(), file=sys.stderr) out = 'failed' t.cancel()
# importing required libraries import numpy as np import pandas as pd from tabula import read_pdf """I had shown here that how we can link the google spreadsheet and make changes without even going to sheet. For further details on how we can use various functions refer the following link 'https://gspread.readthedocs.io/en/latest/'. """ # This is the authenticatication code # We don't have to pay much attention on this code from google.colab import auth auth.authenticate_user() import gspread from oauth2client.client import GoogleCredentials gc = gspread.authorize(GoogleCredentials.get_application_default()) # Using the URL importing the googlesheet # This is an example where I have created a google sheet and put some data # The sheet is stored in the google drive (practice_spreadsheet) wb = gc.open_by_url('https://docs.google.com/spreadsheets/d/1SVM2TySNDLT2FOdE_ifmT4HBnvd6q6pflTeFmGpnz8k/edit#gid=0') # On which worksheet we want to work currently we will work with sheet 1 sheet = wb.worksheet('Sheet1') # Using all the data of the sheet data = sheet.get_all_values() # Converting our data into dataframe df = pd.DataFrame(data)
def get_vision_service(): credentials = GoogleCredentials.get_application_default() return discovery.build('vision', 'v1', credentials=credentials)
def __init__(self, project, zone, instance_group, port, task_type='worker', task_id=0, rpc_layer='grpc', credentials='default', service=None): """Creates a new GCEClusterResolver object. This takes in a few parameters and creates a GCEClusterResolver project. It will then use these parameters to query the GCE API for the IP addresses of each instance in the instance group. Args: project: Name of the GCE project. zone: Zone of the GCE instance group. instance_group: Name of the GCE instance group. port: Port of the listening TensorFlow server (default: 8470) task_type: Name of the TensorFlow job this GCE instance group of VM instances belong to. task_id: The task index for this particular VM, within the GCE instance group. In particular, every single instance should be assigned a unique ordinal index within an instance group manually so that they can be distinguished from each other. rpc_layer: The RPC layer TensorFlow should use to communicate across instances. credentials: GCE Credentials. If nothing is specified, this defaults to GoogleCredentials.get_application_default(). service: The GCE API object returned by the googleapiclient.discovery function. (Default: discovery.build('compute', 'v1')). If you specify a custom service object, then the credentials parameter will be ignored. Raises: ImportError: If the googleapiclient is not installed. """ self._project = project self._zone = zone self._instance_group = instance_group self._task_type = task_type self._task_id = task_id self._rpc_layer = rpc_layer self._port = port self._credentials = credentials if credentials == 'default': if _GOOGLE_API_CLIENT_INSTALLED: self._credentials = GoogleCredentials.get_application_default() if service is None: if not _GOOGLE_API_CLIENT_INSTALLED: raise ImportError( 'googleapiclient must be installed before using the ' 'GCE cluster resolver') self._service = discovery.build('compute', 'v1', credentials=self._credentials) else: self._service = service
def _get_storage_service(credentials): """Get a storage client using the provided credentials or defaults.""" if credentials is None: credentials = GoogleCredentials.get_application_default() return discovery.build('storage', 'v1', credentials=credentials)
def data_processor(job_type): status = 200 message = 'Prcoess Complete ' startTime = datetime.datetime.now() lock_file = True log.info('BUCKET_NAME') log.info(BUCKET_NAME) log.info('ARCHIVE_BUCKET_NAME') log.info(ARCHIVE_BUCKET_NAME) try: bucket = BUCKET_NAME archive_bucket = ARCHIVE_BUCKET_NAME random_number = binascii.hexlify(os.urandom(32)).decode() log.info(' RANDOM NUMBER --- {0}'.format(random_number)) # Get the application default credentials. When running locally, these are # available after running `gcloud init`. When running on compute # engine, these are available from the environment. credentials = GoogleCredentials.get_application_default() # Construct the service object for interacting with the Cloud Storage API - # the 'storage' service, at version 'v1'. # You can browse other available api services and versions here: # https://developers.google.com/api-client-library/python/apis/ service = discovery.build('storage', 'v1', credentials=credentials) # Make a request to buckets.get to retrieve a list of objects in the # specified bucket. req = service.buckets().get(bucket=bucket) resp = req.execute() # print(json.dumps(resp, indent=2)) # Create a request to objects.list to retrieve a list of objects. fields_to_return = \ 'nextPageToken,items(name,size,contentType,metadata(my-key))' req = service.objects().list(bucket=bucket, fields=fields_to_return) file_count = 0 log.info('Process {0} Start time --- {1}'.format(bucket, startTime)) # If you have too many items to list in one request, list_next() will # automatically handle paging with the pageToken. while req: resp = req.execute() # print(json.dumps(resp, indent=2)) if len(resp) == 0: log.info( '############################################################################################' ) log.info('--------- THE BUCKET LIST IS EMPTY --------------') log.info('--------- NO FILES TO PROCESS --------------') log.info(resp) log.info( '############################################################################################' ) else: get_filenames(resp, service, random_number) req = service.objects().list_next(req, resp) except Exception as e: log.error(' Error in getting Bucket Details - {0}'.format(e[0])) message = e[0] status = 500 endTime = datetime.datetime.now() log.info('Process End time --- {0}'.format(endTime)) elapsedTime = endTime - startTime time = 'Total Time to Process all the files -- {0}'.format( divmod(elapsedTime.total_seconds(), 60)) log.info(time) log.info(' ARGS PASSED --- {0}'.format(job_type)) if job_type == 'now': set_scheduler(os.environ.get('SCHEDULER_HOUR'), os.environ.get('SCHEDULER_MIN')) response = dict(data=json.dumps(message), status=status, time=time) return response
def get_a_google_credentials_object(self): return GoogleCredentials(None, None, None, None, None, None, None, None)
def mock_run_flow(flow, storage, args): return GoogleCredentials.get_application_default()
def __connect(self): credentials = GoogleCredentials.get_application_default() self.service = discovery.build('compute', 'v1', credentials=credentials)
def transcribe_file(speech_file, sample_rate, parser): # authenticate with google using credentials in JSON file credentials = GoogleCredentials.get_application_default() client = speech.SpeechClient() # open audio file with io.open(speech_file, 'rb') as audio_file: content = audio_file.read() # send audio file to recognizer audio = types.RecognitionAudio(content=content) config = types.RecognitionConfig( encoding=enums.RecognitionConfig.AudioEncoding.LINEAR16, sample_rate_hertz=sample_rate, language_code='en-US') utterances = [] marker_coordinates = [] # send audio to recognizer response = client.recognize(config, audio) # Each result is for a consecutive portion of the audio. # #Iterate through them to get the transcripts for the entire audio file. for result in response.results: # The first alternative is the most likely one for this portion. application.logger.info('Transcript: {}'.format( result.alternatives[0].transcript)) # send transcribed text to geo parse services_ lat, long, address, geo_json = geo_parse( result.alternatives[0].transcript, parser) # if we get back any geo parse results store as markers for map and append to utterance text if bool(address) and lat is not None: my_point = Point((float(long), float(lat))) my_feature = Feature(geometry=my_point, properties={ 'title': 'Geo Location: {}'.format(address), 'description': 'Transcript: {}'.format( result.alternatives[0].transcript), 'marker-size': 'large', 'marker-color': '#FF0000', 'marker-symbol': 'police' }) # Insert record into DB row = [ str(uuid.uuid4()), float(long), float(lat), 'Geo Location: {}'.format(address), 'Transcript: {}'.format(result.alternatives[0].transcript), str(datetime.datetime.now()) ] db.InsertRow(tablename='security_events', row=row) #use this line to temp export the table for debugging #db.ExportCSV(tablename='security_events') # store lat long as marker coordinates for map marker_coordinates.append(my_feature) # store utterance with geo parsed address after for disaply utterances.append( 'Transcript: {}'.format(result.alternatives[0].transcript) + ' ( ' + '<em style="color:LightGray;">' + 'Geo Location: {}'.format(address) + '</em>' + ' )') elif bool(address) and lat is None: utterances.append( 'Transcript: {}'.format(result.alternatives[0].transcript) + ' ( ' + '<em style="color:LightGray;">' + 'Geo Location: {}'.format(address) + '</em>' + ' )') # if there are no geo parsed results just added text without address else: utterances.append('Transcript: {}'.format( result.alternatives[0].transcript)) return utterances, marker_coordinates
def __init__(self): # Connect to Google Cloud Compute Engine API using the environment's service # account. credentials = GoogleCredentials.get_application_default() self.compute = discovery.build('compute', 'v1', credentials=credentials)
def get(self): """ Uses the check_header parameters to retrieve the email of the already authenticated user. For JupyterHub + Inverting Proxy agent, use X-Inverting-Proxy-User-Id. For JupyterHub + Cloud IAP, X-Goog-IAP-JWT-Assertion. """ if self.authenticator.check_header == "X-Inverting-Proxy-User-Id": b64_email = self.request.headers.get( self.authenticator.check_header, "") # Provides a dummy email to try the code locally. if b64_email == "" and self.authenticator.dummy_email: self.log.info( f'Using a dummy email {self.authenticator.dummy_email}') b64_email = b64encode( self.authenticator.dummy_email.encode('ascii')) if b64_email == "": raise web.HTTPError( 401, f'Missing header {self.authenticator.check_header}') user_email = b64decode(b64_email).decode("ascii") self.log.info(f'user_email is {user_email}') elif self.authenticator.check_header == "X-Goog-IAP-JWT-Assertion": from googleapiclient import discovery from oauth2client.client import GoogleCredentials credentials = GoogleCredentials.get_application_default() compute = discovery.build('compute', 'v1', credentials=credentials, cache_discovery=False) request = compute.backendServices().get( project=self.authenticator.project_id, backendService=self.authenticator.backend_service_name) backend_service = request.execute() backend_service_id = backend_service['id'] self.log.info(f'''self.authenticator.check_header name is {self.authenticator.check_header}''') self.log.info(f'''self.authenticator.check_header value is {self.request.headers.get(self.authenticator.check_header, "")}''') self.log.info(f'''self.authenticator.backend_service_id is {backend_service_id}''') _, user_email, _ = validate_iap_jwt_from_compute_engine( self.request.headers.get(self.authenticator.check_header, ""), self.authenticator.project_number, backend_service_id) if not user_email: raise web.HTTPError(401, 'Can not verify the IAP authentication.') else: raise web.HTTPError(400, 'Mismatch Authentication method and Header.') username, _ = user_email.split("@") user = self.user_from_username(username) # JupyterHub doesn't set the value for that key for some reason. if not hasattr(user, 'json_escaped_name'): setattr(user, 'json_escaped_name', json.dumps(user.name)[1:-1]) self.log.info(f'username is {username}') self.log.info(f'user.name is {user.name}') self.set_login_cookie(user) self.write( self.render_template( self.authenticator.template_to_render, sync=True, user=user, next_url=self.get_next_url(user), ))
def main(event, context): credentials = GoogleCredentials.get_application_default() service = discovery.build('compute', 'v1', credentials=credentials) with open("config.yaml", 'r') as yamlfile: cfg = yaml.full_load(yamlfile) project_configs = cfg['project'] project = project_configs.get('project-id') region = project_configs.get('region') zone = project_configs.get('zone') bucket_configs = cfg['bucket'] bucket = bucket_configs.get('bucket-name') instance_configs = cfg['instance'] instance_prefix_name = instance_configs.get('instance-prefix-name') instance_name = instance_prefix_name + str(randint(0, 100)) machine_type = instance_configs.get('machine-type') startup_script = instance_configs.get('startup-script') startup_script = open(startup_script, 'r') sst = startup_script.read() disk_configs = cfg['disks'] snapShot_name = disk_configs.get('snapshot-name') config_vm = { "kind": "compute#instance", "name": instance_name, "zone": "projects/" + project + "/zones/" + zone, "machineType": "projects/" + project + "/zones/" + zone + "/machineTypes/" + machine_type, "displayDevice": { "enableDisplay": False }, "disks": [{ "kind": "compute#attachedDisk", "type": "PERSISTENT", "boot": True, "mode": "READ_WRITE", "autoDelete": True, "deviceName": instance_name, "source": "projects/" + project + "/zones/" + zone + "/disks/" + instance_name }], "metadata": { "kind": "compute#metadata", "items": [{ "key": "startup-script", "value": sst }, { "key": "bucket", "value": bucket }, { "key": "zone", "value": zone }, { "key": "instance_name", "value": instance_name }, { "key": "file_name", "value": event['name'] }] }, "canIpForward": False, "networkInterfaces": [{ "kind": "compute#networkInterface", "subnetwork": "projects/" + project + "/regions/" + region + "/subnetworks/default", "accessConfigs": [{ "kind": "compute#accessConfig", "name": "External NAT", "type": "ONE_TO_ONE_NAT", "networkTier": "PREMIUM" }], "aliasIpRanges": [] }], "description": "", "labels": {}, "scheduling": { "preemptible": False, "onHostMaintenance": "MIGRATE", "automaticRestart": True, "nodeAffinities": [] }, "deletionProtection": False, "reservationAffinity": { "consumeReservationType": "ANY_RESERVATION" }, "serviceAccounts": [{ "email": "*****@*****.**", "scopes": [ "https://www.googleapis.com/auth/servicecontrol", "https://www.googleapis.com/auth/service.management.readonly", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/monitoring.write", "https://www.googleapis.com/auth/trace.append", "https://www.googleapis.com/auth/devstorage.full_control" ] }] } create_disk(service, project, zone, instance_name, snapShot_name) request = service.instances().insert(project=project, zone=zone, body=config_vm) return request.execute()
from base64 import b64encode import googleapiclient.discovery from oauth2client.client import GoogleCredentials # Settings IMAGE_FILE = "text.png" CREDENTIALS_FILE = "credentials-googleapi.json" # Connect to Google cloud service credentials = GoogleCredentials.from_stream(CREDENTIALS_FILE) service = googleapiclient.discovery.build('vision', 'v1', credentials=credentials) # read file and covert to base64 encoding with open(IMAGE_FILE, "rb") as f: image_data = f.read() encoded_image_data = b64encode(image_data).decode('UTF-8') # Create the request object for Google Vision API batch_request = [{ 'image': { 'content': encoded_image_data }, 'features': [{ 'type': 'TEXT_DETECTION' }] # (asking for extracted text from the image back) }] request = service.images().annotate(body={'requests': batch_request})