def dummy_service_account(): global _DUMMY_SERVICE_ACCOUNT from google.oauth2.service_account import Credentials if _DUMMY_SERVICE_ACCOUNT is None: _DUMMY_SERVICE_ACCOUNT = Credentials.from_service_account_info( _SERVICE_ACCOUNT_JSON ) return _DUMMY_SERVICE_ACCOUNT
def import_sprint_tasks(sprint_tasks, google_sheet_key, google_sheet_index): scope = [ 'https://spreadsheets.google.com/feeds', 'https://www.googleapis.com/auth/drive' ] credentials = Credentials.from_service_account_file('client_secrets.json', scopes=scope) gc = gspread.authorize(credentials) wks = gc.open_by_key(google_sheet_key) worksheet = wks.get_worksheet(int(google_sheet_index)) if worksheet.row_values(1) == []: worksheet.append_row([ 'id', 'title', 'app', 'epic', 'sprint', 'status', 'point', 'import_time' ]) worksheet.append_rows(sprint_tasks)
def multifolderclone(source=None, dest=None, path='accounts', width=2): global account_count global drive global threads stt = time.time() accounts = glob.glob(path + '/*.json') if source == None: source = input("Source Folder ID Missing. Please enter Folder ID of source: ") else: if dest == None: dest = input("Destination Folder ID Missing. Please enter Folder ID of destination: ") else: while len(accounts) == 0: path = input("No service accounts found in current directory. Please enter the path where the accounts are located at: ") accounts = glob.glob(path + '/*.json') print('Copy from ' + source + ' to ' + dest + '.') print('View set to tree (' + str(width) + ').') pbar = progress.bar.Bar("Creating Drive Services", max=len(accounts)) for account in accounts: account_count += 1 credentials = Credentials.from_service_account_file(account, scopes=[ "https://www.googleapis.com/auth/drive" ]) drive.append(googleapiclient.discovery.build("drive", "v3", credentials=credentials)) pbar.next() pbar.finish() threads = threading.BoundedSemaphore(account_count) print('BoundedSemaphore with %d threads' % account_count) try: rcopy(source, dest, "root", "", width) except KeyboardInterrupt: print('Quitting') sys.exit() print('Complete.') hours, rem = divmod((time.time() - stt), 3600) minutes, sec = divmod(rem, 60) print("Elapsed Time:\n{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), sec))
def get_client(project: str, credentials: dict, credentials_secret: str = None) -> bigquery.Client: if credentials_secret is not None: warnings.warn( "The `credentials_secret` argument is deprecated. Use a `Secret` task " "to pass the credentials value at runtime instead.", UserWarning, ) creds = Secret(credentials_secret).get() credentials = Credentials.from_service_account_info(creds) project = project or credentials.project_id if credentials is not None: project = project or credentials.get("project") client = bigquery.Client(project=project, credentials=credentials) else: client = bigquery.Client(project=project) return client
def __init__(self): scopes = [ 'https://spreadsheets.google.com/feeds', 'https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive.file', 'https://www.googleapis.com/auth/drive' ] creds = Credentials.from_service_account_file('creds.json', scopes=scopes) self.gc = gspread.authorize(creds) self.eventsSpreadSheet = self.gc.open('VkEvents') self.questionsSpreadSheet = self.gc.open('VkEventQuestions') self.eventsWorkSheets = {} self.events = Events.Events() self.createEventsTable()
def setupCommand(databaseFileName: str, credentialsFileName: str, bucketName: str) -> None: # Verify credentials and bucket name credentialsJson = json.load(open(credentialsFileName, 'r')) credentials = Credentials.from_service_account_info(credentialsJson) client = storage.Client(project=credentials.project_id, credentials=credentials) client.get_bucket(bucketName) # Open database database = Database(databaseFileName) # Write credentials and bucket name to database database.setPair('credentials', json.dumps(credentialsJson)) database.setPair('bucketName', bucketName) # Close database database.commit() database.close()
def get_google_credentials(): """Obtain the Google credentials object explicitly.""" private_key = os.environ["GOOGLE_PRIVATE_KEY"] private_key_id = os.environ["GOOGLE_PRIVATE_KEY_ID"] signer = RSASigner.from_string(key=private_key, key_id=private_key_id) project_id = os.environ["GOOGLE_PROJECT_ID"] service_account_email = os.environ["GOOGLE_CLIENT_EMAIL"] scopes = ('https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/cloud-platform') token_uri = os.environ["GOOGLE_TOKEN_URI"] credentials = Credentials( signer=signer, service_account_email=service_account_email, token_uri=token_uri, scopes=scopes, project_id=project_id, ) return credentials
def __init__( self, ff_access_keys = None, google_api_key = None, s3UtilsInstance = None, extra_config = DEFAULT_GOOGLE_API_CONFIG ): """Authenticate with Google APIs and initialize sub-class instances.""" if s3UtilsInstance is None: self._s3Utils = s3_utils.s3Utils(env='data') # Google API Keys are stored on production bucket only ATM. else: self._s3Utils = s3UtilsInstance if google_api_key is None: self._api_key = self._s3Utils.get_google_key() if not self._api_key: raise Exception("Failed to get Google API key from S3.") else: self._api_key = google_api_key if not GoogleAPISyncer.validate_api_key_format(self._api_key): raise Exception("Google API Key is in invalid format.") self.extra_config = extra_config self.credentials = Credentials.from_service_account_info( self._api_key, scopes=self.extra_config.get('scopes', DEFAULT_GOOGLE_API_CONFIG['scopes']) ) # These are required only for POSTing/GETing data for TrackingInfo items. if ff_access_keys is None: ff_access_keys = self._s3Utils.get_access_keys() self.server = ff_access_keys['server'] self.access_key = { "key" : ff_access_keys['key'], "secret": ff_access_keys['secret'] } # Init sub-class objects self.analytics = GoogleAPISyncer.AnalyticsAPI(self) self.sheets = GoogleAPISyncer.SheetsAPI(self) self.docs = GoogleAPISyncer.DocsAPI(self)
def insert_schedule(): # サービスアカウントのJSONファイルパス GOOGLE_APPLICATION_CREDENTIALS=os.environ['GOOGLE_CREDENTIALS_FILE'] c = Credentials.from_service_account_file(GOOGLE_APPLICATION_CREDENTIALS) service = build('calendar', 'v3', credentials=c) # Call the Calendar API now = datetime.datetime.utcnow().isoformat() + 'Z' #calendar_id = 'primary' # データを取得したいGoogleカレンダーのIDを指定する calendar_id = os.environ['GOOGLE_CALENDAR_ID'] body = dict() body['summary'] = 'Test summary' body['start'] = {'timeZone': 'Asia/Tokyo', 'dateTime': '2020-03-09T11:00:00+09:00'} body['end'] = {'timeZone': 'Asia/Tokyo', 'dateTime': '2020-03-09T11:30:00+09:00'} body['location'] = 'Kobe' body['guestsCanSeeOtherGuests'] = False body['guestsCanInviteOthers'] = False insert_result = service.events().insert( calendarId=calendar_id, body=body ).execute() print(insert_result) events_result = service.events().list( calendarId=calendar_id, timeMin=now, maxResults=10, singleEvents=True, orderBy='startTime').execute() events = events_result.get('items', []) print(events) if not events: print('No upcoming events found.') for event_c in events: event_id = event_c['id'] start = event_c['start'].get('dateTime', event_c['start'].get('date')) print(event_id, start, event_c['summary']) return events
def main(args): # Load the json format key that you downloaded from the Google API # Console when you created your service account. For p12 keys, use the # from_p12_keyfile method of ServiceAccountCredentials and specify the # service account email address, p12 keyfile, and scopes. service_credentials = Credentials.from_service_account_file( args.service_account_key_path, scopes=[ 'https://www.googleapis.com/auth/admin.directory.device.chromebrowsers' ], subject=args.admin_email) try: http = google_auth_httplib2.AuthorizedHttp(service_credentials, http=Http()) extensions_list = {} base_request_url = 'https://admin.googleapis.com/admin/directory/v1.1beta1/customer/my_customer/devices/chromebrowsers' request_parameters = '' browsers_processed = 0 while True: print('Making request to server ...') data = json.loads( http.request(base_request_url + '?' + request_parameters, 'GET')[1]) browsers_in_data = len(data['browsers']) print('Request returned %s results, analyzing ...' % (browsers_in_data)) ComputeExtensionsList(extensions_list, data) browsers_processed += browsers_in_data if 'nextPageToken' not in data or not data['nextPageToken']: break print('%s browsers processed.' % (browsers_processed)) if (args.max_browsers_to_process is not None and args.max_browsers_to_process <= browsers_processed): print('Stopping at %s browsers processed.' % (browsers_processed)) break request_parameters = ('pageToken={}').format(data['nextPageToken']) finally: print('Analyze results ...') ExtensionListAsCsv(extensions_list, args.extension_list_csv) print("Results written to '%s'" % (args.extension_list_csv))
def sendResponse(JSONObject): if (JSONObject): credentials = Credentials.from_service_account_file("../intent.json") client = dialogflow_v2.SessionsClient(credentials=credentials) session = client.session_path('classify-intents-ujpxuu', 'Testing values') content = JSONObject text_input = dialogflow_v2.types.TextInput(text=content['inputText'], language_code="en") query_input = dialogflow_v2.types.QueryInput(text=text_input) response = client.detect_intent(session=session, query_input=query_input) print('Query text: {}'.format(response.query_result.query_text)) print('Detected intent: {} (confidence: {})\n'.format( response.query_result.intent.display_name, response.query_result.intent_detection_confidence)) confidence = float("{0:.2f}".format( response.query_result.intent_detection_confidence * 100)) if ('Default Welcome Intent' in response.query_result.intent.display_name or 'Default Fallback Intent' in response.query_result.intent.display_name): intentName = 'Others' else: intentName = response.query_result.intent.display_name intentName = intentName.lower().replace(" ", "_") result = { 'inputText': response.query_result.query_text, 'accountHead': intentName, 'confidence': confidence, 'outflow_tags': ["stationery", "office", "supplies"] } return result else: return "Request Failed."
def get_service_account_credentials(private_key): """DEPRECATED: Load service account credentials from key data or key path.""" import google.auth.transport.requests from google.oauth2.service_account import Credentials is_path = os.path.isfile(private_key) try: if is_path: with open(private_key) as f: json_key = json.loads(f.read()) else: # ugly hack: 'private_key' field has new lines inside, # they break json parser, but we need to preserve them json_key = json.loads(private_key.replace("\n", " ")) json_key["private_key"] = json_key["private_key"].replace( " ", "\n" ) if pandas.compat.PY3: json_key["private_key"] = bytes(json_key["private_key"], "UTF-8") credentials = Credentials.from_service_account_info(json_key) credentials = credentials.with_scopes(SCOPES) # Refresh the token before trying to use it. request = google.auth.transport.requests.Request() credentials.refresh(request) return credentials, json_key.get("project_id") except (KeyError, ValueError, TypeError, AttributeError): raise pandas_gbq.exceptions.InvalidPrivateKeyFormat( "Detected private_key as {}. ".format( "path" if is_path else "contents" ) + "Private key is missing or invalid. It should be service " "account private key JSON (file path or string contents) " 'with at least two keys: "client_email" and "private_key". ' "Can be obtained from: https://console.developers.google." "com/permissions/serviceaccounts" )
def __init__( self, project_id=None, service_account_json=None, language_code="en", raise_on_error=False, flip_text_and_intent=False, ): self.project_id = project_id self.language_code = language_code self.raise_on_error = raise_on_error self.flip_text_and_intent = flip_text_and_intent credentials = None if service_account_json: credentials = Credentials.from_service_account_file( service_account_json) self.session_client = dialogflow.SessionsClient( credentials=credentials)
def insert_to_google_sheet(values, sheet, worksheet, index): try: scopes = [ 'https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive' ] credentials = Credentials.from_service_account_file( 'client_secrets.json', scopes=scopes) google_client = gspread.authorize(credentials) sh = google_client.open(sheet) # for i in sh.worksheets(): # print(i.title) ws = sh.worksheet(worksheet) #start at 1 and 1 is col name. ws.insert_row(values, index=index + 2) # return sh.url except Exception as e: print('---upload to google drive failed---') print(e)
def get_dialog(text, session_id=None): project_id = 'dkl-bot-utvfiy' lang = 'ru-RU' try: credentials = Credentials.from_service_account_file( "google-credentials.json") session_client = dialogflow.SessionsClient(credentials=credentials) session = session_client.session_path(project_id, session_id) text_input = dialogflow.types.TextInput(text=text, language_code=lang) query_input = dialogflow.types.QueryInput(text=text_input) response = session_client.detect_intent(session=session, query_input=query_input) except: print('беда ваще') return response.query_result.fulfillment_text
def __init__(self, key: str): scopes = [ 'https://spreadsheets.google.com/feeds', 'https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive.file', 'https://www.googleapis.com/auth/drive' ] directory_path = os.path.dirname(os.path.abspath(__file__)) new_path = os.path.join(directory_path, 'credentials.json') credentials = ServiceAccountCredentials.from_json_keyfile_name( new_path, scopes) data = credentials.serialization_data data['token_uri'] = credentials.token_uri auth = Credentials.from_service_account_info(data, scopes=scopes) self.session = AuthorizedSession(auth) self.id = key self.fetched_data = {} self.requests = []
def _downloadFile(credentials: dict, bucket_name: str, key: str, file_name: str) -> str: """ Downloads a file from cloud storage. :param credentials: The Google cloud storage service credentials retrieved from the Kubernetes secret. :param bucket_name: The name of the bucket. :param key: The key to download the file from the cloud storage. :param file_name: The file that will be downloaded. :return: The location of the downloaded file. """ credentials = ServiceCredentials.from_service_account_info(credentials) gcs_client = StorageClient(credentials.project_id, credentials) bucket = gcs_client.get_bucket(bucket_name) logging.info("Going to download gcs://%s/%s", bucket_name, key) bucket.blob(key).download_to_filename(file_name) logging.info("Backup gcs://%s/%s downloaded to %s", bucket_name, key, file_name) return file_name
def recognize_audio_from_uri( uri: str, credential: Union[str, os.PathLike, None] = None, language_code: str = 'en-US', encoding: enums.RecognitionConfig.AudioEncoding = enums.RecognitionConfig. AudioEncoding.FLAC, sampling_rate_hertz: int = 44100, ) -> types.RecognizeResponse: """ Args: uri (str) : Cloud credential (str, os.PathLike, None) : language_code: encoding (enums.RecognitionConfig.AudioEncoding) : sampling_rate_hertz (int) : Returns: types.RecognizeResponse """ if credential is None: client = SpeechClient() else: credentials = Credentials.from_service_account_file( filename=credential) client = SpeechClient(credentials=credentials) config = types.RecognitionConfig(encoding=encoding, language_code=language_code, sample_rate_hertz=sampling_rate_hertz) audio = types.RecognitionAudio(uri=uri) try: result = client.recognize(config=config, audio=audio) except exceptions.InvalidArgument: print( 'cannot synchronize recognition. switched asynchronized recognition' ) operartion = client.long_running_recognize(config=config, audio=audio) result = operartion.result() return result
def BlobRemoveThread( blobRemoveQueue: queue.SimpleQueue, credentials: str, bucketName: str, isTrialRun: bool, logLevel: str, threadId: int ) -> None: # Set logging level and get logger setLogingLevel(logLevel) logger = logging.getLogger(__name__) # Get GCS bucket credentials = Credentials.from_service_account_info(json.loads(credentials)) client = storage.Client(project=credentials.project_id, credentials=credentials) bucket = client.get_bucket(bucketName) # Create stats variables ticToc = TicToc() while True: # Get task task = blobRemoveQueue.get() if task is None: break # Extract task name: str = task[0] ticToc.tic() if not isTrialRun: try: bucket.delete_blob(name) except: pass logger.info('× {}, elapsed: {:.3f}s, thread: {}'.format( name, ticToc.toc(), threadId ))
def generate_jwt_token_from_impersonated_account(service_account_info: Dict[str, str], audiences: str, issuer: str) -> Tuple[str, int]: """ Using a dictionary containing the information from a Google Cloud service account credentials file, this function impersonates the Company Information API master user service account and signs a JSON Web Token (JWT) used to authenticate the client when accessing the service. :param service_account_info: A dictionary containing all the information found in a Google Cloud service account credentials JSON file. :type service_account_info: Dict[str, str] :param audiences: The intended recipient of the JWT. Found in the Google Endpoint specification. For example, for the company information API, the recipient is 'company-information.api.willowlabs.ai' :type audiences: str :param issuer: The email address of the impersonated API master user service account. :type issuer: str :param jwt_lifetime: The length of time, in seconds, for which the created JWT is valid. :type jwt_lifetime: int :return: A tuple containing the JWT and a POSIX/Unix epoch-style timestamp indicating when the JWT expires. :rtype: Tuple[str, int] """ credentials = Credentials.from_service_account_info(service_account_info, scopes=["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/iam"]) if not credentials.valid: credentials.refresh(Request()) impersonated_credentials = ImpersonatedCredentials(source_credentials=credentials, target_principal=issuer, target_scopes=["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/iam"]) if not impersonated_credentials.valid: impersonated_credentials.refresh(Request()) signer = Signer(Request(), impersonated_credentials, impersonated_credentials.service_account_email) now = int(time.time()) expires = now + MAX_TOKEN_LIFETIME_SECS payload = { 'iat': now, 'exp': expires, 'aud': audiences, 'iss': issuer } return google.auth.jwt.encode(signer, payload).decode("utf-8"), expires
def _get_oauth_token_from_service_account_credentials(self): scopes = ["https://www.googleapis.com/auth/userinfo.email"] assert 'GOOGLE_APPLICATION_CREDENTIALS' in os.environ from google.auth.transport.requests import Request as GoogleAuthRequest from google.oauth2.service_account import Credentials as ServiceAccountCredentials logger.info("Found GOOGLE_APPLICATION_CREDENTIALS environment variable. " "Using service account credentials for authentication.") service_account_credentials_filename = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] if not os.path.isfile(service_account_credentials_filename): msg = 'File "{}" referenced by the GOOGLE_APPLICATION_CREDENTIALS environment variable does not exist' raise Exception(msg.format(service_account_credentials_filename)) credentials = ServiceAccountCredentials.from_service_account_file( service_account_credentials_filename, scopes=scopes ) r = GoogleAuthRequest() credentials.refresh(r) r.session.close() return credentials.token, credentials.expiry
def generaterandomdrive(): global accsf socket.setdefaulttimeout(600) acc_thread.acquire() random_acc = random.choice(accsf) while True: try: credentials = Credentials.from_service_account_file(random_acc, scopes=[ "https://www.googleapis.com/auth/drive" ]) random_drive = build("drive", "v3", credentials=credentials) except HttpError: random_acc = random.choice(accsf) print("#Error SA Error") else: break acc_thread.release() return random_drive
def fallback_ocr(image_path, input_type): ocr_credentials = os.environ['FALLBACK_OCR'] ocr_credentials = Credentials.from_service_account_file(ocr_credentials) result = {} print("Fallback_ocr: Text extraction") if input_type == "pdf": print("Fallback_OCR: extracting pages from pdf for further processing") images = convert_from_bytes(image_path) print("Fallback_OCR: extraction complete with " + str(len(images)) + " images") for index, image in enumerate(images): print("PAGE : ", index + 1) img_byte_arr = io.BytesIO() image.save(img_byte_arr, format='PNG') img_byte_arr = img_byte_arr.getvalue() temp = detect_document(img_byte_arr, ocr_credentials) result[index] = temp else: result = detect_document(image_path, ocr_credentials) return result
def init(self, core: BBotCore): """ Initializebot engine """ super().init(core) self.logger = BBotLoggerAdapter(logging.getLogger('dialogfl_cbe'), self, self.core) self.service_account = json.loads(self.dotbot.chatbot_engine['serviceAccount']) self.platform = None self.available_platforms = { 'google_assistant': 'ACTIONS_ON_GOOGLE', 'facebook': 'FACEBOOK', 'slack': 'SLACK', 'telegram': 'TELEGRAM', 'skype': 'SKYPE' } credentials = Credentials.from_service_account_info(self.service_account) self.session_client = dialogflow.SessionsClient(credentials=credentials)
def get_service(api_name, api_version, scopes, key_file_location): """Get a service that communicates to a Google API. Args: api_name: The name of the api to connect to. api_version: The api version to connect to. scopes: A list auth scopes to authorize for the application. key_file_location: The path to a valid service account JSON key file. Returns: A service that is connected to the specified API. """ credentials = Credentials.from_service_account_file(key_file_location, scopes=scopes) # credentials = ServiceAccountCredentials.from_json_keyfile_name( # key_file_location, scopes=scopes) # Build the service object. service = build(api_name, api_version, credentials=credentials) return service
def get_conn(self): if self.conn is None: params = self.get_connection(self.gcp_conn_id) extra = params.extra_dejson logger = logging.getLogger() logging_level = logger.getEffectiveLevel() logger.setLevel(logging.ERROR) try: if 'authorized_user_file' in extra[ 'extra__google_cloud_platform__key_path']: creds = CredentialsUser.from_authorized_user_file( extra['extra__google_cloud_platform__key_path']) else: creds = Credentials.from_service_account_file( extra['extra__google_cloud_platform__key_path']) self.project_id = creds.project_id self.conn = storage.Client(credentials=creds, project=self.project_id) finally: logger.setLevel(logging_level) return self.conn
def main(): filename = 'download.out' db_filename = 'data.db' days = 7 if settings.file_url: response = requests.get(settings.file_url) response.raise_for_status() with open(filename, 'wb') as f: f.write(response.content) else: assert settings.file_id credentials = Credentials.from_service_account_file('credentials.json') service = build('drive', 'v3', credentials=credentials) download_gdrive_file(service, settings.file_id, filename) unzip(filename, db_filename) raw_data = collect_data(db_filename, days) data = [ dict(value=tag, date=d) for d, tags in raw_data.items() for tag in tags ] exist.post('/attributes/custom/append/', json=data)
def addCalendar(title, times, description): creds = None # The file token.pickle stores the user's access and refresh tokens, and is # created automatically when the authorization flow completes for the first # time. if os.path.exists('token.pickle'): with open('token.pickle', 'rb') as token: creds = pickle.load(token) # If there are no (valid) credentials available, let the user log in. if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: creds.refresh(Request()) else: creds = Credentials.from_service_account_file( './svc.json').with_scopes(SCOPES) # Save the credentials for the next run with open('token.pickle', 'wb') as token: pickle.dump(creds, token) service = build('calendar', 'v3', credentials=creds) event = { "start": { "dateTime": times[0].isoformat() }, "end": { "dateTime": times[1].isoformat() }, "summary": title, "description": description, "transparency": "transparent", "reminders": { "useDefault": False, "overrides": [] # 通知を無効にするにはこのようにする } } # Call the Calendar API # pylint: disable=E1101 service.events().insert(calendarId=CALENDAR_ID, body=event).execute()
def preflight(self): """ Testing items that should be ran before tests are ran. This method returns a new class method for the test. Returns: The class instance which will be used for the unit test. """ global last_test_time diff = time.time() - last_test_time if diff < TEST_DELAY_INTERVAL: time.sleep(diff) last_test_time = time.time() config = dapt.Config(path=conf_path) scope = [ 'https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive' ] creds = Credentials.from_service_account_file(config['sheets-creds-path'], scopes=scope) client = gspread.authorize(creds) sheet = client.open_by_key(config.config['sheets-spreedsheet-id']) worksheet = sheet.get_worksheet(0) data = Database_test_base.INITIAL_TABLE start = gspread.utils.rowcol_to_a1(1, 1) end = gspread.utils.rowcol_to_a1(len(data)+1, len(data[0])+1) range_label = '%s!%s:%s' % (worksheet.title, start, end) sheet.values_update(range_label, params={'valueInputOption': 'RAW'}, body={'values': data}) db = dapt.db.Sheet(config=config) db.connect() return db
def main(): # Google BigQuery Authentication. At first, we'll try to fetch # authentication data from a 'credentials.json' file at our # working directory. If this file doesn't exist, we authenticate # with web login. try: cred_file = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] credentials = Credentials.from_service_account_file(cred_file) except (KeyError, FileNotFoundError): print( "Invalid credentials. Set GOOGLE_APPLICATION_CREDENTIALS to your credentials file." ) exit(1) # Project, dataset and table configuration pandas_gbq.context.credentials = credentials project_id = os.environ["PROJECT_ID"] url_table_id = os.environ["TABLE_ID"] #driver_path='/usr/lib/chromium-browser/chromedriver' miner_name = sys.argv[1] #site_miner = getattr(builders, miner_name, None) site_builder = builders.get(miner_name) #assert site_miner is not None, f"Could not find the miner requested: {sys.argv[0]}" #miner = mining.MiningEngine(ScieloSearchLocations, driver_path=driver_path) site_builder.connect_to_gbq(credentials, project_id, url_table_id, url_schema) #URLBuilder.connect_to_gbq(credentials, project_id, url_table_id, url_schema) #scielo_builder = URLBuilder(miner) # Collect these number of URLs and store them in pd.DataFrame #search_domain = 'https://search.scielo.org/' search_terms = sys.argv[2:-1] limit = int(sys.argv[-1]) site_builder.collect(search_terms, limit) print("Finished with collection", flush=True)
def generate_jwt(claims: Any, google_application_credentials: Optional[Any] = None) -> Any: """Generates a signed JSON Web Token using a Google API Service Account.""" if os.getenv("AUTH0_JWT_TOKEN"): return os.getenv("AUTH0_JWT_TOKEN") if not google_application_credentials: google_application_credentials = os.getenv( "GOOGLE_APPLICATION_CREDENTIALS") if not google_application_credentials: raise ValueError("Please provide GOOGLE_APPLICATION_CREDENTIALS") now = int(time.time()) sa_email = Credentials.from_service_account_file( google_application_credentials).service_account_email payload = { "iat": now, "exp": now + 3600, # iss must match 'issuer' in the security configuration in your # swagger spec (e.g. service account email). It can be any string. "iss": sa_email, # aud must be either your Endpoints service name, or match the value # specified as the 'x-google-audience' in the OpenAPI document. "aud": audience, # sub and email should match the service account's email address "sub": sa_email, "email": sa_email, } claims.update(payload) signer = crypt.RSASigner.from_service_account_file( google_application_credentials) jwt_string = jwt.encode(signer, claims) return jwt_string
'GCLOUD_BUCKET', ] if os.path.exists('settings.json'): with open('settings.json') as f: json_dict = json.load(f) settings = {k: v for k, v in json_dict.items() if k in key_filter} else: settings = {k: os.environ[k] for k in key_filter} return settings _settings = _load_settings() _credentials = Credentials.from_service_account_info({ 'token_uri': 'https://accounts.google.com/o/oauth2/token', 'client_email': _settings['GCLOUD_CLIENT_EMAIL'], 'private_key': _settings['GCLOUD_PRIVATE_KEY'], }) _gcs_client = gcs.Client( project=_settings['GCLOUD_PROJECT_ID'], credentials=_credentials) _loop = asyncio.get_event_loop() _max_workers = 1 _loop.set_default_executor(ProcessPoolExecutor(max_workers=_max_workers)) _manager = multiprocessing.Manager() _jobs_list = _manager.list() _app = web.Application(logger=_logger) _app.router.add_route('POST', '/jobs', _add_job) web.run_app(_app, port=10000)