def dummy_service_account():
global _DUMMY_SERVICE_ACCOUNT
from google.oauth2.service_account import Credentials
if _DUMMY_SERVICE_ACCOUNT is None:
_DUMMY_SERVICE_ACCOUNT = Credentials.from_service_account_info(
_SERVICE_ACCOUNT_JSON
)
return _DUMMY_SERVICE_ACCOUNT
def import_sprint_tasks(sprint_tasks, google_sheet_key, google_sheet_index):
scope = [
'https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/drive'
]
credentials = Credentials.from_service_account_file('client_secrets.json',
scopes=scope)
gc = gspread.authorize(credentials)
wks = gc.open_by_key(google_sheet_key)
worksheet = wks.get_worksheet(int(google_sheet_index))
if worksheet.row_values(1) == []:
worksheet.append_row([
'id', 'title', 'app', 'epic', 'sprint', 'status', 'point',
'import_time'
])
worksheet.append_rows(sprint_tasks)
def multifolderclone(source=None, dest=None, path='accounts', width=2):
global account_count
global drive
global threads
stt = time.time()
accounts = glob.glob(path + '/*.json')
if source == None:
source = input("Source Folder ID Missing. Please enter Folder ID of source: ")
else:
if dest == None:
dest = input("Destination Folder ID Missing. Please enter Folder ID of destination: ")
else:
while len(accounts) == 0:
path = input("No service accounts found in current directory. Please enter the path where the accounts are located at: ")
accounts = glob.glob(path + '/*.json')
print('Copy from ' + source + ' to ' + dest + '.')
print('View set to tree (' + str(width) + ').')
pbar = progress.bar.Bar("Creating Drive Services", max=len(accounts))
for account in accounts:
account_count += 1
credentials = Credentials.from_service_account_file(account, scopes=[
"https://www.googleapis.com/auth/drive"
])
drive.append(googleapiclient.discovery.build("drive", "v3", credentials=credentials))
pbar.next()
pbar.finish()
threads = threading.BoundedSemaphore(account_count)
print('BoundedSemaphore with %d threads' % account_count)
try:
rcopy(source, dest, "root", "", width)
except KeyboardInterrupt:
print('Quitting')
sys.exit()
print('Complete.')
hours, rem = divmod((time.time() - stt), 3600)
minutes, sec = divmod(rem, 60)
print("Elapsed Time:\n{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), sec))
def get_client(project: str,
credentials: dict,
credentials_secret: str = None) -> bigquery.Client:
if credentials_secret is not None:
warnings.warn(
"The `credentials_secret` argument is deprecated. Use a `Secret` task "
"to pass the credentials value at runtime instead.",
UserWarning,
)
creds = Secret(credentials_secret).get()
credentials = Credentials.from_service_account_info(creds)
project = project or credentials.project_id
if credentials is not None:
project = project or credentials.get("project")
client = bigquery.Client(project=project, credentials=credentials)
else:
client = bigquery.Client(project=project)
return client
def __init__(self):
scopes = [
'https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/spreadsheets',
'https://www.googleapis.com/auth/drive.file',
'https://www.googleapis.com/auth/drive'
]
creds = Credentials.from_service_account_file('creds.json',
scopes=scopes)
self.gc = gspread.authorize(creds)
self.eventsSpreadSheet = self.gc.open('VkEvents')
self.questionsSpreadSheet = self.gc.open('VkEventQuestions')
self.eventsWorkSheets = {}
self.events = Events.Events()
self.createEventsTable()
def setupCommand(databaseFileName: str, credentialsFileName: str,
bucketName: str) -> None:
# Verify credentials and bucket name
credentialsJson = json.load(open(credentialsFileName, 'r'))
credentials = Credentials.from_service_account_info(credentialsJson)
client = storage.Client(project=credentials.project_id,
credentials=credentials)
client.get_bucket(bucketName)
# Open database
database = Database(databaseFileName)
# Write credentials and bucket name to database
database.setPair('credentials', json.dumps(credentialsJson))
database.setPair('bucketName', bucketName)
# Close database
database.commit()
database.close()
def get_google_credentials():
"""Obtain the Google credentials object explicitly."""
private_key = os.environ["GOOGLE_PRIVATE_KEY"]
private_key_id = os.environ["GOOGLE_PRIVATE_KEY_ID"]
signer = RSASigner.from_string(key=private_key, key_id=private_key_id)
project_id = os.environ["GOOGLE_PROJECT_ID"]
service_account_email = os.environ["GOOGLE_CLIENT_EMAIL"]
scopes = ('https://www.googleapis.com/auth/bigquery',
'https://www.googleapis.com/auth/cloud-platform')
token_uri = os.environ["GOOGLE_TOKEN_URI"]
credentials = Credentials(
signer=signer,
service_account_email=service_account_email,
token_uri=token_uri,
scopes=scopes,
project_id=project_id,
)
return credentials
def __init__(
self,
ff_access_keys = None,
google_api_key = None,
s3UtilsInstance = None,
extra_config = DEFAULT_GOOGLE_API_CONFIG
):
"""Authenticate with Google APIs and initialize sub-class instances."""
if s3UtilsInstance is None:
self._s3Utils = s3_utils.s3Utils(env='data') # Google API Keys are stored on production bucket only ATM.
else:
self._s3Utils = s3UtilsInstance
if google_api_key is None:
self._api_key = self._s3Utils.get_google_key()
if not self._api_key:
raise Exception("Failed to get Google API key from S3.")
else:
self._api_key = google_api_key
if not GoogleAPISyncer.validate_api_key_format(self._api_key):
raise Exception("Google API Key is in invalid format.")
self.extra_config = extra_config
self.credentials = Credentials.from_service_account_info(
self._api_key,
scopes=self.extra_config.get('scopes', DEFAULT_GOOGLE_API_CONFIG['scopes'])
)
# These are required only for POSTing/GETing data for TrackingInfo items.
if ff_access_keys is None:
ff_access_keys = self._s3Utils.get_access_keys()
self.server = ff_access_keys['server']
self.access_key = {
"key" : ff_access_keys['key'],
"secret": ff_access_keys['secret']
}
# Init sub-class objects
self.analytics = GoogleAPISyncer.AnalyticsAPI(self)
self.sheets = GoogleAPISyncer.SheetsAPI(self)
self.docs = GoogleAPISyncer.DocsAPI(self)
def insert_schedule():
# サービスアカウントのJSONファイルパス
GOOGLE_APPLICATION_CREDENTIALS=os.environ['GOOGLE_CREDENTIALS_FILE']
c = Credentials.from_service_account_file(GOOGLE_APPLICATION_CREDENTIALS)
service = build('calendar', 'v3', credentials=c)
# Call the Calendar API
now = datetime.datetime.utcnow().isoformat() + 'Z'
#calendar_id = 'primary'
# データを取得したいGoogleカレンダーのIDを指定する
calendar_id = os.environ['GOOGLE_CALENDAR_ID']
body = dict()
body['summary'] = 'Test summary'
body['start'] = {'timeZone': 'Asia/Tokyo', 'dateTime': '2020-03-09T11:00:00+09:00'}
body['end'] = {'timeZone': 'Asia/Tokyo', 'dateTime': '2020-03-09T11:30:00+09:00'}
body['location'] = 'Kobe'
body['guestsCanSeeOtherGuests'] = False
body['guestsCanInviteOthers'] = False
insert_result = service.events().insert(
calendarId=calendar_id,
body=body
).execute()
print(insert_result)
events_result = service.events().list(
calendarId=calendar_id, timeMin=now,
maxResults=10, singleEvents=True,
orderBy='startTime').execute()
events = events_result.get('items', [])
print(events)
if not events:
print('No upcoming events found.')
for event_c in events:
event_id = event_c['id']
start = event_c['start'].get('dateTime', event_c['start'].get('date'))
print(event_id, start, event_c['summary'])
return events
def main(args):
# Load the json format key that you downloaded from the Google API
# Console when you created your service account. For p12 keys, use the
# from_p12_keyfile method of ServiceAccountCredentials and specify the
# service account email address, p12 keyfile, and scopes.
service_credentials = Credentials.from_service_account_file(
args.service_account_key_path,
scopes=[
'https://www.googleapis.com/auth/admin.directory.device.chromebrowsers'
],
subject=args.admin_email)
try:
http = google_auth_httplib2.AuthorizedHttp(service_credentials, http=Http())
extensions_list = {}
base_request_url = 'https://admin.googleapis.com/admin/directory/v1.1beta1/customer/my_customer/devices/chromebrowsers'
request_parameters = ''
browsers_processed = 0
while True:
print('Making request to server ...')
data = json.loads(
http.request(base_request_url + '?' + request_parameters, 'GET')[1])
browsers_in_data = len(data['browsers'])
print('Request returned %s results, analyzing ...' % (browsers_in_data))
ComputeExtensionsList(extensions_list, data)
browsers_processed += browsers_in_data
if 'nextPageToken' not in data or not data['nextPageToken']:
break
print('%s browsers processed.' % (browsers_processed))
if (args.max_browsers_to_process is not None and
args.max_browsers_to_process <= browsers_processed):
print('Stopping at %s browsers processed.' % (browsers_processed))
break
request_parameters = ('pageToken={}').format(data['nextPageToken'])
finally:
print('Analyze results ...')
ExtensionListAsCsv(extensions_list, args.extension_list_csv)
print("Results written to '%s'" % (args.extension_list_csv))
def sendResponse(JSONObject):
if (JSONObject):
credentials = Credentials.from_service_account_file("../intent.json")
client = dialogflow_v2.SessionsClient(credentials=credentials)
session = client.session_path('classify-intents-ujpxuu',
'Testing values')
content = JSONObject
text_input = dialogflow_v2.types.TextInput(text=content['inputText'],
language_code="en")
query_input = dialogflow_v2.types.QueryInput(text=text_input)
response = client.detect_intent(session=session,
query_input=query_input)
print('Query text: {}'.format(response.query_result.query_text))
print('Detected intent: {} (confidence: {})\n'.format(
response.query_result.intent.display_name,
response.query_result.intent_detection_confidence))
confidence = float("{0:.2f}".format(
response.query_result.intent_detection_confidence * 100))
if ('Default Welcome Intent'
in response.query_result.intent.display_name
or 'Default Fallback Intent'
in response.query_result.intent.display_name):
intentName = 'Others'
else:
intentName = response.query_result.intent.display_name
intentName = intentName.lower().replace(" ", "_")
result = {
'inputText': response.query_result.query_text,
'accountHead': intentName,
'confidence': confidence,
'outflow_tags': ["stationery", "office", "supplies"]
}
return result
else:
return "Request Failed."
def get_service_account_credentials(private_key):
"""DEPRECATED: Load service account credentials from key data or key path."""
import google.auth.transport.requests
from google.oauth2.service_account import Credentials
is_path = os.path.isfile(private_key)
try:
if is_path:
with open(private_key) as f:
json_key = json.loads(f.read())
else:
# ugly hack: 'private_key' field has new lines inside,
# they break json parser, but we need to preserve them
json_key = json.loads(private_key.replace("\n", " "))
json_key["private_key"] = json_key["private_key"].replace(
" ", "\n"
)
if pandas.compat.PY3:
json_key["private_key"] = bytes(json_key["private_key"], "UTF-8")
credentials = Credentials.from_service_account_info(json_key)
credentials = credentials.with_scopes(SCOPES)
# Refresh the token before trying to use it.
request = google.auth.transport.requests.Request()
credentials.refresh(request)
return credentials, json_key.get("project_id")
except (KeyError, ValueError, TypeError, AttributeError):
raise pandas_gbq.exceptions.InvalidPrivateKeyFormat(
"Detected private_key as {}. ".format(
"path" if is_path else "contents"
)
+ "Private key is missing or invalid. It should be service "
"account private key JSON (file path or string contents) "
'with at least two keys: "client_email" and "private_key". '
"Can be obtained from: https://console.developers.google."
"com/permissions/serviceaccounts"
)
def __init__(
self,
project_id=None,
service_account_json=None,
language_code="en",
raise_on_error=False,
flip_text_and_intent=False,
):
self.project_id = project_id
self.language_code = language_code
self.raise_on_error = raise_on_error
self.flip_text_and_intent = flip_text_and_intent
credentials = None
if service_account_json:
credentials = Credentials.from_service_account_file(
service_account_json)
self.session_client = dialogflow.SessionsClient(
credentials=credentials)
def insert_to_google_sheet(values, sheet, worksheet, index):
try:
scopes = [
'https://www.googleapis.com/auth/spreadsheets',
'https://www.googleapis.com/auth/drive'
]
credentials = Credentials.from_service_account_file(
'client_secrets.json', scopes=scopes)
google_client = gspread.authorize(credentials)
sh = google_client.open(sheet)
# for i in sh.worksheets():
# print(i.title)
ws = sh.worksheet(worksheet)
#start at 1 and 1 is col name.
ws.insert_row(values, index=index + 2)
# return sh.url
except Exception as e:
print('---upload to google drive failed---')
print(e)
def get_dialog(text, session_id=None):
project_id = 'dkl-bot-utvfiy'
lang = 'ru-RU'
try:
credentials = Credentials.from_service_account_file(
"google-credentials.json")
session_client = dialogflow.SessionsClient(credentials=credentials)
session = session_client.session_path(project_id, session_id)
text_input = dialogflow.types.TextInput(text=text, language_code=lang)
query_input = dialogflow.types.QueryInput(text=text_input)
response = session_client.detect_intent(session=session,
query_input=query_input)
except:
print('беда ваще')
return response.query_result.fulfillment_text
def __init__(self, key: str):
scopes = [
'https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/spreadsheets',
'https://www.googleapis.com/auth/drive.file',
'https://www.googleapis.com/auth/drive'
]
directory_path = os.path.dirname(os.path.abspath(__file__))
new_path = os.path.join(directory_path, 'credentials.json')
credentials = ServiceAccountCredentials.from_json_keyfile_name(
new_path, scopes)
data = credentials.serialization_data
data['token_uri'] = credentials.token_uri
auth = Credentials.from_service_account_info(data, scopes=scopes)
self.session = AuthorizedSession(auth)
self.id = key
self.fetched_data = {}
self.requests = []
def _downloadFile(credentials: dict, bucket_name: str, key: str,
file_name: str) -> str:
"""
Downloads a file from cloud storage.
:param credentials: The Google cloud storage service credentials retrieved from the Kubernetes secret.
:param bucket_name: The name of the bucket.
:param key: The key to download the file from the cloud storage.
:param file_name: The file that will be downloaded.
:return: The location of the downloaded file.
"""
credentials = ServiceCredentials.from_service_account_info(credentials)
gcs_client = StorageClient(credentials.project_id, credentials)
bucket = gcs_client.get_bucket(bucket_name)
logging.info("Going to download gcs://%s/%s", bucket_name, key)
bucket.blob(key).download_to_filename(file_name)
logging.info("Backup gcs://%s/%s downloaded to %s", bucket_name, key,
file_name)
return file_name
def recognize_audio_from_uri(
uri: str,
credential: Union[str, os.PathLike, None] = None,
language_code: str = 'en-US',
encoding: enums.RecognitionConfig.AudioEncoding = enums.RecognitionConfig.
AudioEncoding.FLAC,
sampling_rate_hertz: int = 44100,
) -> types.RecognizeResponse:
"""
Args:
uri (str) : Cloud
credential (str, os.PathLike, None) :
language_code:
encoding (enums.RecognitionConfig.AudioEncoding) :
sampling_rate_hertz (int) :
Returns:
types.RecognizeResponse
"""
if credential is None:
client = SpeechClient()
else:
credentials = Credentials.from_service_account_file(
filename=credential)
client = SpeechClient(credentials=credentials)
config = types.RecognitionConfig(encoding=encoding,
language_code=language_code,
sample_rate_hertz=sampling_rate_hertz)
audio = types.RecognitionAudio(uri=uri)
try:
result = client.recognize(config=config, audio=audio)
except exceptions.InvalidArgument:
print(
'cannot synchronize recognition. switched asynchronized recognition'
)
operartion = client.long_running_recognize(config=config, audio=audio)
result = operartion.result()
return result
def BlobRemoveThread(
blobRemoveQueue: queue.SimpleQueue,
credentials: str,
bucketName: str,
isTrialRun: bool,
logLevel: str,
threadId: int
) -> None:
# Set logging level and get logger
setLogingLevel(logLevel)
logger = logging.getLogger(__name__)
# Get GCS bucket
credentials = Credentials.from_service_account_info(json.loads(credentials))
client = storage.Client(project=credentials.project_id, credentials=credentials)
bucket = client.get_bucket(bucketName)
# Create stats variables
ticToc = TicToc()
while True:
# Get task
task = blobRemoveQueue.get()
if task is None:
break
# Extract task
name: str = task[0]
ticToc.tic()
if not isTrialRun:
try:
bucket.delete_blob(name)
except:
pass
logger.info('× {}, elapsed: {:.3f}s, thread: {}'.format(
name,
ticToc.toc(),
threadId
))
def generate_jwt_token_from_impersonated_account(service_account_info: Dict[str, str], audiences: str,
issuer: str) -> Tuple[str, int]:
"""
Using a dictionary containing the information from a Google Cloud service account credentials file, this function
impersonates the Company Information API master user service account and signs a JSON Web Token (JWT) used to
authenticate the client when accessing the service.
:param service_account_info: A dictionary containing all the information found in a Google Cloud service account
credentials JSON file.
:type service_account_info: Dict[str, str]
:param audiences: The intended recipient of the JWT. Found in the Google Endpoint specification. For example,
for the company information API, the recipient is 'company-information.api.willowlabs.ai'
:type audiences: str
:param issuer: The email address of the impersonated API master user service account.
:type issuer: str
:param jwt_lifetime: The length of time, in seconds, for which the created JWT is valid.
:type jwt_lifetime: int
:return: A tuple containing the JWT and a POSIX/Unix epoch-style timestamp indicating when the JWT expires.
:rtype: Tuple[str, int]
"""
credentials = Credentials.from_service_account_info(service_account_info,
scopes=["https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/iam"])
if not credentials.valid:
credentials.refresh(Request())
impersonated_credentials = ImpersonatedCredentials(source_credentials=credentials, target_principal=issuer,
target_scopes=["https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/iam"])
if not impersonated_credentials.valid:
impersonated_credentials.refresh(Request())
signer = Signer(Request(), impersonated_credentials, impersonated_credentials.service_account_email)
now = int(time.time())
expires = now + MAX_TOKEN_LIFETIME_SECS
payload = {
'iat': now,
'exp': expires,
'aud': audiences,
'iss': issuer
}
return google.auth.jwt.encode(signer, payload).decode("utf-8"), expires
def _get_oauth_token_from_service_account_credentials(self):
scopes = ["https://www.googleapis.com/auth/userinfo.email"]
assert 'GOOGLE_APPLICATION_CREDENTIALS' in os.environ
from google.auth.transport.requests import Request as GoogleAuthRequest
from google.oauth2.service_account import Credentials as ServiceAccountCredentials
logger.info("Found GOOGLE_APPLICATION_CREDENTIALS environment variable. "
"Using service account credentials for authentication.")
service_account_credentials_filename = os.environ['GOOGLE_APPLICATION_CREDENTIALS']
if not os.path.isfile(service_account_credentials_filename):
msg = 'File "{}" referenced by the GOOGLE_APPLICATION_CREDENTIALS environment variable does not exist'
raise Exception(msg.format(service_account_credentials_filename))
credentials = ServiceAccountCredentials.from_service_account_file(
service_account_credentials_filename,
scopes=scopes
)
r = GoogleAuthRequest()
credentials.refresh(r)
r.session.close()
return credentials.token, credentials.expiry
def generaterandomdrive():
global accsf
socket.setdefaulttimeout(600)
acc_thread.acquire()
random_acc = random.choice(accsf)
while True:
try:
credentials = Credentials.from_service_account_file(random_acc, scopes=[
"https://www.googleapis.com/auth/drive"
])
random_drive = build("drive", "v3", credentials=credentials)
except HttpError:
random_acc = random.choice(accsf)
print("#Error SA Error")
else:
break
acc_thread.release()
return random_drive
def fallback_ocr(image_path, input_type):
ocr_credentials = os.environ['FALLBACK_OCR']
ocr_credentials = Credentials.from_service_account_file(ocr_credentials)
result = {}
print("Fallback_ocr: Text extraction")
if input_type == "pdf":
print("Fallback_OCR: extracting pages from pdf for further processing")
images = convert_from_bytes(image_path)
print("Fallback_OCR: extraction complete with " + str(len(images)) +
" images")
for index, image in enumerate(images):
print("PAGE : ", index + 1)
img_byte_arr = io.BytesIO()
image.save(img_byte_arr, format='PNG')
img_byte_arr = img_byte_arr.getvalue()
temp = detect_document(img_byte_arr, ocr_credentials)
result[index] = temp
else:
result = detect_document(image_path, ocr_credentials)
return result
def init(self, core: BBotCore):
"""
Initializebot engine
"""
super().init(core)
self.logger = BBotLoggerAdapter(logging.getLogger('dialogfl_cbe'), self, self.core)
self.service_account = json.loads(self.dotbot.chatbot_engine['serviceAccount'])
self.platform = None
self.available_platforms = {
'google_assistant': 'ACTIONS_ON_GOOGLE',
'facebook': 'FACEBOOK',
'slack': 'SLACK',
'telegram': 'TELEGRAM',
'skype': 'SKYPE'
}
credentials = Credentials.from_service_account_info(self.service_account)
self.session_client = dialogflow.SessionsClient(credentials=credentials)
def get_service(api_name, api_version, scopes, key_file_location):
"""Get a service that communicates to a Google API.
Args:
api_name: The name of the api to connect to.
api_version: The api version to connect to.
scopes: A list auth scopes to authorize for the application.
key_file_location: The path to a valid service account JSON key file.
Returns:
A service that is connected to the specified API.
"""
credentials = Credentials.from_service_account_file(key_file_location,
scopes=scopes)
# credentials = ServiceAccountCredentials.from_json_keyfile_name(
# key_file_location, scopes=scopes)
# Build the service object.
service = build(api_name, api_version, credentials=credentials)
return service
def get_conn(self):
if self.conn is None:
params = self.get_connection(self.gcp_conn_id)
extra = params.extra_dejson
logger = logging.getLogger()
logging_level = logger.getEffectiveLevel()
logger.setLevel(logging.ERROR)
try:
if 'authorized_user_file' in extra[
'extra__google_cloud_platform__key_path']:
creds = CredentialsUser.from_authorized_user_file(
extra['extra__google_cloud_platform__key_path'])
else:
creds = Credentials.from_service_account_file(
extra['extra__google_cloud_platform__key_path'])
self.project_id = creds.project_id
self.conn = storage.Client(credentials=creds,
project=self.project_id)
finally:
logger.setLevel(logging_level)
return self.conn
def main():
filename = 'download.out'
db_filename = 'data.db'
days = 7
if settings.file_url:
response = requests.get(settings.file_url)
response.raise_for_status()
with open(filename, 'wb') as f:
f.write(response.content)
else:
assert settings.file_id
credentials = Credentials.from_service_account_file('credentials.json')
service = build('drive', 'v3', credentials=credentials)
download_gdrive_file(service, settings.file_id, filename)
unzip(filename, db_filename)
raw_data = collect_data(db_filename, days)
data = [
dict(value=tag, date=d) for d, tags in raw_data.items() for tag in tags
]
exist.post('/attributes/custom/append/', json=data)
def addCalendar(title, times, description):
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('token.pickle'):
with open('token.pickle', 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
creds = Credentials.from_service_account_file(
'./svc.json').with_scopes(SCOPES)
# Save the credentials for the next run
with open('token.pickle', 'wb') as token:
pickle.dump(creds, token)
service = build('calendar', 'v3', credentials=creds)
event = {
"start": {
"dateTime": times[0].isoformat()
},
"end": {
"dateTime": times[1].isoformat()
},
"summary": title,
"description": description,
"transparency": "transparent",
"reminders": {
"useDefault": False,
"overrides": [] # 通知を無効にするにはこのようにする
}
}
# Call the Calendar API
# pylint: disable=E1101
service.events().insert(calendarId=CALENDAR_ID, body=event).execute()
def preflight(self):
"""
Testing items that should be ran before tests are ran. This method returns a new
class method for the test.
Returns:
The class instance which will be used for the unit test.
"""
global last_test_time
diff = time.time() - last_test_time
if diff < TEST_DELAY_INTERVAL:
time.sleep(diff)
last_test_time = time.time()
config = dapt.Config(path=conf_path)
scope = [
'https://www.googleapis.com/auth/spreadsheets',
'https://www.googleapis.com/auth/drive'
]
creds = Credentials.from_service_account_file(config['sheets-creds-path'], scopes=scope)
client = gspread.authorize(creds)
sheet = client.open_by_key(config.config['sheets-spreedsheet-id'])
worksheet = sheet.get_worksheet(0)
data = Database_test_base.INITIAL_TABLE
start = gspread.utils.rowcol_to_a1(1, 1)
end = gspread.utils.rowcol_to_a1(len(data)+1, len(data[0])+1)
range_label = '%s!%s:%s' % (worksheet.title, start, end)
sheet.values_update(range_label, params={'valueInputOption': 'RAW'}, body={'values': data})
db = dapt.db.Sheet(config=config)
db.connect()
return db
def main():
# Google BigQuery Authentication. At first, we'll try to fetch
# authentication data from a 'credentials.json' file at our
# working directory. If this file doesn't exist, we authenticate
# with web login.
try:
cred_file = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
credentials = Credentials.from_service_account_file(cred_file)
except (KeyError, FileNotFoundError):
print(
"Invalid credentials. Set GOOGLE_APPLICATION_CREDENTIALS to your credentials file."
)
exit(1)
# Project, dataset and table configuration
pandas_gbq.context.credentials = credentials
project_id = os.environ["PROJECT_ID"]
url_table_id = os.environ["TABLE_ID"]
#driver_path='/usr/lib/chromium-browser/chromedriver'
miner_name = sys.argv[1]
#site_miner = getattr(builders, miner_name, None)
site_builder = builders.get(miner_name)
#assert site_miner is not None, f"Could not find the miner requested: {sys.argv[0]}"
#miner = mining.MiningEngine(ScieloSearchLocations, driver_path=driver_path)
site_builder.connect_to_gbq(credentials, project_id, url_table_id,
url_schema)
#URLBuilder.connect_to_gbq(credentials, project_id, url_table_id, url_schema)
#scielo_builder = URLBuilder(miner)
# Collect these number of URLs and store them in pd.DataFrame
#search_domain = 'https://search.scielo.org/'
search_terms = sys.argv[2:-1]
limit = int(sys.argv[-1])
site_builder.collect(search_terms, limit)
print("Finished with collection", flush=True)
def generate_jwt(claims: Any,
google_application_credentials: Optional[Any] = None) -> Any:
"""Generates a signed JSON Web Token using a Google API Service Account."""
if os.getenv("AUTH0_JWT_TOKEN"):
return os.getenv("AUTH0_JWT_TOKEN")
if not google_application_credentials:
google_application_credentials = os.getenv(
"GOOGLE_APPLICATION_CREDENTIALS")
if not google_application_credentials:
raise ValueError("Please provide GOOGLE_APPLICATION_CREDENTIALS")
now = int(time.time())
sa_email = Credentials.from_service_account_file(
google_application_credentials).service_account_email
payload = {
"iat": now,
"exp": now + 3600,
# iss must match 'issuer' in the security configuration in your
# swagger spec (e.g. service account email). It can be any string.
"iss": sa_email,
# aud must be either your Endpoints service name, or match the value
# specified as the 'x-google-audience' in the OpenAPI document.
"aud": audience,
# sub and email should match the service account's email address
"sub": sa_email,
"email": sa_email,
}
claims.update(payload)
signer = crypt.RSASigner.from_service_account_file(
google_application_credentials)
jwt_string = jwt.encode(signer, claims)
return jwt_string
'GCLOUD_BUCKET',
]
if os.path.exists('settings.json'):
with open('settings.json') as f:
json_dict = json.load(f)
settings = {k: v for k, v in json_dict.items() if k in key_filter}
else:
settings = {k: os.environ[k] for k in key_filter}
return settings
_settings = _load_settings()
_credentials = Credentials.from_service_account_info({
'token_uri': 'https://accounts.google.com/o/oauth2/token',
'client_email': _settings['GCLOUD_CLIENT_EMAIL'],
'private_key': _settings['GCLOUD_PRIVATE_KEY'],
})
_gcs_client = gcs.Client(
project=_settings['GCLOUD_PROJECT_ID'], credentials=_credentials)
_loop = asyncio.get_event_loop()
_max_workers = 1
_loop.set_default_executor(ProcessPoolExecutor(max_workers=_max_workers))
_manager = multiprocessing.Manager()
_jobs_list = _manager.list()
_app = web.Application(logger=_logger)
_app.router.add_route('POST', '/jobs', _add_job)
web.run_app(_app, port=10000)
