def test_multistore_file_get_all_keys(self):
# start with no keys
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals([], keys)
# store credentials
credentials = self._create_test_credentials(client_id='client1')
custom_key = {'myapp': 'testing', 'clientid': 'client1'}
store1 = multistore_file.get_credential_storage_custom_key(
FILENAME, custom_key)
store1.put(credentials)
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals([custom_key], keys)
# store more credentials
credentials = self._create_test_credentials(client_id='client2')
string_key = 'string_key'
store2 = multistore_file.get_credential_storage_custom_string_key(
FILENAME, string_key)
store2.put(credentials)
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals(2, len(keys))
self.assertTrue(custom_key in keys)
self.assertTrue({'key': string_key} in keys)
# back to no keys
store1.delete()
store2.delete()
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals([], keys)
def test_multistore_file_get_all_keys(self):
# start with no keys
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals([], keys)
# store credentials
credentials = self._create_test_credentials(client_id='client1')
custom_key = {'myapp': 'testing', 'clientid': 'client1'}
store1 = multistore_file.get_credential_storage_custom_key(
FILENAME, custom_key)
store1.put(credentials)
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals([custom_key], keys)
# store more credentials
credentials = self._create_test_credentials(client_id='client2')
string_key = 'string_key'
store2 = multistore_file.get_credential_storage_custom_string_key(
FILENAME, string_key)
store2.put(credentials)
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals(2, len(keys))
self.assertTrue(custom_key in keys)
self.assertTrue({'key': string_key} in keys)
# back to no keys
store1.delete()
store2.delete()
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals([], keys)
def _AcctountId2StorageKey(self, account_id):
"""Converts account id into storage key."""
all_storage_keys = multistore_file.get_all_credential_keys(
filename=self._store_file)
matching_keys = [
k for k in all_storage_keys if k['account'] == account_id
]
if not matching_keys:
return {'type': 'google-cloud-sdk', 'account': account_id}
# We do not expect any other type keys in the credential store. Just in case
# somehow they occur:
# 1. prefer key with no type
# 2. use google-cloud-sdk type
# 3. use any other
# Also log all cases where type was present but was not google-cloud-sdk.
right_key = matching_keys[0]
for key in matching_keys:
if 'type' in key:
if key['type'] == 'google-cloud-sdk' and 'type' in right_key:
right_key = key
else:
log.file_only_logger.warn(
'Credential store has unknown type [{0}] key for account [{1}]'
.format(key['type'], key['account']))
else:
right_key = key
if 'type' in right_key:
right_key['type'] = 'google-cloud-sdk'
return right_key
def _AcctountId2StorageKey(self, account_id):
"""Converts account id into storage key."""
all_storage_keys = multistore_file.get_all_credential_keys(
filename=self._store_file)
matching_keys = [k for k in all_storage_keys if k['account'] == account_id]
if not matching_keys:
return {'type': 'google-cloud-sdk', 'account': account_id}
# We do not expect any other type keys in the credential store. Just in case
# somehow they occur:
# 1. prefer key with no type
# 2. use google-cloud-sdk type
# 3. use any other
# Also log all cases where type was present but was not google-cloud-sdk.
right_key = matching_keys[0]
for key in matching_keys:
if 'type' in key:
if key['type'] == 'google-cloud-sdk' and 'type' in right_key:
right_key = key
else:
log.file_only_logger.warn(
'Credential store has unknown type [{0}] key for account [{1}]'
.format(key['type'], key['account']))
else:
right_key = key
if 'type' in right_key:
right_key['type'] = 'google-cloud-sdk'
return right_key
def AvailableAccounts():
"""Get all accounts that have credentials stored for the CloudSDK.
This function will also ping the GCE metadata server to see if GCE credentials
are available.
Returns:
[str], List of the accounts.
"""
all_keys = multistore_file.get_all_credential_keys(
filename=config.Paths().credentials_path)
accounts = [
key['account'] for key in all_keys
if key.get('type') == 'google-cloud-sdk'
]
accounts.extend(c_gce.Metadata().Accounts())
devshell_creds = c_devshell.LoadDevshellCredentials()
if devshell_creds:
accounts.append(devshell_creds.devshell_response.user_email)
accounts.sort()
return accounts
def _StorageForAccount(account):
"""Get the oauth2client.multistore_file storage.
Args:
account: str, The account tied to the storage being fetched.
Returns:
oauth2client.client.Storage, A credentials store.
"""
storage_path = config.Paths().credentials_path
parent_dir, unused_name = os.path.split(storage_path)
files.MakeDir(parent_dir)
# We only care about account value of the key, yet store 'type' for backwards
# compatibility. There should be no more than one key in this list.
keys = [
k
for k in multistore_file.get_all_credential_keys(filename=storage_path)
if k['account'] == account
]
if not keys: # New key.
return multistore_file.get_credential_storage_custom_key(
filename=storage_path,
key_dict={
'type': 'google-cloud-sdk',
'account': account
})
# We do not expect any other type keys in the credential store. Just in case
# somehow they occur:
# 1. prefer key with no type
# 2. use google-cloud-sdk type
# 3. use any other
# Also log all cases where type was present but was not google-cloud-sdk.
right_key = keys[0]
for key in keys:
if 'type' in key:
if key['type'] == 'google-cloud-sdk' and 'type' in right_key:
right_key = key
else:
log.file_only_logger.warn(
'Credential store has unknown type [{0}] key for account [{1}]'
.format(key['type'], key['account']))
else:
right_key = key
if 'type' in right_key:
right_key['type'] = 'google-cloud-sdk'
return multistore_file.get_credential_storage_custom_key(
filename=storage_path, key_dict=right_key)
def GetAccounts(self):
"""Overrides."""
all_keys = multistore_file.get_all_credential_keys(
filename=self._store_file)
return {self._StorageKey2AccountId(key) for key in all_keys}
def GetAccounts(self):
"""Overrides."""
all_keys = multistore_file.get_all_credential_keys(
filename=self._store_file)
return {self._StorageKey2AccountId(key) for key in all_keys}
def main():
count = 0
count_files = 0
while True:
log.info("fetcher iteration")
time.sleep(5)
credential_keys = get_all_credential_keys('test_storage.txt')
for key in credential_keys:
user_info = db.query(User).filter(
User.google_id == key['clientId']).first()
if not user_info:
user_info = User(
google_id=key['clientId'],
is_download_data=False)
log.info(
'add new user %s'
' in User table', key['clientId'])
db.add(user_info)
db.commit()
log.info(
'getting documents for user %s',
key['clientId'])
user_info = db.query(User).filter(
User.google_id == key['clientId']).first()
log.info(
'is_download_data = %s',
user_info.is_download_data)
if not user_info.is_download_data:
log.info(
'is_download_data = %s',
user_info.is_download_data)
log.info(
'download files from google id %s',
key['clientId'])
storage = get_credential_storage(
'test_storage.txt',
key['clientId'],
key['userAgent'],
key['scope'])
credentials = storage.get()
log.info(
'get credential for google id %s',
key['clientId'])
while credentials and count == 0:
for document in get_documents(credentials,
key['clientId']):
db.add(document)
count_files = count_files + 1
db.commit()
count = count + 1
break
log.info('download %d files from google id %s',
count_files, key['clientId'])
user_info = db.query(User).filter(
User.google_id == key['clientId']).first()
if user_info:
if not user_info.is_download_data:
user_info.is_download_data = True
log.info(
'download files for user %s'
' is_download_data = %d',
str(key['clientId']),
user_info.is_download_data)
db.commit()
count_files = 0
count = 0
