def _build_arbitrary_client(self,
client_name,
client_version,
force_new=False):
"""
:param client_name: name of the service
:param client_version: version of the client to create
:param force_new: whether to create a new client - useful to create arbitrary clients from facades
:return:
"""
if force_new:
client = discovery.build(client_name,
client_version,
cache_discovery=False,
cache=MemoryCache())
http.set_user_agent(
client._http, get_user_agent()) # force set custom user agent
return client
else:
if not self._client:
client = discovery.build(client_name,
client_version,
cache_discovery=False,
cache=MemoryCache())
http.set_user_agent(
client._http,
get_user_agent()) # force set custom user agent
self._client = client
return self._client
def from_credentials(cls, credentials: credentials.Credentials):
auth_http = google_auth_httplib2.AuthorizedHttp(credentials)
user_agent = '/'.join(['django-cloud-deploy', __version__.__version__])
http.set_user_agent(auth_http, user_agent)
return cls(
discovery.build('cloudresourcemanager',
'v1',
http=auth_http,
cache_discovery=False))
def test_set_user_agent_nested(self):
http = HttpMockSequence([
({'status': '200'}, 'echo_request_headers'),
])
http = set_user_agent(http, "my_app/5.5")
http = set_user_agent(http, "my_library/0.1")
resp, content = http.request("http://example.com")
self.assertEqual('my_app/5.5 my_library/0.1', content['user-agent'])
def expand_projects(self, projects):
ret = []
self.logger.debug('Expanding projects list.',
extra={'projects': projects})
for project in projects:
if project.startswith('projects/'):
project = project[9:]
if project.endswith('/'):
project = project[0:len(project) - 1]
if project.isdecimal():
if len(_PROJECT_NUM_CACHE) == 0:
service = discovery.build('cloudresourcemanager', 'v1')
service._http = http.set_user_agent(
service._http, 'google-pso-tool/pubsub2inbox/1.1.0')
request = service.projects().list()
response = request.execute()
while request:
for p in response['projects']:
_PROJECT_NUM_CACHE[p['projectNumber']] = (
p['projectId'], p['projectNumber'], p['name'],
p['labels'] if 'labels' in p else {})
request = service.projects().list_next(
request, response)
if request:
response = request.execute()
project_parts = project.split('/')
if len(project_parts) == 1:
project_num = project_parts[0]
else:
project_num = project_parts[1]
if project_num not in _PROJECT_NUM_CACHE:
raise UnknownProjectException('Unknown project ID %s!' %
project_num)
ret.append(_PROJECT_NUM_CACHE[project_num])
else:
if '/' not in project and project not in _PROJECT_ID_CACHE:
service = discovery.build('cloudresourcemanager', 'v1')
service._http = http.set_user_agent(
service._http, 'google-pso-tool/pubsub2inbox/1.1.0')
request = service.projects().get(projectId=project)
response = request.execute()
_PROJECT_ID_CACHE[response['projectId']] = (
response['projectId'], response['projectNumber'],
response['name'],
response['labels'] if 'labels' in response else {})
if project in _PROJECT_ID_CACHE:
ret.append(_PROJECT_ID_CACHE[project])
self.logger.debug('Expanding projects list finished.',
extra={'projects': ret})
return ret
def _get_from_metadata(path: str) -> str:
path = f"http://metadata.google.internal/computeMetadata/v1/{path}"
http_client = discovery.build_http()
set_user_agent(http_client, USER_AGENT)
response, content = http_client.request(path, headers={"Metadata-Flavor": "Google", "User-Agent": USER_AGENT})
if response.status != http.client.OK.value:
raise AutoException(f"failed to get metadata for: {path}: {content}")
return content.decode()
def build_request(http, *args, **kwargs):
new_http = httplib2.Http()
if user_agent is not None:
new_http = set_user_agent(new_http, user_agent)
if credentials is not None:
new_http = google_auth_httplib2.AuthorizedHttp(credentials, http=new_http)
return googleapiclient.http.HttpRequest(new_http, *args, **kwargs)
def __init__(self, context, db_driver=None):
self.check_gcs_options()
backup_bucket = CONF.backup_gcs_bucket
backup_credential = CONF.backup_gcs_credential_file
self.gcs_project_id = CONF.backup_gcs_project_id
chunk_size_bytes = CONF.backup_gcs_object_size
sha_block_size_bytes = CONF.backup_gcs_block_size
enable_progress_timer = CONF.backup_gcs_enable_progress_timer
super(GoogleBackupDriver, self).__init__(context, chunk_size_bytes,
sha_block_size_bytes,
backup_bucket,
enable_progress_timer,
db_driver)
credentials = client.GoogleCredentials.from_stream(backup_credential)
self.reader_chunk_size = CONF.backup_gcs_reader_chunk_size
self.writer_chunk_size = CONF.backup_gcs_writer_chunk_size
self.bucket_location = CONF.backup_gcs_bucket_location
self.storage_class = CONF.backup_gcs_storage_class
self.num_retries = CONF.backup_gcs_num_retries
http_user_agent = http.set_user_agent(
httplib2.Http(proxy_info=self.get_gcs_proxy_info()),
CONF.backup_gcs_user_agent)
self.conn = discovery.build('storage',
'v1',
http=http_user_agent,
credentials=credentials)
self.resumable = self.writer_chunk_size != -1
def from_credentials(cls, credentials: credentials.Credentials):
http_client = http.set_user_agent(http.build_http(),
'django-cloud-deploy')
auth_http = google_auth_httplib2.AuthorizedHttp(credentials,
http=http_client)
return cls(
discovery.build('cloudresourcemanager', 'v1', http=auth_http))
def _googleauth(key_file=None, scopes=[], user_agent=None):
"""
Google http_auth helper.
If key_file is not specified, default credentials will be used.
If scopes is specified (and key_file), will be used instead of DEFAULT_SCOPES
:param key_file: path to key file to use. Default is None
:type key_file: ``str``
:param scopes: scopes to set. Default is DEFAUL_SCOPES
:type scopes: ``list``
:param user_agent: User Agent string to use in requests. Default is None.
:type http_auth: ``str`` or None
:return: HTTPLib2 authorized client.
:rtype: :class: `HTTPLib2`
"""
if key_file:
if not scopes:
scopes = DEFAULT_SCOPES
creds = ServiceAccountCredentials.from_json_keyfile_name(key_file,
scopes=scopes)
else:
creds = GoogleCredentials.get_application_default()
http = Http()
if user_agent:
http = set_user_agent(http, user_agent)
http_auth = creds.authorize(http)
return http_auth
def __init__(self, context, db_driver=None):
self.check_gcs_options()
backup_bucket = CONF.backup_gcs_bucket
backup_credential = CONF.backup_gcs_credential_file
self.gcs_project_id = CONF.backup_gcs_project_id
chunk_size_bytes = CONF.backup_gcs_object_size
sha_block_size_bytes = CONF.backup_gcs_block_size
enable_progress_timer = CONF.backup_gcs_enable_progress_timer
super(GoogleBackupDriver,
self).__init__(context, chunk_size_bytes, sha_block_size_bytes,
backup_bucket, enable_progress_timer, db_driver)
credentials = client.GoogleCredentials.from_stream(backup_credential)
self.reader_chunk_size = CONF.backup_gcs_reader_chunk_size
self.writer_chunk_size = CONF.backup_gcs_writer_chunk_size
self.bucket_location = CONF.backup_gcs_bucket_location
self.storage_class = CONF.backup_gcs_storage_class
self.num_retries = CONF.backup_gcs_num_retries
http_user_agent = http.set_user_agent(
httplib2.Http(proxy_info=self.get_gcs_proxy_info()),
CONF.backup_gcs_user_agent)
self.conn = discovery.build('storage',
'v1',
http=http_user_agent,
credentials=credentials)
self.resumable = self.writer_chunk_size != -1
def _build_http(http=None):
"""Construct an http client suitable for googleapiclient usage w/ user agent.
"""
if not http:
http = httplib2.Http(timeout=HTTP_REQUEST_TIMEOUT)
user_agent = 'Python-httplib2/{} (gzip), {}/{}'.format(
httplib2.__version__, 'custodian-gcp', '0.1')
return set_user_agent(http, user_agent)
def get_google_api_auth(module,
scopes=[],
user_agent_product='ansible-python-api',
user_agent_version='NA'):
"""
Authentication for use with google-python-api-client.
Function calls _get_gcp_credentials, which attempts to assemble the credentials from various locations.
Next it attempts to authenticate with Google.
This function returns an httplib2 object that can be provided to the Google Python API client.
For libcloud, don't use this function, use gcp_connect instead. For Google Cloud, See
get_google_cloud_credentials for how to connect.
For more information on Google's client library options for Python, see:
U(https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries)
Google API example:
http_auth, conn_params = gcp_api_auth(module, scopes, user_agent_product, user_agent_version)
service = build('myservice', 'v1', http=http_auth)
...
:param module: initialized Ansible module object
:type module: `class AnsibleModule`
:param scopes: list of scopes
:type scopes: ``list`` of URIs
:param user_agent_product: User agent product. eg: 'ansible-python-api'
:type user_agent_product: ``str``
:param user_agent_version: Version string to append to product. eg: 'NA' or '0.1'
:type user_agent_version: ``str``
:returns: A tuple containing (google authorized) httplib2 request object and a
params dict {'service_account_email': '...', 'credentials_file': '...', 'project_id': ...}
:rtype: ``tuple``
"""
if not HAS_GOOGLE_API_LIB:
module.fail_json(msg="Please install google-api-python-client library")
# TODO(supertom): verify scopes
if not scopes:
scopes = ['https://www.googleapis.com/auth/cloud-platform']
try:
(credentials,
conn_params) = get_google_credentials(module,
scopes,
require_valid_json=True,
check_libcloud=False)
http = set_user_agent(
Http(), '%s-%s' % (user_agent_product, user_agent_version))
http_auth = credentials.authorize(http)
return (http_auth, conn_params)
except Exception as e:
module.fail_json(msg=unexpected_error_msg(e), changed=False)
return (None, None)
def _authorize(self) -> google_auth_httplib2.AuthorizedHttp:
"""
Returns an authorized HTTP object to be used to build a Google cloud
service hook connection.
"""
credentials = self._get_credentials()
http = build_http()
http = set_user_agent(http, "airflow/" + version.version)
authed_http = google_auth_httplib2.AuthorizedHttp(credentials, http=http)
return authed_http
def add_instances(node_chunk):
node_list = node_chunk['nodes']
pg_name = None
if 'pg' in node_chunk:
pg_name = node_chunk['pg']
log.debug(f"node_list:{node_list} pg:{pg_name}")
auth_http = None
if not cfg.google_app_cred_path:
http = set_user_agent(httplib2.Http(),
"Slurm_GCP_Scripts/1.2 (GPN:SchedMD)")
creds = compute_engine.Credentials()
auth_http = google_auth_httplib2.AuthorizedHttp(creds, http=http)
compute = googleapiclient.discovery.build('compute',
'v1',
http=auth_http,
cache_discovery=False)
pid = util.get_pid(node_list[0])
instance_def = cfg.instance_defs[pid]
try:
operation = create_instance(compute, instance_def, node_list, pg_name)
except googleapiclient.errors.HttpError as e:
log.error(
f"failed to add {node_list[0]}*{len(node_list)} to slurm, {e}")
if instance_def.exclusive:
os._exit(1)
down_nodes(node_list, e)
return
result = util.wait_for_operation(compute, cfg.project, operation)
if not result or 'error' in result:
grp_err_msg = result['error']['errors'][0]['message']
log.error(f"group operation failed: {grp_err_msg}")
if instance_def.exclusive:
os._exit(1)
group_ops = util.get_group_operations(compute, cfg.project, result)
failed_nodes = {}
for op in group_ops['items']:
if op['operationType'] != 'insert':
continue
if 'error' in op:
err_msg = op['error']['errors'][0]['message']
failed_node = op['targetLink'].split('/')[-1]
if err_msg not in failed_nodes:
failed_nodes[err_msg] = [failed_node]
else:
failed_nodes[err_msg].append(failed_node)
if failed_nodes:
log.error(f"insert requests failed: {failed_nodes}")
for msg, nodes in failed_nodes.items():
down_nodes(nodes, msg)
def _build_http(http=None):
"""Construct an http client suitable for googleapiclient usage w/ user agent.
"""
if not http:
http = httplib2.Http(
timeout=HTTP_REQUEST_TIMEOUT, ca_certs=HTTPLIB_CA_BUNDLE)
user_agent = 'Python-httplib2/{} (gzip), {}/{}'.format(
httplib2.__version__,
'custodian-gcp',
'0.1')
return set_user_agent(http, user_agent)
def get_service(config, name, version):
credentials = client.OAuth2Credentials(None,
config.get('client_id'),
config.get('client_secret'),
config.get('refresh_token'),
None,
GOOGLE_TOKEN_URI,
None,
revoke_uri=GOOGLE_REVOKE_URI)
http = credentials.authorize(httplib2.Http())
user_agent = config.get('user_agent')
if user_agent:
http = set_user_agent(http, user_agent)
return discovery.build(name, version, http=http, cache_discovery=False)
def create_placement_groups(arg_job_id, vm_count, region):
log.debug(f"Creating PG: {arg_job_id} vm_count:{vm_count} region:{region}")
pg_names = []
pg_ops = []
pg_index = 0
auth_http = None
if not cfg.google_app_cred_path:
http = set_user_agent(httplib2.Http(),
"Slurm_GCP_Scripts/1.2 (GPN:SchedMD)")
creds = compute_engine.Credentials()
auth_http = google_auth_httplib2.AuthorizedHttp(creds, http=http)
compute = googleapiclient.discovery.build('compute',
'v1',
http=auth_http,
cache_discovery=False)
for i in range(vm_count):
if i % PLACEMENT_MAX_CNT:
continue
pg_index += 1
pg_name = f'{cfg.cluster_name}-{arg_job_id}-{pg_index}'
pg_names.append(pg_name)
config = {
'name': pg_name,
'region': region,
'groupPlacementPolicy': {
"collocation": "COLLOCATED",
"vmCount": min(vm_count - i, PLACEMENT_MAX_CNT)
}
}
pg_ops.append(
util.ensure_execute(compute.resourcePolicies().insert(
project=cfg.project, region=region, body=config)))
for operation in pg_ops:
result = util.wait_for_operation(compute, cfg.project, operation)
if result and 'error' in result:
err_msg = result['error']['errors'][0]['message']
log.error(f" placement group operation failed: {err_msg}")
os._exit(1)
return pg_names
def initialize_api(config_path, service_account_file):
"""Initialization of API service and configuration for the Content API.
Args:
config_path: string, relative path to a directory where configuration files
exist.
service_account_file: string, file name of the service account configuration
file.
Returns:
A service object to interface with the Content API.
"""
credentials = _authorize(config_path, service_account_file)
auth_http = google_auth_httplib2.AuthorizedHttp(
credentials,
http=http.set_user_agent(http.build_http(), constants.APPLICATION_NAME))
service = discovery.build(
constants.SERVICE_NAME,
constants.CONTENT_API_VERSION,
http=auth_http,
cache_discovery=False)
return service
def from_service_account_json(cls, service_account_path):
"""Factory to retrieve JSON credentials while creating client.
This method initializes API service and configuration for the Content API
for Shopping.
Args:
service_account_path: String, path of the service account configuration
file.
Returns:
The client created with the retrieved JSON credentials.
"""
credentials = service_account.Credentials.from_service_account_file(
service_account_path, scopes=[_CONTENT_API_SCOPE])
auth_http = google_auth_httplib2.AuthorizedHttp(
credentials,
http=http.set_user_agent(http.build_http(), _APPLICATION_NAME))
service = discovery.build(_SERVICE_NAME,
_SERVICE_VERSION,
http=auth_http,
cache_discovery=False)
return ContentApiClient(service=service)
SCONTROL = '/apps/slurm/current/bin/scontrol'
LOGFILE = '/apps/slurm/log/resume.log'
TOT_REQ_CNT = 1000
# Set to True if the nodes aren't accessible by dns.
UPDATE_NODE_ADDRS = False
instances = {}
operations = {}
retry_list = []
credentials = compute_engine.Credentials()
http = set_user_agent(httplib2.Http(), "Slurm_GCP_Scripts/1.1 (GPN:SchedMD)")
authorized_http = google_auth_httplib2.AuthorizedHttp(credentials, http=http)
# [START wait_for_operation]
def wait_for_operation(compute, project, zone, operation):
print('Waiting for operation to finish...')
while True:
result = compute.zoneOperations().get(
project=project,
zone=zone,
operation=operation).execute()
if result['status'] == 'DONE':
print("done.")
if 'error' in result:
raise Exception(result['error'])
def init(argv, doc, parents=None, sandbox=False):
"""A common initialization routine for the Content API samples.
Args:
argv: list of string, the command-line parameters of the application.
doc: string, description of the application. Usually set to __doc__.
parents: list of argparse.ArgumentParser, additional command-line flags.
sandbox: boolean, whether to use the sandbox API endpoint or not.
Returns:
A tuple of (service, config, flags), where service is the service object,
config is the configuration JSON in Python form, and flags
are the parsed command-line flags.
"""
service = None
sandbox_service = None
flags = None
parent_parsers = []
if parents is not None:
parent_parsers.extend(parents)
parser = argparse.ArgumentParser(
description=doc,
formatter_class=argparse.RawDescriptionHelpFormatter,
parents=parent_parsers)
parser.add_argument(
'--config_path',
metavar='PATH',
default=os.path.expanduser('~/shopping-samples'),
help='configuration directory for the Shopping samples')
parser.add_argument(
'--noconfig',
action='store_true',
help='run samples with no configuration directory')
parser.add_argument(
'--log_file',
metavar='FILE',
help='filename for logging API requests and responses'
)
flags = parser.parse_args(argv[1:])
if flags.log_file:
logging.basicConfig(filename=flags.log_file, level=logging.INFO)
model.dump_request_response = True
config = {}
if not flags.noconfig:
if not os.path.isdir(flags.config_path):
print(
'Configuration directory "%s" does not exist.' % flags.config_path,
file=sys.stderr)
sys.exit(1)
content_path = os.path.join(flags.config_path, 'content')
if not os.path.isdir(content_path):
print(
'Content API configuration directory "%s" does not exist.' %
content_path,
file=sys.stderr)
sys.exit(1)
config_file = os.path.join(content_path, 'merchant-info.json')
if not os.path.isfile(config_file):
print('Configuration file %s does not exist.' % config_file)
print('Falling back to configuration based on authenticated user.')
else:
config = json.load(open(config_file, 'r'))
config['path'] = content_path
credentials = auth.authorize(config)
auth_http = google_auth_httplib2.AuthorizedHttp(
credentials, http=http.set_user_agent(
http.build_http(), _constants.APPLICATION_NAME))
if _constants.ENDPOINT_ENV_VAR in os.environ:
# Strip off everything after the host/port in the URL.
root_url = six.moves.urllib.parse.urljoin(
os.environ[_constants.ENDPOINT_ENV_VAR],
'/')
print('Using non-standard root for API discovery: %s' % root_url)
discovery_url = root_url + '/discovery/v1/apis/{api}/{apiVersion}/rest'
service = discovery.build(
_constants.SERVICE_NAME,
_constants.SERVICE_VERSION,
discoveryServiceUrl=discovery_url,
http=auth_http)
if sandbox:
sandbox_service = discovery.build(
_constants.SERVICE_NAME,
_constants.SANDBOX_SERVICE_VERSION,
discoveryServiceUrl=discovery_url,
http=auth_http)
else:
service = discovery.build(
_constants.SERVICE_NAME, _constants.SERVICE_VERSION, http=auth_http)
if sandbox:
sandbox_service = discovery.build(
_constants.SERVICE_NAME,
_constants.SANDBOX_SERVICE_VERSION,
http=auth_http)
# Now that we have a service object, fill in anything missing from the
# configuration using API calls.
retrieve_remaining_config_from_api(service, config)
return (sandbox_service if sandbox else service, config, flags)
def main():
"""Import multiple users' mbox files to Gmail.
"""
httplib2.debuglevel = args.httplib2debuglevel
# Use args.logging_level if defined.
try:
logging_level = args.logging_level
except AttributeError:
logging_level = 'INFO'
# Default logging to standard output
logging.basicConfig(
level=logging_level,
format=
'%(asctime)s %(levelname)s %(funcName)s@%(filename)s %(message)s',
datefmt='%H:%M:%S')
# More detailed logging to file
file_handler = logging.handlers.RotatingFileHandler(args.log,
maxBytes=1024 * 1024 *
32,
backupCount=8)
file_formatter = logging.Formatter(
'%(asctime)s %(process)d %(levelname)s %(funcName)s '
'(%(filename)s:%(lineno)d) %(message)s')
file_formatter.datefmt = '%Y-%m-%dT%H:%M:%S (%z)'
file_handler.setFormatter(file_formatter)
logging.getLogger().addHandler(file_handler)
logging.info('*** Starting %s %s on Python %s ***', APPLICATION_NAME,
APPLICATION_VERSION, sys.version)
logging.info('Arguments:')
for arg, value in sorted(vars(args).items()):
logging.info('\t%s: %r', arg, value)
number_of_labels_imported_without_error = 0
number_of_labels_imported_with_some_errors = 0
number_of_labels_failed = 0
number_of_messages_imported_without_error = 0
number_of_messages_failed = 0
number_of_users_imported_without_error = 0
number_of_users_imported_with_some_errors = 0
number_of_users_failed = 0
for username in next(os.walk(args.dir))[1]:
try:
logging.info('Processing user %s', username)
try:
credentials = get_credentials(username)
http = credentials.authorize(
set_user_agent(
httplib2.Http(),
'%s-%s' % (APPLICATION_NAME, APPLICATION_VERSION)))
service = discovery.build('gmail', 'v1', http=http)
except Exception:
logging.error("Can't get access token for user %s", username)
raise
try:
results = service.users().labels().list(
userId=username, fields='labels(id,name)').execute(
num_retries=args.num_retries)
labels = results.get('labels', [])
except Exception:
logging.error("Can't get labels for user %s", username)
raise
try:
result = process_mbox_files(username, service, labels)
except Exception:
logging.error("Can't process mbox files for user %s", username)
raise
if result[2] == 0 and result[4] == 0:
number_of_users_imported_without_error += 1
elif result[0] > 0 or result[3] > 0:
number_of_users_imported_with_some_errors += 1
else:
number_of_users_failed += 1
number_of_labels_imported_without_error += result[0]
number_of_labels_imported_with_some_errors += result[1]
number_of_labels_failed += result[2]
number_of_messages_imported_without_error += result[3]
number_of_messages_failed += result[4]
logging.info(
'Done importing user %s. Labels: %d succeeded, %d with some '
'errors, %d failed. Messages: %d succeeded, %d failed.',
username, result[0], result[1], result[2], result[3],
result[4])
except Exception:
number_of_users_failed += 1
logging.exception("Can't process user %s", username)
logging.info("*** Done importing all users from directory '%s'", args.dir)
logging.info('*** Import summary:')
logging.info(' %d users imported with no failures',
number_of_users_imported_without_error)
logging.info(' %d users imported with some failures',
number_of_users_imported_with_some_errors)
logging.info(' %d users failed', number_of_users_failed)
logging.info(' %d labels (mbox files) imported with no failures',
number_of_labels_imported_without_error)
logging.info(' %d labels (mbox files) imported with some failures',
number_of_labels_imported_with_some_errors)
logging.info(' %d labels (mbox files) failed', number_of_labels_failed)
logging.info(' %d messages imported successfully',
number_of_messages_imported_without_error)
logging.info(' %d messages failed\n', number_of_messages_failed)
if (number_of_messages_failed + number_of_labels_failed +
number_of_users_failed > 0):
logging.info('*** Check log file %s for detailed errors.', args.log)
logging.info('Finished.\n\n')
def process(self):
if 'recommendations' not in self.config:
raise NotConfiguredException(
'No Recommender configuration specified in config!')
recommender_config = self.config['recommendations']
for recommender in recommender_config['recommender_types']:
if recommender not in self.recommenders:
raise UnknownRecommenderException(
'Unknown recommender %s specified in config!' %
(recommender))
data = json.loads(self.data)
self.jinja_environment.globals = {
**self.jinja_environment.globals,
**data
}
projects = []
if 'projects' in recommender_config:
projects = self._jinja_var_to_list(recommender_config['projects'],
'projects')
folders = []
if 'folders' in recommender_config:
folders = self._jinja_var_to_list(recommender_config['folders'],
'folders')
organizations = []
if 'organizations' in recommender_config:
organizations = self._jinja_var_to_list(
recommender_config['organizations'], 'organizations')
billing_accounts = []
if 'billingAccounts' in recommender_config:
billing_accounts = self._jinja_var_to_list(
recommender_config['billingAccounts'], 'billing_accounts')
if len(projects) == 0 and len(folders) == 0 and len(
organizations) == 0 and len(billing_accounts) == 0:
raise NotConfiguredException(
'No projects, organizations, folders or billing accounts specified in config!'
)
location_filters = self._jinja_var_to_list(
recommender_config['locations'], 'locations')
if len(location_filters) == 0:
raise NotConfiguredException(
'No location filters specified in config!')
client_info = grpc_client_info.ClientInfo(
user_agent='google-pso-tool/pubsub2inbox/1.1.0')
client = RecommenderClient(client_info=client_info)
credentials, project_id = google.auth.default(
['https://www.googleapis.com/auth/cloud-platform'])
branded_http = google_auth_httplib2.AuthorizedHttp(credentials)
branded_http = http.set_user_agent(
branded_http, 'google-pso-tool/pubsub2inbox/1.1.0')
compute_service = discovery.build('compute', 'v1', http=branded_http)
if len(projects) == 0:
raise NotConfiguredException(
'Please specify at least one project to fetch regions and zones.'
)
all_zones = self.get_zones(compute_service, projects[0],
location_filters)
all_regions = self.get_regions(compute_service, projects[0],
location_filters)
all_locations = all_zones + all_regions
self.logger.debug('Fetched all available locations.',
extra={'locations': all_locations})
parents = []
for project in self.expand_projects(projects):
parents.append(('projects/%s' % project[1], project))
for organization in organizations:
parents.append(('organizations/%s' % organization, [organization]))
for folder in folders:
parents.append(('folder/%s' % folder, [folder]))
for billing_account in billing_accounts:
parents.append(
('billingAccounts/%s' % billing_account, [billing_account]))
self.logger.debug('Determined all parents.',
extra={'parents': parents})
recommendations = {}
recommendations_rollup = {}
if 'fetch_recommendations' in recommender_config:
fetch_recommendations = self._jinja_expand_bool(
recommender_config['fetch_recommendations'])
if fetch_recommendations:
recommendations = self.get_recommendations(
client, recommender_config['recommender_types'], parents,
all_locations, recommender_config['recommendation_filter']
if 'recommendation_filter' in recommender_config else None)
recommendations_rollup = self.rollup_recommendations(
recommendations)
insights = {}
insights_rollup = {}
if 'fetch_insights' in recommender_config:
fetch_insights = self._jinja_expand_bool(
recommender_config['fetch_insights'])
if fetch_insights:
insights = self.get_insights(
client, recommender_config['insight_types'], parents,
all_locations, recommender_config['insight_filter']
if 'insight_filter' in recommender_config else None)
insights_rollup = self.rollup_insights(insights)
self.logger.debug('Fetching recommendations and/or insights finished.')
_ret = {
'recommendations': recommendations,
'recommendations_rollup': recommendations_rollup,
'insights': insights,
'insights_rollup': insights_rollup,
}
if 'vars' in recommender_config:
return {**recommender_config['vars'], **_ret}
return _ret
def _build(self, service_name: str, version: str):
resource = discovery.build(service_name, version, cache_discovery=False)
set_user_agent(resource._http, USER_AGENT)
return resource
def init(argv, doc, parents=None, sandbox=False):
"""A common initialization routine for the Content API samples.
Args:
argv: list of string, the command-line parameters of the application.
doc: string, description of the application. Usually set to __doc__.
parents: list of argparse.ArgumentParser, additional command-line flags.
sandbox: boolean, whether to use the sandbox API endpoint or not.
Returns:
A tuple of (service, config, flags), where service is the service object,
config is the configuration JSON in Python form, and flags
are the parsed command-line flags.
"""
service = None
sandbox_service = None
flags = None
parent_parsers = []
if parents is not None:
parent_parsers.extend(parents)
parser = argparse.ArgumentParser(
description=doc,
formatter_class=argparse.RawDescriptionHelpFormatter,
parents=parent_parsers)
parser.add_argument(
'--config_path',
metavar='PATH',
default=os.path.expanduser('~/shopping-samples'),
help='configuration directory for the Shopping samples')
parser.add_argument('--noconfig',
action='store_true',
help='run samples with no configuration directory')
parser.add_argument('--log_file',
metavar='FILE',
help='filename for logging API requests and responses')
flags = parser.parse_args(argv[1:])
if flags.log_file:
logging.basicConfig(filename=flags.log_file, level=logging.INFO)
model.dump_request_response = True
config = {}
if not flags.noconfig:
if not os.path.isdir(flags.config_path):
print('Configuration directory "%s" does not exist.' %
flags.config_path,
file=sys.stderr)
sys.exit(1)
content_path = os.path.join(flags.config_path, 'content')
if not os.path.isdir(content_path):
print('Content API configuration directory "%s" does not exist.' %
content_path,
file=sys.stderr)
sys.exit(1)
config_file = os.path.join(content_path, 'merchant-info.json')
if not os.path.isfile(config_file):
print('Configuration file %s does not exist.' % config_file)
print('Falling back to configuration based on authenticated user.')
else:
config = json.load(open(config_file, 'r'))
config['path'] = content_path
credentials = auth.authorize(config)
auth_http = google_auth_httplib2.AuthorizedHttp(
credentials,
http=http.set_user_agent(http.build_http(),
_constants.APPLICATION_NAME))
if _constants.ENDPOINT_ENV_VAR in os.environ:
# Strip off everything after the host/port in the URL.
root_url = urlparse.urljoin(os.environ[_constants.ENDPOINT_ENV_VAR],
'/')
print('Using non-standard root for API discovery: %s' % root_url)
discovery_url = root_url + '/discovery/v1/apis/{api}/{apiVersion}/rest'
service = discovery.build(_constants.SERVICE_NAME,
_constants.SERVICE_VERSION,
discoveryServiceUrl=discovery_url,
http=auth_http)
if sandbox:
sandbox_service = discovery.build(
_constants.SERVICE_NAME,
_constants.SANDBOX_SERVICE_VERSION,
discoveryServiceUrl=discovery_url,
http=auth_http)
else:
service = discovery.build(_constants.SERVICE_NAME,
_constants.SERVICE_VERSION,
http=auth_http)
if sandbox:
sandbox_service = discovery.build(
_constants.SERVICE_NAME,
_constants.SANDBOX_SERVICE_VERSION,
http=auth_http)
# Now that we have a service object, fill in anything missing from the
# configuration using API calls.
retrieve_remaining_config_from_api(service, config)
return (sandbox_service if sandbox else service, config, flags)
description=
'Create custom roles by filtering existing permissions or roles')
arg_parser.add_argument('--config',
type=str,
help='Configuration file',
default='config.yaml')
arg_parser.add_argument(
'--terraform',
action='store_true',
help='Output a Terraform compatible custom role definition instead',
default=False)
args = arg_parser.parse_args()
credentials, project_id = google.auth.default(
['https://www.googleapis.com/auth/cloud-platform'])
branded_http = google_auth_httplib2.AuthorizedHttp(credentials)
branded_http = http.set_user_agent(
branded_http, 'google-pso-tool/custom-role-manager/1.0.0')
service = googleapiclient.discovery.build('iam', 'v1', http=branded_http)
with open(args.config) as f:
config = yaml.load(f, Loader=yaml.SafeLoader)
if 'roles' not in config:
logger.error('Roles are not defined in the configuration!')
sys.exit(1)
if 'terraform' in config:
for k, v in config['terraform'].items():
TERRAFORM_TEMPLATES[k] = v
for role in config['roles']:
process_role(logger, service, role, args.terraform)
if TERRAFORM_PRE:
resources = ''
for k, v in TERRAFORM_RESOURCES.items():
if resources == '':
def main():
"""Import multiple users' mbox files to Gmail.
"""
httplib2.debuglevel = args.httplib2debuglevel
# Use args.logging_level if defined.
try:
logging_level = args.logging_level
except AttributeError:
logging_level = 'INFO'
# Default logging to standard output
logging.basicConfig(
level=logging_level,
format='%(asctime)s %(levelname)s %(funcName)s@%(filename)s %(message)s',
datefmt='%H:%M:%S')
# More detailed logging to file
file_handler = logging.handlers.RotatingFileHandler(args.log,
maxBytes=1024 * 1024 * 32,
backupCount=8)
file_formatter = logging.Formatter(
'%(asctime)s %(process)d %(levelname)s %(funcName)s '
'(%(filename)s:%(lineno)d) %(message)s')
file_formatter.datefmt = '%Y-%m-%dT%H:%M:%S (%z)'
file_handler.setFormatter(file_formatter)
logging.getLogger().addHandler(file_handler)
logging.info('*** Starting %s %s on Python %s ***',
APPLICATION_NAME,
APPLICATION_VERSION,
sys.version)
logging.info('Arguments:')
for arg, value in sorted(vars(args).items()):
logging.info('\t%s: %r', arg, value)
number_of_labels_imported_without_error = 0
number_of_labels_imported_with_some_errors = 0
number_of_labels_failed = 0
number_of_messages_imported_without_error = 0
number_of_messages_failed = 0
number_of_users_imported_without_error = 0
number_of_users_imported_with_some_errors = 0
number_of_users_failed = 0
for username in next(os.walk(args.dir))[1]:
try:
logging.info('Processing user %s', username)
try:
credentials = get_credentials(username)
http = credentials.authorize(set_user_agent(
httplib2.Http(),
'%s-%s' % (APPLICATION_NAME, APPLICATION_VERSION)))
service = discovery.build('gmail', 'v1', http=http)
except Exception:
logging.error("Can't get access token for user %s", username)
raise
try:
results = service.users().labels().list(
userId=username,
fields='labels(id,name)').execute(num_retries=args.num_retries)
labels = results.get('labels', [])
except Exception:
logging.error("Can't get labels for user %s", username)
raise
try:
result = process_mbox_files(username, service, labels)
except Exception:
logging.error("Can't process mbox files for user %s", username)
raise
if result[2] == 0 and result[4] == 0:
number_of_users_imported_without_error += 1
elif result[0] > 0 or result[3] > 0:
number_of_users_imported_with_some_errors += 1
else:
number_of_users_failed += 1
number_of_labels_imported_without_error += result[0]
number_of_labels_imported_with_some_errors += result[1]
number_of_labels_failed += result[2]
number_of_messages_imported_without_error += result[3]
number_of_messages_failed += result[4]
logging.info('Done importing user %s. Labels: %d succeeded, %d with some '
'errors, %d failed. Messages: %d succeeded, %d failed.',
username,
result[0],
result[1],
result[2],
result[3],
result[4])
except Exception:
number_of_users_failed += 1
logging.exception("Can't process user %s", username)
logging.info("*** Done importing all users from directory '%s'", args.dir)
logging.info('*** Import summary:')
logging.info(' %d users imported with no failures',
number_of_users_imported_without_error)
logging.info(' %d users imported with some failures',
number_of_users_imported_with_some_errors)
logging.info(' %d users failed',
number_of_users_failed)
logging.info(' %d labels (mbox files) imported with no failures',
number_of_labels_imported_without_error)
logging.info(' %d labels (mbox files) imported with some failures',
number_of_labels_imported_with_some_errors)
logging.info(' %d labels (mbox files) failed',
number_of_labels_failed)
logging.info(' %d messages imported successfully',
number_of_messages_imported_without_error)
logging.info(' %d messages failed\n',
number_of_messages_failed)
if (number_of_messages_failed + number_of_labels_failed +
number_of_users_failed > 0):
logging.info('*** Check log file %s for detailed errors.', args.log)
logging.info('Finished.\n\n')