def _transform(self, resource_from_api):
"""Yield an iterator of loadable iam policies.
Args:
resource_from_api (iterable): Resource manager project list
response.
https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#response-body
Yields:
iterable: Loadable projects, as a per-project dictionary.
"""
for project in (project for d in resource_from_api\
for project in d.get('projects', [])):
yield {
'project_number':
project.get('projectNumber'),
'project_id':
project.get('projectId'),
'project_name':
project.get('name'),
'lifecycle_state':
project.get('lifecycleState'),
'parent_type':
project.get('parent', {}).get('type'),
'parent_id':
project.get('parent', {}).get('id'),
'raw_project':
parser.json_stringify(project),
'create_time':
parser.format_timestamp(project.get('createTime'),
self.MYSQL_DATETIME_FORMAT)
}
def _transform(self, resource_from_api):
"""Create an iterator of instance templates to load into database.
Args:
resource_from_api (dict): Instance templates, keyed by
project id, from GCP API.
Yields:
iterator: instance template properties in a dict.
"""
for (project_id, instance_templates) in resource_from_api.iteritems():
for instance_template in instance_templates:
yield {
'project_id':
project_id,
'id':
instance_template.get('id'),
'creation_timestamp':
parser.format_timestamp(
instance_template.get('creationTimestamp'),
self.MYSQL_DATETIME_FORMAT),
'name':
instance_template.get('name'),
'description':
instance_template.get('description'),
'properties':
parser.json_stringify(
instance_template.get('properties', {})),
'raw_instance_template':
parser.json_stringify(instance_template)
}
def _transform(self, resource_from_api):
"""Yield an iterator of loadable organizations.
Args:
resource_from_api: An iterable of resource manager org search
response.
https://cloud.google.com/resource-manager/reference/rest/v1/organizations/search
https://cloud.google.com/resource-manager/reference/rest/v1/organizations#Organization
Yields:
An iterable of loadable orgs, each org as a dict.
"""
for org in (o for d in resource_from_api for o in d.get(
'organizations', [])):
# org_name is the unique identifier for the org, formatted as
# "organizations/<organization_id>".
org_name = org.get('name')
org_id = org_name[len('%s/' % self.RESOURCE_NAME):]
yield {'org_id': org_id,
'name': org_name,
'display_name': org.get('displayName'),
'lifecycle_state': org.get('lifecycleState'),
'raw_org': parser.json_stringify(org),
'creation_time': parser.format_timestamp(
org.get('creationTime'),
self.MYSQL_DATETIME_FORMAT)}
def _transform(self, resource_from_api):
"""Create an iterator of instance groups to load into database.
Args:
resource_from_api: A dict of instance groups, keyed by
project id, from GCP API.
Yields:
Iterator of instance group properties in a dict.
"""
for (project_id, instance_groups) in resource_from_api.iteritems():
for instance_group in instance_groups:
yield {'project_id': project_id,
'id': instance_group.get('id'),
'creation_timestamp': parser.format_timestamp(
instance_group.get('creationTimestamp'),
self.MYSQL_DATETIME_FORMAT),
'name': instance_group.get('name'),
'description': instance_group.get('description'),
'named_ports': parser.json_stringify(
instance_group.get('namedPorts', [])),
'network': instance_group.get('network'),
'region': instance_group.get('region'),
'size': self._to_int(instance_group.get('size')),
'subnetwork': instance_group.get('subnetwork'),
'zone': instance_group.get('zone'),
'raw_instance_group':
parser.json_stringify(instance_group)}
def _transform(self, resource_from_api):
"""Create an iterator of instance group managers to load into database.
Args:
resource_from_api (dict): Instance group managers, keyed by
project id, from GCP API.
Yields:
iterator: instance group manager properties in a dict.
"""
for (project_id, igms) in resource_from_api.iteritems():
for igm in igms:
yield {'project_id': project_id,
'id': igm.get('id'),
'creation_timestamp': parser.format_timestamp(
igm.get('creationTimestamp'),
self.MYSQL_DATETIME_FORMAT),
'name': igm.get('name'),
'description': igm.get('description'),
'base_instance_name': igm.get('baseInstanceName'),
'current_actions': parser.json_stringify(
igm.get('currentActions', {})),
'instance_group': igm.get('instanceGroup'),
'instance_template': igm.get('instanceTemplate'),
'named_ports': parser.json_stringify(
igm.get('namedPorts', [])),
'region': igm.get('region'),
'target_pools': parser.json_stringify(
igm.get('targetPools', [])),
'target_size': igm.get('targetSize'),
'zone': igm.get('zone'),
'raw_instance_group_manager':
parser.json_stringify(igm)}
def _transform(self, resource_from_api):
"""Transform firewall rules map into loadable format for Cloud SQL.
Loading the project id as project number is not supported by the
GCP firewall api.
Args:
resource_from_api (dict): A dict mapping projects with a list of
their firewall rules.
{project_id1: [firewall_rule1a, firewall_rule1b],
project_id2: [firewall_rule2a, firewall_rule2b],
project_id3: [firewall_rule3a, firewall_rule3b]}
Yields:
iterable: Loadable firewall rules as a per-firewall dictionary.
"""
for project_id, firewall_rules in resource_from_api.iteritems():
for firewall_rule in firewall_rules:
yield {'firewall_rule_id': firewall_rule.get('id'),
'project_id': project_id,
'firewall_rule_name': firewall_rule.get('name'),
'firewall_rule_description':
firewall_rule.get('description'),
'firewall_rule_kind': firewall_rule.get('kind'),
'firewall_rule_network': firewall_rule.get('network'),
'firewall_rule_priority': firewall_rule.get('priority'),
'firewall_rule_direction':
firewall_rule.get('direction'),
'firewall_rule_source_ranges':
parser.json_stringify(
firewall_rule.get('sourceRanges')),
'firewall_rule_destination_ranges':
parser.json_stringify(
firewall_rule.get('destinationRanges')),
'firewall_rule_source_tags':
parser.json_stringify(
firewall_rule.get('sourceTags')),
'firewall_rule_target_tags':
parser.json_stringify(
firewall_rule.get('targetTags')),
'firewall_rule_source_service_accounts':
parser.json_stringify(
firewall_rule.get('sourceServiceAccounts')),
'firewall_rule_target_service_accounts':
parser.json_stringify(
firewall_rule.get('targetServiceAccounts')),
'firewall_rule_allowed':
parser.json_stringify(
firewall_rule.get('allowed')),
'firewall_rule_denied':
parser.json_stringify(
firewall_rule.get('denied')),
'firewall_rule_self_link': firewall_rule.get('selfLink'),
'firewall_rule_create_time': parser.format_timestamp(
firewall_rule.get('creationTimestamp'),
self.MYSQL_DATETIME_FORMAT),
'raw_firewall_rule':
parser.json_stringify(firewall_rule)}
def _transform(firewall_dict, project_id=None, validate=None):
"""Transforms firewall dictionary into FirewallRule.
Args:
firewall_dict (dict): A dictionary with firewall field names matching
the API field names.
project_id (str): A project id string.
validate (bool): Whether to validate this FirewallRule or not.
Returns:
FirewallRule: A FirewallRule created from the input dictionary.
"""
in_dict = {
'firewall_rule_id':
firewall_dict.get('id'),
'firewall_rule_name':
firewall_dict.get('name'),
'firewall_rule_description':
firewall_dict.get('description'),
'firewall_rule_kind':
firewall_dict.get('kind'),
'firewall_rule_network':
firewall_dict.get('network'),
'firewall_rule_priority':
firewall_dict.get('priority'),
'firewall_rule_direction':
firewall_dict.get('direction'),
'firewall_rule_source_ranges':
parser.json_stringify(firewall_dict.get('sourceRanges')),
'firewall_rule_destination_ranges':
parser.json_stringify(firewall_dict.get('destinationRanges')),
'firewall_rule_source_tags':
parser.json_stringify(firewall_dict.get('sourceTags')),
'firewall_rule_target_tags':
parser.json_stringify(firewall_dict.get('targetTags')),
'firewall_rule_source_service_accounts':
parser.json_stringify(firewall_dict.get('sourceServiceAccounts')),
'firewall_rule_target_service_accounts':
parser.json_stringify(firewall_dict.get('targetServiceAccounts')),
'firewall_rule_allowed':
parser.json_stringify(firewall_dict.get('allowed')),
'firewall_rule_denied':
parser.json_stringify(firewall_dict.get('denied')),
'firewall_rule_self_link':
parser.json_stringify(firewall_dict.get('selfLink')),
'firewall_rule_create_time':
parser.format_timestamp(
parser.json_stringify(firewall_dict.get('creationTimestamp')),
FirewallRule.MYSQL_DATETIME_FORMAT),
}
if project_id:
in_dict['project_id'] = project_id
return FirewallRule(validate=validate, **in_dict)
def _transform(self, resource_from_api):
"""Create an iterator of instances to load into database.
Args:
resource_from_api (dict): A dict of instances, keyed by
project id, from GCP API.
Yields:
dict: Instance properties.
"""
for (project_id, instances) in resource_from_api.iteritems():
for instance in instances:
yield {
'project_id':
project_id,
'id':
instance.get('id'),
'creation_timestamp':
parser.format_timestamp(instance.get('creationTimestamp'),
self.MYSQL_DATETIME_FORMAT),
'name':
instance.get('name'),
'description':
instance.get('description'),
'can_ip_forward':
self._to_bool(instance.get('canIpForward', 0)),
'cpu_platform':
instance.get('cpuPlatform'),
'disks':
parser.json_stringify(instance.get('disks', [])),
'machine_type':
instance.get('machineType'),
'metadata':
parser.json_stringify(instance.get('metadata', {})),
'network_interfaces':
parser.json_stringify(instance.get('networkInterfaces',
[])),
'scheduling':
parser.json_stringify(instance.get('scheduling', {})),
'service_accounts':
parser.json_stringify(instance.get('serviceAccounts', [])),
'status':
instance.get('status'),
'status_message':
instance.get('statusMessage'),
'tags':
parser.json_stringify(instance.get('tags')),
'zone':
instance.get('zone'),
'raw_instance':
parser.json_stringify(instance)
}
def _transform(self, resource_from_api):
"""Create an iterator of forwarding rules to load into database.
TODO: truncate the region and target.
Args:
resource_from_api: A dict of forwarding rules, keyed by
project id, from GCP API.
Yields:
Iterator of forwarding rule properties in a dict.
"""
for (project_id, forwarding_rules) in resource_from_api.iteritems():
for rule in forwarding_rules:
yield {
'project_id':
project_id,
'id':
rule.get('id'),
'creation_timestamp':
parser.format_timestamp(rule.get('creationTimestamp'),
self.MYSQL_DATETIME_FORMAT),
'name':
rule.get('name'),
'description':
rule.get('description'),
'region':
rule.get('region'),
'ip_address':
rule.get('IPAddress'),
'ip_protocol':
rule.get('IPProtocol'),
'port_range':
rule.get('portRange'),
'ports':
parser.json_stringify(rule.get('ports', [])),
'target':
rule.get('target'),
'load_balancing_scheme':
rule.get('loadBalancingScheme'),
'subnetwork':
rule.get('subnetwork'),
'network':
rule.get('network'),
'backend_service':
rule.get('backendService'),
'raw_forwarding_rule':
parser.json_stringify(rule)
}
def _transform(self, resource_from_api):
"""Create an iterator of backend services to load into database.
Args:
resource_from_api (dict): Forwarding rules, keyed by
project id, from GCP API.
Yields:
iterator: backend service properties in a dict.
"""
for (project_id, backend_services) in resource_from_api.iteritems():
for backend_service in backend_services:
yield {'project_id': project_id,
'id': backend_service.get('id'),
'creation_timestamp': parser.format_timestamp(
backend_service.get('creationTimestamp'),
self.MYSQL_DATETIME_FORMAT),
'name': backend_service.get('name'),
'description': backend_service.get('description'),
'affinity_cookie_ttl_sec': self._to_int(
backend_service.get('affinityCookieTtlSec')),
'backends': parser.json_stringify(
backend_service.get('backends', [])),
'cdn_policy': parser.json_stringify(
backend_service.get('cdnPolicy', {})),
'connection_draining': parser.json_stringify(
backend_service.get('connectionDraining', {})),
'enable_cdn': self._to_bool(
backend_service.get('enableCDN')),
'health_checks': parser.json_stringify(
backend_service.get('healthChecks', [])),
'iap': parser.json_stringify(
backend_service.get('iap', {})),
'load_balancing_scheme': backend_service.get(
'loadBalancingScheme'),
'port': self._to_int(backend_service.get('port')),
'port_name': backend_service.get('portName'),
'protocol': backend_service.get('protocol'),
'region': backend_service.get('region'),
'session_affinity': backend_service.get(
'sessionAffinity'),
'timeout_sec': backend_service.get('timeoutSec'),
'raw_backend_service':
parser.json_stringify(backend_service)}
