def SetIamPolicy(lake_ref, policy):
"""Set Iam Policy request."""
set_iam_policy_req = dataplex_api.GetMessageModule(
).DataplexProjectsLocationsLakesSetIamPolicyRequest(
resource=lake_ref.RelativeName(),
googleIamV1SetIamPolicyRequest=dataplex_api.GetMessageModule(
).GoogleIamV1SetIamPolicyRequest(policy=policy))
return dataplex_api.GetClientInstance(
).projects_locations_lakes.SetIamPolicy(set_iam_policy_req)
def GenerateSessionSpec(args):
"""Generate SessionSpec From Arguments."""
module = dataplex_api.GetMessageModule()
session_spec = module.GoogleCloudDataplexV1EnvironmentSessionSpec(
enableFastStartup=args.session_enable_fast_startup,
maxIdleDuration=args.session_max_idle_duration)
return session_spec
def Run(self, args):
environment_ref = args.CONCEPTS.environment.Parse()
dataplex_client = dataplex_util.GetClientInstance()
create_req_op = dataplex_client.projects_locations_lakes_environments.Create(
dataplex_util.GetMessageModule(
).DataplexProjectsLocationsLakesEnvironmentsCreateRequest(
environmentId=environment_ref.Name(),
parent=environment_ref.Parent().RelativeName(),
validateOnly=args.validate_only,
googleCloudDataplexV1Environment=environment.
GenerateEnvironmentForCreateRequest(args)))
validate_only = getattr(args, 'validate_only', False)
if validate_only:
log.status.Print('Validation complete.')
return
async_ = getattr(args, 'async_', False)
if not async_:
response = environment.WaitForOperation(create_req_op)
log.CreatedResource(
response.name,
details=
'Environment created in lake [{0}] in project [{1}] with location [{2}]'
.format(environment_ref.lakesId, environment_ref.projectsId,
environment_ref.locationsId))
return response
log.status.Print(
'Creating environment with path [{0}] and operation [{1}].'.format(
environment_ref, create_req_op.name))
return create_req_op
def GenerateCsvOptions(args):
return dataplex_api.GetMessageModule(
).GoogleCloudDataplexV1ZoneDiscoverySpecCsvOptions(
delimiter=args.csv_delimiter,
disableTypeInference=args.csv_disable_type_inference,
encoding=args.csv_encoding,
headerRows=args.csv_header_rows)
def AddIamPolicyBinding(lake_ref, member, role):
"""Add iam policy binding request."""
policy = GetIamPolicy(lake_ref)
iam_util.AddBindingToIamPolicy(
dataplex_api.GetMessageModule().GoogleIamV1Binding, policy, member,
role)
return SetIamPolicy(lake_ref, policy)
def Run(self, args):
update_mask = environment.GenerateUpdateMask(args)
if len(update_mask) < 1:
raise exceptions.HttpException(
'Update commands must specify at least one additional parameter to change.'
)
environment_ref = args.CONCEPTS.environment.Parse()
dataplex_client = dataplex_util.GetClientInstance()
message = dataplex_util.GetMessageModule()
update_req_op = dataplex_client.projects_locations_lakes_environments.Patch(
message.DataplexProjectsLocationsLakesEnvironmentsPatchRequest(
name=environment_ref.RelativeName(),
validateOnly=args.validate_only,
updateMask=u','.join(update_mask),
googleCloudDataplexV1Environment=environment.
GenerateEnvironmentForUpdateRequest(args)))
validate_only = getattr(args, 'validate_only', False)
if validate_only:
log.status.Print('Validation complete.')
return
async_ = getattr(args, 'async_', False)
if not async_:
response = environment.WaitForOperation(update_req_op)
log.UpdatedResource(environment_ref,
details='Operation was successful.')
return response
log.status.Print('Updating [{0}] with operation [{1}].'.format(
environment_ref, update_req_op.name))
return update_req_op
def Run(self, args):
asset_ref = args.CONCEPTS.asset.Parse()
dataplex_client = dataplex_util.GetClientInstance()
create_req_op = dataplex_client.projects_locations_lakes_zones_assets.Create(
dataplex_util.GetMessageModule(
).DataplexProjectsLocationsLakesZonesAssetsCreateRequest(
assetId=asset_ref.Name(),
parent=asset_ref.Parent().RelativeName(),
validateOnly=args.validate_only,
googleCloudDataplexV1Asset=asset.GenerateAssetForCreateRequest(
args)))
validate_only = getattr(args, 'validate_only', False)
if validate_only:
log.status.Print('Validation complete.')
return
async_ = getattr(args, 'async_', False)
if not async_:
asset.WaitForOperation(create_req_op)
log.CreatedResource(asset_ref.Name(),
details='Asset created in [{0}]'.format(
asset_ref.Parent().RelativeName()))
return
log.status.Print('Creating [{0}] with operation [{1}].'.format(
asset_ref, create_req_op.name))
def Run(self, args):
update_mask = lake.GenerateUpdateMask(args)
if len(update_mask) < 1:
raise exceptions.HttpException(
'Update commands must specify at least one additional parameter to change.'
)
lake_ref = args.CONCEPTS.lake.Parse()
dataplex_client = dataplex_util.GetClientInstance()
message = dataplex_util.GetMessageModule()
update_req_op = dataplex_client.projects_locations_lakes.Patch(
message.DataplexProjectsLocationsLakesPatchRequest(
name=lake_ref.RelativeName(),
validateOnly=args.validate_only,
updateMask=u','.join(update_mask),
googleCloudDataplexV1Lake=message.GoogleCloudDataplexV1Lake(
description=args.description,
displayName=args.display_name,
metastore=message.GoogleCloudDataplexV1LakeMetastore(
service=args.metastore_service),
labels=dataplex_util.CreateLabels(
message.GoogleCloudDataplexV1Lake, args))))
validate_only = getattr(args, 'validate_only', False)
if validate_only:
log.status.Print('Validation complete with errors:')
return update_req_op
async_ = getattr(args, 'async_', False)
if not async_:
lake.WaitForOperation(update_req_op)
log.UpdatedResource(lake_ref, details='Operation was sucessful.')
return
log.status.Print('Updating [{0}] with operation [{1}].'.format(
lake_ref, update_req_op.name))
def Run(self, args):
"""Constructs and sends request.
Args:
args: argparse.Namespace, An object that contains the values for the
arguments specified in the .Args() method.
"""
# We extract lake reference, instead of content reference as content id
# is not provided at the time of create content and is auto-generated
# and returned in response.
lake_ref = args.CONCEPTS.lake.Parse()
dataplex_client = dataplex_util.GetClientInstance()
content_response = dataplex_client.projects_locations_lakes_contentitems.Create(
dataplex_util.GetMessageModule(
).DataplexProjectsLocationsLakesContentitemsCreateRequest(
parent=lake_ref.RelativeName(),
validateOnly=args.validate_only,
googleCloudDataplexV1Content=content.
GenerateContentForCreateRequest(args)))
validate_only = getattr(args, 'validate_only', False)
if validate_only:
log.status.Print('Validation complete.')
return
log.CreatedResource(
content_response.name,
details=
'Content created in lake [{0}] in project [{1}] with location [{2}]'
.format(lake_ref.lakesId, lake_ref.projectsId,
lake_ref.locationsId))
def Run(self, args):
lake_ref = args.CONCEPTS.lake.Parse()
dataplex_client = dataplex_util.GetClientInstance()
message = dataplex_util.GetMessageModule()
create_req_op = dataplex_client.projects_locations_lakes.Create(
message.DataplexProjectsLocationsLakesCreateRequest(
lakeId=lake_ref.Name(),
parent=lake_ref.Parent().RelativeName(),
validateOnly=args.validate_only,
googleCloudDataplexV1Lake=message.GoogleCloudDataplexV1Lake(
description=args.description,
displayName=args.display_name,
labels=dataplex_util.CreateLabels(
message.GoogleCloudDataplexV1Lake, args),
metastore=message.GoogleCloudDataplexV1LakeMetastore(
service=args.metastore_service))))
validate_only = getattr(args, 'validate_only', False)
if validate_only:
log.status.Print('Validation complete.')
return
async_ = getattr(args, 'async_', False)
if not async_:
lake.WaitForOperation(create_req_op)
log.CreatedResource(lake_ref.Name(),
details='Lake created in [{0}]'.format(
lake_ref.Parent().RelativeName()))
return
log.status.Print('Creating [{0}] with operation [{1}].'.format(
lake_ref, create_req_op.name))
def GetIamPolicy(lake_ref):
"""Get Iam Policy request."""
get_iam_policy_req = dataplex_api.GetMessageModule(
).DataplexProjectsLocationsLakesGetIamPolicyRequest(
resource=lake_ref.RelativeName())
return dataplex_api.GetClientInstance(
).projects_locations_lakes.GetIamPolicy(get_iam_policy_req)
def GenerateZoneForUpdateRequest(args):
"""Create Zone for Message Update Requests."""
module = dataplex_api.GetMessageModule()
return module.GoogleCloudDataplexV1Zone(
description=args.description,
displayName=args.display_name,
labels=dataplex_api.CreateLabels(module.GoogleCloudDataplexV1Zone,
args),
discoverySpec=GenerateDiscoverySpec(args))
def GenerateDiscoverySpec(args):
return dataplex_api.GetMessageModule(
).GoogleCloudDataplexV1ZoneDiscoverySpec(
enabled=args.discovery_enabled,
includePatterns=args.discovery_include_patterns,
excludePatterns=args.discovery_exclude_patterns,
schedule=args.discovery_schedule,
csvOptions=GenerateCsvOptions(args),
jsonOptions=GenerateJsonOptions(args))
def GenerateSqlScript(args):
"""Creates SQL Script field for Content Message Create/Update Requests."""
module = dataplex_api.GetMessageModule()
query_engine_field = module.GoogleCloudDataplexV1ContentSqlScript
sql_script = module.GoogleCloudDataplexV1ContentSqlScript()
if args.query_engine:
sql_script.engine = query_engine_field.EngineValueValuesEnum(
args.query_engine)
return sql_script
def GenerateNotebook(args):
"""Creates Notebook field for Content Message Create/Update Requests."""
module = dataplex_api.GetMessageModule()
kernel_type_field = module.GoogleCloudDataplexV1ContentNotebook
notebook = module.GoogleCloudDataplexV1ContentNotebook()
if args.kernel_type:
notebook.kernelType = kernel_type_field.KernelTypeValueValuesEnum(
args.kernel_type)
return notebook
def GenerateEnvironmentForUpdateRequest(args):
"""Create Environment for Message Update Requests."""
module = dataplex_api.GetMessageModule()
return module.GoogleCloudDataplexV1Environment(
description=args.description,
displayName=args.display_name,
labels=dataplex_api.CreateLabels(
module.GoogleCloudDataplexV1Environment, args),
infrastructureSpec=GenerateInfrastructureSpec(args),
sessionSpec=GenerateSessionSpec(args))
def GenerateZoneForCreateRequest(args):
"""Create Zone for Message Create Requests."""
module = dataplex_api.GetMessageModule()
return module.GoogleCloudDataplexV1Zone(
description=args.description,
displayName=args.display_name,
labels=dataplex_api.CreateLabels(module.GoogleCloudDataplexV1Zone,
args),
type=module.GoogleCloudDataplexV1Zone.TypeValueValuesEnum(args.type),
discoverySpec=GenerateDiscoverySpec(args),
resourceSpec=module.GoogleCloudDataplexV1ZoneResourceSpec(
locationType=module.GoogleCloudDataplexV1ZoneResourceSpec.
LocationTypeValueValuesEnum(args.resource_location_type)))
def GenerateContentForUpdateRequest(args):
"""Creates Content for Message Update Requests."""
module = dataplex_api.GetMessageModule()
content = module.GoogleCloudDataplexV1Content(
dataText=args.data_text,
description=args.description,
labels=dataplex_api.CreateLabels(module.GoogleCloudDataplexV1Content,
args),
path=args.path)
if args.kernel_type:
content.notebook = GenerateNotebook(args)
if args.query_engine:
content.sqlScript = GenerateSqlScript(args)
return content
def GenerateInfrastructureSpec(args):
"""Generate InfrastructureSpec From Arguments."""
module = dataplex_api.GetMessageModule()
compute_resource = module.GoogleCloudDataplexV1EnvironmentInfrastructureSpecComputeResources(
diskSizeGb=args.compute_disk_size_gb,
nodeCount=args.compute_node_count,
maxNodeCount=args.compute_max_node_count)
os_image_runtime = module.GoogleCloudDataplexV1EnvironmentInfrastructureSpecOsImageRuntime(
imageVersion=args.os_image_version,
javaLibraries=args.os_image_java_libraries,
pythonPackages=args.os_image_python_packages,
properties=args.os_image_properties)
infrastructure_spec = module.GoogleCloudDataplexV1EnvironmentInfrastructureSpec(
compute=compute_resource, osImage=os_image_runtime)
return infrastructure_spec
def GenerateAssetForCreateRequest(args):
"""Create Asset for Message Create Requests."""
module = dataplex_api.GetMessageModule()
resource_spec_field = module.GoogleCloudDataplexV1AssetResourceSpec
resource_spec = module.GoogleCloudDataplexV1AssetResourceSpec(
name=args.resource_name,
type=resource_spec_field.TypeValueValuesEnum(args.resource_type))
request = module.GoogleCloudDataplexV1Asset(
description=args.description,
displayName=args.display_name,
labels=dataplex_api.CreateLabels(module.GoogleCloudDataplexV1Asset,
args),
resourceSpec=resource_spec)
discovery = GenerateDiscoverySpec(args)
if discovery != module.GoogleCloudDataplexV1AssetDiscoverySpec():
setattr(request, 'discoverySpec', discovery)
return request
def Run(self, args):
update_mask = content.GenerateUpdateMask(args)
if len(update_mask) < 1:
raise exceptions.HttpException(
'Update commands must specify at least one additional parameter to change.'
)
content_ref = args.CONCEPTS.content.Parse()
dataplex_client = dataplex_util.GetClientInstance()
dataplex_client.projects_locations_lakes_contentitems.Patch(
dataplex_util.GetMessageModule(
).DataplexProjectsLocationsLakesContentitemsPatchRequest(
name=content_ref.RelativeName(),
validateOnly=args.validate_only,
updateMask=u','.join(update_mask),
googleCloudDataplexV1Content=content.
GenerateContentForUpdateRequest(args)))
validate_only = getattr(args, 'validate_only', False)
if validate_only:
log.status.Print('Validation complete.')
return
log.UpdatedResource(content_ref)
def GenerateDiscoverySpec(args):
"""Create Discovery Spec for Assets."""
module = dataplex_api.GetMessageModule()
discovery_spec = module.GoogleCloudDataplexV1AssetDiscoverySpec(
enabled=args.discovery_enabled,
includePatterns=args.discovery_include_patterns,
excludePatterns=args.discovery_exclude_patterns)
if args.discovery_schedule:
discovery_spec.schedule = args.discovery_schedule
csv_options = GenerateCsvOptions(args)
if csv_options != module.GoogleCloudDataplexV1AssetDiscoverySpecCsvOptions(
):
discovery_spec.csvOptions = csv_options
json_options = GenerateJsonOptions(args)
if json_options != module.GoogleCloudDataplexV1AssetDiscoverySpecJsonOptions(
):
discovery_spec.jsonOptions = json_options
return discovery_spec
def SetIamPolicyFromFile(lake_ref, policy_file):
"""Set iam policy binding request from file."""
policy = iam_util.ParsePolicyFile(
policy_file,
dataplex_api.GetMessageModule().GoogleIamV1Policy)
return SetIamPolicy(lake_ref, policy)
def GenerateJsonOptions(args):
return dataplex_api.GetMessageModule(
).GoogleCloudDataplexV1ZoneDiscoverySpecJsonOptions(
encoding=args.json_encoding,
disableTypeInference=args.json_disable_type_inference)
