def delete_environment(self, name, cascade=False, force=False):
if cascade:
self.sdk.throw_warning(CdpWarning('Cascading Environment deletions currently not supported'))
if force:
self.sdk.throw_warning(CdpWarning('Forced Environment deletions currently not supported'))
return self.sdk.call(
svc='environments', func='delete_environment', ret_field='environment',
environmentName=name
)
def sync_users(self, environments=None):
if isinstance(environments, list):
if len(environments) == 0:
environments = None
elif isinstance(environments, str):
environments = [environments]
else:
self.sdk.throw_error(
CdpError(
"environments must be a list of one or more strings or a string"
))
resp = self.sdk.call(svc='environments',
func='sync_all_users',
ret_error=True,
environmentNames=environments)
if isinstance(resp, CdpError):
if resp.error_code == 'CONFLICT':
operation_match = self.sdk.regex_search(
self.sdk.OPERATION_REGEX, str(resp.violations))
if operation_match is not None:
existing_op_id = operation_match.group(1)
if not self.sdk.strict_errors:
self.sdk.throw_warning(
CdpWarning(
'Sync All Users Operation already running, tracking existing job {0}'
.format(existing_op_id)))
return self.get_sync_status(existing_op_id)
self.sdk.throw_error(resp)
return resp
def create_aws_credential(self, name, role, description, retries=3, delay=2):
resp = self.sdk.call(
svc='environments', func='create_aws_credential', ret_error=True, squelch=[
Squelch(field='violations', value='Credential already exists with name',
warning='Credential with this name already exists', default=None)],
credentialName=name,
roleArn=role,
description=description
)
if isinstance(resp, CdpError):
if retries > 0:
consistency_violations = [
'Unable to verify credential', 'sts:AssumeRole', 'You are not authorized'
]
if any(x in resp.violations['message'] for x in consistency_violations):
retries = retries - 1
self.sdk.throw_warning(
CdpWarning('Got likely AWS IAM eventual consistency error [%s], %d retries left...'
% (resp.violations['message'], retries))
)
self.sdk.sleep(delay)
return self.create_aws_credential(name, role, description, retries, delay)
else:
self.sdk.throw_error(resp)
return resp
def create_aws_environment(self, **kwargs):
# TODO: Rework with named kwargs
resp = self.sdk.call(
svc='environments', func='create_aws_environment', ret_field='environment', ret_error=True,
**kwargs
)
if isinstance(resp, CdpError):
if resp.error_code == 'INVALID_ARGUMENT':
if 'constraintViolations' not in resp.violations:
resp.update(message="Received violation warning:\n%s" % self.sdk.dumps(resp.violations))
self.sdk.throw_warning(CdpWarning(resp.violations['message']))
self.sdk.throw_error(resp)
return resp
def describe_environment(self, name):
resp = self.sdk.call(
svc='environments', func='describe_environment', ret_field='environment', ret_error=True,
environmentName=name
)
if isinstance(resp, CdpError):
if resp.error_code == 'NOT_FOUND':
# Describe will fail in certain fault scenarios early in Environment creation
# We helpfully provide the summary listing as a backup set of information
# If the environment truly does not exist, then this will give the same response
self.sdk.throw_warning(CdpWarning(resp.violations['message']))
return self.summarize_environment(name)
self.sdk.throw_error(resp)
return resp
def resolve_service_crn_from_name(self, name, only_enabled=True):
listing = self.list_services(only_enabled=only_enabled, name=name)
# More than one DF Service may exist with a given name if it was previously uncleanly deleted
if len(listing) == 1:
return listing[0]['crn']
elif len(listing) == 0:
self.sdk.throw_warning(
CdpWarning("No DataFlow Service found matching name %s" %
name))
return None
else:
self.sdk.throw_error(
CdpError("Multiple DataFlow Services found matching name %s" %
name))
def create_azure_credential(self,
name,
subscription,
tenant,
application,
secret,
retries=3,
delay=5):
resp = self.sdk.call(
svc='environments',
func='create_azure_credential',
ret_error=True,
squelch=[
Squelch(field='violations',
value='Credential already exists with name',
warning='Credential with this name already exists',
default=None)
],
credentialName=name,
subscriptionId=subscription,
tenantId=tenant,
appBased={
'applicationId': application,
'secretKey': secret
})
if isinstance(resp, CdpError):
if retries > 0:
consistency_violations = [
'You may have sent your authentication request to the wrong tenant'
]
if any(x in str(resp.violations)
for x in consistency_violations):
retries = retries - 1
self.sdk.throw_warning(
CdpWarning(
'Got likely Azure eventual consistency error [%s], %d retries left...'
% (str(resp.violations), retries)))
self.sdk.sleep(delay)
return self.create_azure_credential(
name, subscription, tenant, application, secret,
retries, delay)
else:
self.sdk.throw_error(resp)
return resp
def list_cluster_templates(self, retries=3, delay=5):
# Intermittent timeout issue in CDP 7.2.10, should be reverted to bare listing in 7.2.12
resp = self.sdk.call(svc='datahub',
func='list_cluster_templates',
ret_field='clusterTemplates',
ret_error=True)
if isinstance(resp, CdpError):
if retries > 0:
if str(resp.status_code
) == '500' and resp.error_code == 'UNKNOWN':
retries = retries - 1
self.sdk.throw_warning(
CdpWarning(
'Got likely CDP Control Plane eventual consistency error, %d retries left...'
% (retries)))
self.sdk.sleep(delay)
return self.list_cluster_templates(retries, delay)
else:
self.sdk.throw_error(resp)
return resp
def get_version_crn_from_flow_definition(self, flow_name, version=None):
summary_list = self.list_flow_definitions(name=flow_name)
if summary_list:
if len(summary_list) == 1:
flow_def = summary_list[0]
kind = flow_def['artifactType']
if kind == 'flow':
detail = self.describe_customflow(flow_def['crn'])
elif kind == 'readyFlow':
detail = self.describe_added_readyflow(flow_def['crn'])
else:
detail = None
self.sdk.throw_error(
CdpError("DataFlow Definition type not supported %s" %
kind))
if version is None:
# versions are sorted descending by default
return detail['versions'][0]['crn']
else:
out = [
x for x in detail['versions']
if x['version'] == version
]
if out:
return out[0]['crn']
else:
self.sdk.throw_error(
CdpError(
"Could not find version %d for DataFlow Definition named %s"
% (version, flow_name)))
else:
self.sdk.throw_error(
CdpError(
"More than one DataFlow Definition found for name %s" %
flow_name))
else:
self.sdk.throw_warning(
CdpWarning("DataFlow Definition not found for name %s" %
flow_name))