def post_connector(connector, name):
options = request.get_json()
connector = importlib.import_module(f"connectors.{connector}")
required_options = {
o['name']: o
for o in connector.CONNECTION_OPTIONS if o.get('required')
}
missing_option_names = set(required_options) - set(options)
if missing_option_names:
missing_options = [required_options[n] for n in missing_option_names]
missing_titles = set(
o.get('title', o['name']) for o in missing_options)
missing_titles_str = '\n - ' + '\n - '.join(missing_titles)
return {
'success':
False,
'errorMessage':
f"Missing required configuration options:{missing_titles_str}",
}
secret_option_names = {
o['name']: o
for o in connector.CONNECTION_OPTIONS if o.get('secret')
}
for opt_name in secret_option_names:
if vault.ENABLED and opt_name in options:
options[opt_name] = vault.encrypt(options[opt_name])
return connector.connect(name, options)
def post_connector(connector, name):
options = request.get_json()
connector = importlib.import_module(f"connectors.{connector}")
required_options = {
o['name']: o
for o in connector.CONNECTION_OPTIONS if o.get('required')
}
missing_option_names = set(required_options) - set(options)
if missing_option_names:
missing_options = [required_options[n] for n in missing_option_names]
missing_titles = set(
o.get('title', o['name']) for o in missing_options)
missing_titles_str = '\n - ' + '\n - '.join(missing_titles)
return {
'success':
False,
'errorMessage':
f"Missing required configuration options:{missing_titles_str}",
}
for o in connector.CONNECTION_OPTIONS:
oname = o['name']
ovalue = options.get(oname)
# lists that are passed in as strings are comma-saparated
if o.get('type') == 'list' and type(ovalue) is str:
options[oname] = None if ovalue is None else ovalue.split(',')
# TODO: fix connection options to support secret ints (probs w/ one for loop)
int_option_names = {
o['name']
for o in connector.CONNECTION_OPTIONS if o.get('type') == 'int'
}
for opt_name in int_option_names:
options[opt_name] = int(options[opt_name])
secret_option_names = {
o['name']
for o in connector.CONNECTION_OPTIONS if o.get('secret')
}
for opt_name in secret_option_names:
if vault.ENABLED and opt_name in options:
options[opt_name] = vault.encrypt(options[opt_name])
return connector.connect(name, options)
def connect(connection_name, options):
connection_type = options['connection_type']
base_name = f"azure_{connection_name}_{connection_type}"
account_name = options['account_name']
container_name = options['container_name']
suffix = options['suffix']
sas_token = options['sas_token']
sas_token_ct = vault.encrypt(sas_token)
comment = f'''
---
module: azure
storage_account: {account_name}
container_name: {container_name}
suffix: {suffix}
sas_token: {sas_token_ct}
sa_user: {USER}
snowflake_account: {ACCOUNT}
database: {DATABASE}
'''
db.create_stage(
name=f'data.{base_name}_STAGE',
url=f"azure://{account_name}.blob.{suffix}/{container_name}",
cloud='azure',
prefix='',
credentials=sas_token,
file_format=FILE_FORMAT)
db.execute(
f'GRANT USAGE ON STAGE data.{base_name}_STAGE TO ROLE {SA_ROLE}')
db.create_table(name=f'data.{base_name}_CONNECTION',
cols=LANDING_TABLES_COLUMNS[connection_type],
comment=comment)
db.execute(
f'GRANT INSERT, SELECT ON data.{base_name}_CONNECTION TO ROLE {SA_ROLE}'
)
pipe_sql = {
'operation':
f'''
COPY INTO DATA.{base_name}_CONNECTION(RAW, HASH_RAW, CALLER_IP_ADDRESS, CATEGORY, CORRELATION_ID, DURATION_MS,
IDENTITY, IDENTITY_AUTHORIZATION, IDENTITY_CLAIMS, LEVEL, LOCATION,
OPERATION_NAME, PROPERTIES, PROPERTIES_ANCESTORS, PROPERTIES_IS_COMPLIANCE_CHECK,
PROPERTIES_POLICIES, PROPERTIES_RESOURCE_LOCAATION, RESOURCE_ID, RESULT_SIGNATURE,
RESULT_TYPE, EVENT_TIME, LOADED_ON)
FROM (
SELECT $1, HASH($1), $1:callerIpAddress::STRING, $1:category::STRING, $1:correlationId::STRING,
$1:durationMs::NUMBER, $1:identity::VARIANT, $1:identity.authorization::VARIANT, $1:identity.claims::VARIANT,
$1:level::STRING, $1:location::STRING, $1:operationName::STRING, $1:properties::VARIANT,
$1:properties.ancestors::STRING, $1:properties.isComplianceCheck::STRING, PARSE_JSON($1:properties.policies),
$1:properties.resourceLocation::STRING, $1:resourceId::STRING, $1:resultSignature::STRING,
$1:resultType::STRING, $1:time::TIMESTAMP_LTZ, CURRENT_TIMESTAMP()
FROM @DATA.{base_name}_STAGE)
''',
'audit':
f'''
COPY INTO data.{base_name}_CONNECTION (RAW, HASH_RAW, CALLER_IP_ADDRESS, CATEGORY, CORRELATION_ID,
DURATION_MS, LEVEL, OPERATION_NAME, OPERATION_VERSION, PROPERTIES,
PROPERTIES_ACTIVITY_DATE_TIME, PROPERTIES_ACTIVITY_DISPLAY_NAME,
PROPERTIES_ADDITIONAL_DETAILS, PROPERTIES_CATEGORY, PROPERTIES_ID,
PROPERTIES_INITIATED_BY, PROPERTIES_LOGGED_BY_SERVICE, PROPERTIES_OPERATION_TYPE,
PROPERTIES_RESULT, PROPERTIES_RESULT_REASON, PROPERTIES_TARGET_RESOURCES,
RESOURCE_ID, RESULT_SIGNATURE, TENANT_ID, EVENT_TIME, LOADED_ON)
FROM (
SELECT $1, HASH($1), $1:callerIpAddress::STRING, $1:category::STRING, $1:correlationId::STRING,
$1:durationMs::NUMBER, $1:level::STRING, $1:operationName::STRING, $1:operationVersion::STRING,
$1:properties::VARIANT, $1:properties.activityDateTime::TIMESTAMP_LTZ,
$1:properties.activityDisplayName::STRING, $1:properties.additionalDetails::VARIANT,
$1:properties.category::STRING, $1:properties.id::STRING, $1:properties.initiatedBy::VARIANT,
$1:properties.loggedByService::STRING, $1:properties.operationType::STRING, $1:properties.result::STRING,
$1:resultReason::STRING, $1:properties.targetResources::VARIANT, $1:resourceId::STRING,
$1:resultSignature::STRING, $1:tenantId::STRING, $1:time::TIMESTAMP_LTZ, CURRENT_TIMESTAMP()
FROM @data.{base_name}_STAGE
)
''',
'signin':
f'''
COPY INTO DATA.{base_name}_CONNECTION (
RAW, HASH_RAW, LEVEL, CALLER_IP_ADDRESS, CATEGORY, CORRELATION_ID, DURATION_MS,
IDENTITY, LOCATION, OPERATION_NAME, OPERATION_VERSION, PROPERTIES,
PROPERTIES_APP_DISPLAY_NAME, PROPERTIES_APP_ID,
PROPERTIES_APPLIED_CONDITIONAL_ACESS_POLICIES, PROPERTIES_AUTHENTICATION_METHODS_USED,
PROPERTIES_AUTHENTICATION_PROCESSING_DETAILS, PROPERTIES_CLIENT_APP_USED,
PROPERTIES_CONDITIONAL_ACCESS_STATUS, PROPERTIES_CREATED_DATE_TIME,
PROPERTIES_DEVICE_DETAIL, PROPERTIES_ID, PROPERTIES_IP_ADDRESS, PROPERTIES_IS_INTERACTIVE, PROPERTIES_LOCATION,
PROPERTIES_MFA_DETAIL, PROPERTIES_NETWORK_LOCATION, PROPERTIES_PROCESSING_TIME_IN_MILLISECONDS,
PROPERTIES_RESOURCE_DISPLAY_NAME, PROPERTIES_RESOURCE_ID, PROPERTIES_RISK_DETAIL,
PROPERTIES_RISK_EVENT_TYPES, PROPERTIES_RISK_LEVEL_AGGREGATED, PROPERTIES_RISK_LEVEL_DURING_SIGNIN,
PROPERTIES_RISK_STATE, PROPERTIES_STATUS, PROPERTIES_TOKEN_ISSUER_TYPE, PROPERTIES_USER_DISPLAY_NAME,
PROPERTIES_USER_ID, PROPERTIES_USER_PRINCIPAL_NAME, RESOURCE_ID, RESULT_DESCRIPTION, RESULT_SIGNATURE,
RESULT_TYPE, TENANT_ID, EVENT_TIME, LOADED_ON
)
FROM (
SELECT $1, HASH($1), $1:Level::NUMBER, $1:callerIpAddress::STRING, $1:category::STRING, $1:correlationId::STRING,
$1:durationMs, $1:identity::STRING, $1:location::STRING, $1:operationName::STRING,
$1:operationVersion::STRING, $1:properties::VARIANT, $1:properties.appDisplayName::STRING,
$1:properties.appId::STRING, $1:properties.appliedConditionalAccessPolicies::VARIANT,
$1:properties.authenticationMethodsUsed::VARIANT, $1:properties.authenticationProcessingDetails::VARIANT,
$1:properties.clientAppUsed::STRING, $1:properties.conditionalAccessStatus::STRING,
$1:properties.createdDateTime::TIMESTAMP_LTZ, $1:properties.deviceDetail::VARIANT, $1:properties.id::STRING,
$1:properties.ipAddress::STRING, $1:properties.isInteractive::BOOLEAN, $1:properties.location::VARIANT,
$1:properties.mfaDetail::VARIANT, $1:properties.networkLocationDetails::VARIANT,
$1:properties.processingTimeInMilliseconds::NUMBER, $1:properties.resourceDisplayName::STRING,
$1:properties.resourceId::STRING, $1:properties.riskDetail::STRING, $1:properties.riskEventTypes::VARIANT,
$1:properties.riskLevelAggregated::STRING, $1:properties.riskLevelDuringSignIn::STRING,
$1:properties.riskState::VARIANT, $1:properties.status::VARIANT, $1:properties.tokenIssuerType::STRING,
$1:properties.userDisplayName::STRING, $1:properties.userId::STRING, $1:properties.userPrincipalName::STRING,
$1:resourceId::STRING, $1:resultDescription::STRING, $1:resultSignature::STRING, $1:resultType::STRING,
$1:tenantId::STRING, $1:time::TIMESTAMP_LTZ,
CURRENT_TIMESTAMP()
FROM @DATA.{base_name}_STAGE
)
'''
}
db.create_pipe(name=f"data.{base_name}_PIPE",
sql=pipe_sql[options['connection_type']],
replace=True)
db.execute(
f'ALTER PIPE data.{base_name}_PIPE SET PIPE_EXECUTION_PAUSED=true')
db.execute(
f'GRANT OWNERSHIP ON PIPE data.{base_name}_PIPE TO ROLE {SA_ROLE}')
return {
'newStage': 'finalized',
'newMessage': 'Table, Stage, and Pipe created'
}