def get_auth_stub(config):
"""
Given a config dict in the format:
{'clientid': ... your ET client ID ...,
'clientsecret': ... your ET client secret ...}
... return an auth stub to be used when making requests.
"""
LOGGER.info("Generating auth stub...")
params = {
'clientid': config['client_id'],
'clientsecret': config['client_secret']
}
if config.get('tenant_subdomain'):
# For S10+ accounts: https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis.meta/mc-apis/your-subdomain-tenant-specific-endpoints.htm
params['authenticationurl'] = ('https://{}.auth.marketingcloudapis.com/v1/requestToken'
.format(config['tenant_subdomain']))
LOGGER.info("Authentication URL is: %s", params['authenticationurl'])
params['soapendpoint'] = ('https://{}.soap.marketingcloudapis.com/Service.asmx'
.format(config['tenant_subdomain']))
# First try V1
try:
LOGGER.info('Trying to authenticate using V1 endpoint')
params['useOAuth2Authentication'] = "False"
auth_stub = FuelSDK.ET_Client(params=params)
transport = HttpAuthenticated(timeout=int(config.get('request_timeout', 3600)))
auth_stub.soap_client.set_options(
transport=transport, timeout=3600)
LOGGER.info("Success.")
return auth_stub
except Exception as e:
LOGGER.info('Failed to auth using V1 endpoint')
if not config.get('tenant_subdomain'):
LOGGER.warning('No tenant_subdomain found, will not attempt to auth with V2 endpoint')
raise e
# Next try V2
# Move to OAuth2: https://help.salesforce.com/articleView?id=mc_rn_january_2019_platform_ip_remove_legacy_package_create_ability.htm&type=5
try:
LOGGER.info('Trying to authenticate using V2 endpoint')
params['useOAuth2Authentication'] = "True"
params['authenticationurl'] = ('https://{}.auth.marketingcloudapis.com'
.format(config['tenant_subdomain']))
LOGGER.info("Authentication URL is: %s", params['authenticationurl'])
auth_stub = FuelSDK.ET_Client(params=params)
transport = HttpAuthenticated(timeout=int(config.get('request_timeout', 3600)))
auth_stub.soap_client.set_options(
transport=transport, timeout=3600)
except Exception as e:
LOGGER.info('Failed to auth using V2 endpoint')
raise e
LOGGER.info("Success.")
return auth_stub
def authenticate(self, client_id=None, client_secret=None, debug=False):
if client_id is None or client_secret is None:
self.client = FuelSDK.ET_Client(debug=debug)
else:
self.client = FuelSDK.ET_Client(params={
'clientid': client_id,
'clientsecret': client_secret
},
debug=debug)
def get_auth_stub(config):
"""
Given a config dict in the format:
{'clientid': ... your ET client ID ...,
'clientsecret': ... your ET client secret ...}
... return an auth stub to be used when making requests.
"""
LOGGER.info("Generating auth stub...")
params = {
'clientid': config['client_id'],
'clientsecret': config['client_secret']
}
if config.get('tenant_subdomain'):
# For S10+ accounts: https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis.meta/mc-apis/your-subdomain-tenant-specific-endpoints.htm
params['authenticationurl'] = (
'https://{}.auth.marketingcloudapis.com/v1/requestToken'.format(
config['tenant_subdomain']))
params['soapendpoint'] = (
'https://{}.soap.marketingcloudapis.com/Service.asmx'.format(
config['tenant_subdomain']))
auth_stub = FuelSDK.ET_Client(params=params)
transport = HttpAuthenticated(
timeout=int(config.get('request_timeout', 900)))
auth_stub.soap_client.set_options(transport=transport)
LOGGER.info("Success.")
return auth_stub
def __init__(self, get_server_wsdl=False, debug=False, params=None):
if debug:
logger_debug.setLevel(logging.DEBUG)
self.client = FuelSDK.ET_Client(get_server_wsdl=get_server_wsdl,
debug=debug,
params=params)
def retrieve_auth_token(account):
print(f'{Fore.YELLOW}==== Retrieving Credentials for {account} ====')
credentials = get_secrets(account)
return FuelSDK.ET_Client(
False, False, {
"clientid": credentials['client_id'],
"clientsecret": credentials['client_secret']
})
def de_create(de_name, folderID):
try:
debug = False
stubObj = f.ET_Client(False, debug)
target_de = str(de_name)
target_folder = int(folderID) #502 #API_GEN
# Create Data Extension
print('Creating Data Extension %s' % target_de)
de = f.ET_DataExtension()
de.auth_stub = stubObj
de.props = {
"Name": target_de,
"CustomerKey": target_de,
"CategoryID": target_folder
}
# de.columns = get_columns(table_name)
de.columns = get_columns_with_datatypes(
table_name) # switched to new version with data types
# de.columns = [{'Name': 'sk', 'FieldType': 'Decimal', 'MaxLength': '38'},
# {'Name': 'id', 'FieldType': 'Text', 'MaxLength': '1024'}]
print("DE_COLUMNS = %s" % de.columns)
properties = de.props
de.search_filter = {
'Property': 'CustomerKey',
'SimpleOperator': 'equals',
'Value': target_de
}
filter = de.search_filter
de_exists = de.get(properties, filter)
if len(de_exists.results) == 0: # If DE does not exist, post
post_response = de.post()
print('Post Status: ' + str(post_response.status))
print('Code: ' + str(post_response.code))
print('Message: ' + str(post_response.message))
print('Results: ' + str(post_response.results))
else:
# pass
# TODO: Drop and Recreate DE - [COMPLETED]
print("Warning: DE exists. Deleting DE %s" % target_de)
delResponse = de.delete()
print('Delete Status: ' + str(delResponse.status))
print('Code: ' + str(delResponse.code))
print("Creating DE %s" % target_de)
post_response = de.post()
print('Post Status: ' + str(post_response.status))
print('Code: ' + str(post_response.code))
except Exception as e:
print('Caught exception: ' + str(e))
print(e)
def load_de(table_name, de_name):
debug = False
stubObj = f.ET_Client(False, debug)
DE_NAME = de_name
FOLDER_ID = 502 # API_GEN
de_to_load = f.ET_DataExtension_Row()
de_to_load.CustomerKey = de_name
de_to_load.auth_stub = stubObj
# de_to_load.props = {"Col1": "Value1", "Col2": "Value2"}
de_to_load.props = fetch_table_data(table_name)
de_loaded_response = de_to_load.post()
def get_subscriberkey(EmailAddress):
headers = {
'content-type': 'application/json',
'Authorization': get_auth_token()
}
if (not os.environ.get('PYTHONHTTPSVERIFY', '')
and getattr(ssl, '_create_unverified_context', None)):
ssl._create_default_https_context = ssl._create_unverified_context
debug = False
stubObj = f.ET_Client(False, debug)
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
# define DE List
# deList = list()
# deList = ['DE_order_confirm']
# deList = ['DE_order_confirm', 'DE_password_reset', 'DE_password_changed',
# 'DE_pdt_print_ready', 'DE_pdt_proof_ready', 'DE_photo_share_receiver',
# 'DE_photo_share_sender', 'DE_share_project_receiver']
# nameOfDE = 'DE_password_reset'
# loop over DE list
de = 'DE_SF_USER_D'
row.CustomerKey = str(de)
row.props = ["USERID"]
# set search filter
row.search_filter = {
'Property': 'CURR_EMAIL_ADDRESS',
'SimpleOperator': 'equals',
'Value': EmailAddress
}
getResponse = row.get()
data = dict()
#
if len(getResponse.results) > 0:
data = {
"name": getResponse.results[0].Properties.Property[0].Name,
"value": getResponse.results[0].Properties.Property[0].Value
}
else:
data = {"Message": str(getResponse.results)}
return data
def get_auth_stub(config):
"""
Given a config dict in the format:
{'clientid': ... your ET client ID ...,
'clientsecret': ... your ET client secret ...}
... return an auth stub to be used when making requests.
"""
LOGGER.info("Generating auth stub...")
auth_stub = FuelSDK.ET_Client(params={
'clientid': config['client_id'],
'clientsecret': config['client_secret']
})
auth_stub.soap_client.set_options(
timeout=int(config.get('request_timeout', 900)))
LOGGER.info("Success.")
return auth_stub
def get_auth_stub(config):
"""
Given a config dict in the format:
{'clientid': ... your ET client ID ...,
'clientsecret': ... your ET client secret ...}
... return an auth stub to be used when making requests.
"""
LOGGER.info("Generating auth stub...")
params = {
'clientid': config['client_id'],
'clientsecret': config['client_secret']
}
if config.get('tenant_subdomain'):
# For S10+ accounts: https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis.meta/mc-apis/your-subdomain-tenant-specific-endpoints.htm
# Move to OAuth2: https://help.salesforce.com/articleView?id=mc_rn_january_2019_platform_ip_remove_legacy_package_create_ability.htm&type=5
if config.get('use_oauth2') == "True":
params['useOAuth2Authentication'] = "True"
params['authenticationurl'] = ('https://{}.auth.marketingcloudapis.com'
.format(config['tenant_subdomain']))
else:
params['useOAuth2Authentication'] = "False"
params['authenticationurl'] = ('https://{}.auth.marketingcloudapis.com/v1/requestToken'
.format(config['tenant_subdomain']))
LOGGER.debug(f"Authentication URL is: {params['authenticationurl']}")
params['soapendpoint'] = ('https://{}.soap.marketingcloudapis.com/Service.asmx'
.format(config['tenant_subdomain']))
auth_stub = FuelSDK.ET_Client(params=params)
transport = HttpAuthenticated(timeout=int(config.get('request_timeout', 900)))
auth_stub.soap_client.set_options(
transport=transport)
LOGGER.info("Success.")
return auth_stub
# Add a require statement to reference the Fuel SDK's functionality:
import FuelSDK
# Next, create an instance of the ET_Client class:
myClient = FuelSDK.ET_Client()
# Create an instance of the object type we want to work with:
list = FuelSDK.ET_List()
# Associate the ET_Client to the object using the auth_stub property:
list.auth_stub = myClient
# Utilize one of the ET_List methods:
response = list.get()
# Print out the results for viewing
print('Post Status: ' + str(response.status))
print('Code: ' + str(response.code))
print('Message: ' + str(response.message))
print('Result Count: ' + str(len(response.results)))
print('Results: ' + str(response.results))
def get(self, emailType, correlationId):
"""
Fetch sends for a given emailType and correlationId
---
tags:
- SFMC
parameters:
- in: path
name: emailType
required: true
description: The emailType of the send, try password_reset.
type: string
- in: path
name: correlationId
required: true
description: The correlationId of the send, try TEST
type: string
responses:
200:
description: Results from Salesforce
"""
# requestToken
token_url = "https://auth.exacttargetapis.com/v1/requestToken"
headers = {'content-type': 'application/json'}
token_data = {
"clientId": config.clientid,
"clientSecret": config.clientsecret
}
response = requests.post(token_url, json=token_data, headers=headers)
token = json.loads(response.text)
# print("access token =" + str(token['accessToken']))
access_token = ''.join(token['accessToken'])
# request token
auth_token = 'Bearer ' + access_token # Bearer Token
# print("auth_token = " + auth_token)
# SSL context verification
if (not os.environ.get('PYTHONHTTPSVERIFY', '')
and getattr(ssl, '_create_unverified_context', None)):
ssl._create_default_https_context = ssl._create_unverified_context
debug = False
stubObj = f.ET_Client(False, debug)
if emailType.lower() == 'password_reset':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
print("{\"name\": %s, \"value\": %s}" %
(q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'order_confirm':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
# print("# records = %s" % len(getResponse.results))
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'password_changed':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'pdt_print_ready':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'pdt_proof_ready':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'photo_share_receiver':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'photo_share_sender':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'share_project_receiver':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'share_project_sender':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'ship_confirm':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'welcome':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
elif emailType.lower() == 'new_prospect_request':
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
nameOfDE = emailType.lower()
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = [
"correlationId", 'EmailAddress', 'SubscriberKey', 'emailType',
'uid', 'timestamp_DT'
]
# correlationId = 'PASS_RESET_TEST_001'
# set search filter
row.search_filter = {
'Property': 'correlationId',
'SimpleOperator': 'equals',
'Value': correlationId
}
getResponse = row.get()
data = list()
data_dict = {}
records_found = len(getResponse.results)
if records_found > 0:
results = getResponse.results
for i, j in enumerate(results):
# print("%s-%s"%(i,j))
for p, q in enumerate(j.Properties.Property):
# print(("%s-%s-%s"%(i,p,q)))
# print("{\"name\": %s, \"value\": %s}" % (q.Name, q.Value))
data_dict.update({"name": q.Name, "value": q.Value})
data.append(data_dict.copy())
response_dict = {
"status": 'success',
"message":
str(records_found) + ' result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
# REF: https://stackoverflow.com/questions/42354001/python-json-object-must-be-str-bytes-or-bytearray-not-dict
else:
data = []
response_dict = {
"status": 'success',
"message": '0 result(s) returned by Salesforce',
"data": data
}
response_str = json.dumps(response_dict)
response = json.loads(response_str)
pass
return response
def __init__(self, sfmc_creds):
"""
Initialize ET_Client object for the client
"""
self.sfmc_client = FuelSDK.ET_Client(params=sfmc_creds)
logging.info("Authentication success with SFMC........")
import time
from collections import Iterable
import FuelSDK
from FuelSDK.data_extension_upsert import SalesforceConnector, DataCollector
client = FuelSDK.ET_Client(params={
'clientid': '',
'clientsecret': '',
'defaultwsdl': "",
'authenticationurl': "",
'baseapiurl': "",
'soapendpoint': "",
'useOAuth2Authentication': '',
'accountid': 0
})
class RedshiftCollector:
def __init__(self, connection):
self._rs_connection = connection
def iterator(self, rate=1000, offset=0, limit=None):
retry_count = 0
fetched_rows = 0
while (limit is None or fetched_rows + offset < limit) and retry_count < 50:
try:
rs_cursor = self._rs_connection.cursor(cursor_factory=psycopg2.extras.DictCursor)
optimize_limit = min(rate, limit - fetched_rows) if limit else 'ALL'
def run_job():
print "Job starts "
print 'current time ' + str(datetime.now())
time = datetime.now()
filename = 'Data/Click_Event/' + 'click_event_' + str(
time.strftime('%Y-%m-%d-%H-%M-%S')) + '.json'
# Create cut-off from and to dates
fromDate = audit_log_extracts(None, None, 1)
toDate = time.strftime('%Y-%m-%dT%H:%M:%S')
try:
debug = False
stubObj = FuelSDK.ET_Client(False, debug)
print '>>> Retrieve Filtered BounceEvents with GetMoreResults'
getClickEvent = FuelSDK.ET_ClickEvent()
getClickEvent.auth_stub = stubObj
getClickEvent.props = [
"SendID", "SubscriberKey", "EventDate", "Client.ID", "EventType",
"BatchID", "TriggeredSendDefinitionObjectID", "PartnerKey"
]
from_filter = {
'Property': 'EventDate',
'SimpleOperator': 'greaterThan',
'DateValue': fromDate
}
to_filter = {
'Property': 'EventDate',
'SimpleOperator': 'lessThan',
'DateValue': toDate
}
getClickEvent.search_filter = {
'LeftOperand': from_filter,
'LogicalOperator': 'AND',
'RightOperand': to_filter
}
getResponse = getClickEvent.get()
file = open(filename, 'w')
for x in getResponse.results:
res = dict(x)
y = dict(res['Client'])
del res['Client']
res['Client'] = str(y['ID'])
res = dict((k.lower(), v) for k, v in res.iteritems())
result = json.dumps(res,
default=myconverter,
sort_keys=True,
indent=4,
separators=(',', ': '))
file.write(str(result))
while getResponse.more_results:
for x in getResponse.results:
res = dict(x)
y = dict(res['Client'])
del res['Client']
res['Client'] = str(y['ID'])
result = json.dumps(res,
default=myconverter,
sort_keys=True,
indent=4,
separators=(',', ': '))
file.write(str(result))
file.close()
toDate = str(
datetime.strptime(toDate, '%Y-%m-%dT%H:%M:%S') +
timedelta(seconds=-1))
fromDate = str(
datetime.strptime(fromDate, '%Y-%m-%dT%H:%M:%S.%f') +
timedelta(seconds=1))
audit_log_extracts(fromDate, toDate, 0)
size = os.path.getsize(filename)
if size == 0:
print "0 byte file "
os.remove(filename)
else:
check_call(['gzip', filename])
upload_file_s3(filename + '.gz')
load_data_dw(filename + '.gz')
print "S3 upload and Redshift integration successful "
os.remove(filename + '.gz')
print "Job Ends "
print 'current time ' + str(datetime.now())
except Exception as e:
print 'Caught exception: ' + e.message
print e
def get_de_row(EmailAddress):
headers = {
'content-type': 'application/json',
'Authorization': get_auth_token()
}
if (not os.environ.get('PYTHONHTTPSVERIFY', '')
and getattr(ssl, '_create_unverified_context', None)):
ssl._create_default_https_context = ssl._create_unverified_context
debug = False
stubObj = f.ET_Client(False, debug)
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
# define DE List
de = 'SentEvent'
row.CustomerKey = str(de)
# get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = ["TriggeredSendCustomerKey", 'EmailAddress']
# correlationID from method param
# correlationID = 'a3e3227f-b8e8-40f2-b5e2-9e899147e2d1'
SubscriberKey = get_subscriberkey(EmailAddress).get('value')
print("SubscriberKey = %s" % SubscriberKey)
# set search filter
row.search_filter = {
'Property': 'SubscriberKey',
'SimpleOperator': 'equals',
'Value': SubscriberKey
}
# start_time = time.time()
getResponse = row.get()
data = list()
if len(getResponse.results) > 0:
record_found = True
for response in getResponse.results[0].Properties.Property:
for property in response:
data.append(property, )
# https://stackoverflow.com/questions/13761054/more-pythonic-way-to-format-a-json-string-from-a-list-of-tuples
data = '{{{}}}'.format(','.join([
'{}:{}'.format(json.dumps(k), json.dumps(v))
for k, v in data
]))
else:
data = {"Message": str(getResponse.results)}
return data
import FuelSDK as f
import os
import ssl
if (not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl, '_create_unverified_context', None)):
ssl._create_default_https_context = ssl._create_unverified_context
debug = False
stubObj = f.ET_Client(False, debug)
row = f.ET_DataExtension_Row()
row.auth_stub = stubObj
# loop over DE list
# deList = list()
# nameOfDE = '_SENT'
nameOfDE = 'password_reset'
# nameOfDE = 'DE_password_changed'
# nameOfDE = 'DE_order_confirm'
# nameOfDE = 'DE_password_reset'
row.CustomerKey = nameOfDE
# TODO: get columns in the DE and append to row props
# row.props = getDeColumns(nameOfDE)
row.props = ["correlationId", 'EmailAddress']
# correlationID from method param
correlationId = 'PASS_RESET_TEST_001'