def get_auth_stub(config): """ Given a config dict in the format: {'clientid': ... your ET client ID ..., 'clientsecret': ... your ET client secret ...} ... return an auth stub to be used when making requests. """ LOGGER.info("Generating auth stub...") params = { 'clientid': config['client_id'], 'clientsecret': config['client_secret'] } if config.get('tenant_subdomain'): # For S10+ accounts: https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis.meta/mc-apis/your-subdomain-tenant-specific-endpoints.htm params['authenticationurl'] = ('https://{}.auth.marketingcloudapis.com/v1/requestToken' .format(config['tenant_subdomain'])) LOGGER.info("Authentication URL is: %s", params['authenticationurl']) params['soapendpoint'] = ('https://{}.soap.marketingcloudapis.com/Service.asmx' .format(config['tenant_subdomain'])) # First try V1 try: LOGGER.info('Trying to authenticate using V1 endpoint') params['useOAuth2Authentication'] = "False" auth_stub = FuelSDK.ET_Client(params=params) transport = HttpAuthenticated(timeout=int(config.get('request_timeout', 3600))) auth_stub.soap_client.set_options( transport=transport, timeout=3600) LOGGER.info("Success.") return auth_stub except Exception as e: LOGGER.info('Failed to auth using V1 endpoint') if not config.get('tenant_subdomain'): LOGGER.warning('No tenant_subdomain found, will not attempt to auth with V2 endpoint') raise e # Next try V2 # Move to OAuth2: https://help.salesforce.com/articleView?id=mc_rn_january_2019_platform_ip_remove_legacy_package_create_ability.htm&type=5 try: LOGGER.info('Trying to authenticate using V2 endpoint') params['useOAuth2Authentication'] = "True" params['authenticationurl'] = ('https://{}.auth.marketingcloudapis.com' .format(config['tenant_subdomain'])) LOGGER.info("Authentication URL is: %s", params['authenticationurl']) auth_stub = FuelSDK.ET_Client(params=params) transport = HttpAuthenticated(timeout=int(config.get('request_timeout', 3600))) auth_stub.soap_client.set_options( transport=transport, timeout=3600) except Exception as e: LOGGER.info('Failed to auth using V2 endpoint') raise e LOGGER.info("Success.") return auth_stub
def authenticate(self, client_id=None, client_secret=None, debug=False): if client_id is None or client_secret is None: self.client = FuelSDK.ET_Client(debug=debug) else: self.client = FuelSDK.ET_Client(params={ 'clientid': client_id, 'clientsecret': client_secret }, debug=debug)
def de_create(de_name, folderID): try: debug = False stubObj = f.ET_Client(False, debug) target_de = str(de_name) target_folder = int(folderID) #502 #API_GEN # Create Data Extension print('Creating Data Extension %s' % target_de) de = f.ET_DataExtension() de.auth_stub = stubObj de.props = { "Name": target_de, "CustomerKey": target_de, "CategoryID": target_folder } # de.columns = get_columns(table_name) de.columns = get_columns_with_datatypes( table_name) # switched to new version with data types # de.columns = [{'Name': 'sk', 'FieldType': 'Decimal', 'MaxLength': '38'}, # {'Name': 'id', 'FieldType': 'Text', 'MaxLength': '1024'}] print("DE_COLUMNS = %s" % de.columns) properties = de.props de.search_filter = { 'Property': 'CustomerKey', 'SimpleOperator': 'equals', 'Value': target_de } filter = de.search_filter de_exists = de.get(properties, filter) if len(de_exists.results) == 0: # If DE does not exist, post post_response = de.post() print('Post Status: ' + str(post_response.status)) print('Code: ' + str(post_response.code)) print('Message: ' + str(post_response.message)) print('Results: ' + str(post_response.results)) else: # pass # TODO: Drop and Recreate DE - [COMPLETED] print("Warning: DE exists. Deleting DE %s" % target_de) delResponse = de.delete() print('Delete Status: ' + str(delResponse.status)) print('Code: ' + str(delResponse.code)) print("Creating DE %s" % target_de) post_response = de.post() print('Post Status: ' + str(post_response.status)) print('Code: ' + str(post_response.code)) except Exception as e: print('Caught exception: ' + str(e)) print(e)
def load_de(table_name, de_name): debug = False stubObj = f.ET_Client(False, debug) DE_NAME = de_name FOLDER_ID = 502 # API_GEN de_to_load = f.ET_DataExtension_Row() de_to_load.CustomerKey = de_name de_to_load.auth_stub = stubObj # de_to_load.props = {"Col1": "Value1", "Col2": "Value2"} de_to_load.props = fetch_table_data(table_name) de_loaded_response = de_to_load.post()
def get_subscriberkey(EmailAddress): headers = { 'content-type': 'application/json', 'Authorization': get_auth_token() } if (not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl, '_create_unverified_context', None)): ssl._create_default_https_context = ssl._create_unverified_context debug = False stubObj = f.ET_Client(False, debug) row = f.ET_DataExtension_Row() row.auth_stub = stubObj # define DE List # deList = list() # deList = ['DE_order_confirm'] # deList = ['DE_order_confirm', 'DE_password_reset', 'DE_password_changed', # 'DE_pdt_print_ready', 'DE_pdt_proof_ready', 'DE_photo_share_receiver', # 'DE_photo_share_sender', 'DE_share_project_receiver'] # nameOfDE = 'DE_password_reset' # loop over DE list de = 'DE_SF_USER_D' row.CustomerKey = str(de) row.props = ["USERID"] # set search filter row.search_filter = { 'Property': 'CURR_EMAIL_ADDRESS', 'SimpleOperator': 'equals', 'Value': EmailAddress } getResponse = row.get() data = dict() # if len(getResponse.results) > 0: data = { "name": getResponse.results[0].Properties.Property[0].Name, "value": getResponse.results[0].Properties.Property[0].Value } else: data = {"Message": str(getResponse.results)} return data
def __init__(self, get_server_wsdl=False, debug=False, params=None): if debug: logger_debug.setLevel(logging.DEBUG) self.client = FuelSDK.ET_Client(get_server_wsdl=get_server_wsdl, debug=debug, params=params)
def check_automation_is_turned_on(automation_name, authentication): search_filter = { 'Property': 'Name', 'SimpleOperator': 'equals', 'Value': automation_name } props = ["Name", "Status", "ScheduledTime", "CustomerKey"] automation = FuelSDK.ET_Get(auth_stub=authentication, obj_type="Automation", props=props, search_filter=search_filter) automation_status = automation.results[0].Status customer_key = automation.results[0].CustomerKey status = get_automation_status(automation_status) if automation_status == 6 or automation_status == 7: print( f'{Fore.GREEN}{automation_name} is set to: {status}{Style.RESET_ALL}' ) check_last_time_ran(automation_name, authentication) else: print( f'{Fore.RED}{automation_name} is currently set to: {status}{Style.RESET_ALL}' ) if status != "Running": check_last_time_ran(automation_name, authentication)
def get_auth_stub(config): """ Given a config dict in the format: {'clientid': ... your ET client ID ..., 'clientsecret': ... your ET client secret ...} ... return an auth stub to be used when making requests. """ LOGGER.info("Generating auth stub...") params = { 'clientid': config['client_id'], 'clientsecret': config['client_secret'] } if config.get('tenant_subdomain'): # For S10+ accounts: https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis.meta/mc-apis/your-subdomain-tenant-specific-endpoints.htm params['authenticationurl'] = ( 'https://{}.auth.marketingcloudapis.com/v1/requestToken'.format( config['tenant_subdomain'])) params['soapendpoint'] = ( 'https://{}.soap.marketingcloudapis.com/Service.asmx'.format( config['tenant_subdomain'])) auth_stub = FuelSDK.ET_Client(params=params) transport = HttpAuthenticated( timeout=int(config.get('request_timeout', 900))) auth_stub.soap_client.set_options(transport=transport) LOGGER.info("Success.") return auth_stub
def retrieve_de(self, args): """ retrieve all rows from data extension. :param string customer_key: data extension's customer key :return: data extension's name array. """ fields = self.describe_de(args) row = FuelSDK.ET_DataExtension_Row() row.auth_stub = self.client row.CustomerKey = args.customer_key row.props = [field['Name'] for field in fields] response = row.get() writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL, lineterminator='\n') writer.writerow(row.props) for result in response.results: row = [] for prop in result.Properties[0]: if prop.Value is None: row.append("") else: row.append(prop.Value.encode("utf-8")) writer.writerow(row)
def search_queries(args): create_query_csv() with open(f"{Path.home()}/sfmc_cli_credentials.json", "r") as f: accounts = json.loads(f.read()) for account in accounts: print(f'{Fore.BLUE}==== Searching in {account["name"]} ====') auth_token = retrieve_auth_token(account["name"]) search_filter = { 'Property': 'QueryText', 'SimpleOperator': 'like', 'Value': args } props = ["Name", "Status", "QueryText"] queries = FuelSDK.ET_Get(auth_stub=auth_token, obj_type="QueryDefinition", props=props, search_filter=search_filter) for result in queries.results: query_name = result.Name automation_name = retrieve_automation_name_for_query( auth_token, query_name) print(f'{account["name"]},{result.Name},{automation_name}') write_to_query_csv( f'{account["name"]},{result.Name},{automation_name}\n') print( f'{Fore.GREEN}==== You can find the csv file at: ~/sfmc_cli_queries_result.csv ====' )
def describe_de(self, args): """ describe data extension with customer key. :param string customer_key: data extension's customer key :return: data extension's name array. """ de_target_fields = [ "Name", "CustomerKey", "DefaultValue", "FieldType", "Scale", "MaxLength", "IsPrimaryKey", "IsRequired", ] deColumn = FuelSDK.ET_DataExtension_Column() deColumn.auth_stub = self.client deColumn.props = de_target_fields deColumn.search_filter = { 'Property': 'DataExtension.CustomerKey', 'SimpleOperator': 'equals', 'Value': args.customer_key } response = deColumn.get() return [ self.convert_field_to_dict(result, de_target_fields) for result in response.results ]
def triggered_send(self, args): sendTrig = FuelSDK.ET_TriggeredSend() sendTrig.auth_stub = self.client sendTrig.props = {"CustomerKey": args.customer_key} if args.attribute_file is None: attributes = {} else: attributes = json.loads(args.attribute_file.read()) sendTrig.subscribers = [{ "EmailAddress": args.email, "SubscriberKey": args.subscriber_key, }] sendTrig.attributes = [{ "Name": key, "Value": val } for key, val in attributes.items()] sendResponse = sendTrig.send() print( json.dumps([{ "StatusCode": result.StatusCode, "StatusMessage": result.StatusMessage, "OrdinalID": result.OrdinalID, "NewID": result.NewID, "ErrorCode": result.ErrorCode if hasattr(result, "ErrorCode") else None, } for result in sendResponse.results]))
def check_last_time_ran(automation_name, authentication): search_filter = { 'Property': 'Name', 'SimpleOperator': 'equals', 'Value': automation_name } props = ['Name', 'CustomerKey', 'ProgramID', 'CompletedTime', 'Status'] instances = FuelSDK.ET_Get(auth_stub=authentication, obj_type='AutomationInstance', props=props, search_filter=search_filter) instances.results.sort(key=lambda automation: automation.CompletedTime) last_instance_completed_time = instances.results[-1].CompletedTime last_instance_status = instances.results[-1].StatusMessage if last_instance_status == "Error": print( f'{automation_name} last ran at {last_instance_completed_time} with status: {Fore.RED}{last_instance_status}{Style.RESET_ALL}' ) else: print( f'{automation_name} last ran at {last_instance_completed_time} with status: {Fore.BLUE}{last_instance_status}{Style.RESET_ALL}' )
def retrieve_bounceevent(self, args): """ retrieve all bounce event with triggered send's customer key. :param string customer_key: data extension's customer key :return: data extension's name array. """ triggeredSendDefinitionObjectID = self.retrieve_triggeredsend(args) getBounceEvent = FuelSDK.ET_BounceEvent() getBounceEvent.auth_stub = self.client getBounceEvent.props = [ "SendID", "SubscriberKey", "EventDate", "Client.ID", "EventType", "BatchID", "TriggeredSendDefinitionObjectID", "ListID", "PartnerKey", "SubscriberID" ] getBounceEvent.search_filter = { 'Property': 'TriggeredSendDefinitionObjectID', 'SimpleOperator': 'equals', 'Value': triggeredSendDefinitionObjectID } getResponse = getBounceEvent.get() writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL, lineterminator='\n') writer.writerow(["EventDate", "SubscriberID"]) for result in getResponse.results: writer.writerow([result.EventDate, result.SubscriberKey]) while getResponse.more_results: getResponse = getBounceEvent.getMoreResults() for result in getResponse.results: writer.writerow([result.EventDate, result.SubscriberKey])
def retrieve_auth_token(account): print(f'{Fore.YELLOW}==== Retrieving Credentials for {account} ====') credentials = get_secrets(account) return FuelSDK.ET_Client( False, False, { "clientid": credentials['client_id'], "clientsecret": credentials['client_secret'] })
def get_de_rows_filter(self, de_name, col_name, filter_value, operation, value_type='date'): """ Get filtered records from data extension based on the filter applied on the column Use value_type != 'date' for filtering values other than date type Operation: SFMC supported operations """ dataextensionrow = FuelSDK.ET_DataExtension_Row() dataextensionrow.auth_stub = self.sfmc_client dataextensionrow.Name = de_name col_names = self._get_col(de_name) if col_name not in col_names: #Check for column presence logging.error( f"{col_name} column not present in data extension {de_name}") return False if value_type.lower() == 'date': logging.debug('Date value selected for filtering') dataextensionrow.search_filter = { 'Property': col_name, 'SimpleOperator': operation, 'DateValue': filter_value } else: logging.debug('Non date value selected for filtering') dataextensionrow.search_filter = { 'Property': col_name, 'SimpleOperator': operation, 'Value': filter_value } df_in_file = pd.DataFrame() for col in col_names: #Get data column wise dataextensionrow.props = [col] results = dataextensionrow.get() col_data = [ i['Properties']['Property'][0]['Value'] for i in results.results ] while results.more_results: results = dataextensionrow.getMoreResults() col_data = col_data + [ i['Properties']['Property'][0]['Value'] for i in results.results ] df_in_file[col] = col_data logging.info( f"Total rows gathered from data extension:{de_name} = {df_in_file.shape[0]}" ) return df_in_file
def get_data_extension_rows(self, customer_key, search_filter=None, property_list=None): de_row = FuelSDK.ET_DataExtension_Row() de_row.auth_stub = self.get_client() de_row.CustomerKey = customer_key if search_filter: de_row.search_filter = search_filter if property_list: de_row.props = property_list return de_row.get()
def getName(self): if self.Name is None: if self.CustomerKey is None: raise Exception('Unable to process DataExtension::Row request due to CustomerKey and Name not being defined on ET_DatExtension::row') else: de = FuelSDK.ET_DataExtension() de.auth_stub = self.auth_stub de.props = ["Name", "CustomerKey"] de.search_filter = {'Property': 'CustomerKey', 'SimpleOperator': 'equals', 'Value': self.CustomerKey} getResponse = de.get() if getResponse.status and len(getResponse.results) == 1 and 'Name' in getResponse.results[0]: self.Name = getResponse.results[0]['Name'] else: raise Exception('Unable to process DataExtension::Row request due to unable to find DataExtension based on CustomerKey')
def _get_col(self, de_name): """ Get all columns for a Data Extension """ dataextensioncol = FuelSDK.ET_DataExtension_Column() dataextensioncol.auth_stub = self.sfmc_client dataextensioncol.props = ["Name"] dataextensioncol.search_filter = { 'Property': 'DataExtension.CustomerKey', 'SimpleOperator': 'equals', 'Value': de_name } response = dataextensioncol.get() return [i['Name'] for i in response.results]
def create_de_row(self, args): """ create data extension row. :param string customer_key: data extension's customer key :param string attributes_json: :return: data extension's name array. """ deRow = FuelSDK.ET_DataExtension_Row() deRow.CustomerKey = args.customer_key deRow.auth_stub = self.client args.attributes = json.loads(args.attribute_file.read()) deRow.props = json.loads(args.attributes_json) deRowResponse = deRow.post() print(json.dumps(deRowResponse.results))
def _update_data_extension_rows(self, de, iterator_range): de_row = FuelSDK.ET_DataExtension_Rows(self.de_external_key) de_row.auth_stub = self.client de_row.Name = self.de_name de_row.props = de sent_successfully = False retry_count = 0 sleep_time = 4 de_logger.debug( f'Sending request range :{iterator_range} for the {retry_count} time' ) while not sent_successfully and retry_count < self.sending_retry: try: if isinstance(de, list): results = de_row.post() else: results = de_row.put() sent_successfully = results.code in range(200, 300) de_logger.debug(f'Result : {results.results}') if not sent_successfully: de_logger.warning( f'request were failed with status code {results.code}, sleeping for :{sleep_time} sec' ) time.sleep(sleep_time) sleep_time *= 2 else: de_logger.info(f'{iterator_range} DE sent successfully') return iterator_range, results.results except Exception as e: de_logger.debug(e) finally: retry_count += 1 de_logger.error( f'Request :{de} failed to sent after {retry_count} retries') return iterator_range, None
def get_auth_stub(config): """ Given a config dict in the format: {'clientid': ... your ET client ID ..., 'clientsecret': ... your ET client secret ...} ... return an auth stub to be used when making requests. """ LOGGER.info("Generating auth stub...") auth_stub = FuelSDK.ET_Client(params={ 'clientid': config['client_id'], 'clientsecret': config['client_secret'] }) auth_stub.soap_client.set_options( timeout=int(config.get('request_timeout', 900))) LOGGER.info("Success.") return auth_stub
def verify_request(self, request_id: str): """ :param request_id: request id :return:boolean of success or failure """ de_row = FuelSDK.ET_Async_StatusResult() de_row.auth_stub = self.client result_fetching_sleep_time = 4 retry_count = 0 while retry_count < 3: r = de_row.get_status(request_id) try: if r.code in range( 200, 300 ) and "status" in r.results and "requestStatus" in r.results[ "status"] and r.results["status"][ "requestStatus"] == 'Complete': de_logger.info( f'request was verified {request_id}, result status {r.results["status"]["resultStatus"]}' ) if r.results["status"]["hasErrors"]: de_logger.warning(r.results) return False return True else: de_logger.debug(r.results) de_logger.debug( f'sleeping for :{result_fetching_sleep_time} sec to get result of request {request_id}' ) time.sleep(result_fetching_sleep_time) except Exception as e: de_logger.warning(e) time.sleep(result_fetching_sleep_time) finally: retry_count += 1 result_fetching_sleep_time *= 2 return False
def get_auth_stub(config): """ Given a config dict in the format: {'clientid': ... your ET client ID ..., 'clientsecret': ... your ET client secret ...} ... return an auth stub to be used when making requests. """ LOGGER.info("Generating auth stub...") params = { 'clientid': config['client_id'], 'clientsecret': config['client_secret'] } if config.get('tenant_subdomain'): # For S10+ accounts: https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis.meta/mc-apis/your-subdomain-tenant-specific-endpoints.htm # Move to OAuth2: https://help.salesforce.com/articleView?id=mc_rn_january_2019_platform_ip_remove_legacy_package_create_ability.htm&type=5 if config.get('use_oauth2') == "True": params['useOAuth2Authentication'] = "True" params['authenticationurl'] = ('https://{}.auth.marketingcloudapis.com' .format(config['tenant_subdomain'])) else: params['useOAuth2Authentication'] = "False" params['authenticationurl'] = ('https://{}.auth.marketingcloudapis.com/v1/requestToken' .format(config['tenant_subdomain'])) LOGGER.debug(f"Authentication URL is: {params['authenticationurl']}") params['soapendpoint'] = ('https://{}.soap.marketingcloudapis.com/Service.asmx' .format(config['tenant_subdomain'])) auth_stub = FuelSDK.ET_Client(params=params) transport = HttpAuthenticated(timeout=int(config.get('request_timeout', 900))) auth_stub.soap_client.set_options( transport=transport) LOGGER.info("Success.") return auth_stub
def _replicate(self, customer_key, keys, parent_category_id, table, partial=False, start=None, end=None, unit=None, replication_key=None): if partial: LOGGER.info("Fetching {} from {} to {}" .format(table, start, end)) cursor = FuelSDK.ET_DataExtension_Row() cursor.auth_stub = self.auth_stub cursor.CustomerKey = customer_key cursor.props = keys if partial: cursor.search_filter = get_date_page(replication_key, start, unit) batch_size = int(self.config.get('batch_size', 2500)) result = request_from_cursor('DataExtensionObject', cursor, batch_size=batch_size) for row in result: row = self.filter_keys_and_parse(row) row['CategoryID'] = parent_category_id self.state = incorporate(self.state, table, replication_key, row.get(replication_key)) singer.write_records(table, [row]) if partial: self.state = incorporate(self.state, table, replication_key, start) save_state(self.state)
def retrieve_subs(self, args): """ retrieve all subscriber rows. :param string customer_key: data extension's customer key :return: data extension's name array. """ getSub = FuelSDK.ET_Subscriber() getSub.auth_stub = self.client response = getSub.get() attributes = [] if (hasattr(response.results[0], 'Attributes')): attributes = [ attr.Name.encode("utf-8") for attr in response.results[0].Attributes ] writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL, lineterminator='\n') header = ["SubscriberID", "EmailAddress", "SubscriberKey"] header.extend(attributes) writer.writerow(header) for result in response.results: field_map = {} if (hasattr(result, 'Attributes')): for field in result.Attributes: field_map[field.Name] = field.Value fields = [result.ID, result.EmailAddress, result.SubscriberKey] for attribute in attributes: val = field_map[attribute] if val is None: fields.append("") else: fields.append(val.encode("utf-8")) writer.writerow(fields)
def retrieve_triggeredsend(self, args): """ retrive a triggered send with customer key. :param string customer_key: data extension's customer key :return: data extension's name array. """ getTS = FuelSDK.ET_TriggeredSend() getTS.auth_stub = self.client getTS.props = [ "CustomerKey", "Name", "TriggeredSendStatus", "ObjectID" ] getTS.search_filter = { 'Property': 'CustomerKey', 'SimpleOperator': 'equals', 'Value': args.customer_key } getResponse = getTS.get() for result in getResponse.results: return result.ObjectID return ""
def describe_all_de(self, args): """ describe all data extension. :param string customer_key: data extension's customer key :return: data extension's name array. """ de = FuelSDK.ET_DataExtension() de.auth_stub = self.client de.props = ["Name", "CustomerKey", "ObjectID"] response = de.get() writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL, lineterminator='\n') writer.writerow(de.props) for result in response.results: writer.writerow([ result.Name.encode("utf-8"), result.CustomerKey.encode("utf-8"), result.ObjectID.encode("utf-8") ])
def get(self): self.getName() ''' if props and props.is_a? Array then @props = props end ''' if self.props is not None and type(self.props) is dict: # pylint:disable=unidiomatic-typecheck self.props = self.props.keys() ''' if filter and filter.is_a? Hash then @filter = filter end ''' # add 'options' parameter to set 'batch_size' obj = FuelSDK.ET_Get(self.auth_stub, "DataExtensionObject[{0}]".format(self.Name), self.props, self.search_filter, self.options) self.last_request_id = obj.request_id return obj
def get(self): ''' if props and props.is_a? Array then @props = props end ''' if self.props is not None and type(self.props) is dict: # pylint:disable=unidiomatic-typecheck self.props = self.props.keys() ''' if filter and filter.is_a? Hash then @filter = filter end ''' ''' fixCustomerKey = False if filter and filter.is_a? Hash then @filter = filter if @filter.has_key?("Property") && @filter["Property"] == "CustomerKey" then @filter["Property"] = "DataExtension.CustomerKey" fixCustomerKey = true end end ''' # add 'options' parameter to set 'batch_size' obj = FuelSDK.ET_Get(self.auth_stub, self.obj, self.props, self.search_filter, self.options) self.last_request_id = obj.request_id ''' if fixCustomerKey then @filter["Property"] = "CustomerKey" end ''' return obj