class CampaignDataAccessObject(DataAccessObject):
SCHEMA = with_properties({
'id': {
'type': ['null', 'string'],
},
'createdDate': {
'type': ['null', 'string'],
},
'modifiedDate': {
'type': ['null', 'string'],
},
'name': {
'type': ['null', 'string'],
},
'description': {
'type': ['null', 'string'],
},
'campaignCode': {
'type': ['null', 'string'],
},
'color': {
'type': ['null', 'string'],
}
})
TABLE = 'campaign'
KEY_PROPERTIES = ['id']
def sync_data(self):
cursor = request(
'Campaign',
FuelSDK.ET_Campaign,
self.auth_stub)
for campaign in cursor:
campaign = self.filter_keys_and_parse(campaign)
singer.write_records(self.__class__.TABLE, [campaign])
class EventDataAccessObject(DataAccessObject):
SCHEMA = with_properties({
'SendID': {
'type': ['null', 'integer'],
'description': 'Contains identifier for a specific send.',
},
'EventDate': {
'type': ['null', 'string'],
'format': 'datetime',
'description': 'Date when a tracking event occurred.',
},
'EventType': {
'type': ['null', 'string'],
'description': 'The type of tracking event',
},
'SubscriberKey': SUBSCRIBER_KEY_FIELD,
})
TABLE = 'event'
KEY_PROPERTIES = ['SendID', 'EventType', 'SubscriberKey', 'EventDate']
def sync_data(self):
table = self.__class__.TABLE
endpoints = {
'sent': FuelSDK.ET_SentEvent,
'click': FuelSDK.ET_ClickEvent,
'open': FuelSDK.ET_OpenEvent,
'bounce': FuelSDK.ET_BounceEvent,
'unsub': FuelSDK.ET_UnsubEvent
}
for event_name, selector in endpoints.items():
search_filter = None
start = get_last_record_value_for_table(self.state, event_name)
if start is None:
start = self.config.get('start_date')
if start is None:
raise RuntimeError('start_date not defined!')
pagination_unit = self.config.get(
'pagination__{}_interval_unit'.format(event_name), 'minutes')
pagination_quantity = self.config.get(
'pagination__{}_interval_quantity'.format(event_name), 10)
unit = {pagination_unit: int(pagination_quantity)}
end = increment_date(start, unit)
while before_now(start):
LOGGER.info("Fetching {} from {} to {}"
.format(event_name, start, end))
search_filter = get_date_page('EventDate', start, unit)
stream = request(event_name,
selector,
self.auth_stub,
search_filter)
for event in stream:
event = self.filter_keys_and_parse(event)
self.state = incorporate(self.state,
event_name,
'EventDate',
event.get('EventDate'))
singer.write_records(table, [event])
self.state = incorporate(self.state,
event_name,
'EventDate',
start)
save_state(self.state)
start = end
end = increment_date(start, unit)
class EmailDataAccessObject(DataAccessObject):
SCHEMA = with_properties({
'CategoryID': {
'type': ['null', 'integer'],
'description': ('Specifies the identifier of the folder '
'containing the email.'),
},
'CharacterSet': {
'type': ['null', 'string'],
'description': ('Indicates encoding used in an email '
'message.'),
},
'ClonedFromID': {
'type': ['null', 'integer'],
'description': ('ID of email message from which the specified '
'email message was created.'),
},
'ContentAreaIDs': {
'type':
'array',
'description': ('Contains information on content areas '
'included in an email message.'),
'items': {
'type': 'integer'
}
},
'ContentCheckStatus': {
'type': ['null', 'string'],
'description': ('Indicates whether content validation has '
'completed for this email message.'),
},
'CreatedDate': CREATED_DATE_FIELD,
'CustomerKey': CUSTOMER_KEY_FIELD,
'EmailType': {
'type': ['null', 'string'],
'description': ('Defines preferred email type.'),
},
'HasDynamicSubjectLine': {
'type':
'boolean',
'description': ('Indicates whether email message contains '
'a dynamic subject line.'),
},
'HTMLBody': {
'type': ['null', 'string'],
'description': ('Contains HTML body of an email message.'),
},
'ID': ID_FIELD,
'IsActive': {
'type': 'boolean',
'description': ('Specifies whether the object is active.')
},
'IsHTMLPaste': {
'type':
'boolean',
'description': ('Indicates whether email message was created '
'via pasted HTML.')
},
'ModifiedDate': MODIFIED_DATE_FIELD,
'Name': {
'type': ['null', 'string'],
'description': 'Name of the object or property.',
},
'ObjectID': OBJECT_ID_FIELD,
'PartnerProperties': CUSTOM_PROPERTY_LIST,
'PreHeader': {
'type': ['null', 'string'],
'description': ('Contains text used in preheader of email '
'message on mobile devices.')
},
'Status': {
'type': ['null', 'string'],
'description': ('Defines status of object. Status of an '
'address.'),
},
'Subject': {
'type': ['null', 'string'],
'description': ('Contains subject area information for a '
'message.'),
},
'SyncTextWithHTML': {
'type':
'boolean',
'description': ('Makes the text version of an email contain '
'the same content as the HTML version.'),
},
'TextBody': {
'type': ['null', 'string'],
'description': ('Contains raw text body of a message.'),
},
'__AdditionalEmailAttribute1': {
'type': ['null', 'string']
},
'__AdditionalEmailAttribute2': {
'type': ['null', 'string']
},
'__AdditionalEmailAttribute3': {
'type': ['null', 'string']
},
'__AdditionalEmailAttribute4': {
'type': ['null', 'string']
},
'__AdditionalEmailAttribute5': {
'type': ['null', 'string']
},
})
TABLE = 'email'
KEY_PROPERTIES = ['ID']
def parse_object(self, obj):
to_return = obj.copy()
content_area_ids = []
for content_area in to_return.get('ContentAreas', []):
content_area_ids.append(content_area.get('ID'))
to_return['EmailID'] = to_return.get('Email', {}).get('ID')
to_return['ContentAreaIDs'] = content_area_ids
return super(EmailDataAccessObject, self).parse_object(to_return)
def sync_data(self):
table = self.__class__.TABLE
selector = FuelSDK.ET_Email
search_filter = None
retrieve_all_since = get_last_record_value_for_table(self.state, table)
if retrieve_all_since is not None:
search_filter = {
'Property': 'ModifiedDate',
'SimpleOperator': 'greaterThan',
'Value': retrieve_all_since
}
stream = request('Email', selector, self.auth_stub, search_filter)
for email in stream:
email = self.filter_keys_and_parse(email)
self.state = incorporate(self.state, table, 'ModifiedDate',
email.get('ModifiedDate'))
singer.write_records(table, [email])
save_state(self.state)
class ListDataAccessObject(DataAccessObject):
SCHEMA = with_properties({
'Category': {
'type': ['null', 'integer'],
'description': 'ID of the folder that an item is located in.',
},
'CreatedDate': CREATED_DATE_FIELD,
'ID': ID_FIELD,
'ModifiedDate': MODIFIED_DATE_FIELD,
'ObjectID': OBJECT_ID_FIELD,
'PartnerProperties': CUSTOM_PROPERTY_LIST,
'ListClassification': {
'type': ['null', 'string'],
'description': ('Specifies the classification for a list.'),
},
'ListName': {
'type': ['null', 'string'],
'description': 'Name of a specific list.',
},
'Description': DESCRIPTION_FIELD,
'SendClassification': {
'type': ['null', 'string'],
'description': ('Indicates the send classification to use '
'as part of a send definition.'),
},
'Type': {
'type': ['null', 'string'],
'description': ('Indicates type of specific list. Valid '
'values include Public, Private, Salesforce, '
'GlobalUnsubscribe, and Master.')
}
})
TABLE = 'list'
KEY_PROPERTIES = ['ID']
def sync_data(self):
table = self.__class__.TABLE
selector = FuelSDK.ET_List
search_filter = None
retrieve_all_since = get_last_record_value_for_table(self.state, table)
if retrieve_all_since is not None:
search_filter = {
'Property': 'ModifiedDate',
'SimpleOperator': 'greaterThan',
'Value': retrieve_all_since
}
stream = request('List',
selector,
self.auth_stub,
search_filter)
for _list in stream:
_list = self.filter_keys_and_parse(_list)
self.state = incorporate(self.state,
table,
'ModifiedDate',
_list.get('ModifiedDate'))
singer.write_records(table, [_list])
save_state(self.state)
class SendDataAccessObject(DataAccessObject):
SCHEMA = with_properties({
'CreatedDate': CREATED_DATE_FIELD,
'EmailID': {
'type': 'integer',
'description': ('Specifies the ID of an email message '
'associated with a send.'),
},
'EmailName': {
'type': 'string',
'description': ('Specifies the name of an email message '
'associated with a send.'),
},
'FromAddress': {
'type': 'string',
'description': ('Indicates From address associated with a '
'object. Deprecated for email send '
'definitions and triggered send '
'definitions.'),
},
'FromName': {
'type': 'string',
'description': ('Specifies the default email message From '
'Name. Deprecated for email send '
'definitions and triggered send '
'definitions.'),
},
'ID': ID_FIELD,
'IsAlwaysOn': {
'type': 'boolean',
'description': ('Indicates whether the request can be '
'performed while the system is is '
'maintenance mode. A value of true '
'indicates the system will process the '
'request.'),
},
'IsMultipart': {
'type': 'boolean',
'description': ('Indicates whether the email is sent with '
'Multipart/MIME enabled.'),
},
'ModifiedDate': MODIFIED_DATE_FIELD,
'PartnerProperties': CUSTOM_PROPERTY_LIST,
'SendDate': {
'type': 'string',
'format': 'date-time',
'description': ('Indicates the date on which a send '
'occurred. Set this value to have a CST '
'(Central Standard Time) value.'),
},
'SentDate': {
'type': 'string',
'format': 'date-time',
'description': ('Indicates date on which a send took '
'place.'),
},
'Status': {
'type': 'string',
'description': ('Defines status of object. Status of an '
'address.'),
},
'Subject': {
'type': 'string',
'description': ('Contains subject area information for '
'a message.'),
}
})
TABLE = 'send'
KEY_PROPERTIES = ['ID']
def parse_object(self, obj):
to_return = obj.copy()
to_return['EmailID'] = to_return.get('Email', {}).get('ID')
return super(SendDataAccessObject, self).parse_object(to_return)
def sync_data(self):
table = self.__class__.TABLE
selector = FuelSDK.ET_Send
search_filter = None
retrieve_all_since = get_last_record_value_for_table(self.state, table)
if retrieve_all_since is not None:
search_filter = {
'Property': 'ModifiedDate',
'SimpleOperator': 'greaterThan',
'Value': retrieve_all_since
}
stream = request('Send',
selector,
self.auth_stub,
search_filter)
for send in stream:
send = self.filter_keys_and_parse(send)
self.state = incorporate(self.state,
table,
'ModifiedDate',
send.get('ModifiedDate'))
singer.write_records(table, [send])
save_state(self.state)
class ListSendDataAccessObject(DataAccessObject):
SCHEMA = with_properties({
'CreatedDate': CREATED_DATE_FIELD,
'CustomerKey': CUSTOMER_KEY_FIELD,
'ExistingUndeliverables': {
'type': ['null', 'integer'],
'description': ('Indicates whether bounces occurred on previous '
'send.'),
},
'ExistingUnsubscribes': {
'type': ['null', 'integer'],
'description': ('Indicates whether unsubscriptions occurred on '
'previous send.'),
},
'ForwardedEmails': {
'type': ['null', 'integer'],
'description': ('Number of emails forwarded for a send.'),
},
'HardBounces': {
'type': ['null', 'integer'],
'description': ('Indicates number of hard bounces associated '
'with a send.'),
},
'InvalidAddresses': {
'type': ['null', 'integer'],
'description': ('Specifies the number of invalid addresses '
'associated with a send.'),
},
'ListID': {
'type': ['null', 'integer'],
'description': 'List associated with the send.',
},
'ID': ID_FIELD,
'MissingAddresses': {
'type': ['null', 'integer'],
'description': ('Specifies number of missing addresses '
'encountered within a send.'),
},
'ModifiedDate': MODIFIED_DATE_FIELD,
'NumberDelivered': {
'type': ['null', 'integer'],
'description': ('Number of sent emails that did not bounce.'),
},
'NumberSent': {
'type': ['null', 'integer'],
'description': ('Number of emails actually sent as part of an '
'email send. This number reflects all of the sent '
'messages and may include bounced messages.'),
},
'ObjectID': OBJECT_ID_FIELD,
'OtherBounces': {
'type': ['null', 'integer'],
'description': 'Specifies number of Other-type bounces in a send.',
},
'PartnerProperties': CUSTOM_PROPERTY_LIST,
'SendID': {
'type': ['null', 'integer'],
'description': 'Contains identifier for a specific send.',
},
'SoftBounces': {
'type': ['null', 'integer'],
'description': ('Indicates number of soft bounces associated with '
'a specific send.'),
},
'UniqueClicks': {
'type': ['null', 'integer'],
'description': 'Indicates number of unique clicks on message.',
},
'UniqueOpens': {
'type': ['null', 'integer'],
'description': ('Indicates number of unique opens resulting from '
'a triggered send.'),
},
'Unsubscribes': {
'type': ['null', 'integer'],
'description': ('Indicates the number of unsubscribe events '
'associated with a send.'),
},
})
TABLE = 'list_send'
KEY_PROPERTIES = ['ListID', 'SendID']
def parse_object(self, obj):
to_return = obj.copy()
to_return['ListID'] = to_return.get('List', {}).get('ID')
return super(ListSendDataAccessObject, self).parse_object(to_return)
def sync_data(self):
table = self.__class__.TABLE
selector = FuelSDK.ET_ListSend
search_filter = None
retrieve_all_since = get_last_record_value_for_table(self.state, table)
if retrieve_all_since is not None:
search_filter = {
'Property': 'ModifiedDate',
'SimpleOperator': 'greaterThan',
'Value': retrieve_all_since
}
stream = request('ListSend',
selector,
self.auth_stub,
search_filter)
for list_send in stream:
list_send = self.filter_keys_and_parse(list_send)
self.state = incorporate(self.state,
table,
'ModifiedDate',
list_send.get('ModifiedDate'))
singer.write_records(table, [list_send])
save_state(self.state)
SCHEMA = with_properties({
'Addresses': {
'type':
'array',
'description': ('Indicates addresses belonging to a subscriber, '
'used to create, retrieve, update or delete an '
'email or SMS Address for a given subscriber.'),
'items': {
'type': 'object',
'properties': {
'Address': {
'type': 'string'
},
'AddressType': {
'type': 'string'
},
'AddressStatus': {
'type': 'string'
}
}
}
},
'Attributes': CUSTOM_PROPERTY_LIST,
'CreatedDate': CREATED_DATE_FIELD,
'CustomerKey': CUSTOMER_KEY_FIELD,
'EmailAddress': {
'type': ['null', 'string'],
'description': ('Contains the email address for a subscriber. '
'Indicates the data extension field contains '
'email address data.'),
},
'EmailTypePreference': {
'type': ['null', 'string'],
'description': 'The format in which email should be sent'
},
'ID': ID_FIELD,
'ListIDs': {
'type': 'array',
'description': 'Defines list IDs a subscriber resides on.',
'items': {
'type': 'string'
}
},
'Locale': {
'type': ['null', 'string'],
'description': ('Contains the locale information for an Account. '
'If no location is set, Locale defaults to en-US '
'(English in United States).'),
},
'ModifiedDate': MODIFIED_DATE_FIELD,
'ObjectID': OBJECT_ID_FIELD,
'PartnerKey': {
'type': ['null', 'string'],
'description': ('Unique identifier provided by partner for an '
'object, accessible only via API.'),
},
'PartnerProperties': CUSTOM_PROPERTY_LIST,
'PartnerType': {
'type': ['null', 'string'],
'description': 'Defines partner associated with a subscriber.'
},
'PrimaryEmailAddress': {
'type': ['null', 'string'],
'description': 'Indicates primary email address for a subscriber.'
},
'PrimarySMSAddress': {
'type': ['null', 'string'],
'description': ('Indicates primary SMS address for a subscriber. '
'Used to create and update SMS Address for a '
'given subscriber.'),
},
'PrimarySMSPublicationStatus': {
'type': ['null', 'string'],
'description': 'Indicates the subscriber\'s modality status.',
},
'Status': {
'type': 'string',
'description': 'Defines status of object. Status of an address.',
},
'SubscriberKey': SUBSCRIBER_KEY_FIELD,
'SubscriberTypeDefinition': {
'type': ['null', 'string'],
'description': ('Specifies if a subscriber resides in an '
'integration, such as Salesforce or Microsoft '
'Dynamics CRM'),
},
'UnsubscribedDate': {
'type': ['null', 'string'],
'description': ('Represents date subscriber unsubscribed '
'from a list.'),
}
})
class FolderDataAccessObject(DataAccessObject):
SCHEMA = with_properties({
'AllowChildren': {
'type':
'boolean',
'description': ('Specifies whether a data folder can have '
'child data folders.'),
},
'ContentType': {
'type':
'string',
'description': ('Defines the type of content contained '
'within a folder.'),
},
'CreatedDate': CREATED_DATE_FIELD,
'CustomerKey': CUSTOMER_KEY_FIELD,
'Description': DESCRIPTION_FIELD,
'ID': ID_FIELD,
'ModifiedDate': MODIFIED_DATE_FIELD,
'Name': {
'type': ['null', 'string'],
'description': 'Name of the object or property.',
},
'ObjectID': OBJECT_ID_FIELD,
'ParentFolder': {
'type': ['null', 'integer'],
'description': ('Specifies the parent folder for a data '
'folder.'),
},
'PartnerProperties': CUSTOM_PROPERTY_LIST,
'Type': {
'type': ['null', 'string'],
'description': ('Indicates type of specific list. Valid '
'values include Public, Private, Salesforce, '
'GlobalUnsubscribe, and Master.')
}
})
TABLE = 'folder'
KEY_PROPERTIES = ['ID']
def parse_object(self, obj):
to_return = obj.copy()
to_return['ParentFolder'] = to_return.get('ParentFolder', {}).get('ID')
return super(FolderDataAccessObject, self).parse_object(to_return)
def sync_data(self):
table = self.__class__.TABLE
selector = FuelSDK.ET_Folder
search_filter = None
retrieve_all_since = get_last_record_value_for_table(self.state, table)
if retrieve_all_since is not None:
search_filter = {
'Property': 'ModifiedDate',
'SimpleOperator': 'greaterThan',
'Value': retrieve_all_since
}
stream = request('Folder', selector, self.auth_stub, search_filter)
for folder in stream:
folder = self.filter_keys_and_parse(folder)
self.state = incorporate(self.state, table, 'ModifiedDate',
folder.get('ModifiedDate'))
singer.write_records(table, [folder])
save_state(self.state)
class ContentAreaDataAccessObject(DataAccessObject):
SCHEMA = with_properties({
'BackgroundColor': {
'type': ['null', 'string'],
'description': 'Indicates background color of content area',
},
'BorderColor': {
'type': ['null', 'string'],
'description': ('Indicates color of border surrounding '
'content area'),
},
'BorderWidth': {
'type': ['null', 'integer'],
'description': ('Indicates pixel width of border '
'surrounding content area'),
},
'CategoryID': {
'type': ['null', 'integer'],
'description': 'Specifies the identifier of the folder.',
},
'Cellpadding': {
'type': ['null', 'integer'],
'description': ('Indicates pixel value of padding '
'around content area'),
},
'Cellspacing': {
'type': ['null', 'integer'],
'description': ('Indicates pixel value of spacing '
'for content area'),
},
'Content': {
'type': ['null', 'string'],
'description': ('Identifies content contained in '
'a content area.'),
},
'CreatedDate': CREATED_DATE_FIELD,
'CustomerKey': CUSTOMER_KEY_FIELD,
'FontFamily': {
'type': ['null', 'string'],
'description': 'Indicates font family used in content area',
},
'HasFontSize': {
'type':
'boolean',
'description': ('Indicates whether the content area includes '
'a specified font size or not'),
},
'ID': ID_FIELD,
'IsBlank': {
'type':
'boolean',
'description': ('Indicates if specified content area '
'contains no content.'),
},
'IsDynamicContent': {
'type':
'boolean',
'description': ('Indicates if specific content area '
'contains dynamic content.'),
},
'IsLocked': {
'type':
'boolean',
'description': ('Indicates if specific email content area '
'within an Enterprise or Enterprise 2.0 '
'account is locked and cannot be changed by '
'subaccounts.'),
},
'IsSurvey': {
'type':
'boolean',
'description': ('Indicates whether a specific content area '
'contains survey questions.'),
},
'Key': {
'type': ['null', 'string'],
'description': ('Specifies key associated with content area '
'in HTML body. Relates to the Email object '
'via a custom type.'),
},
'ModifiedDate': MODIFIED_DATE_FIELD,
'Name': {
'type': ['null', 'string'],
'description': 'Name of the object or property.',
},
'ObjectID': OBJECT_ID_FIELD,
'PartnerProperties': CUSTOM_PROPERTY_LIST,
'Width': {
'type': ['null', 'integer'],
'description': 'Indicates pixel width of content area',
},
})
TABLE = 'content_area'
KEY_PROPERTIES = ['ID']
def sync_data(self):
table = self.__class__.TABLE
selector = FuelSDK.ET_ContentArea
search_filter = None
retrieve_all_since = get_last_record_value_for_table(self.state, table)
if retrieve_all_since is not None:
search_filter = {
'Property': 'ModifiedDate',
'SimpleOperator': 'greaterThan',
'Value': retrieve_all_since
}
stream = request('ContentAreaDataAccessObject', selector,
self.auth_stub, search_filter)
for content_area in stream:
content_area = self.filter_keys_and_parse(content_area)
self.state = incorporate(self.state, table, 'ModifiedDate',
content_area.get('ModifiedDate'))
singer.write_records(table, [content_area])
save_state(self.state)
class ListSubscriberDataAccessObject(DataAccessObject):
SCHEMA = with_properties({
'ID': ID_FIELD,
'CreatedDate': CREATED_DATE_FIELD,
'ModifiedDate': MODIFIED_DATE_FIELD,
'ObjectID': OBJECT_ID_FIELD,
'PartnerProperties': CUSTOM_PROPERTY_LIST,
'ListID': {
'type': ['null', 'integer'],
'description': ('Defines identification for a list the '
'subscriber resides on.'),
},
'Status': {
'type': ['null', 'string'],
'description': ('Defines status of object. Status of '
'an address.'),
},
'SubscriberKey': SUBSCRIBER_KEY_FIELD,
})
TABLE = 'list_subscriber'
KEY_PROPERTIES = ['SubscriberKey', 'ListID']
def __init__(self, config, state, auth_stub, catalog):
super(ListSubscriberDataAccessObject, self).__init__(
config, state, auth_stub, catalog)
self.replicate_subscriber = False
self.subscriber_catalog = None
def _get_all_subscribers_list(self):
"""
Find the 'All Subscribers' list via the SOAP API, and return it.
"""
result = request('List', FuelSDK.ET_List, self.auth_stub, {
'Property': 'ListName',
'SimpleOperator': 'equals',
'Value': 'All Subscribers',
})
lists = list(result)
if len(lists) != 1:
msg = ('Found {} all subscriber lists, expected one!'
.format(len(lists)))
raise RuntimeError(msg)
return sudsobj_to_dict(lists[0])
def sync_data(self):
table = self.__class__.TABLE
subscriber_dao = SubscriberDataAccessObject(
self.config,
self.state,
self.auth_stub,
self.subscriber_catalog)
start = get_last_record_value_for_table(self.state, table)
if start is None:
start = self.config.get('start_date')
pagination_unit = self.config.get(
'pagination__list_subscriber_interval_unit', 'days')
pagination_quantity = self.config.get(
'pagination__list_subsctiber_interval_quantity', 1)
unit = {pagination_unit: int(pagination_quantity)}
end = increment_date(start, unit)
all_subscribers_list = self._get_all_subscribers_list()
while before_now(start):
stream = request('ListSubscriber',
FuelSDK.ET_List_Subscriber,
self.auth_stub,
_get_list_subscriber_filter(
all_subscribers_list,
start, unit))
batch_size = 100
if self.replicate_subscriber:
subscriber_dao.write_schema()
for list_subscribers_batch in partition_all(stream, batch_size):
for list_subscriber in list_subscribers_batch:
list_subscriber = self.filter_keys_and_parse(
list_subscriber)
if list_subscriber.get('ModifiedDate'):
self.state = incorporate(
self.state,
table,
'ModifiedDate',
list_subscriber.get('ModifiedDate'))
singer.write_records(table, [list_subscriber])
if self.replicate_subscriber:
subscriber_keys = list(map(
_get_subscriber_key, list_subscribers_batch))
subscriber_dao.pull_subscribers_batch(subscriber_keys)
save_state(self.state)
start = end
end = increment_date(start, unit)
class LinkSendDataAccessObject(DataAccessObject):
REPLICATION_METHOD = "FULL_TABLE"
SCHEMA = with_properties({
'ID': ID_FIELD,
'SendID': {
'type': ['null', 'integer'],
'description': 'Contains identifier for a specific send.',
},
'ClientID': {
'type': ['null', 'integer'],
'description': 'Contains identifier for a specific client.',
},
'PartnerKey': {
'type': ['null', 'integer'],
'description': '',
},
'TotalClicks': {
'type': ['null', 'integer'],
'description': '',
},
'UniqueClicks': {
'type': ['null', 'integer'],
'description': '',
},
'URL': {
'type': ['null', 'string'],
'description': '',
},
'Alias': {
'type': ['null', 'string'],
'description': '',
},
'LinkID': {
'type': ['null', 'integer'],
'description': 'Contains identifier for a specific link.',
}
})
TABLE = 'link_send'
KEY_PROPERTIES = ['SendID']
def parse_object(self, obj):
to_return = obj.copy()
to_return['ClientID'] = to_return.get('Client', {}).get('ID')
to_return['LinkID'] = to_return.get('Link', {}).get('ID')
to_return['Alias'] = to_return.get('Link', {}).get('Alias')
to_return['URL'] = to_return.get('Link', {}).get('URL')
to_return['TotalClicks'] = to_return.get('Link', {}).get('TotalClicks')
to_return['UniqueClicks'] = to_return.get('Link',
{}).get('UniqueClicks')
return super(LinkSendDataAccessObject, self).parse_object(to_return)
def sync_data(self):
pass
def sync_data_by_sendID(self, sendId):
if not sendId:
return
table = self.__class__.TABLE
_filter = {}
if sendId:
_filter = {
'Property': 'SendID',
'SimpleOperator': 'equals',
'Value': sendId
}
else:
LOGGER.info('No send id here, moving on')
return
stream = request(self.__class__.TABLE, ET_LinkSend, self.auth_stub,
_filter)
for link_send in stream:
link_send = self.filter_keys_and_parse(link_send)
singer.write_records(table, [link_send])
class EventDataAccessObject(DataAccessObject):
SCHEMA = with_properties({
'SendID': {
'type': ['null', 'integer'],
'description': 'Contains identifier for a specific send.',
},
'EventDate': {
'type': ['null', 'string'],
'format': 'datetime',
'description': 'Date when a tracking event occurred.',
},
'EventType': {
'type': ['null', 'string'],
'description': 'The type of tracking event',
},
'BatchID': {
'type': ['null', 'integer'],
'description':
'Ties triggered send sent events to other events (like clicks and opens that occur at a later date and time)',
},
'CorrelationID': {
'type': ['null', 'string'],
'description':
'Identifies correlation of objects across several requests.',
},
'URL': {
'type': ['null', 'string'],
'description': 'URL that was clicked.',
},
'SubscriberKey': SUBSCRIBER_KEY_FIELD,
})
TABLE = 'event'
KEY_PROPERTIES = ['SendID', 'EventType', 'SubscriberKey', 'EventDate']
def sync_data(self):
table = self.__class__.TABLE
search_filter = None
start = get_last_record_value_for_table(self.state, self.event_name,
self.config.get('start_date'))
if start is None:
start = self.config.get('start_date')
if start is None:
raise RuntimeError('start_date not defined!')
pagination_unit = self.config.get(
'pagination__{}_interval_unit'.format(self.event_name), 'minutes')
pagination_quantity = self.config.get(
'pagination__{}_interval_quantity'.format(self.event_name), 10)
unit = {pagination_unit: int(pagination_quantity)}
end = increment_date(start, unit)
while before_now(start):
LOGGER.info("Fetching {} from {} to {}".format(
self.event_name, start, end))
search_filter = get_date_page('EventDate', start, unit)
stream = request(self.event_name, self.selector, self.auth_stub,
search_filter)
for event in stream:
event = self.filter_keys_and_parse(event)
self.state = incorporate(self.state, self.event_name,
'EventDate', event.get('EventDate'))
if event.get('SubscriberKey') is None:
LOGGER.info(
"SubscriberKey is NULL so ignoring {} record with SendID: {} and EventDate: {}"
.format(self.event_name, event.get('SendID'),
event.get('EventDate')))
continue
event = self.remove_sensitive_data(event)
singer.write_records(table, [event])
self.state = incorporate(self.state, self.event_name, 'EventDate',
start)
save_state(self.state)
start = end
end = increment_date(start, unit)
