class CampaignDataAccessObject(DataAccessObject): SCHEMA = with_properties({ 'id': { 'type': ['null', 'string'], }, 'createdDate': { 'type': ['null', 'string'], }, 'modifiedDate': { 'type': ['null', 'string'], }, 'name': { 'type': ['null', 'string'], }, 'description': { 'type': ['null', 'string'], }, 'campaignCode': { 'type': ['null', 'string'], }, 'color': { 'type': ['null', 'string'], } }) TABLE = 'campaign' KEY_PROPERTIES = ['id'] def sync_data(self): cursor = request( 'Campaign', FuelSDK.ET_Campaign, self.auth_stub) for campaign in cursor: campaign = self.filter_keys_and_parse(campaign) singer.write_records(self.__class__.TABLE, [campaign])
class EventDataAccessObject(DataAccessObject): SCHEMA = with_properties({ 'SendID': { 'type': ['null', 'integer'], 'description': 'Contains identifier for a specific send.', }, 'EventDate': { 'type': ['null', 'string'], 'format': 'datetime', 'description': 'Date when a tracking event occurred.', }, 'EventType': { 'type': ['null', 'string'], 'description': 'The type of tracking event', }, 'SubscriberKey': SUBSCRIBER_KEY_FIELD, }) TABLE = 'event' KEY_PROPERTIES = ['SendID', 'EventType', 'SubscriberKey', 'EventDate'] def sync_data(self): table = self.__class__.TABLE endpoints = { 'sent': FuelSDK.ET_SentEvent, 'click': FuelSDK.ET_ClickEvent, 'open': FuelSDK.ET_OpenEvent, 'bounce': FuelSDK.ET_BounceEvent, 'unsub': FuelSDK.ET_UnsubEvent } for event_name, selector in endpoints.items(): search_filter = None start = get_last_record_value_for_table(self.state, event_name) if start is None: start = self.config.get('start_date') if start is None: raise RuntimeError('start_date not defined!') pagination_unit = self.config.get( 'pagination__{}_interval_unit'.format(event_name), 'minutes') pagination_quantity = self.config.get( 'pagination__{}_interval_quantity'.format(event_name), 10) unit = {pagination_unit: int(pagination_quantity)} end = increment_date(start, unit) while before_now(start): LOGGER.info("Fetching {} from {} to {}" .format(event_name, start, end)) search_filter = get_date_page('EventDate', start, unit) stream = request(event_name, selector, self.auth_stub, search_filter) for event in stream: event = self.filter_keys_and_parse(event) self.state = incorporate(self.state, event_name, 'EventDate', event.get('EventDate')) singer.write_records(table, [event]) self.state = incorporate(self.state, event_name, 'EventDate', start) save_state(self.state) start = end end = increment_date(start, unit)
class EmailDataAccessObject(DataAccessObject): SCHEMA = with_properties({ 'CategoryID': { 'type': ['null', 'integer'], 'description': ('Specifies the identifier of the folder ' 'containing the email.'), }, 'CharacterSet': { 'type': ['null', 'string'], 'description': ('Indicates encoding used in an email ' 'message.'), }, 'ClonedFromID': { 'type': ['null', 'integer'], 'description': ('ID of email message from which the specified ' 'email message was created.'), }, 'ContentAreaIDs': { 'type': 'array', 'description': ('Contains information on content areas ' 'included in an email message.'), 'items': { 'type': 'integer' } }, 'ContentCheckStatus': { 'type': ['null', 'string'], 'description': ('Indicates whether content validation has ' 'completed for this email message.'), }, 'CreatedDate': CREATED_DATE_FIELD, 'CustomerKey': CUSTOMER_KEY_FIELD, 'EmailType': { 'type': ['null', 'string'], 'description': ('Defines preferred email type.'), }, 'HasDynamicSubjectLine': { 'type': 'boolean', 'description': ('Indicates whether email message contains ' 'a dynamic subject line.'), }, 'HTMLBody': { 'type': ['null', 'string'], 'description': ('Contains HTML body of an email message.'), }, 'ID': ID_FIELD, 'IsActive': { 'type': 'boolean', 'description': ('Specifies whether the object is active.') }, 'IsHTMLPaste': { 'type': 'boolean', 'description': ('Indicates whether email message was created ' 'via pasted HTML.') }, 'ModifiedDate': MODIFIED_DATE_FIELD, 'Name': { 'type': ['null', 'string'], 'description': 'Name of the object or property.', }, 'ObjectID': OBJECT_ID_FIELD, 'PartnerProperties': CUSTOM_PROPERTY_LIST, 'PreHeader': { 'type': ['null', 'string'], 'description': ('Contains text used in preheader of email ' 'message on mobile devices.') }, 'Status': { 'type': ['null', 'string'], 'description': ('Defines status of object. Status of an ' 'address.'), }, 'Subject': { 'type': ['null', 'string'], 'description': ('Contains subject area information for a ' 'message.'), }, 'SyncTextWithHTML': { 'type': 'boolean', 'description': ('Makes the text version of an email contain ' 'the same content as the HTML version.'), }, 'TextBody': { 'type': ['null', 'string'], 'description': ('Contains raw text body of a message.'), }, '__AdditionalEmailAttribute1': { 'type': ['null', 'string'] }, '__AdditionalEmailAttribute2': { 'type': ['null', 'string'] }, '__AdditionalEmailAttribute3': { 'type': ['null', 'string'] }, '__AdditionalEmailAttribute4': { 'type': ['null', 'string'] }, '__AdditionalEmailAttribute5': { 'type': ['null', 'string'] }, }) TABLE = 'email' KEY_PROPERTIES = ['ID'] def parse_object(self, obj): to_return = obj.copy() content_area_ids = [] for content_area in to_return.get('ContentAreas', []): content_area_ids.append(content_area.get('ID')) to_return['EmailID'] = to_return.get('Email', {}).get('ID') to_return['ContentAreaIDs'] = content_area_ids return super(EmailDataAccessObject, self).parse_object(to_return) def sync_data(self): table = self.__class__.TABLE selector = FuelSDK.ET_Email search_filter = None retrieve_all_since = get_last_record_value_for_table(self.state, table) if retrieve_all_since is not None: search_filter = { 'Property': 'ModifiedDate', 'SimpleOperator': 'greaterThan', 'Value': retrieve_all_since } stream = request('Email', selector, self.auth_stub, search_filter) for email in stream: email = self.filter_keys_and_parse(email) self.state = incorporate(self.state, table, 'ModifiedDate', email.get('ModifiedDate')) singer.write_records(table, [email]) save_state(self.state)
class ListDataAccessObject(DataAccessObject): SCHEMA = with_properties({ 'Category': { 'type': ['null', 'integer'], 'description': 'ID of the folder that an item is located in.', }, 'CreatedDate': CREATED_DATE_FIELD, 'ID': ID_FIELD, 'ModifiedDate': MODIFIED_DATE_FIELD, 'ObjectID': OBJECT_ID_FIELD, 'PartnerProperties': CUSTOM_PROPERTY_LIST, 'ListClassification': { 'type': ['null', 'string'], 'description': ('Specifies the classification for a list.'), }, 'ListName': { 'type': ['null', 'string'], 'description': 'Name of a specific list.', }, 'Description': DESCRIPTION_FIELD, 'SendClassification': { 'type': ['null', 'string'], 'description': ('Indicates the send classification to use ' 'as part of a send definition.'), }, 'Type': { 'type': ['null', 'string'], 'description': ('Indicates type of specific list. Valid ' 'values include Public, Private, Salesforce, ' 'GlobalUnsubscribe, and Master.') } }) TABLE = 'list' KEY_PROPERTIES = ['ID'] def sync_data(self): table = self.__class__.TABLE selector = FuelSDK.ET_List search_filter = None retrieve_all_since = get_last_record_value_for_table(self.state, table) if retrieve_all_since is not None: search_filter = { 'Property': 'ModifiedDate', 'SimpleOperator': 'greaterThan', 'Value': retrieve_all_since } stream = request('List', selector, self.auth_stub, search_filter) for _list in stream: _list = self.filter_keys_and_parse(_list) self.state = incorporate(self.state, table, 'ModifiedDate', _list.get('ModifiedDate')) singer.write_records(table, [_list]) save_state(self.state)
class SendDataAccessObject(DataAccessObject): SCHEMA = with_properties({ 'CreatedDate': CREATED_DATE_FIELD, 'EmailID': { 'type': 'integer', 'description': ('Specifies the ID of an email message ' 'associated with a send.'), }, 'EmailName': { 'type': 'string', 'description': ('Specifies the name of an email message ' 'associated with a send.'), }, 'FromAddress': { 'type': 'string', 'description': ('Indicates From address associated with a ' 'object. Deprecated for email send ' 'definitions and triggered send ' 'definitions.'), }, 'FromName': { 'type': 'string', 'description': ('Specifies the default email message From ' 'Name. Deprecated for email send ' 'definitions and triggered send ' 'definitions.'), }, 'ID': ID_FIELD, 'IsAlwaysOn': { 'type': 'boolean', 'description': ('Indicates whether the request can be ' 'performed while the system is is ' 'maintenance mode. A value of true ' 'indicates the system will process the ' 'request.'), }, 'IsMultipart': { 'type': 'boolean', 'description': ('Indicates whether the email is sent with ' 'Multipart/MIME enabled.'), }, 'ModifiedDate': MODIFIED_DATE_FIELD, 'PartnerProperties': CUSTOM_PROPERTY_LIST, 'SendDate': { 'type': 'string', 'format': 'date-time', 'description': ('Indicates the date on which a send ' 'occurred. Set this value to have a CST ' '(Central Standard Time) value.'), }, 'SentDate': { 'type': 'string', 'format': 'date-time', 'description': ('Indicates date on which a send took ' 'place.'), }, 'Status': { 'type': 'string', 'description': ('Defines status of object. Status of an ' 'address.'), }, 'Subject': { 'type': 'string', 'description': ('Contains subject area information for ' 'a message.'), } }) TABLE = 'send' KEY_PROPERTIES = ['ID'] def parse_object(self, obj): to_return = obj.copy() to_return['EmailID'] = to_return.get('Email', {}).get('ID') return super(SendDataAccessObject, self).parse_object(to_return) def sync_data(self): table = self.__class__.TABLE selector = FuelSDK.ET_Send search_filter = None retrieve_all_since = get_last_record_value_for_table(self.state, table) if retrieve_all_since is not None: search_filter = { 'Property': 'ModifiedDate', 'SimpleOperator': 'greaterThan', 'Value': retrieve_all_since } stream = request('Send', selector, self.auth_stub, search_filter) for send in stream: send = self.filter_keys_and_parse(send) self.state = incorporate(self.state, table, 'ModifiedDate', send.get('ModifiedDate')) singer.write_records(table, [send]) save_state(self.state)
class ListSendDataAccessObject(DataAccessObject): SCHEMA = with_properties({ 'CreatedDate': CREATED_DATE_FIELD, 'CustomerKey': CUSTOMER_KEY_FIELD, 'ExistingUndeliverables': { 'type': ['null', 'integer'], 'description': ('Indicates whether bounces occurred on previous ' 'send.'), }, 'ExistingUnsubscribes': { 'type': ['null', 'integer'], 'description': ('Indicates whether unsubscriptions occurred on ' 'previous send.'), }, 'ForwardedEmails': { 'type': ['null', 'integer'], 'description': ('Number of emails forwarded for a send.'), }, 'HardBounces': { 'type': ['null', 'integer'], 'description': ('Indicates number of hard bounces associated ' 'with a send.'), }, 'InvalidAddresses': { 'type': ['null', 'integer'], 'description': ('Specifies the number of invalid addresses ' 'associated with a send.'), }, 'ListID': { 'type': ['null', 'integer'], 'description': 'List associated with the send.', }, 'ID': ID_FIELD, 'MissingAddresses': { 'type': ['null', 'integer'], 'description': ('Specifies number of missing addresses ' 'encountered within a send.'), }, 'ModifiedDate': MODIFIED_DATE_FIELD, 'NumberDelivered': { 'type': ['null', 'integer'], 'description': ('Number of sent emails that did not bounce.'), }, 'NumberSent': { 'type': ['null', 'integer'], 'description': ('Number of emails actually sent as part of an ' 'email send. This number reflects all of the sent ' 'messages and may include bounced messages.'), }, 'ObjectID': OBJECT_ID_FIELD, 'OtherBounces': { 'type': ['null', 'integer'], 'description': 'Specifies number of Other-type bounces in a send.', }, 'PartnerProperties': CUSTOM_PROPERTY_LIST, 'SendID': { 'type': ['null', 'integer'], 'description': 'Contains identifier for a specific send.', }, 'SoftBounces': { 'type': ['null', 'integer'], 'description': ('Indicates number of soft bounces associated with ' 'a specific send.'), }, 'UniqueClicks': { 'type': ['null', 'integer'], 'description': 'Indicates number of unique clicks on message.', }, 'UniqueOpens': { 'type': ['null', 'integer'], 'description': ('Indicates number of unique opens resulting from ' 'a triggered send.'), }, 'Unsubscribes': { 'type': ['null', 'integer'], 'description': ('Indicates the number of unsubscribe events ' 'associated with a send.'), }, }) TABLE = 'list_send' KEY_PROPERTIES = ['ListID', 'SendID'] def parse_object(self, obj): to_return = obj.copy() to_return['ListID'] = to_return.get('List', {}).get('ID') return super(ListSendDataAccessObject, self).parse_object(to_return) def sync_data(self): table = self.__class__.TABLE selector = FuelSDK.ET_ListSend search_filter = None retrieve_all_since = get_last_record_value_for_table(self.state, table) if retrieve_all_since is not None: search_filter = { 'Property': 'ModifiedDate', 'SimpleOperator': 'greaterThan', 'Value': retrieve_all_since } stream = request('ListSend', selector, self.auth_stub, search_filter) for list_send in stream: list_send = self.filter_keys_and_parse(list_send) self.state = incorporate(self.state, table, 'ModifiedDate', list_send.get('ModifiedDate')) singer.write_records(table, [list_send]) save_state(self.state)
SCHEMA = with_properties({ 'Addresses': { 'type': 'array', 'description': ('Indicates addresses belonging to a subscriber, ' 'used to create, retrieve, update or delete an ' 'email or SMS Address for a given subscriber.'), 'items': { 'type': 'object', 'properties': { 'Address': { 'type': 'string' }, 'AddressType': { 'type': 'string' }, 'AddressStatus': { 'type': 'string' } } } }, 'Attributes': CUSTOM_PROPERTY_LIST, 'CreatedDate': CREATED_DATE_FIELD, 'CustomerKey': CUSTOMER_KEY_FIELD, 'EmailAddress': { 'type': ['null', 'string'], 'description': ('Contains the email address for a subscriber. ' 'Indicates the data extension field contains ' 'email address data.'), }, 'EmailTypePreference': { 'type': ['null', 'string'], 'description': 'The format in which email should be sent' }, 'ID': ID_FIELD, 'ListIDs': { 'type': 'array', 'description': 'Defines list IDs a subscriber resides on.', 'items': { 'type': 'string' } }, 'Locale': { 'type': ['null', 'string'], 'description': ('Contains the locale information for an Account. ' 'If no location is set, Locale defaults to en-US ' '(English in United States).'), }, 'ModifiedDate': MODIFIED_DATE_FIELD, 'ObjectID': OBJECT_ID_FIELD, 'PartnerKey': { 'type': ['null', 'string'], 'description': ('Unique identifier provided by partner for an ' 'object, accessible only via API.'), }, 'PartnerProperties': CUSTOM_PROPERTY_LIST, 'PartnerType': { 'type': ['null', 'string'], 'description': 'Defines partner associated with a subscriber.' }, 'PrimaryEmailAddress': { 'type': ['null', 'string'], 'description': 'Indicates primary email address for a subscriber.' }, 'PrimarySMSAddress': { 'type': ['null', 'string'], 'description': ('Indicates primary SMS address for a subscriber. ' 'Used to create and update SMS Address for a ' 'given subscriber.'), }, 'PrimarySMSPublicationStatus': { 'type': ['null', 'string'], 'description': 'Indicates the subscriber\'s modality status.', }, 'Status': { 'type': 'string', 'description': 'Defines status of object. Status of an address.', }, 'SubscriberKey': SUBSCRIBER_KEY_FIELD, 'SubscriberTypeDefinition': { 'type': ['null', 'string'], 'description': ('Specifies if a subscriber resides in an ' 'integration, such as Salesforce or Microsoft ' 'Dynamics CRM'), }, 'UnsubscribedDate': { 'type': ['null', 'string'], 'description': ('Represents date subscriber unsubscribed ' 'from a list.'), } })
class FolderDataAccessObject(DataAccessObject): SCHEMA = with_properties({ 'AllowChildren': { 'type': 'boolean', 'description': ('Specifies whether a data folder can have ' 'child data folders.'), }, 'ContentType': { 'type': 'string', 'description': ('Defines the type of content contained ' 'within a folder.'), }, 'CreatedDate': CREATED_DATE_FIELD, 'CustomerKey': CUSTOMER_KEY_FIELD, 'Description': DESCRIPTION_FIELD, 'ID': ID_FIELD, 'ModifiedDate': MODIFIED_DATE_FIELD, 'Name': { 'type': ['null', 'string'], 'description': 'Name of the object or property.', }, 'ObjectID': OBJECT_ID_FIELD, 'ParentFolder': { 'type': ['null', 'integer'], 'description': ('Specifies the parent folder for a data ' 'folder.'), }, 'PartnerProperties': CUSTOM_PROPERTY_LIST, 'Type': { 'type': ['null', 'string'], 'description': ('Indicates type of specific list. Valid ' 'values include Public, Private, Salesforce, ' 'GlobalUnsubscribe, and Master.') } }) TABLE = 'folder' KEY_PROPERTIES = ['ID'] def parse_object(self, obj): to_return = obj.copy() to_return['ParentFolder'] = to_return.get('ParentFolder', {}).get('ID') return super(FolderDataAccessObject, self).parse_object(to_return) def sync_data(self): table = self.__class__.TABLE selector = FuelSDK.ET_Folder search_filter = None retrieve_all_since = get_last_record_value_for_table(self.state, table) if retrieve_all_since is not None: search_filter = { 'Property': 'ModifiedDate', 'SimpleOperator': 'greaterThan', 'Value': retrieve_all_since } stream = request('Folder', selector, self.auth_stub, search_filter) for folder in stream: folder = self.filter_keys_and_parse(folder) self.state = incorporate(self.state, table, 'ModifiedDate', folder.get('ModifiedDate')) singer.write_records(table, [folder]) save_state(self.state)
class ContentAreaDataAccessObject(DataAccessObject): SCHEMA = with_properties({ 'BackgroundColor': { 'type': ['null', 'string'], 'description': 'Indicates background color of content area', }, 'BorderColor': { 'type': ['null', 'string'], 'description': ('Indicates color of border surrounding ' 'content area'), }, 'BorderWidth': { 'type': ['null', 'integer'], 'description': ('Indicates pixel width of border ' 'surrounding content area'), }, 'CategoryID': { 'type': ['null', 'integer'], 'description': 'Specifies the identifier of the folder.', }, 'Cellpadding': { 'type': ['null', 'integer'], 'description': ('Indicates pixel value of padding ' 'around content area'), }, 'Cellspacing': { 'type': ['null', 'integer'], 'description': ('Indicates pixel value of spacing ' 'for content area'), }, 'Content': { 'type': ['null', 'string'], 'description': ('Identifies content contained in ' 'a content area.'), }, 'CreatedDate': CREATED_DATE_FIELD, 'CustomerKey': CUSTOMER_KEY_FIELD, 'FontFamily': { 'type': ['null', 'string'], 'description': 'Indicates font family used in content area', }, 'HasFontSize': { 'type': 'boolean', 'description': ('Indicates whether the content area includes ' 'a specified font size or not'), }, 'ID': ID_FIELD, 'IsBlank': { 'type': 'boolean', 'description': ('Indicates if specified content area ' 'contains no content.'), }, 'IsDynamicContent': { 'type': 'boolean', 'description': ('Indicates if specific content area ' 'contains dynamic content.'), }, 'IsLocked': { 'type': 'boolean', 'description': ('Indicates if specific email content area ' 'within an Enterprise or Enterprise 2.0 ' 'account is locked and cannot be changed by ' 'subaccounts.'), }, 'IsSurvey': { 'type': 'boolean', 'description': ('Indicates whether a specific content area ' 'contains survey questions.'), }, 'Key': { 'type': ['null', 'string'], 'description': ('Specifies key associated with content area ' 'in HTML body. Relates to the Email object ' 'via a custom type.'), }, 'ModifiedDate': MODIFIED_DATE_FIELD, 'Name': { 'type': ['null', 'string'], 'description': 'Name of the object or property.', }, 'ObjectID': OBJECT_ID_FIELD, 'PartnerProperties': CUSTOM_PROPERTY_LIST, 'Width': { 'type': ['null', 'integer'], 'description': 'Indicates pixel width of content area', }, }) TABLE = 'content_area' KEY_PROPERTIES = ['ID'] def sync_data(self): table = self.__class__.TABLE selector = FuelSDK.ET_ContentArea search_filter = None retrieve_all_since = get_last_record_value_for_table(self.state, table) if retrieve_all_since is not None: search_filter = { 'Property': 'ModifiedDate', 'SimpleOperator': 'greaterThan', 'Value': retrieve_all_since } stream = request('ContentAreaDataAccessObject', selector, self.auth_stub, search_filter) for content_area in stream: content_area = self.filter_keys_and_parse(content_area) self.state = incorporate(self.state, table, 'ModifiedDate', content_area.get('ModifiedDate')) singer.write_records(table, [content_area]) save_state(self.state)
class ListSubscriberDataAccessObject(DataAccessObject): SCHEMA = with_properties({ 'ID': ID_FIELD, 'CreatedDate': CREATED_DATE_FIELD, 'ModifiedDate': MODIFIED_DATE_FIELD, 'ObjectID': OBJECT_ID_FIELD, 'PartnerProperties': CUSTOM_PROPERTY_LIST, 'ListID': { 'type': ['null', 'integer'], 'description': ('Defines identification for a list the ' 'subscriber resides on.'), }, 'Status': { 'type': ['null', 'string'], 'description': ('Defines status of object. Status of ' 'an address.'), }, 'SubscriberKey': SUBSCRIBER_KEY_FIELD, }) TABLE = 'list_subscriber' KEY_PROPERTIES = ['SubscriberKey', 'ListID'] def __init__(self, config, state, auth_stub, catalog): super(ListSubscriberDataAccessObject, self).__init__( config, state, auth_stub, catalog) self.replicate_subscriber = False self.subscriber_catalog = None def _get_all_subscribers_list(self): """ Find the 'All Subscribers' list via the SOAP API, and return it. """ result = request('List', FuelSDK.ET_List, self.auth_stub, { 'Property': 'ListName', 'SimpleOperator': 'equals', 'Value': 'All Subscribers', }) lists = list(result) if len(lists) != 1: msg = ('Found {} all subscriber lists, expected one!' .format(len(lists))) raise RuntimeError(msg) return sudsobj_to_dict(lists[0]) def sync_data(self): table = self.__class__.TABLE subscriber_dao = SubscriberDataAccessObject( self.config, self.state, self.auth_stub, self.subscriber_catalog) start = get_last_record_value_for_table(self.state, table) if start is None: start = self.config.get('start_date') pagination_unit = self.config.get( 'pagination__list_subscriber_interval_unit', 'days') pagination_quantity = self.config.get( 'pagination__list_subsctiber_interval_quantity', 1) unit = {pagination_unit: int(pagination_quantity)} end = increment_date(start, unit) all_subscribers_list = self._get_all_subscribers_list() while before_now(start): stream = request('ListSubscriber', FuelSDK.ET_List_Subscriber, self.auth_stub, _get_list_subscriber_filter( all_subscribers_list, start, unit)) batch_size = 100 if self.replicate_subscriber: subscriber_dao.write_schema() for list_subscribers_batch in partition_all(stream, batch_size): for list_subscriber in list_subscribers_batch: list_subscriber = self.filter_keys_and_parse( list_subscriber) if list_subscriber.get('ModifiedDate'): self.state = incorporate( self.state, table, 'ModifiedDate', list_subscriber.get('ModifiedDate')) singer.write_records(table, [list_subscriber]) if self.replicate_subscriber: subscriber_keys = list(map( _get_subscriber_key, list_subscribers_batch)) subscriber_dao.pull_subscribers_batch(subscriber_keys) save_state(self.state) start = end end = increment_date(start, unit)
class LinkSendDataAccessObject(DataAccessObject): REPLICATION_METHOD = "FULL_TABLE" SCHEMA = with_properties({ 'ID': ID_FIELD, 'SendID': { 'type': ['null', 'integer'], 'description': 'Contains identifier for a specific send.', }, 'ClientID': { 'type': ['null', 'integer'], 'description': 'Contains identifier for a specific client.', }, 'PartnerKey': { 'type': ['null', 'integer'], 'description': '', }, 'TotalClicks': { 'type': ['null', 'integer'], 'description': '', }, 'UniqueClicks': { 'type': ['null', 'integer'], 'description': '', }, 'URL': { 'type': ['null', 'string'], 'description': '', }, 'Alias': { 'type': ['null', 'string'], 'description': '', }, 'LinkID': { 'type': ['null', 'integer'], 'description': 'Contains identifier for a specific link.', } }) TABLE = 'link_send' KEY_PROPERTIES = ['SendID'] def parse_object(self, obj): to_return = obj.copy() to_return['ClientID'] = to_return.get('Client', {}).get('ID') to_return['LinkID'] = to_return.get('Link', {}).get('ID') to_return['Alias'] = to_return.get('Link', {}).get('Alias') to_return['URL'] = to_return.get('Link', {}).get('URL') to_return['TotalClicks'] = to_return.get('Link', {}).get('TotalClicks') to_return['UniqueClicks'] = to_return.get('Link', {}).get('UniqueClicks') return super(LinkSendDataAccessObject, self).parse_object(to_return) def sync_data(self): pass def sync_data_by_sendID(self, sendId): if not sendId: return table = self.__class__.TABLE _filter = {} if sendId: _filter = { 'Property': 'SendID', 'SimpleOperator': 'equals', 'Value': sendId } else: LOGGER.info('No send id here, moving on') return stream = request(self.__class__.TABLE, ET_LinkSend, self.auth_stub, _filter) for link_send in stream: link_send = self.filter_keys_and_parse(link_send) singer.write_records(table, [link_send])
class EventDataAccessObject(DataAccessObject): SCHEMA = with_properties({ 'SendID': { 'type': ['null', 'integer'], 'description': 'Contains identifier for a specific send.', }, 'EventDate': { 'type': ['null', 'string'], 'format': 'datetime', 'description': 'Date when a tracking event occurred.', }, 'EventType': { 'type': ['null', 'string'], 'description': 'The type of tracking event', }, 'BatchID': { 'type': ['null', 'integer'], 'description': 'Ties triggered send sent events to other events (like clicks and opens that occur at a later date and time)', }, 'CorrelationID': { 'type': ['null', 'string'], 'description': 'Identifies correlation of objects across several requests.', }, 'URL': { 'type': ['null', 'string'], 'description': 'URL that was clicked.', }, 'SubscriberKey': SUBSCRIBER_KEY_FIELD, }) TABLE = 'event' KEY_PROPERTIES = ['SendID', 'EventType', 'SubscriberKey', 'EventDate'] def sync_data(self): table = self.__class__.TABLE search_filter = None start = get_last_record_value_for_table(self.state, self.event_name, self.config.get('start_date')) if start is None: start = self.config.get('start_date') if start is None: raise RuntimeError('start_date not defined!') pagination_unit = self.config.get( 'pagination__{}_interval_unit'.format(self.event_name), 'minutes') pagination_quantity = self.config.get( 'pagination__{}_interval_quantity'.format(self.event_name), 10) unit = {pagination_unit: int(pagination_quantity)} end = increment_date(start, unit) while before_now(start): LOGGER.info("Fetching {} from {} to {}".format( self.event_name, start, end)) search_filter = get_date_page('EventDate', start, unit) stream = request(self.event_name, self.selector, self.auth_stub, search_filter) for event in stream: event = self.filter_keys_and_parse(event) self.state = incorporate(self.state, self.event_name, 'EventDate', event.get('EventDate')) if event.get('SubscriberKey') is None: LOGGER.info( "SubscriberKey is NULL so ignoring {} record with SendID: {} and EventDate: {}" .format(self.event_name, event.get('SendID'), event.get('EventDate'))) continue event = self.remove_sensitive_data(event) singer.write_records(table, [event]) self.state = incorporate(self.state, self.event_name, 'EventDate', start) save_state(self.state) start = end end = increment_date(start, unit)