def sync_data(self):
table = self.__class__.TABLE
endpoints = {
'sent': FuelSDK.ET_SentEvent,
'click': FuelSDK.ET_ClickEvent,
'open': FuelSDK.ET_OpenEvent,
'bounce': FuelSDK.ET_BounceEvent,
'unsub': FuelSDK.ET_UnsubEvent
}
for event_name, selector in endpoints.items():
search_filter = None
start = get_last_record_value_for_table(self.state, event_name)
if start is None:
start = self.config.get('start_date')
if start is None:
raise RuntimeError('start_date not defined!')
pagination_unit = self.config.get(
'pagination__{}_interval_unit'.format(event_name), 'minutes')
pagination_quantity = self.config.get(
'pagination__{}_interval_quantity'.format(event_name), 10)
unit = {pagination_unit: int(pagination_quantity)}
end = increment_date(start, unit)
while before_now(start):
LOGGER.info("Fetching {} from {} to {}"
.format(event_name, start, end))
search_filter = get_date_page('EventDate', start, unit)
stream = request(event_name,
selector,
self.auth_stub,
search_filter)
for event in stream:
event = self.filter_keys_and_parse(event)
self.state = incorporate(self.state,
event_name,
'EventDate',
event.get('EventDate'))
singer.write_records(table, [event])
self.state = incorporate(self.state,
event_name,
'EventDate',
start)
save_state(self.state)
start = end
end = increment_date(start, unit)
def _get_list_subscriber_filter(_list, start, unit):
return {
'LogicalOperator': 'AND',
'LeftOperand': {
'Property': 'ListID',
'SimpleOperator': 'equals',
'Value': _list.get('ID'),
},
'RightOperand': get_date_page('ModifiedDate', start, unit)
}
def sync_data(self):
table = self.__class__.TABLE
search_filter = None
start = get_last_record_value_for_table(self.state, self.event_name,
self.config.get('start_date'))
if start is None:
start = self.config.get('start_date')
if start is None:
raise RuntimeError('start_date not defined!')
pagination_unit = self.config.get(
'pagination__{}_interval_unit'.format(self.event_name), 'minutes')
pagination_quantity = self.config.get(
'pagination__{}_interval_quantity'.format(self.event_name), 10)
unit = {pagination_unit: int(pagination_quantity)}
end = increment_date(start, unit)
while before_now(start):
LOGGER.info("Fetching {} from {} to {}".format(
self.event_name, start, end))
search_filter = get_date_page('EventDate', start, unit)
stream = request(self.event_name, self.selector, self.auth_stub,
search_filter)
for event in stream:
event = self.filter_keys_and_parse(event)
self.state = incorporate(self.state, self.event_name,
'EventDate', event.get('EventDate'))
if event.get('SubscriberKey') is None:
LOGGER.info(
"SubscriberKey is NULL so ignoring {} record with SendID: {} and EventDate: {}"
.format(self.event_name, event.get('SendID'),
event.get('EventDate')))
continue
event = self.remove_sensitive_data(event)
singer.write_records(table, [event])
self.state = incorporate(self.state, self.event_name, 'EventDate',
start)
save_state(self.state)
start = end
end = increment_date(start, unit)
def _replicate(self, customer_key, keys,
parent_category_id, table,
partial=False, start=None,
end=None, unit=None, replication_key=None):
if partial:
LOGGER.info("Fetching {} from {} to {}"
.format(table, start, end))
cursor = FuelSDK.ET_DataExtension_Row()
cursor.auth_stub = self.auth_stub
cursor.CustomerKey = customer_key
cursor.props = keys
if partial:
cursor.search_filter = get_date_page(replication_key,
start,
unit)
batch_size = int(self.config.get('batch_size', 2500))
result = request_from_cursor('DataExtensionObject', cursor,
batch_size=batch_size)
for row in result:
row = self.filter_keys_and_parse(row)
row['CategoryID'] = parent_category_id
self.state = incorporate(self.state,
table,
replication_key,
row.get(replication_key))
singer.write_records(table, [row])
if partial:
self.state = incorporate(self.state,
table,
replication_key,
start)
save_state(self.state)
def sync_data(self):
table = self.__class__.TABLE
endpoints = {
'sent': FuelSDK.ET_SentEvent,
'click': FuelSDK.ET_ClickEvent,
'open': FuelSDK.ET_OpenEvent,
'bounce': FuelSDK.ET_BounceEvent,
'unsub': FuelSDK.ET_UnsubEvent
}
for event_name, selector in endpoints.items():
search_filter = None
# pass config to return start date if not bookmark is found
start = get_last_record_value_for_table(self.state, event_name, self.config)
if start is None:
raise RuntimeError('start_date not defined!')
pagination_unit = self.config.get(
'pagination__{}_interval_unit'.format(event_name), 'minutes')
pagination_quantity = self.config.get(
'pagination__{}_interval_quantity'.format(event_name), 10)
unit = {pagination_unit: int(pagination_quantity)}
end = increment_date(start, unit)
while before_now(start):
LOGGER.info("Fetching {} from {} to {}"
.format(event_name, start, end))
search_filter = get_date_page('EventDate', start, unit)
stream = request(event_name,
selector,
self.auth_stub,
search_filter,
batch_size=self.batch_size)
catalog_copy = copy.deepcopy(self.catalog)
for event in stream:
event = self.filter_keys_and_parse(event)
self.state = incorporate(self.state,
event_name,
'EventDate',
event.get('EventDate'))
if event.get('SubscriberKey') is None:
LOGGER.info("SubscriberKey is NULL so ignoring {} record with SendID: {} and EventDate: {}"
.format(event_name,
event.get('SendID'),
event.get('EventDate')))
continue
self.write_records_with_transform(event, catalog_copy, table)
self.state = incorporate(self.state,
event_name,
'EventDate',
start)
save_state(self.state)
start = end
end = increment_date(start, unit)