def test_no_error_1(self, mocked_logger, mocked_post_request):
json = {
"accessToken": "access_token_123",
"expiresIn": time.time() + 3600,
"legacyToken": "legacyToken_123"
}
mocked_post_request.return_value = get_response(json)
config = {
"client_id": "client_id_123",
"client_secret": "client_secret_123",
"tenant_subdomain": "",
"start_date": "2019-01-01T00:00:00Z",
"request_timeout": "900",
"batch_size": 2500
}
_client.get_auth_stub(config)
# as "tenant_subdomain" is not provided, auth_stub will be generated from v1
mocked_logger.assert_called_with("Success.")
def test_error_6(self, mocked_ET_Client):
mocked_ET_Client.side_effect = requests.exceptions.ConnectionError(
"Connection Error")
config = {
"client_id": "client_id_123",
"client_secret": "client_secret_123",
"tenant_subdomain": "tenant_subdomain_123",
"start_date": "2019-01-01T00:00:00Z",
"request_timeout": "900",
"batch_size": 2500
}
try:
_client.get_auth_stub(config)
except Exception as e:
# as "tenant_subdomain" is provided, error will be raised after call from v2
self.assertEquals(
str(e),
"Connection Error. Please check your 'client_id', 'client_secret' or 'tenant_subdomain'."
)
def test_error_4(self, mocked_post_request):
json = {"error": "Client authentication failed."}
mocked_post_request.return_value = get_response(json)
config = {
"client_id": "",
"client_secret": "",
"tenant_subdomain": "tenant_subdomain_123",
"start_date": "2019-01-01T00:00:00Z",
"request_timeout": "900",
"batch_size": 2500
}
try:
_client.get_auth_stub(config)
except Exception as e:
# as "client_secret" and "client_id" is not provided and
# "tenant_subdomain" is provided, error will be raised after call from v2
self.assertEquals(
str(e),
"clientid or clientsecret is null: clientid and clientsecret must be passed when instantiating ET_Client or must be provided in environment variables / config file. Please check your 'client_id', 'client_secret' or 'tenant_subdomain'."
)
def test_error_2(self, mocked_post_request):
json = {"error": "Client authentication failed."}
mocked_post_request.return_value = get_response(json)
config = {
"client_id": "client_id_123",
"client_secret": "client_secret_123",
"tenant_subdomain": "tenant_subdomain_123",
"start_date": "2019-01-01T00:00:00Z",
"request_timeout": "900",
"batch_size": 2500
}
try:
_client.get_auth_stub(config)
except Exception as e:
# as "tenant_subdomain" is provided, error will be raised after call from v2
self.assertEquals(
str(e),
"Unable to validate App Keys(ClientID/ClientSecret) provided: "
+ str(json) +
". Please check your 'client_id', 'client_secret' or 'tenant_subdomain'."
)
def do_discover(args):
LOGGER.info("Starting discovery.")
config = load_config(args.config)
state = load_state(args.state)
auth_stub = get_auth_stub(config)
catalog = []
for available_stream_accessor in AVAILABLE_STREAM_ACCESSORS:
stream_accessor = available_stream_accessor(config, state, auth_stub,
None)
catalog += stream_accessor.generate_catalog()
print(json.dumps({'streams': catalog}))
def do_discover(args):
LOGGER.info("Starting discovery.")
config = args.config
state = args.state
auth_stub = get_auth_stub(config)
catalog = []
for available_stream_accessor in AVAILABLE_STREAM_ACCESSORS:
# option to sync data extension
if not config.get('discover_data_extension', False) and \
available_stream_accessor is DataExtensionDataAccessObject:
continue
stream_accessor = available_stream_accessor(config, state, auth_stub,
None)
catalog += stream_accessor.generate_catalog()
print(json.dumps({'streams': catalog}, indent=4))
def do_sync(args):
LOGGER.info("Starting sync.")
config = load_config(args.config)
state = load_state(args.state)
catalog = load_catalog(args.properties)
auth_stub = get_auth_stub(config)
stream_accessors = []
subscriber_selected = False
subscriber_catalog = None
list_subscriber_selected = False
for stream_catalog in catalog.get('streams'):
stream_accessor = None
if not _is_selected(stream_catalog.get('schema', {})):
LOGGER.info("'{}' is not marked selected, skipping.".format(
stream_catalog.get('stream')))
continue
if SubscriberDataAccessObject.matches_catalog(stream_catalog):
subscriber_selected = True
subscriber_catalog = stream_catalog
LOGGER.info("'subscriber' selected, will replicate via "
"'list_subscriber'")
continue
if ListSubscriberDataAccessObject.matches_catalog(stream_catalog):
list_subscriber_selected = True
for available_stream_accessor in AVAILABLE_STREAM_ACCESSORS:
if available_stream_accessor.matches_catalog(stream_catalog):
stream_accessors.append(
available_stream_accessor(config, state, auth_stub,
stream_catalog))
break
if subscriber_selected and not list_subscriber_selected:
LOGGER.fatal('Cannot replicate `subscriber` without '
'`list_subscriber`. Please select `list_subscriber` '
'and try again.')
exit(1)
for stream_accessor in stream_accessors:
if isinstance(stream_accessor, ListSubscriberDataAccessObject) and \
subscriber_selected:
stream_accessor.replicate_subscriber = True
stream_accessor.subscriber_catalog = subscriber_catalog
try:
stream_accessor.state = state
stream_accessor.sync()
state = stream_accessor.state
except Exception:
LOGGER.error('Failed to sync endpoint, moving on!')
save_state(state)
def do_sync(args):
LOGGER.info("Starting sync.")
config = args.config
state = args.state
catalog = args.properties
success = True
auth_stub = get_auth_stub(config)
stream_accessors = []
subscriber_selected = False
subscriber_catalog = None
list_subscriber_selected = False
for stream_catalog in catalog.get('streams'):
stream_accessor = None
if not _is_selected(stream_catalog):
LOGGER.info("'{}' is not marked selected, skipping."
.format(stream_catalog.get('stream')))
continue
# for 'subscriber' stream if it is selected, add values for 'subscriber_catalog' and
# 'subscriber_selected', and it will replicated via 'list_subscribers' stream
# The 'subscribers' stream is the child stream of 'list_subscribers'
# When we sync 'list_subscribers', it makes the list of subscriber's
# 'SubscriberKey' that were returned as part of 'list_subscribers' records
# and pass that list to 'subscribers' stream and thus 'subscribers' stream
# will only sync records of subscribers that are present in the list.
# Hence, for different start dates the 'SubscriberKey' list will differ and
# thus 'subscribers' records will also be different for different start dates.
if SubscriberDataAccessObject.matches_catalog(stream_catalog):
subscriber_selected = True
subscriber_catalog = stream_catalog
LOGGER.info("'subscriber' selected, will replicate via "
"'list_subscriber'")
continue
if ListSubscriberDataAccessObject.matches_catalog(stream_catalog):
list_subscriber_selected = True
for available_stream_accessor in AVAILABLE_STREAM_ACCESSORS:
if available_stream_accessor.matches_catalog(stream_catalog):
stream_accessors.append(available_stream_accessor(
config, state, auth_stub, stream_catalog))
break
# do not replicate 'subscriber' stream without selecting 'list_subscriber' stream
if subscriber_selected and not list_subscriber_selected:
LOGGER.fatal('Cannot replicate `subscriber` without '
'`list_subscriber`. Please select `list_subscriber` '
'and try again.')
sys.exit(1)
for stream_accessor in stream_accessors:
if isinstance(stream_accessor, ListSubscriberDataAccessObject) and \
subscriber_selected:
stream_accessor.replicate_subscriber = True
stream_accessor.subscriber_catalog = subscriber_catalog
try:
stream_accessor.state = state
stream_accessor.sync()
state = stream_accessor.state
except Exception as e:
LOGGER.exception(e)
LOGGER.error('Failed to sync endpoint, moving on!')
success = False
save_state(state)
return success
