def test_no_error_1(self, mocked_logger, mocked_post_request): json = { "accessToken": "access_token_123", "expiresIn": time.time() + 3600, "legacyToken": "legacyToken_123" } mocked_post_request.return_value = get_response(json) config = { "client_id": "client_id_123", "client_secret": "client_secret_123", "tenant_subdomain": "", "start_date": "2019-01-01T00:00:00Z", "request_timeout": "900", "batch_size": 2500 } _client.get_auth_stub(config) # as "tenant_subdomain" is not provided, auth_stub will be generated from v1 mocked_logger.assert_called_with("Success.")
def test_error_6(self, mocked_ET_Client): mocked_ET_Client.side_effect = requests.exceptions.ConnectionError( "Connection Error") config = { "client_id": "client_id_123", "client_secret": "client_secret_123", "tenant_subdomain": "tenant_subdomain_123", "start_date": "2019-01-01T00:00:00Z", "request_timeout": "900", "batch_size": 2500 } try: _client.get_auth_stub(config) except Exception as e: # as "tenant_subdomain" is provided, error will be raised after call from v2 self.assertEquals( str(e), "Connection Error. Please check your 'client_id', 'client_secret' or 'tenant_subdomain'." )
def test_error_4(self, mocked_post_request): json = {"error": "Client authentication failed."} mocked_post_request.return_value = get_response(json) config = { "client_id": "", "client_secret": "", "tenant_subdomain": "tenant_subdomain_123", "start_date": "2019-01-01T00:00:00Z", "request_timeout": "900", "batch_size": 2500 } try: _client.get_auth_stub(config) except Exception as e: # as "client_secret" and "client_id" is not provided and # "tenant_subdomain" is provided, error will be raised after call from v2 self.assertEquals( str(e), "clientid or clientsecret is null: clientid and clientsecret must be passed when instantiating ET_Client or must be provided in environment variables / config file. Please check your 'client_id', 'client_secret' or 'tenant_subdomain'." )
def test_error_2(self, mocked_post_request): json = {"error": "Client authentication failed."} mocked_post_request.return_value = get_response(json) config = { "client_id": "client_id_123", "client_secret": "client_secret_123", "tenant_subdomain": "tenant_subdomain_123", "start_date": "2019-01-01T00:00:00Z", "request_timeout": "900", "batch_size": 2500 } try: _client.get_auth_stub(config) except Exception as e: # as "tenant_subdomain" is provided, error will be raised after call from v2 self.assertEquals( str(e), "Unable to validate App Keys(ClientID/ClientSecret) provided: " + str(json) + ". Please check your 'client_id', 'client_secret' or 'tenant_subdomain'." )
def do_discover(args): LOGGER.info("Starting discovery.") config = load_config(args.config) state = load_state(args.state) auth_stub = get_auth_stub(config) catalog = [] for available_stream_accessor in AVAILABLE_STREAM_ACCESSORS: stream_accessor = available_stream_accessor(config, state, auth_stub, None) catalog += stream_accessor.generate_catalog() print(json.dumps({'streams': catalog}))
def do_discover(args): LOGGER.info("Starting discovery.") config = args.config state = args.state auth_stub = get_auth_stub(config) catalog = [] for available_stream_accessor in AVAILABLE_STREAM_ACCESSORS: # option to sync data extension if not config.get('discover_data_extension', False) and \ available_stream_accessor is DataExtensionDataAccessObject: continue stream_accessor = available_stream_accessor(config, state, auth_stub, None) catalog += stream_accessor.generate_catalog() print(json.dumps({'streams': catalog}, indent=4))
def do_sync(args): LOGGER.info("Starting sync.") config = load_config(args.config) state = load_state(args.state) catalog = load_catalog(args.properties) auth_stub = get_auth_stub(config) stream_accessors = [] subscriber_selected = False subscriber_catalog = None list_subscriber_selected = False for stream_catalog in catalog.get('streams'): stream_accessor = None if not _is_selected(stream_catalog.get('schema', {})): LOGGER.info("'{}' is not marked selected, skipping.".format( stream_catalog.get('stream'))) continue if SubscriberDataAccessObject.matches_catalog(stream_catalog): subscriber_selected = True subscriber_catalog = stream_catalog LOGGER.info("'subscriber' selected, will replicate via " "'list_subscriber'") continue if ListSubscriberDataAccessObject.matches_catalog(stream_catalog): list_subscriber_selected = True for available_stream_accessor in AVAILABLE_STREAM_ACCESSORS: if available_stream_accessor.matches_catalog(stream_catalog): stream_accessors.append( available_stream_accessor(config, state, auth_stub, stream_catalog)) break if subscriber_selected and not list_subscriber_selected: LOGGER.fatal('Cannot replicate `subscriber` without ' '`list_subscriber`. Please select `list_subscriber` ' 'and try again.') exit(1) for stream_accessor in stream_accessors: if isinstance(stream_accessor, ListSubscriberDataAccessObject) and \ subscriber_selected: stream_accessor.replicate_subscriber = True stream_accessor.subscriber_catalog = subscriber_catalog try: stream_accessor.state = state stream_accessor.sync() state = stream_accessor.state except Exception: LOGGER.error('Failed to sync endpoint, moving on!') save_state(state)
def do_sync(args): LOGGER.info("Starting sync.") config = args.config state = args.state catalog = args.properties success = True auth_stub = get_auth_stub(config) stream_accessors = [] subscriber_selected = False subscriber_catalog = None list_subscriber_selected = False for stream_catalog in catalog.get('streams'): stream_accessor = None if not _is_selected(stream_catalog): LOGGER.info("'{}' is not marked selected, skipping." .format(stream_catalog.get('stream'))) continue # for 'subscriber' stream if it is selected, add values for 'subscriber_catalog' and # 'subscriber_selected', and it will replicated via 'list_subscribers' stream # The 'subscribers' stream is the child stream of 'list_subscribers' # When we sync 'list_subscribers', it makes the list of subscriber's # 'SubscriberKey' that were returned as part of 'list_subscribers' records # and pass that list to 'subscribers' stream and thus 'subscribers' stream # will only sync records of subscribers that are present in the list. # Hence, for different start dates the 'SubscriberKey' list will differ and # thus 'subscribers' records will also be different for different start dates. if SubscriberDataAccessObject.matches_catalog(stream_catalog): subscriber_selected = True subscriber_catalog = stream_catalog LOGGER.info("'subscriber' selected, will replicate via " "'list_subscriber'") continue if ListSubscriberDataAccessObject.matches_catalog(stream_catalog): list_subscriber_selected = True for available_stream_accessor in AVAILABLE_STREAM_ACCESSORS: if available_stream_accessor.matches_catalog(stream_catalog): stream_accessors.append(available_stream_accessor( config, state, auth_stub, stream_catalog)) break # do not replicate 'subscriber' stream without selecting 'list_subscriber' stream if subscriber_selected and not list_subscriber_selected: LOGGER.fatal('Cannot replicate `subscriber` without ' '`list_subscriber`. Please select `list_subscriber` ' 'and try again.') sys.exit(1) for stream_accessor in stream_accessors: if isinstance(stream_accessor, ListSubscriberDataAccessObject) and \ subscriber_selected: stream_accessor.replicate_subscriber = True stream_accessor.subscriber_catalog = subscriber_catalog try: stream_accessor.state = state stream_accessor.sync() state = stream_accessor.state except Exception as e: LOGGER.exception(e) LOGGER.error('Failed to sync endpoint, moving on!') success = False save_state(state) return success