def sync(config, state, catalog):
errors_encountered = False
selected_stream_ids = get_selected_streams(catalog)
client = GAClient(config)
# Loop over streams in catalog
for stream in catalog['streams']:
stream_id = stream['tap_stream_id']
stream_schema = stream['schema']
stream_metadata = metadata.to_map(stream['metadata'])
key_properties = metadata.get(stream_metadata, (),
"table-key-properties")
if stream_id in selected_stream_ids:
LOGGER.info('Syncing stream: ' + stream_id)
try:
report_definition = ReportsHelper.get_report_definition(stream)
results = client.process_stream(report_definition)
# we write the schema message after we are sure that we could
# fetch records without errors
singer.write_schema(stream_id, stream_schema, key_properties)
singer.write_records(stream_id, results)
except TapGaInvalidArgumentError as e:
errors_encountered = True
LOGGER.error(
"Skipping stream: '{}' due to invalid report definition.".
format(stream_id))
LOGGER.debug("Error: '{}'.".format(e))
except TapGaRateLimitError as e:
errors_encountered = True
LOGGER.error(
"Skipping stream: '{}' due to Rate Limit Errors.".format(
stream_id))
LOGGER.debug("Error: '{}'.".format(e))
except TapGaQuotaExceededError as e:
errors_encountered = True
LOGGER.error(
"Skipping stream: '{}' due to Quota Exceeded Errors.".
format(stream_id))
LOGGER.debug("Error: '{}'.".format(e))
except TapGaAuthenticationError as e:
LOGGER.error(
"Stopping execution while processing '{}' due to Authentication Errors."
.format(stream_id))
LOGGER.debug("Error: '{}'.".format(e))
sys.exit(1)
except TapGaUnknownError as e:
LOGGER.error(
"Stopping execution while processing '{}' due to Unknown Errors."
.format(stream_id))
LOGGER.debug("Error: '{}'.".format(e))
sys.exit(1)
else:
LOGGER.info('Skipping unselected stream: ' + stream_id)
# If we encountered errors, exit with 1
if errors_encountered:
sys.exit(1)
return
def sync(config, state, catalog):
errors_encountered = False
selected_stream_ids = get_selected_streams(catalog)
client = GAClient(config)
# Loop over streams in catalog
for stream in catalog['streams']:
stream_id = stream['tap_stream_id']
stream_schema = stream['schema']
if state and stream_id in state:
client.start_date = state[stream_id]
stream_metadata = metadata.to_map(stream['metadata'])
key_properties = update_key_properties(stream_schema, stream_metadata)
if stream_id in selected_stream_ids:
LOGGER.info('Syncing stream: ' + stream_id)
try:
singer.write_schema(stream_id, stream_schema, key_properties)
report_definition = ReportsHelper.get_report_definition(stream)
for page, date in client.process_stream(report_definition):
singer.write_records(stream_id, page)
if date is not None: # we need to update all dates that are not "golden", even if it's the start date
singer.write_state({stream_id: date})
except TapGaInvalidArgumentError as e:
errors_encountered = True
LOGGER.error(
"Skipping stream: '{}' due to invalid report definition.".
format(stream_id))
LOGGER.debug("Error: '{}'.".format(e))
except TapGaRateLimitError as e:
errors_encountered = True
LOGGER.error(
"Skipping stream: '{}' due to Rate Limit Errors.".format(
stream_id))
LOGGER.debug("Error: '{}'.".format(e))
except TapGaQuotaExceededError as e:
errors_encountered = True
LOGGER.error(
"Skipping stream: '{}' due to Quota Exceeded Errors.".
format(stream_id))
LOGGER.debug("Error: '{}'.".format(e))
except TapGaAuthenticationError as e:
LOGGER.error(
"Stopping execution while processing '{}' due to Authentication Errors."
.format(stream_id))
LOGGER.debug("Error: '{}'.".format(e))
sys.exit(1)
except TapGaUnknownError as e:
LOGGER.error(
"Stopping execution while processing '{}' due to Unknown Errors."
.format(stream_id))
LOGGER.debug("Error: '{}'.".format(e))
sys.exit(1)
else:
LOGGER.info('Skipping unselected stream: ' + stream_id)
# If we encountered errors, exit with 1
if errors_encountered:
sys.exit(1)
return