def sync_date(dates_to_fetch,
subscription_schema_,
timezone=TIMEZONE,
state_date_format=STATE_DATE_FORMAT,
config=CONFIG):
if len(dates_to_fetch) == 0:
LOGGER.info('Subscription stream is up to date.')
else:
singer.write_schema('itunes_subscriptions', subscription_schema_,
list())
for next_date in dates_to_fetch:
next_date = pendulum.parse(next_date, tz=timezone)
LOGGER.info('Starting sync for {}'.format(
next_date.format(state_date_format)))
# store list of itc arguments
itc_args = {
'access_token': config['accessToken'],
'version': config['version'],
'mode': config['mode'],
'target_date': next_date.format(ITUNES_REQUEST_DATE_FORMAT),
'vendor': config['vendor'],
'endpoint': config['endpoint_sales'],
'account': config['account'],
'report_version': config['report_version']
}
# Get data for the given date.
data, _ = itc_reporter.get_subscription_report(**itc_args)
# Transform specific data to make sure they are encoded correctly from
# string to int or float.
to_transform = utils.get_special_schema_types(
subscription_schema_['properties'])
the_headers, the_data = utils.data_extraction(data)
utils.process_records(the_headers,
the_data,
next_date,
to_transform['int_keys'],
to_transform['float_keys'],
schema='itunes_subscriptions')
if ARGS.state:
prior_state = utils.load_tap_setting(ARGS.state)
prior_state['subscription'] = next_date.format(
state_date_format)
utils.write_state(prior_state, ARGS.state)
def find_state():
try:
stream_states = utils.load_tap_setting(ARGS.state)
state = stream_states['subscription']
LOGGER.info('Found starting state for subscription stream: {}'.format(
state.format(STATE_DATE_FORMAT)))
except TypeError:
state = CONFIG['start_date']
LOGGER.info(
'No starting state found. Using config start date: {}'.format(
state.format(STATE_DATE_FORMAT)))
return state
import argparse
import pendulum
import singer
import tap_itunes.itc_reporter as itc_reporter
import tap_itunes.utils as utils
PARSER = argparse.ArgumentParser()
PARSER.add_argument('-c', '--config', help='Config file', required=True)
PARSER.add_argument('-s', '--state', help='State file')
ARGS = PARSER.parse_args()
LOGGER = singer.get_logger()
CONFIG = utils.load_tap_setting(path=ARGS.config)
STATE_DATE_FORMAT = '%Y-%m-%d'
ITUNES_REQUEST_DATE_FORMAT = '%Y%m%d'
TIMEZONE = 'America/Los_Angeles'
SUBSCRIPTION_SCHEMA = utils.load_schema('subscription')
# Determine if any streams need to be updated.
def find_state():
try:
stream_states = utils.load_tap_setting(ARGS.state)
state = stream_states['subscription']
LOGGER.info('Found starting state for subscription stream: {}'.format(
state.format(STATE_DATE_FORMAT)))
except TypeError:
state = CONFIG['start_date']