def main():
# define required config file keys
required_config_keys = ['client_id', 'client_secret', 'start_date']
# check if required keys are in the config file
args = singer.parse_args(required_config_keys)
# get the input
config = args.config
catalog = args.catalog or Catalog([])
state = args.state
# instatiate the client
client = BillwerkClient(config)
if args.properties and not args.catalog:
raise Exception(
"DEPRECATED: Use of the 'properties' parameter is not supported. Please use --catalog instead"
)
if args.discover:
LOGGER.info("Starting discovery mode")
catalog = do_discover()
write_catalog(catalog)
else:
LOGGER.info('Starting sync mode')
do_sync(client, config, state, catalog)
def main():
args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
catalog = args.catalog if args.catalog else discover()
if args.discover:
write_catalog(catalog)
else:
sync(args.config, args.state, catalog)
def main():
args = get_args()
catalog = args.catalog if args.catalog else discover()
if args.discover:
write_catalog(catalog)
else:
sync(args.config, args.state, catalog)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--catalog',
help='Preliminary catalog file',
required=True)
parser.add_argument('-c',
'--config',
help='Stream & field selector file',
required=True)
args = parser.parse_args()
catalog_filename = args.catalog
selector_filename = args.config
LOGGER.info('Beginning selection of streams and fields')
catalog = ExtendedCatalog.load(catalog_filename)
selector = SchemaSelector.load(selector_filename)
for stream_idx, catalog_stream in enumerate(catalog.streams):
if not selector.is_selected_stream(catalog_stream.stream):
continue
# Obtain primary key(s)
key_properties = selector.get_key_properties_for_stream(
catalog_stream.stream)
# Turn on replication for stream and set key properties
catalog.set_stream_as_selected(catalog_stream.stream,
key_properties)
for meta_idx, metadata in enumerate(catalog_stream.metadata):
if not is_field_metadata(metadata):
continue
# Check if field is selected
breadcrumb = metadata.get('breadcrumb', [])
field = get_field_from_breadcrumb(breadcrumb)
is_selected_field = selector.is_selected_field_for_stream(
catalog_stream.stream, field)
if not is_selected_field:
continue
catalog.set_field_as_selected(catalog_stream.stream, field)
LOGGER.debug((f'Selecting {field} in {catalog_stream.stream}'))
# Write to stdout
write_catalog(catalog)
LOGGER.info('Completed selection of streams and fields')
def main():
parsed_args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
catalog = parsed_args.catalog if parsed_args.catalog else discover()
if parsed_args.discover:
write_catalog(catalog)
else:
with OutreachClient(parsed_args.config) as client:
check_auth(client)
sync(client, parsed_args.config, catalog, parsed_args.state,
parsed_args.config['start_date'])
def main():
args = singer.utils.parse_args([])
is_sandbox = args.config.get('sandbox')
if isinstance(is_sandbox, str):
is_sandbox = args.config.get('sandbox') == 'true'
catalog = args.catalog if args.catalog else discover(is_sandbox)
if args.discover:
write_catalog(catalog)
else:
sync(args.config, args.state, catalog)
def main():
required_config_keys = ['start_date', 'consumer_key', 'consumer_secret', 'access_token', 'access_token_secret']
args = singer.parse_args(required_config_keys)
config = args.config # pylint:disable=unused-variable
client = TrelloClient(config) # pylint:disable=unused-variable
catalog = args.catalog or Catalog([])
state = args.state # pylint:disable=unused-variable
if args.properties and not args.catalog:
raise Exception("DEPRECATED: Use of the 'properties' parameter is not supported. Please use --catalog instead")
if args.discover:
LOGGER.info("Starting discovery mode")
catalog = do_discover()
write_catalog(catalog)
else:
do_sync(client, config, state, catalog)
def main():
required_config_keys = ['start_date']
args = singer.parse_args(required_config_keys)
config = args.config
client = AdrollClient(args.config_path, config)
catalog = args.catalog or Catalog([])
state = args.state
if args.properties and not args.catalog:
raise Exception("DEPRECATED: Use of the 'properties' parameter is not supported. Please use --catalog instead")
if args.discover:
LOGGER.info("Starting discovery mode")
catalog = do_discover()
write_catalog(catalog)
else:
LOGGER.info("Starting sync mode")
do_sync(client, config, state, catalog)
def main():
required_config_keys = ["system_id"]
args = utils.parse_args(required_config_keys)
config = args.config
client = _load_gbfs_client(config.get("system_id"))
catalog = args.catalog or Catalog([])
state = args.state
if args.discover:
LOGGER.info("Starting discovery mode")
catalog = do_discover(client)
write_catalog(catalog)
return
LOGGER.info("Starting sync mode")
do_sync(client, config, state, catalog)
if hasattr(args, "state_path"):
_write_state(args.state_path, state)
def main():
# Parse command line arguments
args = utils.parse_args(REQUIRED_CONFIG_KEYS)
client = Client(args.config)
# If discover flag was passed, run discovery mode and dump output to stdout
if args.discover:
LOGGER.info("Starting discovery mode")
catalog = discover(client)
write_catalog(catalog)
# Otherwise run in sync mode
else:
LOGGER.info("Starting sync mode")
if args.catalog:
catalog = args.catalog
else:
catalog = discover(client)
sync(client, args.config, args.state, catalog)
def main():
required_config_keys = ['start_date']
args = singer.parse_args(required_config_keys)
config = args.config
freshdesk_client = FreshdeskClient(args.config_path, config)
catalog = args.catalog or Catalog([])
state = args.state
if args.properties and not args.catalog:
raise Exception("DEPRECATED: Use of the 'properties' parameter is not supported. Please use --catalog instead")
if args.discover:
LOGGER.info("Starting discovery mode")
catalog = discover(freshdesk_client)
write_catalog(catalog)
else:
LOGGER.info("Starting sync mode")
config, state = parse_args(REQUIRED_CONFIG_KEYS)
CONFIG.update(config)
STATE.update(state)
sync(freshdesk_client, config, state, catalog)
def test_write_empty_catalog(self):
catalog = Catalog([])
write_catalog(catalog)
def test_write_catalog_with_streams(self):
catalog = Catalog([CatalogEntry(tap_stream_id='a',schema=Schema(),metadata=[])])
write_catalog(catalog)
def do_discover(client, config):
"""
Make request to discover.py and write result to stdout.
"""
catalog = discover(client, config, get_view_ids(config))
write_catalog(catalog)
def do_discover(client, profile_id):
"""
Make request to discover.py and write result to stdout.
"""
catalog = discover(client, profile_id)
write_catalog(catalog)
