def main(): # define required config file keys required_config_keys = ['client_id', 'client_secret', 'start_date'] # check if required keys are in the config file args = singer.parse_args(required_config_keys) # get the input config = args.config catalog = args.catalog or Catalog([]) state = args.state # instatiate the client client = BillwerkClient(config) if args.properties and not args.catalog: raise Exception( "DEPRECATED: Use of the 'properties' parameter is not supported. Please use --catalog instead" ) if args.discover: LOGGER.info("Starting discovery mode") catalog = do_discover() write_catalog(catalog) else: LOGGER.info('Starting sync mode') do_sync(client, config, state, catalog)
def main(): args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) catalog = args.catalog if args.catalog else discover() if args.discover: write_catalog(catalog) else: sync(args.config, args.state, catalog)
def main(): args = get_args() catalog = args.catalog if args.catalog else discover() if args.discover: write_catalog(catalog) else: sync(args.config, args.state, catalog)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--catalog', help='Preliminary catalog file', required=True) parser.add_argument('-c', '--config', help='Stream & field selector file', required=True) args = parser.parse_args() catalog_filename = args.catalog selector_filename = args.config LOGGER.info('Beginning selection of streams and fields') catalog = ExtendedCatalog.load(catalog_filename) selector = SchemaSelector.load(selector_filename) for stream_idx, catalog_stream in enumerate(catalog.streams): if not selector.is_selected_stream(catalog_stream.stream): continue # Obtain primary key(s) key_properties = selector.get_key_properties_for_stream( catalog_stream.stream) # Turn on replication for stream and set key properties catalog.set_stream_as_selected(catalog_stream.stream, key_properties) for meta_idx, metadata in enumerate(catalog_stream.metadata): if not is_field_metadata(metadata): continue # Check if field is selected breadcrumb = metadata.get('breadcrumb', []) field = get_field_from_breadcrumb(breadcrumb) is_selected_field = selector.is_selected_field_for_stream( catalog_stream.stream, field) if not is_selected_field: continue catalog.set_field_as_selected(catalog_stream.stream, field) LOGGER.debug((f'Selecting {field} in {catalog_stream.stream}')) # Write to stdout write_catalog(catalog) LOGGER.info('Completed selection of streams and fields')
def main(): parsed_args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) catalog = parsed_args.catalog if parsed_args.catalog else discover() if parsed_args.discover: write_catalog(catalog) else: with OutreachClient(parsed_args.config) as client: check_auth(client) sync(client, parsed_args.config, catalog, parsed_args.state, parsed_args.config['start_date'])
def main(): args = singer.utils.parse_args([]) is_sandbox = args.config.get('sandbox') if isinstance(is_sandbox, str): is_sandbox = args.config.get('sandbox') == 'true' catalog = args.catalog if args.catalog else discover(is_sandbox) if args.discover: write_catalog(catalog) else: sync(args.config, args.state, catalog)
def main(): required_config_keys = ['start_date', 'consumer_key', 'consumer_secret', 'access_token', 'access_token_secret'] args = singer.parse_args(required_config_keys) config = args.config # pylint:disable=unused-variable client = TrelloClient(config) # pylint:disable=unused-variable catalog = args.catalog or Catalog([]) state = args.state # pylint:disable=unused-variable if args.properties and not args.catalog: raise Exception("DEPRECATED: Use of the 'properties' parameter is not supported. Please use --catalog instead") if args.discover: LOGGER.info("Starting discovery mode") catalog = do_discover() write_catalog(catalog) else: do_sync(client, config, state, catalog)
def main(): required_config_keys = ['start_date'] args = singer.parse_args(required_config_keys) config = args.config client = AdrollClient(args.config_path, config) catalog = args.catalog or Catalog([]) state = args.state if args.properties and not args.catalog: raise Exception("DEPRECATED: Use of the 'properties' parameter is not supported. Please use --catalog instead") if args.discover: LOGGER.info("Starting discovery mode") catalog = do_discover() write_catalog(catalog) else: LOGGER.info("Starting sync mode") do_sync(client, config, state, catalog)
def main(): required_config_keys = ["system_id"] args = utils.parse_args(required_config_keys) config = args.config client = _load_gbfs_client(config.get("system_id")) catalog = args.catalog or Catalog([]) state = args.state if args.discover: LOGGER.info("Starting discovery mode") catalog = do_discover(client) write_catalog(catalog) return LOGGER.info("Starting sync mode") do_sync(client, config, state, catalog) if hasattr(args, "state_path"): _write_state(args.state_path, state)
def main(): # Parse command line arguments args = utils.parse_args(REQUIRED_CONFIG_KEYS) client = Client(args.config) # If discover flag was passed, run discovery mode and dump output to stdout if args.discover: LOGGER.info("Starting discovery mode") catalog = discover(client) write_catalog(catalog) # Otherwise run in sync mode else: LOGGER.info("Starting sync mode") if args.catalog: catalog = args.catalog else: catalog = discover(client) sync(client, args.config, args.state, catalog)
def main(): required_config_keys = ['start_date'] args = singer.parse_args(required_config_keys) config = args.config freshdesk_client = FreshdeskClient(args.config_path, config) catalog = args.catalog or Catalog([]) state = args.state if args.properties and not args.catalog: raise Exception("DEPRECATED: Use of the 'properties' parameter is not supported. Please use --catalog instead") if args.discover: LOGGER.info("Starting discovery mode") catalog = discover(freshdesk_client) write_catalog(catalog) else: LOGGER.info("Starting sync mode") config, state = parse_args(REQUIRED_CONFIG_KEYS) CONFIG.update(config) STATE.update(state) sync(freshdesk_client, config, state, catalog)
def test_write_empty_catalog(self): catalog = Catalog([]) write_catalog(catalog)
def test_write_catalog_with_streams(self): catalog = Catalog([CatalogEntry(tap_stream_id='a',schema=Schema(),metadata=[])]) write_catalog(catalog)
def do_discover(client, config): """ Make request to discover.py and write result to stdout. """ catalog = discover(client, config, get_view_ids(config)) write_catalog(catalog)
def do_discover(client, profile_id): """ Make request to discover.py and write result to stdout. """ catalog = discover(client, profile_id) write_catalog(catalog)