def main():
args = parse_args()
CONFIG.update(args.config)
# Overwrite config specs with commandline args
args_dict = args.__dict__
for arg in args_dict.keys():
if arg in CONFIG.keys() and args_dict[arg] is None:
continue
CONFIG[arg] = args_dict[arg]
if not CONFIG.get("end_datetime"):
CONFIG["end_datetime"] = datetime.datetime.utcnow().isoformat()
singer_utils.check_config(CONFIG, REQUIRED_CONFIG_KEYS)
if not CONFIG.get("start_datetime") and not CONFIG.get("state"):
LOGGER.error("state or start_datetime must be specified")
return
# If discover flag was passed, run discovery mode and dump output to stdout
if args.discover:
catalog = discover(CONFIG)
print(json.dumps(catalog, indent=2))
# Otherwise run in sync mode
elif args.catalog:
catalog = args.catalog
sync(CONFIG, args.state, catalog)
else:
LOGGER.critical("Catalog file not specified")
return
def main():
# Parse command line arguments
args = parse_args(REQUIRED_CONFIG_KEYS)
# Check API credential keys
check_config(args.config["api_credentials"], REQUIRED_API_CREDENTIAL_KEYS)
# Set global configuration variables
TAP_CONFIG.api_credentials = args.config["api_credentials"]
TAP_CONFIG.api_version = args.config.get("api_version",
DEFAULT_API_VERSION)
TAP_CONFIG.staging = args.config.get("staging", False)
TAP_CONFIG.api_url = args.config.get("api_url")
TAP_CONFIG.start_date = args.config["start_date"]
# If discover flag was passed, run discovery mode and dump output to stdout
if args.discover:
catalog = discover()
catalog.dump()
# Otherwise run in sync mode
else:
if args.catalog:
catalog = args.catalog
else:
catalog = discover()
TAP_CONFIG.catalog = catalog
sync(args.config, args.state, catalog)
def parse_args(required_config_keys):
# fork function to be able to grab path of state file
'''Parse standard command-line args.
Parses the command-line arguments mentioned in the SPEC and the
BEST_PRACTICES documents:
-c,--config Config file
-s,--state State file
-d,--discover Run in discover mode
-p,--properties Properties file: DEPRECATED, please use --catalog instead
--catalog Catalog file
Returns the parsed args object from argparse. For each argument that
point to JSON files (config, state, properties), we will automatically
load and parse the JSON file.
'''
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', help='Config file', required=True)
parser.add_argument('-s', '--state', help='State file')
parser.add_argument(
'-p',
'--properties',
help='Property selections: DEPRECATED, Please use --catalog instead')
parser.add_argument('--catalog', help='Catalog file')
parser.add_argument('-d',
'--discover',
action='store_true',
help='Do schema discovery')
args = parser.parse_args()
if args.config:
setattr(args, 'config_path', args.config)
args.config = utils.load_json(args.config)
if args.state:
setattr(args, 'state_path', args.state)
args.state_file = args.state
args.state = utils.load_json(args.state)
else:
args.state_file = None
args.state = {}
if args.properties:
setattr(args, 'properties_path', args.properties)
args.properties = utils.load_json(args.properties)
if args.catalog:
setattr(args, 'catalog_path', args.catalog)
args.catalog = Catalog.load(args.catalog)
utils.check_config(args.config, required_config_keys)
return args
def parse_args(required_config_keys):
parser = argparse.ArgumentParser()
parser.add_argument(
'-c', '--config',
help='Config file',
required=True)
parser.add_argument(
'-s', '--state',
help='state file')
parser.add_argument(
'-p', '--properties',
help='Property selections: DEPRECATED, Please use --catalog instead')
parser.add_argument(
'--catalog',
help='Catalog file')
parser.add_argument(
'-d', '--discover',
action='store_true',
help='Do schema discovery')
args = parser.parse_args()
if args.config:
setattr(args, 'config_path', args.config)
args.config = utils.load_json(args.config)
if args.state:
setattr(args, 'state_path', args.state)
args.state_file = args.state
args.state = utils.load_json(args.state)
else:
args.state_file = None
args.state = {}
if args.properties:
setattr(args, 'properties_path', args.properties)
args.properties = utils.load_json(args.properties)
if args.catalog:
setattr(args, 'catalog_path', args.catalog)
args.catalog = Catalog.load(args.catalog)
utils.check_config(args.config, required_config_keys)
return args
def main():
# Parse command line arguments
args = utils.parse_args(REQUIRED_CONFIG_KEYS)
# Check API credential keys
utils.check_config(args.config['api_credentials'],
REQUIRED_API_CREDENTIAL_KEYS)
configs.api_credentials = args.config['api_credentials']
# If discover flag was passed, run discovery mode and dump output to stdout
if args.discover:
catalog = discover()
catalog.dump()
# Otherwise run in sync mode
else:
if args.catalog:
catalog = args.catalog
else:
catalog = discover()
configs.catalog = catalog
sync(args.config, args.state, catalog)
def parse_args(spec_file, required_config_keys):
''' This is to replace singer's default utils.parse_args()
https://github.com/singer-io/singer-python/blob/master/singer/utils.py
Parse standard command-line args.
Parses the command-line arguments mentioned in the SPEC and the
BEST_PRACTICES documents:
-c,--config Config file
-s,--state State file
-d,--discover Run in discover mode
--catalog Catalog file
Returns the parsed args object from argparse. For each argument that
point to JSON files (config, state, properties), we will automatically
load and parse the JSON file.
'''
# Read default spec file
default_spec = {}
default_spec_file = get_abs_path("default_spec.json")
with open(default_spec_file, "r") as f:
default_spec.update(json.load(f))
# Read spec file
with open(spec_file, "r") as f:
SPEC.update(json.load(f))
# TODO: What about the fields other than arg
for a in default_spec["args"]:
if SPEC["args"].get(a) is None:
SPEC["args"][a] = default_spec["args"][a]
parser = argparse.ArgumentParser(SPEC["application"])
parser.add_argument("spec_file", type=str, help="Specification file")
# Capture additional args
for arg in SPEC["args"].keys():
parser.add_argument("--" + arg,
type=TYPES[SPEC["args"][arg]["type"]],
default=SPEC["args"][arg].get("default"),
help=SPEC["args"][arg].get("help"),
required=SPEC["args"][arg].get("required", False))
# Default arguments
parser.add_argument('-c', '--config', help='Config file', required=True)
"""
parser.add_argument(
"--schema_dir",
type=str,
help="Path to the schema directory.",
required=True)
"""
parser.add_argument('-s', '--state', help='State file')
parser.add_argument('--catalog', help='Catalog file')
parser.add_argument('-d',
'--discover',
action='store_true',
help='Do schema discovery')
parser.add_argument('-i',
'--infer_schema',
action='store_true',
help='Do infer schema')
parser.add_argument(
"--url",
type=str,
help="REST API endpoint with {params}. Required in config.")
args = parser.parse_args()
if args.config:
args.config = utils.load_json(args.config)
if args.state:
args.state = utils.load_json(args.state)
else:
args.state = {}
if args.catalog and os.path.isfile(args.catalog):
args.catalog = Catalog.load(args.catalog)
utils.check_config(args.config, required_config_keys)
return args
def parse_args(required_config_keys):
''' This is to replace singer's default utils.parse_args()
https://github.com/singer-io/singer-python/blob/master/singer/utils.py
Parse standard command-line args.
Parses the command-line arguments mentioned in the SPEC and the
BEST_PRACTICES documents:
-c,--config Config file
-s,--state State file
-d,--discover Run in discover mode
-p,--properties Properties file: DEPRECATED, please use --catalog instead
--catalog Catalog file
Returns the parsed args object from argparse. For each argument that
point to JSON files (config, state, properties), we will automatically
load and parse the JSON file.
'''
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', help='Config file', required=True)
parser.add_argument('-s', '--state', help='State file')
parser.add_argument(
'-p',
'--properties',
help='Property selections: DEPRECATED, Please use --catalog instead')
parser.add_argument('--catalog', help='Catalog file')
parser.add_argument('-d',
'--discover',
action='store_true',
help='Do schema discovery')
# Capture additional args
parser.add_argument(
"--start_at",
type=str,
default=None,
help=
"Inclusive start date time in ISO8601-Date-String format: 2019-04-11T00:00:00Z"
)
parser.add_argument(
"--end_at",
type=str,
default=None,
help=
"Exclusive end date time in ISO8601-Date-String format: 2019-04-12T00:00:00Z"
)
parser.add_argument(
"--modified_items_only",
type=str,
default=None,
help=
"When True, use v1 API to fetch the modified items between start_at and end_at"
)
parser.add_argument(
"--schema_dir",
type=str,
default=None,
help="Path to schemas dir. Default schemas are used when not specified."
)
args = parser.parse_args()
if args.config:
args.config = utils.load_json(args.config)
if args.state:
args.state = utils.load_json(args.state)
else:
args.state = {}
if args.properties:
args.properties = utils.load_json(args.properties)
if args.catalog:
args.catalog = Catalog.load(args.catalog)
utils.check_config(args.config, required_config_keys)
return args
def parse_args(spec_file, required_config_keys):
''' This is to replace singer's default utils.parse_args()
https://github.com/singer-io/singer-python/blob/master/singer/utils.py
Parse standard command-line args.
Parses the command-line arguments mentioned in the SPEC and the
BEST_PRACTICES documents:
-c,--config Config file
-s,--state State file
-d,--discover Run in discover mode
--catalog Catalog file
Returns the parsed args object from argparse. For each argument that
point to JSON files (config, state, properties), we will automatically
load and parse the JSON file.
'''
# Read default spec file
default_spec = {}
default_spec_file = get_abs_path("default_spec.json")
with open(default_spec_file, "r") as f:
default_spec.update(json.load(f))
SPEC.update(default_spec)
custom_spec = {}
# Overwrite with the custom spec file
with open(spec_file, "r") as f:
custom_spec.update(json.load(f))
SPEC["application"] = custom_spec.get("application", SPEC["application"])
if custom_spec.get("args"):
SPEC["args"].update(custom_spec.get("args"))
parser = argparse.ArgumentParser(SPEC["application"])
parser.add_argument("spec_file", type=str, help="Specification file")
# Capture additional args
for arg in SPEC["args"].keys():
parser.add_argument("--" + arg,
type=TYPES[SPEC["args"][arg]["type"]],
default=SPEC["args"][arg].get("default"),
help=SPEC["args"][arg].get("help"),
required=SPEC["args"][arg].get("required", False))
# Default singer arguments, commands, and required args
parser.add_argument('-c', '--config', help='Config file', required=True)
parser.add_argument('-s', '--state', help='State file')
parser.add_argument('--catalog', help='Catalog file')
# commands
parser.add_argument('-d',
'--discover',
action='store_true',
help='Do schema discovery')
parser.add_argument('-i',
'--infer_schema',
action='store_true',
help='Do infer schema')
parser.add_argument("--offline",
"-o",
action="store_true",
help="Offline test mode")
args = parser.parse_args()
if args.config:
args.config = utils.load_json(args.config)
if args.state:
args.state = utils.load_json(args.state)
else:
args.state = {}
if args.catalog and os.path.isfile(args.catalog):
args.catalog = Catalog.load(args.catalog)
utils.check_config(args.config, required_config_keys)
return args