def setUp(self):
self.catalog = Catalog.load('test_catalog.json')
with open('test_state.json') as f:
self.state = json.load(f)
self.start_date = "2019-11-12T06:47:14.000000Z"
def parse_args(required_config_keys):
parser = argparse.ArgumentParser()
parser.add_argument(
'-c', '--config',
help='Config file',
required=True)
parser.add_argument(
'-s', '--state',
help='state file')
parser.add_argument(
'-p', '--properties',
help='Property selections: DEPRECATED, Please use --catalog instead')
parser.add_argument(
'--catalog',
help='Catalog file')
parser.add_argument(
'-d', '--discover',
action='store_true',
help='Do schema discovery')
args = parser.parse_args()
if args.config:
setattr(args, 'config_path', args.config)
args.config = utils.load_json(args.config)
if args.state:
setattr(args, 'state_path', args.state)
args.state_file = args.state
args.state = utils.load_json(args.state)
else:
args.state_file = None
args.state = {}
if args.properties:
setattr(args, 'properties_path', args.properties)
args.properties = utils.load_json(args.properties)
if args.catalog:
setattr(args, 'catalog_path', args.catalog)
args.catalog = Catalog.load(args.catalog)
utils.check_config(args.config, required_config_keys)
return args
def parse_args(required_config_keys):
'''Parse standard command-line args.
-c,--config Config file
-s,--state State file
-d,--discover Run in discover mode
-a,--select_all Select all streams and fields for discover mode
--catalog Catalog file
Returns the parsed args object from argparse. For each argument that
point to JSON files (config, state, properties), we will automatically
load and parse the JSON file.
'''
parser = argparse.ArgumentParser()
parser.add_argument('--config', '-c', help='Config file', required=True)
parser.add_argument('--state', '-s', help='State file')
parser.add_argument('--catalog', help='Catalog file')
parser.add_argument('--discover',
'-d',
action='store_true',
help='Do schema discovery')
parser.add_argument('--select_all',
'-a',
action='store_true',
help='Select all streams and fields in discover mode')
args = parser.parse_args()
if args.config:
setattr(args, 'config_path', args.config)
args.config = load_json(args.config)
if args.state:
setattr(args, 'state_path', args.state)
args.state = load_json(args.state)
else:
args.state = {}
if args.catalog:
setattr(args, 'catalog_path', args.catalog)
args.catalog = Catalog.load(args.catalog)
if args.select_all and not args.discover:
parser.error('Select all only available for discovery mode')
check_config(args.config, required_config_keys)
return args