def test_array_of_strings_overridden(self):
parser = JobConfigSchemaAsArgsParser.from_description(self.job_config_schema, 'testjob')
out = parser.parse(['--divisions', 'a'])
expected = {
'divisions': ['a'], 'boolwithdefaultfalse': False, 'boolwithdefaulttrue': True
}
self.assertEqual(expected, out)
def test_array_of_strings_default(self) -> None:
parser = JobConfigSchemaAsArgsParser.from_description(self.job_config_schema, 'testjob')
out = parser.parse([])
expected = {
'divisions': ['candidates', 'counties', 'townships', 'precincts'],
'boolwithdefaultfalse': False, 'boolwithdefaulttrue': True
}
self.assertEqual(expected, out)
def test_enum_cannot_be_invalid(self) -> None:
parser = JobConfigSchemaAsArgsParser.from_description(self.job_config_schema, 'testjob')
with self.assertRaises(SystemExit) as r:
parser.parse(['--enumwithnulloption', 'x'])
self.assertEqual(str(r.exception.__context__),
"argument --enumwithnulloption: invalid choice: 'x' "
"(choose from 'a', 'b')")
def test_enum_can_be_set(self) -> None:
parser = JobConfigSchemaAsArgsParser.from_description(self.job_config_schema, 'testjob')
out = parser.parse(['--enumwithnulloption', 'a'])
expected = {
'divisions': ['candidates', 'counties', 'townships', 'precincts'],
'boolwithdefaultfalse': False, 'boolwithdefaulttrue': True,
'enumwithnulloption': 'a'
}
self.assertEqual(expected, out)
def test_boolean_no_default_false(self) -> None:
parser = JobConfigSchemaAsArgsParser.from_description(self.job_config_schema, 'testjob')
out = parser.parse(['--no_boolwithnodefault'])
expected = {
'divisions': ['candidates', 'counties', 'townships', 'precincts'],
'boolwithdefaultfalse': False, 'boolwithdefaulttrue': True,
'boolwithnodefault': False,
}
self.assertEqual(expected, out)
def test_dict_of_dict(self):
parser = JobConfigSchemaAsArgsParser.from_description(self.job_config_schema, 'testjob')
out = parser.parse(['--nested_dict.object_key', 'a'])
expected = {
'nested_dict': {
'object_key': 'a'
},
'divisions': ['candidates', 'counties', 'townships', 'precincts'],
'boolwithdefaultfalse': False, 'boolwithdefaulttrue': True
}
self.assertEqual(expected, out)
def test_bad_syntax_4(self):
bad_job_config_schema = {
"type": "object",
"properties": 123
}
parser = JobConfigSchemaAsArgsParser.from_description(bad_job_config_schema, 'testjob')
with self.assertRaises(TypeError) as r:
parser.parse([])
self.assertEqual(str(r.exception),
"Did not understand 123 in "
"{'type': 'object', 'properties': 123}")
def test_bad_syntax_1(self):
bad_job_config_schema = {
"type": "object",
"properties": odict[
'divisions': 123 # should be another json schema, not a number...
],
}
parser = JobConfigSchemaAsArgsParser.from_description(bad_job_config_schema, 'testjob')
with self.assertRaises(TypeError) as r:
parser.parse([])
self.assertEqual(str(r.exception),
'Did not understand [123] in [odict[divisions: 123]]')
def test_bad_syntax_2(self):
bad_job_config_schema = {
"type": "object",
"properties": odict[
'divisions': {
'type': 'array',
'items': 123,
},
],
}
parser = JobConfigSchemaAsArgsParser.from_description(bad_job_config_schema, 'testjob')
with self.assertRaises(TypeError) as r:
parser.parse([])
self.assertEqual(str(r.exception),
"Did not understand [123] in "
"[odict[divisions: {'type': 'array', 'items': 123}]]")
def populate_subparser(bootstrap_session: 'Session',
sub_parser: argparse.ArgumentParser,
source_method_name: str,
target_method_name: str,
subjob_name: str) -> JobConfig:
source_method = getattr(bootstrap_session.records.sources, source_method_name)
target_method = getattr(bootstrap_session.records.targets, target_method_name)
job_config_schema = {
"type": "object",
"properties": odict[
'source': method_to_json_schema(source_method), # type: ignore
'target': method_to_json_schema(target_method), # type: ignore
],
"required": ["source", "target"],
}
JobConfigSchemaAsArgsParser(config_json_schema=job_config_schema,
argument_parser=sub_parser).configure_arg_parser()
return job_config_schema
def main() -> None:
# https://github.com/googleapis/google-auth-library-python/issues/271
import warnings
warnings.filterwarnings("ignore",
"Your application has authenticated using end user credentials")
# skip in-memory sources/targets like dataframes that don't make
# sense from the command-line
source_method_name_by_cli_name = {
'table': 'table',
'gsheet': 'google_sheet',
'recordsdir': 'directory_from_url',
'url': 'data_url',
'file': 'local_file'
}
target_method_name_by_cli_name = {
'gsheet': 'google_sheet',
'table': 'table',
'recordsdir': 'directory_from_url',
'url': 'data_url',
'file': 'local_file',
'spectrum': 'spectrum',
}
sources = source_method_name_by_cli_name.keys()
targets = target_method_name_by_cli_name.keys()
description = 'Move tabular data ("records") from one place to another'
parser = argparse.ArgumentParser(description=description,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
pi_config_schema =\
method_signature_to_json_schema(ProcessingInstructions.__init__,
special_handling={},
parameters_to_ignore=['self'])
JobConfigSchemaAsArgsParser(config_json_schema=pi_config_schema,
argument_parser=parser).configure_arg_parser()
# https://stackoverflow.com/questions/15405636/pythons-argparse-to-show-programs-version-with-prog-and-version-string-formatt
parser.add_argument('-V', '--version', action='version', version="%(prog)s ("+__version__+")")
subparsers = parser.add_subparsers(help='subcommand_help')
from records_mover import Session
bootstrap_session = Session()
for source in sources:
for target in targets:
name = f"{source}2{target}"
sub_parser = subparsers.add_parser(name, help=f"Copy from {source} to {target}")
source_method_name = source_method_name_by_cli_name[source]
target_method_name = target_method_name_by_cli_name[target]
job_config_schema = \
populate_subparser(bootstrap_session,
sub_parser, source_method_name, target_method_name,
subjob_name=name)
sub_parser.set_defaults(func=make_job_fn(source_method_name=source_method_name,
target_method_name=target_method_name,
name=name,
job_config_schema=job_config_schema))
args = parser.parse_args()
raw_config = vars(args)
func = getattr(args, 'func', None)
if func is None:
parser.print_help()
else:
set_stream_logging()
try:
func(raw_config)
except Exception:
# This is logged above using a redacting logger
sys.exit(1)