def parse_cdr_metadata_args(args=None):
fields = resources.fields_for(METADATA_TABLE)
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
'--component',
required=True,
help='Job specification for adding data to metadata table',
choices=list(JOB_COMPONENTS))
parser.add_argument('--project_id',
required=True,
help='Identifies the dataset to copy metadata from')
parser.add_argument('--target_dataset',
default=True,
help='Identifies the dataset to copy metadata to')
parser.add_argument('--source_dataset',
default=None,
help='Identifies the dataset to copy metadata from')
for field in fields:
parser.add_argument(f'--{field[NAME]}',
default=None,
help=f'{field[DESCRIPTION]}')
cdr_metadata_args, unknown_args = parser.parse_known_args(args)
custom_args = clean_cdr._get_kwargs(unknown_args)
return cdr_metadata_args, custom_args
def parse_rdr_args(raw_args=None):
parser = ArgumentParser(
description='Arguments pertaining to cleaning an RDR dataset.')
parser.add_argument('--run_as',
action='store',
dest='run_as_email',
help='Service account email address to impersonate',
required=True)
parser.add_argument(
'--release_tag',
action='store',
dest='release_tag',
help='Release tag for naming and labeling the cleaned dataset with.',
required=True)
parser.add_argument('--curation_project',
action='store',
dest='curation_project_id',
help='Curation project to load the RDR data into.',
required=True)
parser.add_argument('--rdr_dataset',
action='store',
dest='rdr_dataset',
help='RDR dataset to backup and clean.',
required=True)
parser.add_argument('-l',
'--console_log',
dest='console_log',
action='store_true',
required=False,
help='Log to the console as well as to a file.')
parser.add_argument(
'--truncation_date',
action='store',
dest='truncation_date',
required=False,
help=('date to truncate the RDR data to. The cleaning rule defaults '
'to the current date if unset.'))
parser.add_argument(
'--export_date',
action='store',
dest='export_date',
required=False,
help=
'Raw rdr export date. store_pid_rid_mappings default to current date if unset'
)
common_args, unknown_args = parser.parse_known_args(raw_args)
custom_args = clean_cdr._get_kwargs(unknown_args)
return common_args, custom_args
def test_get_kwargs(self):
expected = {'k1': 'v1', 'k2': 'v2'}
actual_result = cc._get_kwargs(['--k1', 'v1', '--k2', 'v2'])
self.assertEqual(expected, actual_result)
with self.assertRaises(RuntimeError) as c:
cc._get_kwargs(['--k1', 'v1', '--f1'])
with self.assertRaises(RuntimeError) as c:
cc._get_kwargs(['--', 'v1'])
with self.assertRaises(RuntimeError) as c:
cc._get_kwargs(['--k1', '--k2'])
def parse_deid_args(args=None):
parser = argparse.ArgumentParser(
description='Parse deid command line arguments')
parser.add_argument('-c',
'--credentials_filepath',
dest='credentials_filepath',
action='store',
default='',
help='file path to credentials for GCP to access BQ',
required=False)
parser.add_argument(
'--run_as',
dest='target_principal',
action='store',
help='Email address of service account to impersonate.',
required=True)
parser.add_argument('-p',
'--project_id',
action='store',
dest='project_id',
help=('Project associated with the '
'input dataset.'),
required=True)
parser.add_argument('-t',
'--tier',
action='store',
dest='tier',
help='controlled or registered tier',
required=True,
choices=TIER_LIST)
parser.add_argument('-i',
'--idataset',
action='store',
dest='idataset',
help='Name of the input dataset',
required=True)
parser.add_argument(
'-r',
'--release_tag',
action='store',
dest='release_tag',
help='release tag for dataset in the format of YYYYq#r#',
required=True,
type=validate_release_tag_param)
parser.add_argument('-d',
'--deid_stage',
action='store',
dest='deid_stage',
help='deid stage (deid, base or clean)',
required=True,
choices=DEID_STAGE_LIST)
parser.add_argument('-l',
'--console_log',
dest='console_log',
action='store_true',
required=False,
help='Log to the console as well as to a file.')
common_args, unknown_args = parser.parse_known_args(args)
custom_args = clean_cdr._get_kwargs(unknown_args)
return common_args, custom_args