def parse_command_line(sys_argv): """ parse_command_line parses the arguments from the command line other than the name of the file Args: sys_argv -- sys.argv Returns: a namespace of arguments """ parser = get_base_parser() parser = add_load_args(parser) parser.add_argument("--date", help="either 'yesterday' or YYYY-MM-DD \ if there is no date, s3_to_redshift checks back 5 days") # skip the file name, parse everything after return parser.parse_args(sys_argv[1:])
def parse_command_line(sys_argv): """ parse_command_line parses the arguments from the command line other than the name of the file Args: sys_argv -- sys.argv Returns: a namespace of arguments """ parser = get_base_parser() parser.add_argument("--date", help="'yesterday' or 'twodaysago' or YYYY-MM-DD", required=True) # skip the file name, parse everything after return parser.parse_args(sys_argv[1:])
def parse_command_line(sys_argv): """ parse_command_line parses the arguments from the command line other than the name of the file Args: sys_argv -- sys.argv Returns: a namespace of arguments """ parser = get_base_parser() parser = add_load_args(parser) parser.add_argument( "--date", help="either 'yesterday' or YYYY-MM-DD \ if there is no date, s3_to_redshift checks back 5 days" ) # skip the file name, parse everything after return parser.parse_args(sys_argv[1:])
def parse_command_line(sys_argv): """ parse_command_line parses the arguments from the command line other than the name of the file Args: sys_argv -- sys.argv Returns: a namespace of arguments """ parser = get_base_parser() parser.add_argument( "--date", help="'yesterday' or 'twodaysago' or YYYY-MM-DD", required=True ) # skip the file name, parse everything after return parser.parse_args(sys_argv[1:])
def parse_command_line(sys_argv): """ parse_command_line takes in sys.argv and parses the arguments other than the first (since it's the program name). It starts with a parser from get_default_parser having the following arguments: -r, --run-local --io_yaml --conig --config-override --private and adds the following to the arguments: -s, --start_date -e, --end_date -p, --private db_file Returns: an ArgParse.Namespace """ parser = get_base_parser() parser = add_load_args(parser) parser.description = """ This program takes arguments that are the union of arguments for s3_to_psv and s3_to_redshift. It processes each date using the yaml schema file specified in the pipeline_io.yaml file as 'pipeline.yaml_schema_file'. The resulting tables will be found in the 'dev' database in the redshift cluster defined in the config override file in the 'redshift_host' variable.""" parser.add_argument('-s', '--start_date', help='YYYY-MM-DD', required=True, action=DateAction) parser.add_argument('-e', '--end_date', help='YYYY-MM-DD', required=True, action=DateAction) parser.add_argument( "-p", "--pool-size", type=int, help="processes to run simultaneously an int in [1,num cpu's]", required=False, default=1, ) parser.add_argument( "--exceed-max-processes", action="store_true", help="use this to run more processes than cpu's", ) parser.add_argument("--dry-run", action="store_true", help="use this to print out the steps we will take") stepper_type = parser.add_mutually_exclusive_group(required=False) stepper_type.add_argument( "--serial-stepper", action="store_true", help="use the serial stepper (alternates between et & load)") stepper_type.add_argument( "--load-polling-interval", type=int, default=0, help="interval in seconds between attempts to find data to load, \ default is 0 (meaning no polling)") etl_type = parser.add_mutually_exclusive_group(required=False) etl_type.add_argument( "--load-only", action="store_true", help="use this to do only a load step", ) etl_type.add_argument( "--et-only", action="store_true", help="use this to do only an et step", ) return parser.parse_args(sys_argv[1:])
def parse_command_line(sys_argv): """ parse_command_line takes in sys.argv and parses the arguments other than the first (since it's the program name). It starts with a parser from get_default_parser having the following arguments: -r, --run-local --io_yaml --conig --config-override --private and adds the following to the arguments: -s, --start_date -e, --end_date -p, --private db_file Returns: an ArgParse.Namespace """ parser = get_base_parser() parser = add_load_args(parser) parser.description = """ This program takes arguments that are the union of arguments for s3_to_psv and s3_to_redshift. It processes each date using the yaml schema file specified in the pipeline_io.yaml file as 'pipeline.yaml_schema_file'. The resulting tables will be found in the 'dev' database in the redshift cluster defined in the config override file in the 'redshift_host' variable.""" parser.add_argument( '-s', '--start_date', help='YYYY-MM-DD', required=True, action=DateAction ) parser.add_argument( '-e', '--end_date', help='YYYY-MM-DD', required=True, action=DateAction ) parser.add_argument( "-p", "--pool-size", type=int, help="processes to run simultaneously an int in [1,num cpu's]", required=False, default=1, ) parser.add_argument( "--exceed-max-processes", action="store_true", help="use this to run more processes than cpu's", ) parser.add_argument( "--dry-run", action="store_true", help="use this to print out the steps we will take" ) stepper_type = parser.add_mutually_exclusive_group(required=False) stepper_type.add_argument( "--serial-stepper", action="store_true", help="use the serial stepper (alternates between et & load)" ) stepper_type.add_argument( "--load-polling-interval", type=int, default=0, help="interval in seconds between attempts to find data to load, \ default is 0 (meaning no polling)" ) etl_type = parser.add_mutually_exclusive_group(required=False) etl_type.add_argument( "--load-only", action="store_true", help="use this to do only a load step", ) etl_type.add_argument( "--et-only", action="store_true", help="use this to do only an et step", ) return parser.parse_args(sys_argv[1:])