def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--region', required=True, help='E.g. \'us_nd\'') parser.add_argument('--file-type', required=True, choices=[file_type.value for file_type in GcsfsDirectIngestFileType], help='Defines whether we should move raw files or generated ingest_view files') parser.add_argument('--dry-run', default=True, type=str_to_bool, help='Runs copy in dry-run mode, only prints the file copies it would do.') parser.add_argument('--start-date-bound', help='The lower bound date to start from, inclusive. For partial copying of ingested files. ' 'E.g. 2019-09-23.') parser.add_argument('--end-date-bound', help='The upper bound date to end at, inclusive. For partial copying of ingested files. ' 'E.g. 2019-09-23.') args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(message)s') CopyFilesFromProdToStagingController( region_code=args.region, file_type=GcsfsDirectIngestFileType(args.file_type), start_date_bound=args.start_date_bound, end_date_bound=args.end_date_bound, dry_run=args.dry_run).run()
def main(): """Runs the move_state_files_to_deprecated script.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--file-type', required=True, choices=[file_type.value for file_type in GcsfsDirectIngestFileType], help= 'Defines whether we should move raw files or generated ingest_view files' ) parser.add_argument('--region', required=True, help='E.g. \'us_nd\'') parser.add_argument( '--dry-run', default=True, type=str_to_bool, help= 'Runs move in dry-run mode, only prints the file moves it would do.') parser.add_argument( '--start-date-bound', help= 'The lower bound date to start from, inclusive. For partial moving of ingested files. ' 'E.g. 2019-09-23.') parser.add_argument( '--end-date-bound', help= 'The upper bound date to end at, inclusive. For partial moving of ingested files. ' 'E.g. 2019-09-23.') parser.add_argument( '--project-id', help='The id for this particular project, E.g. \'recidiviz-123\'') parser.add_argument( '--file-filter', default=None, help= 'Regex name filter - when set, will only move files that match this regex.' ) args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(message)s') MoveFilesToDeprecatedController(file_type=GcsfsDirectIngestFileType( args.file_type), region_code=args.region, start_date_bound=args.start_date_bound, end_date_bound=args.end_date_bound, project_id=args.project_id, dry_run=args.dry_run, file_filter=args.file_filter).run()
def main() -> None: """Executes the main flow of the script.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument("--region", required=True, help="E.g. 'us_nd'") parser.add_argument( "--file-type", required=True, choices=[file_type.value for file_type in GcsfsDirectIngestFileType], help="Defines whether we should move raw files or generated ingest_view files", ) parser.add_argument( "--dry-run", default=True, type=str_to_bool, help="Runs copy in dry-run mode, only prints the file copies it would do.", ) parser.add_argument( "--start-date-bound", help="The lower bound date to start from, inclusive. For partial copying of ingested files. " "E.g. 2019-09-23.", ) parser.add_argument( "--end-date-bound", help="The upper bound date to end at, inclusive. For partial copying of ingested files. " "E.g. 2019-09-23.", ) args = parser.parse_args() logging.basicConfig(level=logging.INFO, format="%(message)s") CopyFilesFromProdToStagingController( region_code=args.region, file_type=GcsfsDirectIngestFileType(args.file_type), start_date_bound=args.start_date_bound, end_date_bound=args.end_date_bound, dry_run=args.dry_run, ).run()
def main(): """Runs the move_state_files_to_storage script.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--project-id', required=True, help='Which project\'s files should be moved (e.g. recidiviz-123).') parser.add_argument('--region', required=True, help='E.g. \'us_nd\'') parser.add_argument( '--file-type-to-move', required=True, choices=[file_type.value for file_type in GcsfsDirectIngestFileType], help='Defines what type of files to move out of storage.') parser.add_argument( '--destination-file-type', required=True, choices=[ file_type.value for file_type in { GcsfsDirectIngestFileType.RAW_DATA, GcsfsDirectIngestFileType.INGEST_VIEW } ], help= 'Defines what type the files should be after they have been moved. Must match ' 'file-type-to-move unless file-type-to-move is \'unspecified\'.') parser.add_argument( '--start-date-bound', help= 'The lower bound date to start from, inclusive. For partial replays of ingested files. ' 'E.g. 2019-09-23.') parser.add_argument( '--end-date-bound', help= 'The upper bound date to end at, inclusive. For partial replays of ingested files. ' 'E.g. 2019-09-23.') parser.add_argument( '--dry-run', default=True, type=str_to_bool, help= 'Runs move in dry-run mode, only prints the file moves it would do.') parser.add_argument( '--file-filter', default=None, help= 'Regex name filter - when set, will only move files that match this regex.' ) args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(message)s') MoveFilesFromStorageController( project_id=args.project_id, region=args.region, file_type_to_move=GcsfsDirectIngestFileType(args.file_type_to_move), destination_file_type=GcsfsDirectIngestFileType( args.destination_file_type), start_date_bound=args.start_date_bound, end_date_bound=args.end_date_bound, dry_run=args.dry_run, file_filter=args.file_filter).run_move()
def main() -> None: """Runs the move_state_files_to_deprecated script.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( "--file-type", required=True, choices=[file_type.value for file_type in GcsfsDirectIngestFileType], help="Defines whether we should move raw files or generated ingest_view files", ) parser.add_argument("--region", required=True, help="E.g. 'us_nd'") parser.add_argument( "--ingest-instance", required=True, choices=[instance.value for instance in DirectIngestInstance], help="Defines which ingest instance we should be deprecating files for.", ) parser.add_argument( "--dry-run", default=True, type=str_to_bool, help="Runs move in dry-run mode, only prints the file moves it would do.", ) parser.add_argument( "--start-date-bound", help="The lower bound date to start from, inclusive. For partial moving of ingested files. " "E.g. 2019-09-23.", ) parser.add_argument( "--end-date-bound", help="The upper bound date to end at, inclusive. For partial moving of ingested files. " "E.g. 2019-09-23.", ) parser.add_argument( "--project-id", help="The id for this particular project, E.g. 'recidiviz-123'" ) parser.add_argument( "--file-filter", default=None, help="Regex name filter - when set, will only move files that match this regex.", ) args = parser.parse_args() logging.basicConfig(level=logging.INFO, format="%(message)s") MoveFilesToDeprecatedController( file_type=GcsfsDirectIngestFileType(args.file_type), region_code=args.region, ingest_instance=DirectIngestInstance(args.ingest_instance), start_date_bound=args.start_date_bound, end_date_bound=args.end_date_bound, project_id=args.project_id, dry_run=args.dry_run, file_filter=args.file_filter, ).run()