def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--region', required=True, help='E.g. \'us_nd\'')
parser.add_argument('--file-type', required=True,
choices=[file_type.value for file_type in GcsfsDirectIngestFileType],
help='Defines whether we should move raw files or generated ingest_view files')
parser.add_argument('--dry-run', default=True, type=str_to_bool,
help='Runs copy in dry-run mode, only prints the file copies it would do.')
parser.add_argument('--start-date-bound',
help='The lower bound date to start from, inclusive. For partial copying of ingested files. '
'E.g. 2019-09-23.')
parser.add_argument('--end-date-bound',
help='The upper bound date to end at, inclusive. For partial copying of ingested files. '
'E.g. 2019-09-23.')
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format='%(message)s')
CopyFilesFromProdToStagingController(
region_code=args.region,
file_type=GcsfsDirectIngestFileType(args.file_type),
start_date_bound=args.start_date_bound,
end_date_bound=args.end_date_bound,
dry_run=args.dry_run).run()
def main():
"""Runs the move_state_files_to_deprecated script."""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--file-type',
required=True,
choices=[file_type.value for file_type in GcsfsDirectIngestFileType],
help=
'Defines whether we should move raw files or generated ingest_view files'
)
parser.add_argument('--region', required=True, help='E.g. \'us_nd\'')
parser.add_argument(
'--dry-run',
default=True,
type=str_to_bool,
help=
'Runs move in dry-run mode, only prints the file moves it would do.')
parser.add_argument(
'--start-date-bound',
help=
'The lower bound date to start from, inclusive. For partial moving of ingested files. '
'E.g. 2019-09-23.')
parser.add_argument(
'--end-date-bound',
help=
'The upper bound date to end at, inclusive. For partial moving of ingested files. '
'E.g. 2019-09-23.')
parser.add_argument(
'--project-id',
help='The id for this particular project, E.g. \'recidiviz-123\'')
parser.add_argument(
'--file-filter',
default=None,
help=
'Regex name filter - when set, will only move files that match this regex.'
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format='%(message)s')
MoveFilesToDeprecatedController(file_type=GcsfsDirectIngestFileType(
args.file_type),
region_code=args.region,
start_date_bound=args.start_date_bound,
end_date_bound=args.end_date_bound,
project_id=args.project_id,
dry_run=args.dry_run,
file_filter=args.file_filter).run()
def main() -> None:
"""Executes the main flow of the script."""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument("--region", required=True, help="E.g. 'us_nd'")
parser.add_argument(
"--file-type",
required=True,
choices=[file_type.value for file_type in GcsfsDirectIngestFileType],
help="Defines whether we should move raw files or generated ingest_view files",
)
parser.add_argument(
"--dry-run",
default=True,
type=str_to_bool,
help="Runs copy in dry-run mode, only prints the file copies it would do.",
)
parser.add_argument(
"--start-date-bound",
help="The lower bound date to start from, inclusive. For partial copying of ingested files. "
"E.g. 2019-09-23.",
)
parser.add_argument(
"--end-date-bound",
help="The upper bound date to end at, inclusive. For partial copying of ingested files. "
"E.g. 2019-09-23.",
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(message)s")
CopyFilesFromProdToStagingController(
region_code=args.region,
file_type=GcsfsDirectIngestFileType(args.file_type),
start_date_bound=args.start_date_bound,
end_date_bound=args.end_date_bound,
dry_run=args.dry_run,
).run()
def main():
"""Runs the move_state_files_to_storage script."""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--project-id',
required=True,
help='Which project\'s files should be moved (e.g. recidiviz-123).')
parser.add_argument('--region', required=True, help='E.g. \'us_nd\'')
parser.add_argument(
'--file-type-to-move',
required=True,
choices=[file_type.value for file_type in GcsfsDirectIngestFileType],
help='Defines what type of files to move out of storage.')
parser.add_argument(
'--destination-file-type',
required=True,
choices=[
file_type.value for file_type in {
GcsfsDirectIngestFileType.RAW_DATA,
GcsfsDirectIngestFileType.INGEST_VIEW
}
],
help=
'Defines what type the files should be after they have been moved. Must match '
'file-type-to-move unless file-type-to-move is \'unspecified\'.')
parser.add_argument(
'--start-date-bound',
help=
'The lower bound date to start from, inclusive. For partial replays of ingested files. '
'E.g. 2019-09-23.')
parser.add_argument(
'--end-date-bound',
help=
'The upper bound date to end at, inclusive. For partial replays of ingested files. '
'E.g. 2019-09-23.')
parser.add_argument(
'--dry-run',
default=True,
type=str_to_bool,
help=
'Runs move in dry-run mode, only prints the file moves it would do.')
parser.add_argument(
'--file-filter',
default=None,
help=
'Regex name filter - when set, will only move files that match this regex.'
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format='%(message)s')
MoveFilesFromStorageController(
project_id=args.project_id,
region=args.region,
file_type_to_move=GcsfsDirectIngestFileType(args.file_type_to_move),
destination_file_type=GcsfsDirectIngestFileType(
args.destination_file_type),
start_date_bound=args.start_date_bound,
end_date_bound=args.end_date_bound,
dry_run=args.dry_run,
file_filter=args.file_filter).run_move()
def main() -> None:
"""Runs the move_state_files_to_deprecated script."""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument(
"--file-type",
required=True,
choices=[file_type.value for file_type in GcsfsDirectIngestFileType],
help="Defines whether we should move raw files or generated ingest_view files",
)
parser.add_argument("--region", required=True, help="E.g. 'us_nd'")
parser.add_argument(
"--ingest-instance",
required=True,
choices=[instance.value for instance in DirectIngestInstance],
help="Defines which ingest instance we should be deprecating files for.",
)
parser.add_argument(
"--dry-run",
default=True,
type=str_to_bool,
help="Runs move in dry-run mode, only prints the file moves it would do.",
)
parser.add_argument(
"--start-date-bound",
help="The lower bound date to start from, inclusive. For partial moving of ingested files. "
"E.g. 2019-09-23.",
)
parser.add_argument(
"--end-date-bound",
help="The upper bound date to end at, inclusive. For partial moving of ingested files. "
"E.g. 2019-09-23.",
)
parser.add_argument(
"--project-id", help="The id for this particular project, E.g. 'recidiviz-123'"
)
parser.add_argument(
"--file-filter",
default=None,
help="Regex name filter - when set, will only move files that match this regex.",
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(message)s")
MoveFilesToDeprecatedController(
file_type=GcsfsDirectIngestFileType(args.file_type),
region_code=args.region,
ingest_instance=DirectIngestInstance(args.ingest_instance),
start_date_bound=args.start_date_bound,
end_date_bound=args.end_date_bound,
project_id=args.project_id,
dry_run=args.dry_run,
file_filter=args.file_filter,
).run()