def build_export_pipeline(client_config, checkpoint_service, args) -> Pipeline:
"""
All export jobs
export_instance_profiles -> export_users -> export_groups -> export_secrets -> export_clusters -> export_instance_pools -> export_jobs
-> log_workspace_items -> export_workspace_acls
-> export_notebooks
-> export_metastore -> export_metastore_table_acls
"""
skip_tasks = args.skip_tasks
source_info_file = os.path.join(client_config['export_dir'], "source_info.txt")
with open(source_info_file, 'w') as f:
f.write(client_config['url'])
completed_pipeline_steps = checkpoint_service.get_checkpoint_key_set(
wmconstants.WM_EXPORT, wmconstants.MIGRATION_PIPELINE_OBJECT_TYPE)
pipeline = Pipeline(client_config['export_dir'], completed_pipeline_steps, args.dry_run)
export_instance_profiles = pipeline.add_task(InstanceProfileExportTask(client_config, checkpoint_service, wmconstants.INSTANCE_PROFILES in skip_tasks))
export_users = pipeline.add_task(UserExportTask(client_config, checkpoint_service, wmconstants.USERS in skip_tasks), [export_instance_profiles])
export_groups = pipeline.add_task(GroupExportTask(client_config, checkpoint_service, wmconstants.GROUPS in skip_tasks), [export_users])
workspace_item_log_export = pipeline.add_task(WorkspaceItemLogExportTask(client_config, args, checkpoint_service, wmconstants.WORKSPACE_ITEM_LOG in skip_tasks), [export_groups])
export_workspace_acls = pipeline.add_task(WorkspaceACLExportTask(client_config, checkpoint_service, wmconstants.WORKSPACE_ACLS in skip_tasks), [workspace_item_log_export])
export_notebooks = pipeline.add_task(NotebookExportTask(client_config, checkpoint_service, wmconstants.NOTEBOOKS in skip_tasks), [workspace_item_log_export])
export_secrets = pipeline.add_task(SecretExportTask(client_config, args, checkpoint_service, wmconstants.SECRETS in skip_tasks), [export_groups])
export_clusters = pipeline.add_task(ClustersExportTask(client_config, args, checkpoint_service, wmconstants.CLUSTERS in skip_tasks), [export_secrets])
export_instance_pools = pipeline.add_task(InstancePoolsExportTask(client_config, args, checkpoint_service, wmconstants.INSTANCE_POOLS in skip_tasks), [export_clusters])
export_jobs = pipeline.add_task(JobsExportTask(client_config, args, checkpoint_service, wmconstants.JOBS in skip_tasks), [export_instance_pools])
export_metastore = pipeline.add_task(MetastoreExportTask(client_config, checkpoint_service, args, wmconstants.METASTORE in skip_tasks), [export_groups])
export_metastore_table_acls = pipeline.add_task(MetastoreTableACLExportTask(client_config, args, checkpoint_service, wmconstants.METASTORE_TABLE_ACLS in skip_tasks), [export_metastore])
# FinishExport task is never skipped
finish_export = pipeline.add_task(FinishExportTask(client_config),
[export_workspace_acls, export_notebooks, export_jobs,
export_metastore_table_acls])
return pipeline
def build_import_pipeline(client_config, checkpoint_service, args) -> Pipeline:
"""
All import jobs
import_instance_profiles -> import_users -> import_groups -> import_secrets -> import_clusters -> import_instance_pools -> import_jobs
-> log_workspace_items -> import_notebooks -> import_workspace_acls
-> import_metastore -> import_metastore_table_acls
"""
skip_tasks = args.skip_tasks
source_info_file = os.path.join(client_config['export_dir'], "source_info.txt")
with open(source_info_file, 'r') as f:
source_url = f.readline()
confirm = input(f"Import from `{source_url}` into `{client_config['url']}`? (y/N) ")
if confirm.lower() not in ["y", "yes"]:
raise RuntimeError("User aborted import pipeline. Exiting..")
completed_pipeline_steps = checkpoint_service.get_checkpoint_key_set(
wmconstants.WM_IMPORT, wmconstants.MIGRATION_PIPELINE_OBJECT_TYPE)
pipeline = Pipeline(client_config['export_dir'], completed_pipeline_steps, args.dry_run)
import_instance_profiles = pipeline.add_task(InstanceProfileImportTask(client_config, checkpoint_service, wmconstants.INSTANCE_PROFILES in skip_tasks))
import_users = pipeline.add_task(UserImportTask(client_config, checkpoint_service, wmconstants.USERS in skip_tasks), [import_instance_profiles])
import_groups = pipeline.add_task(GroupImportTask(client_config, checkpoint_service, wmconstants.GROUPS in skip_tasks), [import_users])
import_notebooks = pipeline.add_task(NotebookImportTask(client_config, checkpoint_service, args, wmconstants.NOTEBOOKS in skip_tasks), [import_groups])
import_workspace_acls = pipeline.add_task(WorkspaceACLImportTask(client_config, checkpoint_service, wmconstants.WORKSPACE_ACLS in skip_tasks), [import_notebooks])
import_secrets = pipeline.add_task(SecretImportTask(client_config, checkpoint_service, wmconstants.SECRETS in skip_tasks), [import_groups])
import_clusters = pipeline.add_task(ClustersImportTask(client_config, args, checkpoint_service, wmconstants.CLUSTERS in skip_tasks), [import_secrets])
import_instance_pools = pipeline.add_task(InstancePoolsImportTask(client_config, args, checkpoint_service, wmconstants.INSTANCE_POOLS in skip_tasks), [import_clusters])
import_jobs = pipeline.add_task(JobsImportTask(client_config, args, checkpoint_service, wmconstants.JOBS in skip_tasks), [import_instance_pools])
import_metastore = pipeline.add_task(MetastoreImportTask(client_config, checkpoint_service, args, wmconstants.METASTORE in skip_tasks), [import_groups])
import_metastore_table_acls = pipeline.add_task(MetastoreTableACLImportTask(client_config, args, checkpoint_service, wmconstants.METASTORE_TABLE_ACLS in skip_tasks), [import_metastore])
return pipeline