def get_migration_file_configs(migration):
full_plan = {plan.db_name: plan for plan in migration.shard_plan}
missing_files = get_missing_files_by_node_and_source(
migration.source_couch_config, full_plan
)
migration_file_configs = {}
for node, source_file_list in missing_files.items():
target_host = node.split('@')[1]
files_for_node = []
for source, file_list in source_file_list.items():
source_host = source.split('@')[1]
files_for_node.append(
SourceFiles(
source_host=source_host,
source_dir=migration.couchdb2_data_dir,
target_dir=migration.couchdb2_data_dir,
files=[f.filename for f in file_list]
)
)
if files_for_node:
migration_file_configs[target_host] = files_for_node
return migration_file_configs
def get_migration_file_configs(migration):
full_plan = {plan.db_name: plan for plan in migration.shard_plan}
missing_files = get_missing_files_by_node_and_source(
migration.source_couch_config, full_plan
)
migration_file_configs = {}
for node, source_file_list in missing_files.items():
target_host = node.split('@')[1]
files_for_node = []
for source, file_list in source_file_list.items():
source_host = source.split('@')[1]
files_for_node.append(
SourceFiles(
source_host=source_host,
source_dir=migration.couchdb2_data_dir,
target_dir=migration.couchdb2_data_dir,
files=[f.filename for f in file_list]
)
)
if files_for_node:
migration_file_configs[target_host] = files_for_node
return migration_file_configs
def generate_rsync_lists(migration):
full_plan = {plan.db_name: plan for plan in migration.shard_plan}
missing_files = get_missing_files_by_node_and_source(
migration.source_couch_config, full_plan)
paths_by_host = defaultdict(list)
for node, source_file_list in missing_files.items():
node_ip = node.split('@')[1]
node_files_path = os.path.join(migration.rsync_files_path, node_ip)
if not os.path.exists(node_files_path):
os.makedirs(node_files_path)
files_for_node = []
for source, file_list in source_file_list.items():
source_ip = source.split('@')[1]
files = sorted([f.filename for f in file_list])
filename = '{}__files'.format(source_ip)
path = os.path.join(node_files_path, filename)
with open(path, 'w') as f:
f.write('{}\n'.format('\n'.join(files)))
files_for_node.append((source_ip, filename))
paths_by_host[node_ip].append(path)
if files_for_node:
# create rsync script
rsync_script = _render_template(
'couchdb_rsync.sh.j2', {
'rsync_file_list':
files_for_node,
'couch_data_dir':
'/opt/data/couchdb2/',
'rsync_file_root':
os.path.join('/tmp', RSYNC_FILE_LIST_FOLDER_NAME)
})
with open(os.path.join(node_files_path, COUCHDB_RSYNC_SCRIPT),
'w') as f:
f.write(rsync_script)
return paths_by_host