def cp_scripts_configs(user, ssh_key, ns_ids, lns_ids, remote_folder, config_file, workload_config_file, node_config_folder, remote_node_config_file): """ #!/bin/sh user=$1 ssh_key=$2 pl_ns=$3 pl_lns=$4 config_file=$5 workload_config_file=$6 node_config_folder=$7 remote_folder=$8 remote_node_config=$9 echo "Copy scripts to NS and LNS ..." cat $pl_ns $pl_lns | parallel -j+200 ssh -i $ssh_key -oStrictHostKeyChecking=no -oConnectTimeout=60 $user@{} "mkdir -p $remote_folder" cat $pl_ns $pl_lns | parallel -j+200 scp -i $ssh_key -oStrictHostKeyChecking=no -oConnectTimeout=60 name_server.py argparse.py local_name_server.py exp_config.py $config_file $workload_config_file $user@{}:$remote_folder cat $pl_ns $pl_lns | parallel -j+200 scp -i $ssh_key -oStrictHostKeyChecking=no -oConnectTimeout=60 $node_config_folder/config_{} $user@{}:$remote_folder/$remote_node_config echo "Copied!" """ ns_lns_ids = {} ns_lns_ids.update(ns_ids) ns_lns_ids.update(lns_ids) # step 1: mkdir remote folders for each node cmds = [] for ns, ns_host in ns_lns_ids.items(): node_folder = os.path.join(remote_folder, str(ns)) cmd = 'ssh -i ' + ssh_key + ' -oStrictHostKeyChecking=no -oConnectTimeout=60 ' + user + '@' + ns_host + \ ' "mkdir -p ' + node_folder + '" ' cmds.append(cmd) run_cmds(cmds) # step 2: copy files (except node config) cmds = [] for ns, ns_host in ns_lns_ids.items(): node_folder = os.path.join(remote_folder, str(ns)) cmd = 'scp -i ' + ssh_key + ' -oStrictHostKeyChecking=no -oConnectTimeout=60 ' \ ' name_server.py argparse.py local_name_server.py exp_config.py ' + \ config_file + ' ' + workload_config_file + ' ' + user + '@' + ns_host + ':' + node_folder cmds.append(cmd) run_cmds(cmds) # step 3: copy node config file cmds = [] for ns, ns_host in ns_lns_ids.items(): node_folder = os.path.join(remote_folder, str(ns)) node_config_local = os.path.join(node_config_folder, str(ns)) cmd = 'scp -i ' + ssh_key + ' -oStrictHostKeyChecking=no -oConnectTimeout=60 ' + \ node_config_local + ' ' + user + '@' + ns_host + ':' + os.path.join(node_folder, remote_node_config_file) cmds.append(cmd) run_cmds(cmds)
def get_log(user, ssh_key, ns_ids, lns_ids, local_output_folder, remote_log_folder): """#!/bin/sh user=$1 ssh_key=$2 pl_ns=$3 pl_lns=$4 local_output_folder=$5 remote_log_folder=$6 #nozip="$1" date echo "Remove earlier log ..." rm -rf log/* mkdir -p log echo "Create local directories ..." cat $pl_lns | parallel -j+100 mkdir log/log_lns_{} & cat $pl_ns | parallel -j+100 mkdir log/log_ns_{} & #if [ "$nozip" != "--nozip" ]; then echo "Gzip logs at LNS and NS ..." # remove console output files, delete log folder cat $pl_ns $pl_lns | parallel -j+100 ssh -i $ssh_key -oStrictHostKeyChecking=no -oConnectTimeout=60 $user@{} "gzip $remote_log_folder/* $remote_log_folder/log/*" #echo "Gzip logs at NS ..." #date #echo "Copying config file from LNS ..." #cat $pl_lns | parallel -j+100 scp -i $ssh_key -r -oStrictHostKeyChecking=no -oConnectTimeout=60 $user@{}:pl_config log/log_lns_{} & echo "Copying logs from LNS ..." #echo "Copying gnrs_stat.xml files only ..." cat $pl_lns | parallel -j+100 scp -i $ssh_key -r -oStrictHostKeyChecking=no -oConnectTimeout=60 $user@{}:$remote_log_folder/* log/log_lns_{} #date #echo "Copying config file from NS ..." #cat $pl_ns | parallel -j+100 scp -i $ssh_key -r -oStrictHostKeyChecking=no -oConnectTimeout=60 $user@{}:pl_config log/log_ns_{} & date echo "Copying logs from NS ..." cat $pl_ns | parallel -j+100 scp -i $ssh_key -r -oStrictHostKeyChecking=no -oConnectTimeout=60 -r $user@{}:$remote_log_folder/* log/log_ns_{} #cat pl_ns | parallel -j+100 scp -i auspice.pem -r -oStrictHostKeyChecking=no -oConnectTimeout=60 ec2-user@{}:ping_output log/log_ns_{} #cat pl_lns | parallel -j+100 scp -i auspice.pem -r -oStrictHostKeyChecking=no -oConnectTimeout=60 ec2-user@{}:ping_output log/log_lns_{} # OLD #cat pl_ns | parallel -j+100 scp -i auspice.pem -r -oStrictHostKeyChecking=no -oConnectTimeout=10 ec2-user@{}:log log/log_ns_{} #For GNRS-westy #cat pl_lns | parallel -j+100 scp -i auspice.pem -oStrictHostKeyChecking=no -oConnectTimeout=10 -r ec2-user@{}:log log/log_lns_{} if [ "$local_output_folder" != "" ]; then mkdir -p $local_output_folder mv log/* $local_output_folder fi echo "Output copied to: " $local_output_folder """ # local_output_folder = os.path.join(local_output_folder, 'log') # print '*********** GET LOGS BEGIN >>>>>> ', local_output_folder os.system('rm -rf ' + local_output_folder) os.system('mkdir -p ' + local_output_folder) ns_lns_ids = {} ns_lns_ids.update(ns_ids) ns_lns_ids.update(lns_ids) print ns_ids print lns_ids print 'Create NS log folders ...' cmds = [] for node_id in ns_ids: folder = os.path.join(local_output_folder, get_ns_folder(node_id)) cmds.append('mkdir -p ' + folder) run_cmds(cmds) print 'Create LNS log folders ...' cmds = [] for node_id in lns_ids: folder = os.path.join(local_output_folder, get_lns_folder(node_id)) cmds.append('mkdir -p ' + folder) run_cmds(cmds) print 'Gzip remote folders ... ' cmds = [] for node_id, ns_host in ns_lns_ids.items(): remote_node_folder = os.path.join(remote_log_folder, str(node_id)) cmd = 'ssh -i ' + ssh_key + ' -oStrictHostKeyChecking=no -oConnectTimeout=60 ' + user + '@' + ns_host + \ ' "gzip ' + remote_node_folder + '/* ' + remote_node_folder + '/log/*" ' cmds.append(cmd) run_cmds(cmds) print 'Copy LNS logs ...' cmds = [] for node_id, ns_host in ns_ids.items(): remote_node_folder = os.path.join(remote_log_folder, str(node_id)) local_node_folder = os.path.join(local_output_folder, get_ns_folder(node_id)) cmd = 'scp -i ' + ssh_key + ' -r -oStrictHostKeyChecking=no -oConnectTimeout=60 ' + user + '@' + ns_host + ':' + \ remote_node_folder + '/* ' + local_node_folder cmds.append(cmd) run_cmds(cmds) print 'Copy NS logs ...' cmds = [] for node_id, ns_host in lns_ids.items(): remote_node_folder = os.path.join(remote_log_folder, str(node_id)) local_node_folder = os.path.join(local_output_folder, get_lns_folder(node_id)) cmd = 'scp -i ' + ssh_key + ' -r -oStrictHostKeyChecking=no -oConnectTimeout=60 ' + user + '@' + ns_host + ':' + \ remote_node_folder + '/* ' + local_node_folder cmds.append(cmd) run_cmds(cmds)