def stop_mpd_daemon(verbose):
mpdkill_cmd = 'mpdallexit'
error_code = run_wait_command(mpdkill_cmd, verbose)
if error_code > 0 and verbose:
print >>sys.stderr, 'Error killing MPD daemon'
def launch_mpd_daemon(num_machines, node_filename, verbose):
mpdboot_cmd = "mpdboot --totalnum=%d -f %s" % (num_machines, node_filename)
error_code = run_wait_command(mpdboot_cmd, verbose)
if error_code > 0:
exit_with_error("Error launching mpd daemon")
def launch_mpd_daemon(num_machines, node_filename, verbose):
mpdboot_cmd = 'mpdboot --totalnum=%d -f %s' % (num_machines, node_filename)
error_code = run_wait_command(mpdboot_cmd, verbose)
if error_code > 0:
exit_with_error('Error launching mpd daemon')
def stop_mpd_daemon(verbose):
mpdkill_cmd = "mpdallexit"
error_code = run_wait_command(mpdkill_cmd, verbose)
if error_code > 0 and verbose:
print >> sys.stderr, "Error killing MPD daemon"
def rsync_files(src_location,
dest_location,
rsync_opts,
wait_time,
max_retry_count,
verbose=False):
if type(rsync_opts) is ListType:
opts_string = ' '.join(rsync_opts)
else:
opts_string = rsync_opts
retry = 0
error_code = BOGUS_ERROR_CODE # bogus code to get loop started
while retry < max_retry_count:
if retry > 0:
time.sleep(wait_time)
rsync_command = RSYNC_BIN + " " + opts_string + " " + src_location + " " + dest_location
if verbose:
print "Running rsync command: %s" % rsync_command
error_code = run_wait_command(rsync_command, verbose)
if verbose:
# In case we need to debug which error codes are not
# being used as errors
print "rsync error code: %d" % error_code
sys.stdout.flush()
if error_code not in RSYNC_RETRY_EXIT_CODES:
break
retry += 1
if error_code > 0 or retry > 1:
error_strings = []
if error_code not in RSYNC_RETRY_EXIT_CODES:
error_strings.append("rsync failed after %d tries" % retry)
error_strings.append("rsync exit code: %d" % error_code)
else:
error_strings.append("rsync succeeded after %d tries" % retry)
error_strings.append("rsync source: %s" % src_location)
error_strings.append("rsync destination: %s" % dest_location)
if error_code not in RSYNC_RETRY_EXIT_CODES:
exit_with_error(error_strings)
else:
for err_line in error_strings:
print >> sys.stderr, err_line
def check_for_remote_dir(remote_machine, remote_dir):
if remote_machine != None and len(remote_machine) > 0:
test_command = '%s %s test -d "%s"' % (SSH_EXEC, remote_machine, remote_dir)
print "Sanity checking on %s for existance of: %s" % (remote_machine, remote_dir)
else:
test_command = 'test -d "%s"' % (remote_dir)
print "Sanity checking for existance of: %s" % (remote_dir)
error_code = run_wait_command(test_command, True)
if error_code == 0:
does_exist = True
else:
does_exist = False
return does_exist
def rsync_files(src_location, dest_location, rsync_opts, wait_time, max_retry_count, verbose=False):
if type(rsync_opts) is ListType:
opts_string = ' '.join(rsync_opts)
else:
opts_string = rsync_opts
retry = 0
error_code = BOGUS_ERROR_CODE # bogus code to get loop started
while retry < max_retry_count:
if retry > 0:
time.sleep(wait_time)
rsync_command = RSYNC_BIN + " " + opts_string + " " + src_location + " " + dest_location
if verbose:
print "Running rsync command: %s" % rsync_command
error_code = run_wait_command(rsync_command, verbose)
if verbose:
# In case we need to debug which error codes are not
# being used as errors
print "rsync error code: %d" % error_code
sys.stdout.flush()
if error_code not in RSYNC_RETRY_EXIT_CODES:
break
retry += 1
if error_code > 0 or retry > 1:
error_strings = []
if error_code not in RSYNC_RETRY_EXIT_CODES:
error_strings.append( "rsync failed after %d tries" % retry )
error_strings.append( "rsync exit code: %d" % error_code )
else:
error_strings.append( "rsync succeeded after %d tries" % retry )
error_strings.append( "rsync source: %s" % src_location )
error_strings.append( "rsync destination: %s" % dest_location )
if error_code not in RSYNC_RETRY_EXIT_CODES:
exit_with_error(error_strings)
else:
for err_line in error_strings:
print >>sys.stderr, err_line
def check_for_remote_dir(remote_machine, remote_dir):
if remote_machine != None and len(remote_machine) > 0:
test_command = '%s %s test -d "%s"' % (SSH_EXEC, remote_machine,
remote_dir)
print "Sanity checking on %s for existance of: %s" % (remote_machine,
remote_dir)
else:
test_command = 'test -d "%s"' % (remote_dir)
print "Sanity checking for existance of: %s" % (remote_dir)
error_code = run_wait_command(test_command, True)
if error_code == 0:
does_exist = True
else:
does_exist = False
return does_exist
clean_existing_files(CLEAN_FILE_LIST, verbose=VERBOSE_RUN_JOB)
# Fix ABSCO directory
if VERBOSE_RUN_JOB:
print 'Setting absco_path in %s in %s' % (ABSCO_DIR, RUN_FILENAME)
set_absco_path(RUN_FILENAME, ABSCO_DIR, REMOTE_DIR)
# Attempt to set up mpd for parallel mode
if parallel_mode:
node_filename = None
if os.environ.has_key('PBS_NODEFILE') and len(os.environ['PBS_NODEFILE']) > 0:
node_filename = os.environ['PBS_NODEFILE']
elif os.environ.has_key('LSB_MCPU_HOSTS') and len(os.environ['LSB_MCPU_HOSTS']) > 0:
node_filename = 'lsf_machine_file.tmp'
run_wait_command('machinefile.lsf > %s' % node_filename, verbose=True)
if node_filename == None:
exit_with_error('No node filename could be found in enviromental variables', REMOTE_DIR)
if not os.path.exists(node_filename):
exit_with_error('Node filename does not exist: "%s"' % node_filename, REMOTE_DIR)
(num_machines, num_nodes) = determine_node_setup(node_filename)
print 'Using %d nodes on %d machines' % (num_nodes, num_machines)
if PARALLEL_LAUNCH_MPD[CLUSTER_HOSTNAME]:
if VERBOSE_RUN_JOB:
print 'Launching mpd daemon'
clean_existing_files(CLEAN_FILE_LIST, verbose=VERBOSE_RUN_JOB)
# Fix ABSCO directory
if VERBOSE_RUN_JOB:
print "Setting absco_path in %s in %s" % (ABSCO_DIR, RUN_FILENAME)
set_absco_path(RUN_FILENAME, ABSCO_DIR, REMOTE_DIR)
# Attempt to set up mpd for parallel mode
if parallel_mode:
node_filename = None
if os.environ.has_key("PBS_NODEFILE") and len(os.environ["PBS_NODEFILE"]) > 0:
node_filename = os.environ["PBS_NODEFILE"]
elif os.environ.has_key("LSB_MCPU_HOSTS") and len(os.environ["LSB_MCPU_HOSTS"]) > 0:
node_filename = "lsf_machine_file.tmp"
run_wait_command("machinefile.lsf > %s" % node_filename, verbose=True)
if node_filename == None:
exit_with_error("No node filename could be found in enviromental variables", REMOTE_DIR)
if not os.path.exists(node_filename):
exit_with_error('Node filename does not exist: "%s"' % node_filename, REMOTE_DIR)
(num_machines, num_nodes) = determine_node_setup(node_filename)
print "Using %d nodes on %d machines" % (num_nodes, num_machines)
if PARALLEL_LAUNCH_MPD[CLUSTER_HOSTNAME]:
if VERBOSE_RUN_JOB:
print "Launching mpd daemon"
print '%s:' % inp_key_name
print '%s -> %s' % (ext_fullname, local_fullname)
print ''
inp_sect_obj.Set_Keyword_Value(inp_key_name, local_fullname)
if len(external_copy_list) == 0:
print >>sys.stderr, 'No valid files found in config files for copying'
sys.exit(1)
# Check that the files to copy all exist on the foreign system
test_expressions = ' -a '.join([ '"(" -e "%s" ")"' % remote_file for remote_file in external_copy_list ])
test_command = 'ssh -q %s test \'%s\'' % (cmd_options.remote_machine, test_expressions)
print 'Testing for file existance on machine: %s' % cmd_options.remote_machine
error_code = run_wait_command(test_command, False)
if error_code != 0:
print >>sys.stderr, 'One or more of the following remote files do not exist on machine: %s' % cmd_options.remote_machine
for remote_file in external_copy_list:
print >>sys.stderr, remote_file
print >>sys.stderr, 'One or more of the previous remote files do not exist on machine: %s' % cmd_options.remote_machine
sys.exit(error_code)
else:
print 'Verified existance of %d remote files' % len(external_copy_list)
# Make destination directory if it does not exist
if not os.path.exists(external_inp_dir):
os.makedirs(external_inp_dir)
# Copy files from remote system