def stop_mpd_daemon(verbose): mpdkill_cmd = 'mpdallexit' error_code = run_wait_command(mpdkill_cmd, verbose) if error_code > 0 and verbose: print >>sys.stderr, 'Error killing MPD daemon'
def launch_mpd_daemon(num_machines, node_filename, verbose): mpdboot_cmd = "mpdboot --totalnum=%d -f %s" % (num_machines, node_filename) error_code = run_wait_command(mpdboot_cmd, verbose) if error_code > 0: exit_with_error("Error launching mpd daemon")
def launch_mpd_daemon(num_machines, node_filename, verbose): mpdboot_cmd = 'mpdboot --totalnum=%d -f %s' % (num_machines, node_filename) error_code = run_wait_command(mpdboot_cmd, verbose) if error_code > 0: exit_with_error('Error launching mpd daemon')
def stop_mpd_daemon(verbose): mpdkill_cmd = "mpdallexit" error_code = run_wait_command(mpdkill_cmd, verbose) if error_code > 0 and verbose: print >> sys.stderr, "Error killing MPD daemon"
def rsync_files(src_location, dest_location, rsync_opts, wait_time, max_retry_count, verbose=False): if type(rsync_opts) is ListType: opts_string = ' '.join(rsync_opts) else: opts_string = rsync_opts retry = 0 error_code = BOGUS_ERROR_CODE # bogus code to get loop started while retry < max_retry_count: if retry > 0: time.sleep(wait_time) rsync_command = RSYNC_BIN + " " + opts_string + " " + src_location + " " + dest_location if verbose: print "Running rsync command: %s" % rsync_command error_code = run_wait_command(rsync_command, verbose) if verbose: # In case we need to debug which error codes are not # being used as errors print "rsync error code: %d" % error_code sys.stdout.flush() if error_code not in RSYNC_RETRY_EXIT_CODES: break retry += 1 if error_code > 0 or retry > 1: error_strings = [] if error_code not in RSYNC_RETRY_EXIT_CODES: error_strings.append("rsync failed after %d tries" % retry) error_strings.append("rsync exit code: %d" % error_code) else: error_strings.append("rsync succeeded after %d tries" % retry) error_strings.append("rsync source: %s" % src_location) error_strings.append("rsync destination: %s" % dest_location) if error_code not in RSYNC_RETRY_EXIT_CODES: exit_with_error(error_strings) else: for err_line in error_strings: print >> sys.stderr, err_line
def check_for_remote_dir(remote_machine, remote_dir): if remote_machine != None and len(remote_machine) > 0: test_command = '%s %s test -d "%s"' % (SSH_EXEC, remote_machine, remote_dir) print "Sanity checking on %s for existance of: %s" % (remote_machine, remote_dir) else: test_command = 'test -d "%s"' % (remote_dir) print "Sanity checking for existance of: %s" % (remote_dir) error_code = run_wait_command(test_command, True) if error_code == 0: does_exist = True else: does_exist = False return does_exist
def rsync_files(src_location, dest_location, rsync_opts, wait_time, max_retry_count, verbose=False): if type(rsync_opts) is ListType: opts_string = ' '.join(rsync_opts) else: opts_string = rsync_opts retry = 0 error_code = BOGUS_ERROR_CODE # bogus code to get loop started while retry < max_retry_count: if retry > 0: time.sleep(wait_time) rsync_command = RSYNC_BIN + " " + opts_string + " " + src_location + " " + dest_location if verbose: print "Running rsync command: %s" % rsync_command error_code = run_wait_command(rsync_command, verbose) if verbose: # In case we need to debug which error codes are not # being used as errors print "rsync error code: %d" % error_code sys.stdout.flush() if error_code not in RSYNC_RETRY_EXIT_CODES: break retry += 1 if error_code > 0 or retry > 1: error_strings = [] if error_code not in RSYNC_RETRY_EXIT_CODES: error_strings.append( "rsync failed after %d tries" % retry ) error_strings.append( "rsync exit code: %d" % error_code ) else: error_strings.append( "rsync succeeded after %d tries" % retry ) error_strings.append( "rsync source: %s" % src_location ) error_strings.append( "rsync destination: %s" % dest_location ) if error_code not in RSYNC_RETRY_EXIT_CODES: exit_with_error(error_strings) else: for err_line in error_strings: print >>sys.stderr, err_line
clean_existing_files(CLEAN_FILE_LIST, verbose=VERBOSE_RUN_JOB) # Fix ABSCO directory if VERBOSE_RUN_JOB: print 'Setting absco_path in %s in %s' % (ABSCO_DIR, RUN_FILENAME) set_absco_path(RUN_FILENAME, ABSCO_DIR, REMOTE_DIR) # Attempt to set up mpd for parallel mode if parallel_mode: node_filename = None if os.environ.has_key('PBS_NODEFILE') and len(os.environ['PBS_NODEFILE']) > 0: node_filename = os.environ['PBS_NODEFILE'] elif os.environ.has_key('LSB_MCPU_HOSTS') and len(os.environ['LSB_MCPU_HOSTS']) > 0: node_filename = 'lsf_machine_file.tmp' run_wait_command('machinefile.lsf > %s' % node_filename, verbose=True) if node_filename == None: exit_with_error('No node filename could be found in enviromental variables', REMOTE_DIR) if not os.path.exists(node_filename): exit_with_error('Node filename does not exist: "%s"' % node_filename, REMOTE_DIR) (num_machines, num_nodes) = determine_node_setup(node_filename) print 'Using %d nodes on %d machines' % (num_nodes, num_machines) if PARALLEL_LAUNCH_MPD[CLUSTER_HOSTNAME]: if VERBOSE_RUN_JOB: print 'Launching mpd daemon'
clean_existing_files(CLEAN_FILE_LIST, verbose=VERBOSE_RUN_JOB) # Fix ABSCO directory if VERBOSE_RUN_JOB: print "Setting absco_path in %s in %s" % (ABSCO_DIR, RUN_FILENAME) set_absco_path(RUN_FILENAME, ABSCO_DIR, REMOTE_DIR) # Attempt to set up mpd for parallel mode if parallel_mode: node_filename = None if os.environ.has_key("PBS_NODEFILE") and len(os.environ["PBS_NODEFILE"]) > 0: node_filename = os.environ["PBS_NODEFILE"] elif os.environ.has_key("LSB_MCPU_HOSTS") and len(os.environ["LSB_MCPU_HOSTS"]) > 0: node_filename = "lsf_machine_file.tmp" run_wait_command("machinefile.lsf > %s" % node_filename, verbose=True) if node_filename == None: exit_with_error("No node filename could be found in enviromental variables", REMOTE_DIR) if not os.path.exists(node_filename): exit_with_error('Node filename does not exist: "%s"' % node_filename, REMOTE_DIR) (num_machines, num_nodes) = determine_node_setup(node_filename) print "Using %d nodes on %d machines" % (num_nodes, num_machines) if PARALLEL_LAUNCH_MPD[CLUSTER_HOSTNAME]: if VERBOSE_RUN_JOB: print "Launching mpd daemon"
print '%s:' % inp_key_name print '%s -> %s' % (ext_fullname, local_fullname) print '' inp_sect_obj.Set_Keyword_Value(inp_key_name, local_fullname) if len(external_copy_list) == 0: print >>sys.stderr, 'No valid files found in config files for copying' sys.exit(1) # Check that the files to copy all exist on the foreign system test_expressions = ' -a '.join([ '"(" -e "%s" ")"' % remote_file for remote_file in external_copy_list ]) test_command = 'ssh -q %s test \'%s\'' % (cmd_options.remote_machine, test_expressions) print 'Testing for file existance on machine: %s' % cmd_options.remote_machine error_code = run_wait_command(test_command, False) if error_code != 0: print >>sys.stderr, 'One or more of the following remote files do not exist on machine: %s' % cmd_options.remote_machine for remote_file in external_copy_list: print >>sys.stderr, remote_file print >>sys.stderr, 'One or more of the previous remote files do not exist on machine: %s' % cmd_options.remote_machine sys.exit(error_code) else: print 'Verified existance of %d remote files' % len(external_copy_list) # Make destination directory if it does not exist if not os.path.exists(external_inp_dir): os.makedirs(external_inp_dir) # Copy files from remote system