def run_local_synctool():
if not synctool_param.SYNCTOOL_CMD:
stderr('%s: error: synctool_cmd has not been defined in %s' % (os.path.basename(sys.argv[0]),
synctool_param.CONF_FILE))
sys.exit(-1)
cmd_arr = shlex.split(synctool_param.SYNCTOOL_CMD) + PASS_ARGS
synctool_lib.run_with_nodename(cmd_arr, synctool_param.NODENAME)
def worker_pkg(rank, args): '''runs ssh + synctool-pkg to the nodes in parallel''' (nodes, ssh_cmd_arr, pkg_cmd_arr) = args node = nodes[rank] nodename = NODESET.get_nodename_from_interface(node) # run 'ssh node pkg_cmd' ssh_cmd_arr.append(node) ssh_cmd_arr.extend(pkg_cmd_arr) synctool_lib.run_with_nodename(ssh_cmd_arr, nodename)
def worker_synctool(rank, args):
'''runs rsync of $masterdir to the nodes and ssh+synctool in parallel'''
(nodes, rsync_cmd_arr, ssh_cmd_arr, synctool_cmd_arr) = args
node = nodes[rank]
nodename = NODESET.get_nodename_from_interface(node)
if rsync_cmd_arr != None:
# rsync masterdir to the node
rsync_cmd_arr.append('%s:%s/' % (node, synctool_param.MASTERDIR))
synctool_lib.run_with_nodename(rsync_cmd_arr, nodename)
# run 'ssh node synctool_cmd'
ssh_cmd_arr.append(node)
ssh_cmd_arr.extend(synctool_cmd_arr)
synctool_lib.run_with_nodename(ssh_cmd_arr, nodename)
def worker_scp(rank, args):
"""runs scp (remote copy) to node"""
if synctool_lib.DRY_RUN: # got here for nothing
return
(nodes, scp_cmd_arr, files_str) = args
node = nodes[rank]
nodename = NODESET.get_nodename_from_interface(node)
# note that the fileset already had been added to scp_cmd_arr
if DESTDIR:
scp_cmd_arr.append("%s:%s" % (node, DESTDIR))
else:
scp_cmd_arr.append("%s:" % node)
synctool_lib.run_with_nodename(scp_cmd_arr, nodename)
def worker_ssh(rank, args): if synctool_lib.DRY_RUN: # got here for nothing return (nodes, ssh_cmd_arr, remote_cmd_arr) = args node = nodes[rank] nodename = NODESET.get_nodename_from_interface(node) if nodename == synctool_param.NODENAME: # is this node the local node? Then do not use ssh ssh_cmd_arr = [] else: ssh_cmd_arr.append(node) ssh_cmd_arr.extend(remote_cmd_arr) # execute ssh+remote command and show output with the nodename synctool_lib.run_with_nodename(ssh_cmd_arr, nodename)
def upload(interface, upload_filename, upload_suffix=None):
'''copy a file from a node into the overlay/ tree'''
if not synctool_param.SCP_CMD:
stderr('%s: error: scp_cmd has not been defined in %s' % (os.path.basename(sys.argv[0]),
synctool_param.CONF_FILE))
sys.exit(-1)
if upload_filename[0] != '/':
stderr('error: the filename to upload must be an absolute path')
sys.exit(-1)
trimmed_upload_fn = upload_filename[1:] # remove leading slash
import synctool_overlay
# make the known groups lists
synctool_config.remove_ignored_groups()
synctool_param.MY_GROUPS = synctool_config.get_my_groups()
synctool_param.ALL_GROUPS = synctool_config.make_all_groups()
if upload_suffix and not upload_suffix in synctool_param.ALL_GROUPS:
stderr("no such group '%s'" % upload_suffix)
sys.exit(-1)
# shadow DRY_RUN because that var can not be used correctly here
if '-f' in PASS_ARGS or '--fix' in PASS_ARGS:
dry_run = False
else:
dry_run = True
if not synctool_lib.QUIET:
stdout('DRY RUN, not uploading any files')
terse(synctool_lib.TERSE_DRYRUN, 'not uploading any files')
node = NODESET.get_nodename_from_interface(interface)
# pretend that the current node is now the given node;
# this is needed for find() to find the most optimal reference for the file
orig_NODENAME = synctool_param.NODENAME
synctool_param.NODENAME = node
synctool_config.insert_group(node, node)
orig_MY_GROUPS = synctool_param.MY_GROUPS[:]
synctool_param.MY_GROUPS = synctool_config.get_my_groups()
# see if file is already in the repository
(obj, err) = synctool_overlay.find_terse(synctool_overlay.OV_OVERLAY, upload_filename)
if err == synctool_overlay.OV_FOUND_MULTIPLE:
# multiple source possible
# possibilities have already been printed
sys.exit(1)
if err == synctool_overlay.OV_NOT_FOUND:
# no source path found
if string.find(upload_filename, '...') >= 0:
stderr("%s is not in the repository, don't know what to map this path to\n"
"Please give the full path instead of a terse path, or touch the source file\n"
"in the repository first and try again"
% os.path.basename(upload_filename))
sys.exit(1)
# it wasn't a terse path, throw a source path together
# This picks the first overlay dir as default source, which may not be correct
# but it is a good guess
repos_filename = os.path.join(synctool_param.OVERLAY_DIRS[0], trimmed_upload_fn)
if upload_suffix:
repos_filename = repos_filename + '._' + upload_suffix
else:
repos_filename = repos_filename + '._' + node # use _nodename as default suffix
else:
if upload_suffix:
# remove the current group suffix an add the specified suffix to the filename
arr = string.split(obj.src_path, '.')
if len(arr) > 1 and arr[-1][0] == '_':
repos_filename = string.join(arr[:-1], '.')
repos_filename = repos_filename + '._' + upload_suffix
else:
repos_filename = obj.src_path
synctool_param.NODENAME = orig_NODENAME
synctool_param.MY_GROUPS = orig_MY_GROUPS
verbose('%s:%s uploaded as %s' % (node, upload_filename, repos_filename))
terse(synctool_lib.TERSE_UPLOAD, repos_filename)
unix_out('%s %s:%s %s' % (synctool_param.SCP_CMD, interface, upload_filename, repos_filename))
if dry_run:
stdout('would be uploaded as %s' % synctool_lib.prettypath(repos_filename))
else:
# first check if the directory in the repository exists
repos_dir = os.path.dirname(repos_filename)
stat = synctool_stat.SyncStat(repos_dir)
if not stat.exists():
verbose('making directory %s' % synctool_lib.prettypath(repos_dir))
unix_out('mkdir -p %s' % repos_dir)
synctool_lib.mkdir_p(repos_dir)
# make scp command array
scp_cmd_arr = shlex.split(synctool_param.SCP_CMD)
scp_cmd_arr.append('%s:%s' % (interface, upload_filename))
scp_cmd_arr.append(repos_filename)
synctool_lib.run_with_nodename(scp_cmd_arr, NODESET.get_nodename_from_interface(interface))
if os.path.isfile(repos_filename):
stdout('uploaded %s' % synctool_lib.prettypath(repos_filename))
