def _start_cluster_simulation(self, subdomains, cluster=None):
"""Starts a simulation on a cluster of nodes."""
if cluster is None:
try:
cluster = imp.load_source('cluster', self.config.cluster_spec)
except IOError:
cluster = imp.load_source('cluster',
os.path.expanduser('~/.sailfish/{0}'.format(self.config.cluster_spec)))
self._cluster_gateways = []
self._node_subdomains = split_subdomains_between_nodes(cluster.nodes, subdomains)
import execnet
for _, node in zip(self._node_subdomains, cluster.nodes):
self._cluster_gateways.append(execnet.makegateway(node.host))
# Copy files to remote nodes if necessary.
if self.config.cluster_sync:
local, dest = self.config.cluster_sync.split(':')
assert dest[0] != '/', 'Only relative paths are supported on remote nodes.'
rsync = execnet.RSync(local)
for gw in self._cluster_gateways:
rsync.add_target(gw, dest)
rsync.send()
subdomain_id_to_addr = {}
for node_id, subdomains in enumerate(self._node_subdomains):
for subdomain in subdomains:
subdomain_id_to_addr[subdomain.id] = cluster.nodes[node_id].addr
self._cluster_channels = []
import sys
for i, (node, gw) in enumerate(zip(cluster.nodes, self._cluster_gateways)):
# Assign specific GPUs from this node, as defined by the cluster
# config file.
node_config = copy.copy(self.config)
node_config.gpus = cluster.nodes[i].gpus
for k, v in node.settings.items():
setattr(node_config, k, v)
self._cluster_channels.append(
gw.remote_exec(_start_cluster_machine_master,
args=pickle.dumps((node_config, self._node_subdomains[i])),
main_script=sys.argv[0],
lb_class_name=self._lb_class.__name__,
subdomain_addr_map=subdomain_id_to_addr,
iface=node.iface))
ports = {}
for channel in self._cluster_channels:
data = channel.receive()
# If a string is received, print it to help with debugging.
if type(data) is str:
print(data)
else:
ports.update(data)
for channel in self._cluster_channels:
channel.send(ports)
def _ship(self, host):
head = host.rpc.git_current_head()
if head is None:
bundle_range = self.branch
else:
head = head.decode("ascii")
bundle_range = "{head}..{branch}".format(
head=head, branch=self.branch)
fd, bundle_file = tempfile.mkstemp()
os.close(fd)
out, err = cmd(
"git bundle create {file} {range}".format(
file=bundle_file, range=bundle_range),
acceptable_returncodes=[0, 128])
if "create empty bundle" in err:
return
change_size = os.stat(bundle_file).st_size
output.annotate(
"Sending {} bytes of changes".format(change_size), debug=True)
rsync = execnet.RSync(bundle_file, verbose=False)
rsync.add_target(host.gateway,
host.remote_repository + "/batou-bundle.git")
rsync.send()
os.unlink(bundle_file)
output.annotate("Unbundling changes", debug=True)
host.rpc.git_unbundle_code()
def update(self, host):
env = self.environment
blacklist = ['.batou', 'work', '.git', '.hg', '.vagrant', '.kitchen',
'.batou-lock']
for candidate in os.listdir(env.base_dir):
if candidate in blacklist:
continue
source = os.path.join(env.base_dir, candidate)
target = os.path.join(host.remote_base, candidate)
output.annotate("rsync: {} -> {}".format(source, target),
debug=True)
rsync = execnet.RSync(source, verbose=False)
rsync.add_target(host.gateway, target, delete=True)
rsync.send()
def _ship(self, host):
heads = host.rpc.hg_current_heads()
if not heads:
raise ValueError("Remote repository did not find any heads. "
"Can not continue creating a bundle.")
fd, bundle_file = tempfile.mkstemp()
os.close(fd)
bases = " ".join("--base {}".format(x) for x in heads)
cmd("hg -qy bundle {} {}".format(bases, bundle_file),
acceptable_returncodes=[0, 1])
change_size = os.stat(bundle_file).st_size
if not change_size:
return
output.annotate(
"Sending {} bytes of changes".format(change_size), debug=True)
rsync = execnet.RSync(bundle_file, verbose=False)
rsync.add_target(host.gateway,
host.remote_repository + "/batou-bundle.hg")
rsync.send()
os.unlink(bundle_file)
output.annotate("Unbundling changes", debug=True)
host.rpc.hg_unbundle_code()
