def _start_cluster_simulation(self, subdomains, cluster=None): """Starts a simulation on a cluster of nodes.""" if cluster is None: try: cluster = imp.load_source('cluster', self.config.cluster_spec) except IOError: cluster = imp.load_source('cluster', os.path.expanduser('~/.sailfish/{0}'.format(self.config.cluster_spec))) self._cluster_gateways = [] self._node_subdomains = split_subdomains_between_nodes(cluster.nodes, subdomains) import execnet for _, node in zip(self._node_subdomains, cluster.nodes): self._cluster_gateways.append(execnet.makegateway(node.host)) # Copy files to remote nodes if necessary. if self.config.cluster_sync: local, dest = self.config.cluster_sync.split(':') assert dest[0] != '/', 'Only relative paths are supported on remote nodes.' rsync = execnet.RSync(local) for gw in self._cluster_gateways: rsync.add_target(gw, dest) rsync.send() subdomain_id_to_addr = {} for node_id, subdomains in enumerate(self._node_subdomains): for subdomain in subdomains: subdomain_id_to_addr[subdomain.id] = cluster.nodes[node_id].addr self._cluster_channels = [] import sys for i, (node, gw) in enumerate(zip(cluster.nodes, self._cluster_gateways)): # Assign specific GPUs from this node, as defined by the cluster # config file. node_config = copy.copy(self.config) node_config.gpus = cluster.nodes[i].gpus for k, v in node.settings.items(): setattr(node_config, k, v) self._cluster_channels.append( gw.remote_exec(_start_cluster_machine_master, args=pickle.dumps((node_config, self._node_subdomains[i])), main_script=sys.argv[0], lb_class_name=self._lb_class.__name__, subdomain_addr_map=subdomain_id_to_addr, iface=node.iface)) ports = {} for channel in self._cluster_channels: data = channel.receive() # If a string is received, print it to help with debugging. if type(data) is str: print(data) else: ports.update(data) for channel in self._cluster_channels: channel.send(ports)
def _ship(self, host): head = host.rpc.git_current_head() if head is None: bundle_range = self.branch else: head = head.decode("ascii") bundle_range = "{head}..{branch}".format( head=head, branch=self.branch) fd, bundle_file = tempfile.mkstemp() os.close(fd) out, err = cmd( "git bundle create {file} {range}".format( file=bundle_file, range=bundle_range), acceptable_returncodes=[0, 128]) if "create empty bundle" in err: return change_size = os.stat(bundle_file).st_size output.annotate( "Sending {} bytes of changes".format(change_size), debug=True) rsync = execnet.RSync(bundle_file, verbose=False) rsync.add_target(host.gateway, host.remote_repository + "/batou-bundle.git") rsync.send() os.unlink(bundle_file) output.annotate("Unbundling changes", debug=True) host.rpc.git_unbundle_code()
def update(self, host): env = self.environment blacklist = ['.batou', 'work', '.git', '.hg', '.vagrant', '.kitchen', '.batou-lock'] for candidate in os.listdir(env.base_dir): if candidate in blacklist: continue source = os.path.join(env.base_dir, candidate) target = os.path.join(host.remote_base, candidate) output.annotate("rsync: {} -> {}".format(source, target), debug=True) rsync = execnet.RSync(source, verbose=False) rsync.add_target(host.gateway, target, delete=True) rsync.send()
def _ship(self, host): heads = host.rpc.hg_current_heads() if not heads: raise ValueError("Remote repository did not find any heads. " "Can not continue creating a bundle.") fd, bundle_file = tempfile.mkstemp() os.close(fd) bases = " ".join("--base {}".format(x) for x in heads) cmd("hg -qy bundle {} {}".format(bases, bundle_file), acceptable_returncodes=[0, 1]) change_size = os.stat(bundle_file).st_size if not change_size: return output.annotate( "Sending {} bytes of changes".format(change_size), debug=True) rsync = execnet.RSync(bundle_file, verbose=False) rsync.add_target(host.gateway, host.remote_repository + "/batou-bundle.hg") rsync.send() os.unlink(bundle_file) output.annotate("Unbundling changes", debug=True) host.rpc.hg_unbundle_code()