class SwissFelCluster: def __init__(self, cores=8, memory="24 GB", workers=5): self.cluster = SLURMCluster(cores=cores, memory=memory) self.client = Client(self.cluster) self.ip = socket.gethostbyname(socket.gethostname()) self.dashboard_port_scheduler = self.client._scheduler_identity.get( "services")["dashboard"] self.username = getpass.getuser() def _repr_html_(self): return self.client._repr_html_() def scale_workers(self, N_workers): self.cluster.scale(N_workers) def create_dashboard_tunnel(self, ssh_host="ra"): print( "type following commant in a terminal, if port is taken, change first number in command." ) print(" ".join([ f"jupdbport={self.dashboard_port_scheduler}", "&&", "ssh", "-f", "-L", f"$jupdbport:{self.ip}:{self.dashboard_port_scheduler}", f"{self.username}@{ssh_host}", "sleep 10", "&&", "firefox", "http://localhost:$jupdbport", ]))
if __name__ == '__main__': ex_path = sys.argv[1] tf_path = sys.argv[2] net_out_path = sys.argv[3] meta_out_path = sys.argv[4] start_time = time.time() expression_matrix = pd.read_csv(ex_path, sep='\t') tf_names = load_tf_names(tf_path) gene_names = expression_matrix.columns client = Client(LocalCluster()) print(client._repr_html_()) network_graph, meta_graph = create_graph( expression_matrix.as_matrix(), gene_names, tf_names, "GBM", SGBM_KWARGS, client=client, # broadcast! early_stop_window_length=25, include_meta=True) # Good! a, b = client.persist([network_graph, meta_graph]) network_df = a.compute(sync=True) meta_df = b.compute(sync=True)