def exchange1d_nonblocking(cart1d, u, left, right): req = [MPI.Request() for k in range(4)] u_left, u_right, u_left_ghost, u_right_ghost = assign_aliases(u) req[0] = cart1d.Irecv(u_right_ghost, source=right) req[1] = cart1d.Irecv(u_left_ghost, source=left) req[2] = cart1d.Isend(u_left, dest=left) req[3] = cart1d.Isend(u_right, dest=right) return req
def spawn(self): """ Spawn MPI processes for and execute each of the managed targets. """ if self._is_parent: # Find the path to the mpi_backend.py script (which should be in the # same directory as this module: parent_dir = os.path.dirname(__file__) mpi_backend_path = os.path.join(parent_dir, 'mpi_backend.py') # Spawn processes: self._intercomm = MPI.COMM_SELF.Spawn(sys.executable, args=[mpi_backend_path], maxprocs=len(self)) # First, transmit twiggy logging emitters to spawned processes so # that they can configure their logging facilities: for i in self._targets.keys(): self._intercomm.send(twiggy.emitters, i) # Next, serialize the routing table ONCE and then transmit it to all # of the child nodes: try: routing_table = self.routing_table except: routing_table = RoutingTable() self.log_warning( 'Routing Table is null, using empty routing table.') self._intercomm.bcast(routing_table, root=MPI.ROOT) # Transmit class to instantiate, globals required by the class, and # the constructor arguments; the backend will wait to receive # them and then start running the targets on the appropriate nodes. req = MPI.Request() r_list = [] for i in self._targets.keys(): target_globals = all_global_vars(self._targets[i]) # Serializing atexit with dill appears to fail in virtualenvs # sometimes if atexit._exithandlers contains an unserializable function: if 'atexit' in target_globals: del target_globals['atexit'] data = (self._targets[i], target_globals, self._kwargs[i]) r_list.append(self._intercomm.isend(data, i)) # Need to clobber data to prevent all_global_vars from # including it in its output: del data req.Waitall(r_list)
def wait(self): """ Wait for execution to complete. """ # Start listening for control messages: r_ctrl = [] try: d = self.intercomm.irecv(source=MPI.ANY_SOURCE, tag=self._ctrl_tag) except TypeError: # irecv() in mpi4py 1.3.1 stable uses 'dest' instead of 'source': d = self.intercomm.irecv(dest=MPI.ANY_SOURCE, tag=self._ctrl_tag) r_ctrl.append(d) workers = range(len(self)) req = MPI.Request() while True: # Check for control messages from workers: flag, msg_list = req.testall(r_ctrl) if flag: msg = msg_list[0] if msg[0] == 'done': self.log_info('removing %s from worker list' % msg[1]) workers.remove(msg[1]) # Additional control messages from the workers are processed # here: else: self.process_worker_msg(msg) # Get new control messages: r_ctrl = [] try: d = self.intercomm.irecv(source=MPI.ANY_SOURCE, tag=self._ctrl_tag) except TypeError: # irecv() in mpi4py 1.3.1 stable uses 'dest' instead of 'source': d = self.intercomm.irecv(dest=MPI.ANY_SOURCE, tag=self._ctrl_tag) r_ctrl.append(d) if not workers: self.log_info('finished running manager') break
def naive_linear_regression(n_procs, n_samples, n_features, input_dir, n_stragglers, is_real_data, params, add_delay, update_rule): assert update_rule in ('GD', 'AGD') comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() n_workers = n_procs - 1 rounds = params[0] #beta=np.zeros(n_features) beta = np.random.randn(n_features) # Loading data on workers if (rank): if not is_real_data: X_current = load_data(input_dir + str(rank) + ".dat") y = load_data(input_dir + "label.dat") else: X_current = load_sparse_csr(input_dir + str(rank)) y = load_data(input_dir + "label.dat") rows_per_worker = X_current.shape[0] y_current = y[(rank - 1) * rows_per_worker:rank * rows_per_worker] # Initializing relevant variables if (rank): predy = X_current.dot(beta) #g = -X_current.T.dot(np.divide(y_current,np.exp(np.multiply(predy,y_current))+1)) g = -2 * X_current.T.dot(y_current - predy) send_req = MPI.Request() recv_reqs = [] else: msgBuffers = [np.zeros(n_features) for i in range(n_procs - 1)] g = np.zeros(n_features) betaset = np.zeros((rounds, n_features)) timeset = np.zeros(rounds) worker_timeset = np.zeros((rounds, n_procs - 1)) request_set = [] recv_reqs = [] cnt_completed = 0 status = MPI.Status() eta0 = params[2] # ----- learning rate schedule alpha = params[1] # --- coefficient of l2 regularization utemp = np.zeros(n_features) # for accelerated gradient descent # Posting all Irecv requests for master and workers if (rank): for i in range(rounds): req = comm.Irecv([beta, MPI.DOUBLE], source=0, tag=i) recv_reqs.append(req) else: for i in range(rounds): recv_reqs = [] for j in range(1, n_procs): req = comm.Irecv([msgBuffers[j - 1], MPI.DOUBLE], source=j, tag=i) recv_reqs.append(req) request_set.append(recv_reqs) ######################################################################################################## comm.Barrier() if rank == 0: orig_start_time = time.time() print("---- Starting Naive Iterations ----") for i in range(rounds): if rank == 0: if (i % 10 == 0): print("\t >>> At Iteration %d" % (i)) start_time = time.time() for l in range(1, n_procs): comm.Isend([beta, MPI.DOUBLE], dest=l, tag=i) g[:] = 0 cnt_completed = 0 while cnt_completed < n_procs - 1: req_done = MPI.Request.Waitany(request_set[i], status) src = status.Get_source() worker_timeset[i, src - 1] = time.time() - start_time request_set[i].pop(req_done) g += msgBuffers[src - 1] # add the partial gradients cnt_completed += 1 grad_multiplier = eta0[i] / n_samples # ---- update step for gradient descent if update_rule == "GD": np.subtract((1 - 2 * alpha * eta0[i]) * beta, grad_multiplier * g, out=beta) elif update_rule == "AGD": # ---- updates for accelerated gradient descent theta = 2.0 / (i + 2.0) ytemp = (1 - theta) * beta + theta * utemp betatemp = ytemp - grad_multiplier * g - (2 * alpha * eta0[i]) * beta utemp = beta + (betatemp - beta) * (1 / theta) beta[:] = betatemp else: raise Exception("Error update rule") timeset[i] = time.time() - start_time betaset[i, :] = beta else: recv_reqs[i].Wait() # sendTestBuf = send_req.test() # if not sendTestBuf[0]: # send_req.Cancel() predy = X_current.dot(beta) # TODO: gradient of linear regression #g = X_current.T.dot(np.divide(y_current,np.exp(np.multiply(predy,y_current))+1)) g = X_current.T.dot(y_current - predy) g *= -2 ########################################## straggler simulation ################################################### if add_delay == 1: np.random.seed(seed=i) #straggler_indices = np.random.choice([t for t in range(1, n_workers+1)], n_stragglers, replace=False) #if rank in straggler_indices: # time.sleep(time_sleep) artificial_delays = np.random.exponential(0.5, n_workers) delay = artificial_delays[rank - 1] time.sleep(delay) ################################################################################################################### send_req = comm.Isend([g, MPI.DOUBLE], dest=0, tag=i) ##################################################################################################### comm.Barrier() if rank == 0: elapsed_time = time.time() - orig_start_time print("Total Time Elapsed: %.3f" % (elapsed_time)) # Load all training data if not is_real_data: X_train = load_data(input_dir + "1.dat") print(">> Loaded 1") for j in range(2, n_procs - 1): X_temp = load_data(input_dir + str(j) + ".dat") X_train = np.vstack((X_train, X_temp)) print(">> Loaded " + str(j)) else: X_train = load_sparse_csr(input_dir + "1") for j in range(2, n_procs - 1): X_temp = load_sparse_csr(input_dir + str(j)) X_train = sps.vstack((X_train, X_temp)) y_train = load_data(input_dir + "label.dat") y_train = y_train[0:X_train.shape[0]] # Load all testing data y_test = load_data(input_dir + "label_test.dat") if not is_real_data: X_test = load_data(input_dir + "test_data.dat") else: X_test = load_sparse_csr(input_dir + "test_data") n_train = X_train.shape[0] n_test = X_test.shape[0] training_loss = np.zeros(rounds) testing_loss = np.zeros(rounds) auc_loss = np.zeros(rounds) from sklearn.metrics import roc_curve, auc for i in range(rounds): beta = np.squeeze(betaset[i, :]) predy_train = X_train.dot(beta) predy_test = X_test.dot(beta) training_loss[i] = calculate_mse(y_train, predy_train, n_train) testing_loss[i] = calculate_mse(y_test, predy_test, n_test) # TODOs: for linear regressiuon there is no fp tp any more, change to loss #fpr, tpr, thresholds = roc_curve(y_test,predy_test, pos_label=1) #auc_loss[i] = auc(fpr,tpr) print( "Iteration %d: Train Loss = %.6f, Test Loss = %.6f, Total time taken =%5.3f" % (i, training_loss[i], testing_loss[i], timeset[i])) output_dir = input_dir + "results/" if not os.path.exists(output_dir): os.makedirs(output_dir) #save_vector(training_loss, output_dir+"naive_acc_training_loss.dat") #save_vector(testing_loss, output_dir+"naive_acc_testing_loss.dat") #save_vector(auc_loss, output_dir+"naive_acc_auc.dat") #save_vector(timeset, output_dir+"naive_acc_timeset.dat") #save_matrix(worker_timeset, output_dir+"naive_acc_worker_timeset.dat") print(">>> Done") comm.Barrier()
def setUp(self): self.REQUESTS = [MPI.Request() for i in range(5)] self.STATUSES = [MPI.Status() for i in range(5)]
def setUp(self): self.REQUEST = MPI.Request() self.STATUS = MPI.Status()
def _sync(self): """ Send output data and receive input data. """ if self.time_sync: start = time.time() requests = [] # For each destination module, extract elements from the current # module's port data array, copy them to a contiguous array, and # transmit the latter: for dest_id, dest_rank in zip(self._out_ids, self._out_ranks): # Copy data into destination buffer: if self._out_buf['gpot'][dest_id] is not None: set_by_inds(self._out_buf['gpot'][dest_id], self._out_port_dict_ids['gpot'][dest_id], self.data['gpot'], 'src') if not self.time_sync: self.log_info('gpot data sent to %s: %s' % \ (dest_id, str(self._out_buf['gpot'][dest_id]))) r = MPI.COMM_WORLD.Isend([ self._out_buf_int['gpot'][dest_id], self._out_buf_mtype['gpot'][dest_id] ], dest_rank, GPOT_TAG) requests.append(r) if self._out_buf['spike'][dest_id] is not None: set_by_inds(self._out_buf['spike'][dest_id], self._out_port_dict_ids['spike'][dest_id], self.data['spike'], 'src') if not self.time_sync: self.log_info('spike data sent to %s: %s' % \ (dest_id, str(self._out_buf['spike'][dest_id]))) r = MPI.COMM_WORLD.Isend([ self._out_buf_int['spike'][dest_id], self._out_buf_mtype['spike'][dest_id] ], dest_rank, SPIKE_TAG) requests.append(r) if not self.time_sync: self.log_info('sending to %s' % dest_id) if not self.time_sync: self.log_info('sent all data from %s' % self.id) # For each source module, receive elements and copy them into the # current module's port data array: for src_id, src_rank in zip(self._in_ids, self._in_ranks): if self._in_buf['gpot'][src_id] is not None: r = MPI.COMM_WORLD.Irecv([ self._in_buf_int['gpot'][src_id], self._in_buf_mtype['gpot'][src_id] ], source=src_rank, tag=GPOT_TAG) requests.append(r) if self._in_buf['spike'][src_id] is not None: r = MPI.COMM_WORLD.Irecv([ self._in_buf_int['spike'][src_id], self._in_buf_mtype['spike'][src_id] ], source=src_rank, tag=SPIKE_TAG) requests.append(r) if not self.time_sync: self.log_info('receiving from %s' % src_id) if requests: req = MPI.Request() req.Waitall(requests) if not self.time_sync: self.log_info('all data were received by %s' % self.id) # Copy received elements into the current module's data array: for src_id in self._in_ids: if self._in_buf['gpot'][src_id] is not None: if not self.time_sync: self.log_info('gpot data received from %s: %s' % \ (src_id, str(self._in_buf['gpot'][src_id]))) set_by_inds_from_inds( self.data['gpot'], self._in_port_dict_ids['gpot'][src_id], self._in_buf['gpot'][src_id], self._in_port_dict_buf_ids['gpot'][src_id]) if self._in_buf['spike'][src_id] is not None: if not self.time_sync: self.log_info('spike data received from %s: %s' % \ (src_id, str(self._in_buf['spike'][src_id]))) set_by_inds_from_inds( self.data['spike'], self._in_port_dict_ids['spike'][src_id], self._in_buf['spike'][src_id], self._in_port_dict_buf_ids['spike'][src_id]) # Save timing data: if self.time_sync: stop = time.time() n_gpot = 0 n_spike = 0 for src_id in self._in_ids: n_gpot += len(self._in_buf['gpot'][src_id]) n_spike += len(self._in_buf['spike'][src_id]) self.log_info('sent timing data to master') self.intercomm.isend(['sync_time', (self.rank, self.steps, start, stop, n_gpot*self.pm['gpot'].dtype.itemsize+\ n_spike*self.pm['spike'].dtype.itemsize)], dest=0, tag=self._ctrl_tag) else: self.log_info('saved all data received by %s' % self.id)
def _sync(self): """ Send output data and receive input data. """ if self.time_sync: start = time.time() req = MPI.Request() requests = [] # For each destination module, extract elements from the current # module's port data array, copy them to a contiguous array, and # transmit the latter: for dest_id, dest_rank in zip(self._out_ids, self._out_ranks): # Get source ports in current module that are connected to the # destination module: data_gpot = self.pm['gpot'].get_by_inds( self._out_port_dict_ids['gpot'][dest_id]) data_spike = self.pm['spike'].get_by_inds( self._out_port_dict_ids['spike'][dest_id]) if not self.time_sync: self.log_info('gpot data being sent to %s: %s' % \ (dest_id, str(data_gpot))) self.log_info('spike data being sent to %s: %s' % \ (dest_id, str(data_spike))) r = MPI.COMM_WORLD.Isend( [data_gpot, dtype_to_mpi(data_gpot.dtype)], dest_rank, GPOT_TAG) requests.append(r) r = MPI.COMM_WORLD.Isend( [data_spike, dtype_to_mpi(data_spike.dtype)], dest_rank, SPIKE_TAG) requests.append(r) if not self.time_sync: self.log_info('sending to %s' % dest_id) if not self.time_sync: self.log_info('sent all data from %s' % self.id) # For each source module, receive elements and copy them into the # current module's port data array: received_gpot = [] received_spike = [] ind_in_gpot_list = [] ind_in_spike_list = [] for src_id, src_rank in zip(self._in_ids, self._in_ranks): r = MPI.COMM_WORLD.Irecv( [self.data_in['gpot'][src_id], dtype_to_mpi(data_gpot.dtype)], source=src_rank, tag=GPOT_TAG) requests.append(r) r = MPI.COMM_WORLD.Irecv([ self.data_in['spike'][src_id], dtype_to_mpi(data_spike.dtype) ], source=src_rank, tag=SPIKE_TAG) requests.append(r) if not self.time_sync: self.log_info('receiving from %s' % src_id) req.Waitall(requests) if not self.time_sync: self.log_info('received all data received by %s' % self.id) # Copy received elements into the current module's data array: for src_id in self._in_ids: ind_in_gpot = self._in_port_dict_ids['gpot'][src_id] self.pm['gpot'].set_by_inds(ind_in_gpot, self.data_in['gpot'][src_id]) ind_in_spike = self._in_port_dict_ids['spike'][src_id] self.pm['spike'].set_by_inds(ind_in_spike, self.data_in['spike'][src_id]) # Save timing data: if self.time_sync: stop = time.time() n_gpot = 0 n_spike = 0 for src_id in self._in_ids: n_gpot += len(self.data_in['gpot'][src_id]) n_spike += len(self.data_in['spike'][src_id]) self.log_info('sent timing data to master') self.intercomm.isend(['sync_time', (self.rank, self.steps, start, stop, n_gpot*self.pm['gpot'].dtype.itemsize+\ n_spike*self.pm['spike'].dtype.itemsize)], dest=0, tag=self._ctrl_tag) else: self.log_info('saved all data received by %s' % self.id)
def avoidstragg_logistic_regression(n_procs, n_samples, n_features, input_dir, n_stragglers, is_real_data, params): comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() rounds = params[0] beta=np.zeros(n_features) # Loading data on workers if (rank): if not is_real_data: X_current = load_data(input_dir+str(rank)+".dat") y = load_data(input_dir+"label.dat") else: X_current = load_sparse_csr(input_dir+str(rank)) y = load_data(input_dir+"label.dat") rows_per_worker = X_current.shape[0] y_current=y[(rank-1)*rows_per_worker:rank*rows_per_worker] # Initializing relevant variables if (rank): predy = X_current.dot(beta) g = -X_current.T.dot(np.divide(y_current,np.exp(np.multiply(predy,y_current))+1)) send_req = MPI.Request() recv_reqs = [] else: msgBuffers = [np.zeros(n_features) for i in range(n_procs-1)] g=np.zeros(n_features) betaset = np.zeros((rounds, n_features)) timeset = np.zeros(rounds) worker_timeset=np.zeros((rounds, n_procs-1)) request_set = [] recv_reqs = [] send_set = [] cnt_completed = 0 completed_workers = np.ndarray(n_procs-1,dtype=bool) status = MPI.Status() alpha = params[1] # --- coefficient of l2 regularization eta_sequence = params[2] # --- learning rate schedule utemp = np.zeros(n_features) # for accelerated gradient descent # Posting all Irecv requests for master and workers if (rank): for i in range(rounds): req = comm.Irecv([beta, MPI.DOUBLE], source=0, tag=i) recv_reqs.append(req) else: for i in range(rounds): recv_reqs = [] for j in range(1,n_procs): req = comm.Irecv([msgBuffers[j-1], MPI.DOUBLE], source=j, tag=i) recv_reqs.append(req) request_set.append(recv_reqs) ########################################################################################## comm.Barrier() if rank==0: orig_start_time = time.time() print("---- Starting AvoidStragg Iterations with " +str(n_stragglers) + " stragglers ----") for i in range(rounds): if rank==0: if(i%10 == 0): print("\t >>> At Iteration %d" %(i)) start_time = time.time() g[:]=0.0 cnt_completed = 0 completed_workers[:]=False send_set[:] = [] for l in range(1,n_procs): sreq = comm.Isend([beta, MPI.DOUBLE], dest = l, tag = i) send_set.append(sreq) while cnt_completed < n_procs-1-n_stragglers: req_done = MPI.Request.Waitany(request_set[i], status) src = status.Get_source() worker_timeset[i,src-1]=time.time()-start_time request_set[i].pop(req_done) g += msgBuffers[src-1] # add the partial gradients cnt_completed += 1 completed_workers[src-1] = True grad_multiplier = eta_sequence[i]/(n_samples*(n_procs-1-n_stragglers)/(n_procs-1)) # ---- update step for gradient descent # np.subtract((1-2*alpha*eta_sequence[i])*beta , grad_multiplier*g, out=beta) # ---- updates for accelerated gradient descent theta = 2.0/(i+2.0) ytemp = (1-theta)*beta + theta*utemp betatemp = ytemp - grad_multiplier*g - (2*alpha*eta_sequence[i])*beta utemp = beta + (betatemp-beta)*(1/theta) beta[:] = betatemp timeset[i] = time.time() - start_time betaset[i,:] = beta ind_set = [l for l in range(1,n_procs) if not completed_workers[l-1]] for l in ind_set: worker_timeset[i,l-1]=-1 #MPI.Request.Waitall(send_set) #MPI.Request.Waitall(request_set[i]) else: recv_reqs[i].Wait() sendTestBuf = send_req.test() if not sendTestBuf[0]: send_req.Cancel() #print("Worker " + str(rank) + " cancelled send request for Iteration " + str(i)) predy = X_current.dot(beta) g = X_current.T.dot(np.divide(y_current,np.exp(np.multiply(predy,y_current))+1)) g *= -1 send_req = comm.Isend([g, MPI.DOUBLE], dest=0, tag=i) ######################################################################################### comm.Barrier() if rank==0: elapsed_time= time.time() - orig_start_time print ("Total Time Elapsed: %.3f" %(elapsed_time)) # Load all training data if not is_real_data: X_train = load_data(input_dir+"1.dat") for j in range(2,n_procs-1): X_temp = load_data(input_dir+str(j)+".dat") X_train = np.vstack((X_train, X_temp)) else: X_train = load_sparse_csr(input_dir+"1") for j in range(2,n_procs-1): X_temp = load_sparse_csr(input_dir+str(j)) X_train = sps.vstack((X_train, X_temp)) y_train = load_data(input_dir+"label.dat") y_train = y_train[0:X_train.shape[0]] # Load all testing data y_test = load_data(input_dir + "label_test.dat") if not is_real_data: X_test = load_data(input_dir+"test_data.dat") else: X_test = load_sparse_csr(input_dir+"test_data") n_train = X_train.shape[0] n_test = X_test.shape[0] training_loss = np.zeros(rounds) testing_loss = np.zeros(rounds) auc_loss = np.zeros(rounds) from sklearn.metrics import roc_curve, auc for i in range(rounds): beta = np.squeeze(betaset[i,:]) predy_train = X_train.dot(beta) predy_test = X_test.dot(beta) training_loss[i] = calculate_loss(y_train, predy_train, n_train) testing_loss[i] = calculate_loss(y_test, predy_test, n_test) fpr, tpr, thresholds = roc_curve(y_test,predy_test, pos_label=1) auc_loss[i] = auc(fpr,tpr) print("Iteration %d: Train Loss = %5.3f, Test Loss = %5.3f, AUC = %5.3f, Total time taken =%5.3f"%(i, training_loss[i], testing_loss[i], auc_loss[i], timeset[i])) output_dir = input_dir + "results/" if not os.path.exists(output_dir): os.makedirs(output_dir) save_vector(training_loss, output_dir+"avoidstragg_acc_%d_training_loss.dat"%(n_stragglers)) save_vector(testing_loss, output_dir+"avoidstragg_acc_%d_testing_loss.dat"%(n_stragglers)) save_vector(auc_loss, output_dir+"avoidstragg_acc_%d_auc.dat"%(n_stragglers)) save_vector(timeset, output_dir+"avoidstragg_acc_%d_timeset.dat"%(n_stragglers)) save_matrix(worker_timeset, output_dir+"avoidstragg_acc_%d_worker_timeset.dat"%(n_stragglers)) print(">>> Done") comm.Barrier()
def __init__(self, sel, sel_in, sel_out, sel_gpot, sel_spike, data_gpot, data_spike, columns=['interface', 'io', 'type'], ctrl_tag=CTRL_TAG, gpot_tag=GPOT_TAG, spike_tag=SPIKE_TAG, id=None, device=None, routing_table=None, rank_to_id=None, debug=False, time_sync=False): super(Module, self).__init__(ctrl_tag) self.debug = debug self.time_sync = time_sync self.device = device self._gpot_tag = gpot_tag self._spike_tag = spike_tag # Require several necessary attribute columns: if 'interface' not in columns: raise ValueError('interface column required') if 'io' not in columns: raise ValueError('io column required') if 'type' not in columns: raise ValueError('type column required') # Manually register the file close method associated with MPIOutput # so that it is called by atexit before MPI.Finalize() (if the file is # closed after MPI.Finalize() is called, an error will occur): for k, v in iteritems(twiggy.emitters): if isinstance(v._output, MPIOutput): atexit.register(v._output.close) # Ensure that the input and output port selectors respectively # select mutually exclusive subsets of the set of all ports exposed by # the module: if not SelectorMethods.is_in(sel_in, sel): raise ValueError( 'input port selector not in selector of all ports') if not SelectorMethods.is_in(sel_out, sel): raise ValueError( 'output port selector not in selector of all ports') if not SelectorMethods.are_disjoint(sel_in, sel_out): raise ValueError('input and output port selectors not disjoint') # Ensure that the graded potential and spiking port selectors # respectively select mutually exclusive subsets of the set of all ports # exposed by the module: if not SelectorMethods.is_in(sel_gpot, sel): raise ValueError('gpot port selector not in selector of all ports') if not SelectorMethods.is_in(sel_spike, sel): raise ValueError( 'spike port selector not in selector of all ports') if not SelectorMethods.are_disjoint(sel_gpot, sel_spike): raise ValueError('gpot and spike port selectors not disjoint') # Save routing table and mapping between MPI ranks and module IDs: self.routing_table = routing_table self.rank_to_id = rank_to_id # Generate a unique ID if none is specified: if id is None: self.id = uid() else: # If a unique ID was specified and the routing table is not empty # (i.e., there are connections between multiple modules), the id # must be a node in the routing table: if routing_table is not None and len(routing_table.ids) and \ not routing_table.has_node(id): raise ValueError('routing table must contain specified ' 'module ID: {}'.format(id)) self.id = id # Reformat logger name: LoggerMixin.__init__(self, 'mod %s' % self.id) # Create module interface given the specified ports: self.interface = Interface(sel, columns) # Set the interface ID to 0; we assume that a module only has one interface: self.interface[sel, 'interface'] = 0 # Set the port attributes: self.interface[sel_in, 'io'] = 'in' self.interface[sel_out, 'io'] = 'out' self.interface[sel_gpot, 'type'] = 'gpot' self.interface[sel_spike, 'type'] = 'spike' # Find the input and output ports: self.in_ports = self.interface.in_ports().to_tuples() self.out_ports = self.interface.out_ports().to_tuples() # Find the graded potential and spiking ports: self.gpot_ports = self.interface.gpot_ports().to_tuples() self.spike_ports = self.interface.spike_ports().to_tuples() self.in_gpot_ports = self.interface.in_ports().gpot_ports().to_tuples() self.in_spike_ports = self.interface.in_ports().spike_ports( ).to_tuples() self.out_gpot_ports = self.interface.out_ports().gpot_ports( ).to_tuples() self.out_spike_ports = self.interface.out_ports().spike_ports( ).to_tuples() # Set up mapper between port identifiers and their associated data: if len(data_gpot) != len(self.gpot_ports): raise ValueError('incompatible gpot port data array length') if len(data_spike) != len(self.spike_ports): raise ValueError('incompatible spike port data array length') self.data = {} self.data['gpot'] = data_gpot self.data['spike'] = data_spike self.pm = {} self.pm['gpot'] = PortMapper(sel_gpot, self.data['gpot'], make_copy=False) self.pm['spike'] = PortMapper(sel_spike, self.data['spike'], make_copy=False) # MPI Request object for resolving asynchronous transfers: self.req = MPI.Request()
def coded_logistic_regression(n_procs, n_samples, n_features, input_dir, n_stragglers, is_real_data, params): comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() rounds = params[0] n_workers = n_procs - 1 rows_per_worker = n_samples / (n_procs - 1) # Loading the data if (rank): if not is_real_data: y = load_data(input_dir + "label.dat") X_current = np.zeros([(1 + n_stragglers) * rows_per_worker, n_features]) y_current = np.zeros((1 + n_stragglers) * rows_per_worker) for i in range(1 + n_stragglers): X_current[i * rows_per_worker:(i + 1) * rows_per_worker, :] = load_data(input_dir + str( (rank - 1 + i) % n_workers + 1) + ".dat") y_current[i * rows_per_worker:(i + 1) * rows_per_worker] = y[( (rank - 1 + i) % n_workers) * rows_per_worker:( (rank - 1 + i) % n_workers + 1) * rows_per_worker] else: y_current = np.zeros((1 + n_stragglers) * rows_per_worker) y = load_data(input_dir + "label.dat") for i in range(1 + n_stragglers): if i == 0: X_current = load_sparse_csr(input_dir + str( (rank - 1 + i) % n_workers + 1)) else: X_temp = load_sparse_csr(input_dir + str((rank - 1 + i) % n_workers + 1)) X_current = sps.vstack((X_current, X_temp)) y_current[i * rows_per_worker:(i + 1) * rows_per_worker] = y[( (rank - 1 + i) % n_workers) * rows_per_worker:( (rank - 1 + i) % n_workers + 1) * rows_per_worker] # Initializing relevant variables B = np.zeros((n_workers, n_workers)) beta = np.zeros(n_features) if (rank): predy = X_current.dot(beta) g = -X_current.T.dot( np.divide(y_current, np.exp(np.multiply(predy, y_current)) + 1)) send_req = MPI.Request() recv_reqs = [] else: B = getB(n_workers, n_stragglers) # A = np.zeros((int(sp.binom(n_workers,n_stragglers)),n_workers)) # A=getA(B,n_workers,n_stragglers) msgBuffers = np.array( [np.zeros(n_features) for i in range(n_procs - 1)]) g = np.zeros(n_features) A_row = np.zeros((1, n_procs - 1)) betaset = np.zeros((rounds, n_features)) timeset = np.zeros(rounds) worker_timeset = np.zeros((rounds, n_procs - 1)) request_set = [] recv_reqs = [] send_set = [] cnt_completed = 0 completed_workers = np.ndarray(n_procs - 1, dtype=bool) status = MPI.Status() eta0 = params[2] # ----- learning rate alpha = params[1] # --- coefficient of l2 regularization utemp = np.zeros(n_features) # for accelerated gradient descent B = comm.bcast(B, root=0) # Setting up y_current_mod on all workers if (rank): y_current_mod = np.zeros((1 + n_stragglers) * rows_per_worker) for i in range(1 + n_stragglers): y_current_mod[i * rows_per_worker:(i + 1) * rows_per_worker] = B[rank - 1, ( (rank - 1 + i) % n_workers )] * y_current[i * rows_per_worker:(i + 1) * rows_per_worker] # Posting all Irecv requests for master and workers if (rank): for i in range(rounds): req = comm.Irecv([beta, MPI.DOUBLE], source=0, tag=i) recv_reqs.append(req) else: for i in range(rounds): recv_reqs = [] for j in range(1, n_procs): req = comm.Irecv([msgBuffers[j - 1], MPI.DOUBLE], source=j, tag=i) recv_reqs.append(req) request_set.append(recv_reqs) ####################################################################################################### comm.Barrier() if rank == 0: print("---- Starting Coded Iterations for " + str(n_stragglers) + " stragglers ----") orig_start_time = time.time() for i in range(rounds): if rank == 0: if (i % 10 == 0): print("\t >>> At Iteration %d" % (i)) A_row[:] = 0 send_set[:] = [] completed_workers[:] = False cnt_completed = 0 start_time = time.time() for l in range(1, n_procs): sreq = comm.Isend([beta, MPI.DOUBLE], dest=l, tag=i) send_set.append(sreq) while cnt_completed < n_procs - 1 - n_stragglers: req_done = MPI.Request.Waitany(request_set[i], status) src = status.Get_source() worker_timeset[i, src - 1] = time.time() - start_time request_set[i].pop(req_done) cnt_completed += 1 completed_workers[src - 1] = True completed_ind_set = [ l for l in range(n_procs - 1) if completed_workers[l] ] A_row[0, completed_ind_set] = np.linalg.lstsq( B[completed_ind_set, :].T, np.ones(n_workers))[0] g = np.squeeze(np.dot(A_row, msgBuffers)) # case_idx = calculate_indexA(completed_stragglers) # g = np.dot(A[case_idx,ind_set],tmpBuff) grad_multiplier = eta0[i] / n_samples # ---- update step for gradient descent # np.subtract((1-2*alpha*eta0[i])*beta , grad_multiplier*g, out=beta) # ---- updates for accelerated gradient descent theta = 2.0 / (i + 2.0) ytemp = (1 - theta) * beta + theta * utemp betatemp = ytemp - grad_multiplier * g - (2 * alpha * eta0[i]) * beta utemp = beta + (betatemp - beta) * (1 / theta) beta[:] = betatemp timeset[i] = time.time() - start_time betaset[i, :] = beta ind_set = [ l for l in range(n_procs - 1) if not completed_workers[l] ] for l in ind_set: worker_timeset[i, l] = -1 else: recv_reqs[i].Wait() sendTestBuf = send_req.test() if not sendTestBuf[0]: send_req.Cancel() #print("Worker " + str(rank) + " cancelled send request for Iteration " + str(i)) predy = X_current.dot(beta) g = X_current.T.dot( np.divide(y_current_mod, np.exp(np.multiply(predy, y_current)) + 1)) g *= -1 send_req = comm.Isend([g, MPI.DOUBLE], dest=0, tag=i) ##################################################################################################### comm.Barrier() if rank == 0: elapsed_time = time.time() - orig_start_time print("Total Time Elapsed: %.3f" % (elapsed_time)) # Load all training data if not is_real_data: X_train = load_data(input_dir + "1.dat") print(">> Loaded 1") for j in range(2, n_procs - 1): X_temp = load_data(input_dir + str(j) + ".dat") X_train = np.vstack((X_train, X_temp)) print(">> Loaded " + str(j)) else: X_train = load_sparse_csr(input_dir + "1") for j in range(2, n_procs - 1): X_temp = load_sparse_csr(input_dir + str(j)) X_train = sps.vstack((X_train, X_temp)) y_train = load_data(input_dir + "label.dat") y_train = y_train[0:X_train.shape[0]] # Load all testing data y_test = load_data(input_dir + "label_test.dat") if not is_real_data: X_test = load_data(input_dir + "test_data.dat") else: X_test = load_sparse_csr(input_dir + "test_data") n_train = X_train.shape[0] n_test = X_test.shape[0] training_loss = np.zeros(rounds) testing_loss = np.zeros(rounds) auc_loss = np.zeros(rounds) from sklearn.metrics import roc_curve, auc for i in range(rounds): beta = np.squeeze(betaset[i, :]) predy_train = X_train.dot(beta) predy_test = X_test.dot(beta) training_loss[i] = calculate_loss(y_train, predy_train, n_train) testing_loss[i] = calculate_loss(y_test, predy_test, n_test) fpr, tpr, thresholds = roc_curve(y_test, predy_test, pos_label=1) auc_loss[i] = auc(fpr, tpr) print( "Iteration %d: Train Loss = %5.3f, Test Loss = %5.3f, AUC = %5.3f, Total time taken =%5.3f" % (i, training_loss[i], testing_loss[i], auc_loss[i], timeset[i])) output_dir = input_dir + "results/" if not os.path.exists(output_dir): os.makedirs(output_dir) save_vector( training_loss, output_dir + "coded_acc_%d_training_loss.dat" % (n_stragglers)) save_vector( testing_loss, output_dir + "coded_acc_%d_testing_loss.dat" % (n_stragglers)) save_vector(auc_loss, output_dir + "coded_acc_%d_auc.dat" % (n_stragglers)) save_vector(timeset, output_dir + "coded_acc_%d_timeset.dat" % (n_stragglers)) save_matrix( worker_timeset, output_dir + "coded_acc_%d_worker_timeset.dat" % (n_stragglers)) print(">>> Done") comm.Barrier()
def spawn(self, **kwargs): """ Spawn MPI processes for and execute each of the managed targets. Parameters ---------- kwargs: dict options for the `info` argument in mpi spawn process. see https://www.open-mpi.org/doc/v4.0/man3/MPI_Comm_spawn.3.php """ # Typcially MPI must be have intialized before spawning. if not MPI.Is_initialized(): MPI.Init() if self._is_parent: # Find the path to the mpi_backend.py script (which should be in the # same directory as this module: parent_dir = os.path.dirname(__file__) mpi_backend_path = os.path.join(parent_dir, 'mpi_backend.py') # Set spawn option. Due to --oversubscribe, we will use none in binding info = Info.Create() info.Set('bind_to', "none") for k, v in kwargs.items(): info.Set(k, v) # Spawn processes: self._intercomm = MPI.COMM_SELF.Spawn(sys.executable, args=[mpi_backend_path], maxprocs=len(self), info=info) # First, transmit twiggy logging emitters to spawned processes so # that they can configure their logging facilities: for i in self._targets: self._intercomm.send(twiggy.emitters, i) # Next, serialize the routing table ONCE and then transmit it to all # of the child nodes: try: routing_table = self.routing_table except: routing_table = RoutingTable() self.log_warning( 'Routing Table is null, using empty routing table.') self._intercomm.bcast(routing_table, root=MPI.ROOT) # Transmit class to instantiate, globals required by the class, and # the constructor arguments; the backend will wait to receive # them and then start running the targets on the appropriate nodes. req = MPI.Request() r_list = [] for i in self._targets: target_globals = all_global_vars(self._targets[i]) # Serializing atexit with dill appears to fail in virtualenvs # sometimes if atexit._exithandlers contains an unserializable function: if 'atexit' in target_globals: del target_globals['atexit'] data = (self._targets[i], target_globals, self._kwargs[i]) r_list.append(self._intercomm.isend(data, i)) # Need to clobber data to prevent all_global_vars from # including it in its output: del data req.Waitall(r_list)
def controller(lower, upper): #Set up the basic MPI stuff comm = MPI.COMM_WORLD nproc = comm.Get_size() rank = comm.Get_rank() #Setup values for array of flags length = upper - lower flags = numpy.zeros(length) #Offset of last dispatched value current_val = 0 #Number of in-flight work packets inflight = 0 precheck_num = 0 #How many primes to process precheck_to = 20 #Arrays holding data per worker: #Value last sent to worker vals_in_use = numpy.zeros(nproc - 1) #Workers stats - how many processed in how long processed = numpy.zeros(nproc - 1) start_time = numpy.zeros(nproc - 1) cum_time = numpy.zeros(nproc - 1) end_time = numpy.zeros(nproc - 1) #Some things need to have the correct type BEFORE the MPI calls info = MPI.Status() request = MPI.Request() while True: #Use non-blocking commands although this variant could just as well use blocking #and not post the recieve until after it did the pre-check # Unlike normal MPI, irecv here takes a buffer size only and # the actual result is returned by the wait # First param is buffer size in bytes. request = comm.irecv(4, source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG) #Do some work before waiting if precheck_num < precheck_to: precheck_flags(lower, length, flags, precheck_num) precheck_num = precheck_num + 1 result = request.wait(status=info) if info.tag > 0: # Capture stats end_time[info.source - 1] = time.time() cum_time[info.source - 1] = cum_time[info.source - 1] + ( end_time[info.source - 1] - start_time[info.source - 1]) processed[info.source - 1] = processed[info.source - 1] + 1 offset = vals_in_use[info.source - 1] - lower #Store result #Cheat - if prime mark as 2, (True + 1), else as composite, 1 (False+1) flags[int(offset)] = result + 1 inflight = inflight - 1 if current_val < length: #If there is still work to do, reply with next package #Skip any values that have already been checked i.e. are 1 or 2 #The precheck_flags routine may mark some things as composite while flags[current_val] != 0 and current_val < length: current_val = current_val + 1 vals_in_use[info.source - 1] = lower + current_val #print("Dispatching ", lower+current_val) start_time[info.source - 1] = time.time() current_val = current_val + 1 comm.send(vals_in_use[info.source - 1], dest=info.source, tag=1) inflight = inflight + 1 else: #No more work, shut down the worker comm.send(1, dest=info.source, tag=0) if inflight == 0: #Nothing is in flight, all done break #Summarize findings for i in range(0, nproc - 1): print("Worker ", i, " processed ", int(processed[i - 1]), " packets in ", cum_time[i - 1], "s") #Total the number of elements marked prime (==2) and divide by 2 to get number print("Found ", int(numpy.sum(flags[flags == 2]) / 2), " primes")
def diffusion(U, S, t, tt): # shortcuts discretization = data.discretization domain = data.domain alpha = discretization.alpha beta = discretization.beta nx = domain.nx ny = domain.ny # we initialize the strucutre for saving the requests # and the statuses statuses = [MPI.Status()] * 8 requests = [MPI.Request()] * 8 comm_cart = domain.comm_cart num_requests = 0 # !! non-blocking communication !! # send the North boundary to the north neigbour if domain.neighbour_north >= 0: # set tag to be the sender's rank # post receive requests[num_requests] = comm_cart.Irecv([data.bndN, MPI.DOUBLE], domain.neighbour_north, domain.neighbour_north) num_requests += 1 # pack north buffer data.buffN[0, :] = U[ny-1, :] requests[num_requests] = comm_cart.Isend([data.buffN, MPI.DOUBLE], domain.neighbour_north, domain.rank) num_requests += 1 # same for South if domain.neighbour_south >= 0: # set tag to be the sender's rank # post receive requests[num_requests] = comm_cart.Irecv([data.bndS, MPI.DOUBLE], domain.neighbour_south, domain.neighbour_south) num_requests += 1 # pack south buffer data.buffS[0, :] = U[0, :] requests[num_requests] = comm_cart.Isend([data.buffS, MPI.DOUBLE], domain.neighbour_south, domain.rank) num_requests += 1 # same for East if domain.neighbour_east >= 0: # set tag to be the sender's rank # post receive requests[num_requests] = comm_cart.Irecv([data.bndE, MPI.DOUBLE], domain.neighbour_east, domain.neighbour_east) num_requests += 1 # pack east buffer data.buffE[0, :] = U[:, nx-1] requests[num_requests] = comm_cart.Isend([data.buffE, MPI.DOUBLE], domain.neighbour_east, domain.rank) num_requests += 1 # same for West if domain.neighbour_west >= 0: # set tag to be the sender's rank # post receive requests[num_requests] = comm_cart.Irecv([data.bndW, MPI.DOUBLE], domain.neighbour_west, domain.neighbour_west) num_requests += 1 # pack west buffer data.buffW[0, :] = U[:, 0] requests[num_requests] = comm_cart.Isend([data.buffW, MPI.DOUBLE], domain.neighbour_west, domain.rank) num_requests += 1 srow = domain.ny - 1 scol = domain.nx - 1 # the non-blocking communication allows the communications to take places # and while waiting we can calculate the interior grid points for each domain # srow and scol NOT INCLUDED in the slice operator (work like range()) # S is y^(l+1) # U is x^(l) # data.x_old is x^(l-1) S[1:srow, 1:scol] = ( -(4.0 + alpha) * U[1:srow, 1:scol] + U[1-1:srow-1, 1:scol] + U[1+1:srow+1, 1:scol] + U[1:srow, 1-1:scol-1] + U[1:srow, 1+1:scol+1] + beta * U[1:srow, 1:scol] * (1.0 - U[1:srow, 1:scol]) + alpha * data.x_old[1:srow, 1:scol] ) # wait for all communication to succeed before calculating the boundaries of each subdomain MPI.Request.Waitall(requests, statuses) # east boundary srow = domain.ny - 1 scol = domain.nx - 1 S[1:srow, scol] = ( -(4.0 + alpha) * U[1:srow, scol] + U[1-1:srow-1, scol] + U[1+1:srow+1, scol] + U[1:srow, scol-1] + data.bndE[0, 1:srow] + beta * U[1:srow, scol] * (1.0 - U[1:srow, scol]) + alpha * data.x_old[1:srow, scol] ) srow = domain.ny - 1 scol = 0 # west boundary S[1:srow, scol] = ( -(4.0 + alpha) * U[1:srow, scol] + U[1:srow, scol+1] + U[1-1:srow-1, scol] + U[1+1:srow+1, scol] + alpha * data.x_old[1:srow, scol] + data.bndW[0, 1:srow] + beta * U[1:srow, scol] * (1.0 - U[1:srow, scol]) ) # North boundary srow = domain.ny - 1 # NW corner scol = 0 S[srow, scol] = ( -(4.0 + alpha) * U[srow, scol] + U[srow-1, scol] + data.bndN[0][scol] + data.bndW[0, srow] + U[srow, scol+1] + beta * U[srow, scol] * (1.0 - U[srow, scol]) + alpha * data.x_old[srow, scol]) # north boundary scol = domain.nx - 1 S[srow, 1:scol] = ( -(4.0 + alpha) * U[srow, 1:scol] + U[srow, 1-1:scol-1] + U[srow, 1+1:scol+1] + U[srow-1, 1:scol] + alpha * data.x_old[srow, 1:scol] + data.bndN[0, 1:scol] + beta * U[srow, 1:scol] * (1.0 - U[srow, 1:scol]) ) # NE corner scol = domain.nx - 1 S[srow, scol] = ( -(4.0 + alpha) * U[srow, scol] + U[srow, scol-1] + U[srow-1, scol] + alpha * data.x_old[srow, scol] + data.bndE[0, srow] + data.bndN[0, scol] + beta * U[srow, scol] * (1.0 - U[srow, scol]) ) # South boundary srow = 0 # SW corner scol = 0 S[srow, scol] = ( -(4.0 + alpha) * U[srow, scol] + U[srow, scol+1] + U[srow+1, scol] + alpha * data.x_old[srow, scol] + data.bndW[0, srow] + data.bndS[0, scol] + beta * U[srow, scol] * (1.0 - U[srow, scol]) ) # south boundary scol = domain.nx - 1 S[srow, 1:scol] = ( -(4.0 + alpha) * U[srow, 1:scol] + U[srow, 1-1:scol-1] + U[srow, 1+1:scol+1] + U[srow+1, 1:scol] + alpha * data.x_old[srow, 1:scol] + data.bndS[0, 1:scol] + beta * U[srow, 1:scol] * (1.0 - U[srow, 1:scol]) ) # SE corner scol = domain.nx - 1 S[srow, scol] = ( -(4.0 + alpha) * U[srow, scol] + U[srow, scol-1] + U[srow+1, scol] + alpha * data.x_old[srow, scol] + data.bndE[0, srow] + data.bndS[0, scol] + beta * U[srow, scol] * (1.0 - U[srow, scol]) ) # Statistics # Update the flop counts # 8 flops per point data.flops_count += ( + 12 * (nx - 2) * (ny - 2) # interior points + 11 * (nx - 2 + ny - 2) # all boundaries points + 11 * 4 ) # corner points
def replication_logistic_regression(n_procs, n_samples, n_features, input_dir, n_stragglers, is_real_data, params, add_delay, update_rule): assert update_rule in ('GD', 'AGD') comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() n_workers = n_procs - 1 if (n_workers % (n_stragglers + 1)): print("Error: n_workers must be multiple of n_stragglers+1!") sys.exit(0) rounds = params[0] #beta=np.zeros(n_features) beta = np.random.randn(n_features) #rows_per_worker=n_samples/(n_procs-1) rows_per_worker = n_samples // (n_procs - 1) n_groups = n_workers / (n_stragglers + 1) # Loading the data if (rank): if not is_real_data: X_current = np.zeros( ((1 + n_stragglers) * rows_per_worker, n_features)) y_current = np.zeros((1 + n_stragglers) * rows_per_worker) y = load_data(input_dir + "label.dat") for i in range(1 + n_stragglers): a = (rank - 1) / (n_stragglers + 1) # index of group b = (rank - 1) % (n_stragglers + 1 ) # position inside the group idx = int((n_stragglers + 1) * a + (b + i) % (n_stragglers + 1)) X_current[i * rows_per_worker:(i + 1) * rows_per_worker, :] = load_data(input_dir + str(idx + 1) + ".dat") y_current[i * rows_per_worker:(i + 1) * rows_per_worker] = y[idx * rows_per_worker:(idx + 1) * rows_per_worker] else: y_current = np.zeros((1 + n_stragglers) * rows_per_worker) y = load_data(input_dir + "label.dat") for i in range(1 + n_stragglers): a = (rank - 1) / (n_stragglers + 1) # index of group b = (rank - 1) % (n_stragglers + 1 ) # position inside the group idx = int((n_stragglers + 1) * a + (b + i) % (n_stragglers + 1)) if i == 0: X_current = load_sparse_csr(input_dir + str(idx + 1)) else: X_temp = load_sparse_csr(input_dir + str(idx + 1)) X_current = sps.vstack((X_current, X_temp)) y_current[i * rows_per_worker:(i + 1) * rows_per_worker] = y[idx * rows_per_worker:(idx + 1) * rows_per_worker] # Initializing relevant variables if (rank): predy = X_current.dot(beta) g = -X_current.T.dot( np.divide(y_current, np.exp(np.multiply(predy, y_current)) + 1)) send_req = MPI.Request() recv_reqs = [] else: msgBuffers = [np.zeros(n_features) for i in range(n_procs - 1)] g = np.zeros(n_features) betaset = np.zeros((rounds, n_features)) timeset = np.zeros(rounds) worker_timeset = np.zeros((rounds, n_procs - 1)) request_set = [] recv_reqs = [] send_set = [] cnt_groups = 0 completed_groups = np.ndarray(n_groups, dtype=bool) completed_workers = np.ndarray(n_procs - 1, dtype=bool) status = MPI.Status() eta0 = params[2] # ----- learning rate alpha = params[1] # --- coefficient of l2 regularization utemp = np.zeros(n_features) # for accelerated gradient descent # Posting all Irecv requests for master and workers if (rank): for i in range(rounds): req = comm.Irecv([beta, MPI.DOUBLE], source=0, tag=i) recv_reqs.append(req) else: for i in range(rounds): recv_reqs = [] for j in range(1, n_procs): req = comm.Irecv([msgBuffers[j - 1], MPI.DOUBLE], source=j, tag=i) recv_reqs.append(req) request_set.append(recv_reqs) ########################################################################################### comm.Barrier() if rank == 0: print("---- Starting Replication Iterations for " + str(n_stragglers) + " stragglers" + "simulated delay " + str(add_delay) + "-------") orig_start_time = time.time() for i in range(rounds): if rank == 0: if (i % 10 == 0): print("\t >>> At Iteration %d" % (i)) send_set[:] = [] g[:] = 0 completed_groups[:] = False completed_workers[:] = False cnt_groups = 0 cnt_workers = 0 start_time = time.time() # bcast model step for l in range(1, n_procs): sreq = comm.Isend([beta, MPI.DOUBLE], dest=l, tag=i) send_set.append(sreq) while cnt_groups < n_groups: req_done = MPI.Request.Waitany(request_set[i], status) src = status.Get_source() worker_timeset[i, src - 1] = time.time() - start_time request_set[i].pop(req_done) completed_workers[src - 1] = True groupid = (src - 1) / (n_stragglers + 1) if (not completed_groups[groupid]): completed_groups[groupid] = True g += msgBuffers[src - 1] cnt_groups += 1 grad_multiplier = eta0[i] / n_samples # ---- update step for gradient descent if update_rule == "GD": np.subtract((1 - 2 * alpha * eta0[i]) * beta, grad_multiplier * g, out=beta) elif update_rule == "AGD": # ---- updates for accelerated gradient descent theta = 2.0 / (i + 2.0) ytemp = (1 - theta) * beta + theta * utemp betatemp = ytemp - grad_multiplier * g - (2 * alpha * eta0[i]) * beta utemp = beta + (betatemp - beta) * (1 / theta) beta[:] = betatemp else: raise Exception("Error update rule") timeset[i] = time.time() - start_time betaset[i, :] = beta ind_set = [ l for l in range(1, n_procs) if not completed_workers[l - 1] ] for l in ind_set: worker_timeset[i, l - 1] = -1 MPI.Request.Waitall(send_set) MPI.Request.Waitall(request_set[i]) else: recv_reqs[i].Wait() sendTestBuf = send_req.test() if not sendTestBuf[0]: send_req.Cancel() #print("Worker " + str(rank) + " cancelled send request for Iteration " + str(i)) predy = X_current.dot(beta) g = X_current.T.dot( np.divide(y_current, np.exp(np.multiply(predy, y_current)) + 1)) g *= -1 ########################################## straggler simulation ################################################### if add_delay == 1: np.random.seed(seed=i) #straggler_indices = np.random.choice([t for t in range(1, n_workers+1)], n_stragglers, replace=False) #if rank in straggler_indices: # time.sleep(time_sleep) artificial_delays = np.random.exponential(0.5, n_workers) delay = artificial_delays[rank - 1] time.sleep(delay) ################################################################################################################### send_req = comm.Isend([g, MPI.DOUBLE], dest=0, tag=i) ############################################################################################# comm.Barrier() if rank == 0: elapsed_time = time.time() - orig_start_time print("Total Time Elapsed: %.3f" % (elapsed_time)) # Load all training data if not is_real_data: X_train = load_data(input_dir + "1.dat") for j in range(2, n_procs - 1): X_temp = load_data(input_dir + str(j) + ".dat") X_train = np.vstack((X_train, X_temp)) else: X_train = load_sparse_csr(input_dir + "1") for j in range(2, n_procs - 1): X_temp = load_sparse_csr(input_dir + str(j)) X_train = sps.vstack((X_train, X_temp)) y_train = load_data(input_dir + "label.dat") y_train = y_train[0:X_train.shape[0]] # Load all testing data y_test = load_data(input_dir + "label_test.dat") if not is_real_data: X_test = load_data(input_dir + "test_data.dat") else: X_test = load_sparse_csr(input_dir + "test_data") n_train = X_train.shape[0] n_test = X_test.shape[0] training_loss = np.zeros(rounds) testing_loss = np.zeros(rounds) auc_loss = np.zeros(rounds) from sklearn.metrics import roc_curve, auc for i in range(rounds): beta = np.squeeze(betaset[i, :]) predy_train = X_train.dot(beta) predy_test = X_test.dot(beta) training_loss[i] = calculate_loss(y_train, predy_train, n_train) testing_loss[i] = calculate_loss(y_test, predy_test, n_test) fpr, tpr, thresholds = roc_curve(y_test, predy_test, pos_label=1) auc_loss[i] = auc(fpr, tpr) print( "Iteration %d: Train Loss = %5.3f, Test Loss = %5.3f, AUC = %5.3f, Total time taken =%5.3f" % (i, training_loss[i], testing_loss[i], auc_loss[i], timeset[i])) output_dir = input_dir + "results/" if not os.path.exists(output_dir): os.makedirs(output_dir) save_vector( training_loss, output_dir + "replication_acc_%d_training_loss.dat" % (n_stragglers)) save_vector( testing_loss, output_dir + "replication_acc_%d_testing_loss.dat" % (n_stragglers)) save_vector(auc_loss, output_dir + "replication_acc_%d_auc.dat" % (n_stragglers)) save_vector( timeset, output_dir + "replication_acc_%d_timeset.dat" % (n_stragglers)) save_matrix( worker_timeset, output_dir + "replication_acc_%d_worker_timeset.dat" % (n_stragglers)) print(">>> Done") comm.Barrier()
def _sync(self): """ Send output data and receive input data. """ if self.time_sync: start = time.time() req = MPI.Request() requests = [] # Transmit the entire port data array to each destination module: dest_ids = self.routing_table.dest_ids(self.id) for dest_id in dest_ids: dest_rank = self.rank_to_id[:dest_id] r = MPI.COMM_WORLD.Isend( [self._data_int['gpot'], self._data_mtype['gpot']], dest_rank, GPOT_TAG) requests.append(r) r = MPI.COMM_WORLD.Isend( [self._data_int['spike'], self._data_mtype['spike']], dest_rank, SPIKE_TAG) requests.append(r) if not self.time_sync: self.log_info('sending to %s' % dest_id) if not self.time_sync: self.log_info('sent all data from %s' % self.id) # For each source module, receive elements and copy them into the # current module's port data array: src_ids = self.routing_table.src_ids(self.id) for src_id in src_ids: src_rank = self.rank_to_id[:src_id] r = MPI.COMM_WORLD.Irecv([ self._in_buf_int['gpot'][src_id], self._in_buf_mtype['gpot'][src_id] ], source=src_rank, tag=GPOT_TAG) requests.append(r) r = MPI.COMM_WORLD.Irecv([ self._in_buf_int['spike'][src_id], self._in_buf_mtype['spike'][src_id] ], source=src_rank, tag=SPIKE_TAG) requests.append(r) if not self.time_sync: self.log_info('receiving from %s' % src_id) req.Waitall(requests) if not self.time_sync: self.log_info('received all data received by %s' % self.id) # Copy received elements into the current module's data array: n_gpot = 0 n_spike = 0 for src_id in src_ids: ind_from_gpot = self._from_port_dict_ids['gpot'][src_id] ind_in_gpot = self._in_port_dict_ids['gpot'][src_id] set_by_inds_from_inds(self.data['gpot'], ind_in_gpot, self._in_buf['gpot'][src_id], ind_from_gpot) n_gpot += len(self._in_buf['gpot'][src_id]) ind_from_spike = self._from_port_dict_ids['spike'][src_id] ind_in_spike = self._in_port_dict_ids['spike'][src_id] set_by_inds_from_inds(self.data['spike'], ind_in_spike, self._in_buf['spike'][src_id], ind_from_spike) n_spike += len(self._in_buf['spike'][src_id]) # Save timing data: if self.time_sync: stop = time.time() #self.log_info('sent timing data to master') self.intercomm.isend(['time', (self.rank, self.steps, start, stop, n_gpot*self.pm['gpot'].dtype.itemsize+\ n_spike*self.pm['spike'].dtype.itemsize)], dest=0, tag=self._ctrl_tag) else: self.log_info('saved all data received by %s' % self.id)
def run(self, steps=0): """ Main body of worker process. """ #self.pre_run() self.catch_exception_run(self.pre_run) self.pbar = tqdm(desc=self.progressbar_name(), position=self.rank) self.log_info('running body of worker %s' % self.rank) # Start listening for control messages from parent process: if self.manager: r_ctrl = [] try: d = self.intercomm.irecv(source=0, tag=self._ctrl_tag) except TypeError: # irecv() in mpi4py 1.3.1 stable uses 'dest' instead of 'source': d = self.intercomm.irecv(dest=0, tag=self._ctrl_tag) r_ctrl.append(d) req = MPI.Request() running = False self.steps = 0 if not self.manager: self.max_steps = steps self.pbar.total = self.max_steps running = True while True: if self.manager: # Handle control messages (this assumes that only one control # message will arrive at a time): flag, msg_list = req.testall(r_ctrl) if flag: msg = msg_list[0] # Start executing work method: if msg[0] == 'start': self.log_info('starting') running = True # Stop executing work method:: elif msg[0] == 'stop': if self.max_steps == float('inf'): self.log_info('stopping') running = False else: self.log_info('max steps set - not stopping') # Set maximum number of execution steps: elif msg[0] == 'steps': if msg[1] == 'inf': self.max_steps = float('inf') else: self.max_steps = int(msg[1]) self.pbar.total = self.max_steps self.log_info('setting maximum steps to %s' % self.max_steps) # Quit: elif msg[0] == 'quit': # if self.max_steps == float('inf'): self.log_info('quitting') break # else: # self.log_info('max steps set - not quitting') # Get next message: r_ctrl = [] try: d = self.intercomm.irecv(source=0, tag=self._ctrl_tag) except TypeError: # irecv() in mpi4py 1.3.1 stable uses 'dest' instead of 'source': d = self.intercomm.irecv(dest=0, tag=self._ctrl_tag) r_ctrl.append(d) # Execute work method; the work method may send data back to the master # as a serialized control message containing two elements, e.g., # self.intercomm.isend(['foo', str(self.rank)], # dest=0, tag=self._ctrl_tag) if running: self.do_work() self.steps += 1 self.pbar.update() self.log_info('execution step: %s' % self.steps) # Leave loop if maximum number of steps has been reached: if self.steps >= self.max_steps: running = False self.log_info('maximum steps reached') break #self.post_run() self.catch_exception_run(self.post_run) if not self.post_run_complete: self._finalize()
def spawn(self, part_map): """ Spawn MPI processes for and execute each of the managed targets. Parameters ---------- part_map : dict Maps GPU ID to list of target MPI ranks. """ if self._is_parent: # The number of GPUs over which the targets are partitioned may not # exceed the actual number of supported devices: n_part_gpus = len(part_map.keys()) n_avail_gpus = 0 drv.init() for i in xrange(drv.Device.count()): # MPS requires Tesla/Quadro GPUs with compute capability 3.5 or greater: if mps_avail: d = drv.Device(i) if d.compute_capability() >= (3, 5) and \ re.search('Tesla|Quadro', d.name()): n_avail_gpus += 1 else: n_avail_gpus += 1 if n_part_gpus > n_avail_gpus: raise RuntimeError('partition size (%s) exceeds ' 'number of available GPUs (%s)' % \ (n_part_gpus, n_avail_gpus)) # Start MPS control daemons (this assumes that the available GPUs # are numbered consecutively from 0 onwards - as are the elements of # part_map.keys()): if self._mps_man: self._mps_man.start() self.log_info('starting MPS') # Find the path to the mpi_backend.py script (which should be in the # same directory as this module: import neurokernel.mpi parent_dir = os.path.dirname(neurokernel.mpi.__file__) mpi_backend_path = os.path.join(parent_dir, 'mpi_backend.py') # Check that the union ranks in the partition correspond exactly to # those of the targets added to the manager: n_targets = len(self._targets.keys()) if set(self._targets.keys()) != \ set([t for t in itertools.chain.from_iterable(part_map.values())]): raise ValueError('partition must contain all target ranks') # Invert mapping of GPUs to MPI ranks: rank_to_gpu_map = { rank: gpu for gpu in part_map.keys() for rank in part_map[gpu] } # Set MPS pipe directory: info = MPI.Info.Create() if self._mps_man: mps_dir = self._mps_man.get_mps_dir( self._mps_man.get_mps_ctrl_proc()) info.Set('env', 'CUDA_MPS_PIPE_DIRECTORY=%s' % mps_dir) # Spawn processes: self._intercomm = MPI.COMM_SELF.Spawn(sys.executable, args=[mpi_backend_path], maxprocs=n_targets, info=info) # First, transmit twiggy logging emitters to spawned processes so # that they can configure their logging facilities: for i in self._targets.keys(): self._intercomm.send(twiggy.emitters, i) # Next, serialize the routing table ONCE and then transmit it to all # of the child nodes: self._intercomm.bcast(self.routing_table, root=MPI.ROOT) # Transmit class to instantiate, globals required by the class, and # the constructor arguments; the backend will wait to receive # them and then start running the targets on the appropriate nodes. req = MPI.Request() r_list = [] for i in self._targets.keys(): target_globals = all_global_vars(self._targets[i]) # Serializing atexit with dill appears to fail in virtualenvs # sometimes if atexit._exithandlers contains an unserializable function: if 'atexit' in target_globals: del target_globals['atexit'] data = (self._targets[i], target_globals, self._kwargs[i]) r_list.append(self._intercomm.isend(data, i)) # Need to clobber data to prevent all_global_vars from # including it in its output: del data req.Waitall(r_list)
def partial_replication_logistic_regression(n_procs, n_samples, n_features, input_dir, n_stragglers, n_partitions, is_real_data, params): comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() rounds = params[0] n_workers = n_procs - 1 if (n_workers % (n_stragglers + 1)): print("Error: n_workers must be multiple of n_stragglers+1!") sys.exit(0) rows_per_worker = n_samples // ((n_partitions - n_stragglers) * n_workers ) # per partition num of samples n_groups = n_workers / (n_stragglers + 1) n_separate = n_partitions - n_stragglers - 1 sep_lim = n_separate * rows_per_worker # Loading the data if (rank): if not is_real_data: y = load_data(input_dir + "label.dat") X_current = np.zeros([n_partitions * rows_per_worker, n_features]) y_current = np.zeros(n_partitions * rows_per_worker) for i in range(n_separate): idx = i + n_separate * (rank - 1) X_current[i * rows_per_worker:(i + 1) * rows_per_worker, :] = load_data(input_dir + str(idx + 1) + ".dat") y_current[i * rows_per_worker:(i + 1) * rows_per_worker] = y[idx * rows_per_worker:(idx + 1) * rows_per_worker] for i in range(n_separate, n_partitions): a = (rank - 1) / (n_stragglers + 1) # index of group b = i - n_separate # position inside the group idx = n_separate * n_workers + a * (n_stragglers + 1) + b X_current[i * rows_per_worker:(i + 1) * rows_per_worker, :] = load_data(input_dir + str(idx + 1) + ".dat") y_current[i * rows_per_worker:(i + 1) * rows_per_worker] = y[idx * rows_per_worker:(idx + 1) * rows_per_worker] else: y = load_data(input_dir + "label.dat") y_current = np.zeros(n_partitions * rows_per_worker) for i in range(n_separate): idx = i + n_separate * (rank - 1) y_current[i * rows_per_worker:(i + 1) * rows_per_worker] = y[idx * rows_per_worker:(idx + 1) * rows_per_worker] if i == 0: X_current = load_sparse_csr(input_dir + str(idx + 1)) else: X_temp = load_sparse_csr(input_dir + str(idx + 1)) X_current = sps.vstack((X_current, X_temp)) for i in range(n_separate, n_partitions): a = (rank - 1) / (n_stragglers + 1) # index of group b = i - n_separate # position inside the group idx = n_separate * n_workers + a * (n_stragglers + 1) + b y_current[i * rows_per_worker:(i + 1) * rows_per_worker] = y[idx * rows_per_worker:(idx + 1) * rows_per_worker] X_temp = load_sparse_csr(input_dir + str(idx + 1)) X_current = sps.vstack((X_current, X_temp)) X_current = X_current.tocsr() # Initializing relevant variables beta = np.zeros(n_features) if (rank): predy = X_current.dot(beta) g_firstpart = -X_current.T.dot( np.divide(y_current, np.exp(np.multiply(predy, y_current)) + 1)) g_secondpart = -X_current.T.dot( np.divide(y_current, np.exp(np.multiply(predy, y_current)) + 1)) send_req1 = MPI.Request() send_req2 = MPI.Request() recv_reqs = [] else: print('Stragglers are allowed to be atmost %.2f times slower' % (n_partitions * 1.0 / (n_partitions - n_stragglers - 1))) msgBuffers_firstparts = [ np.zeros(n_features) for i in range(n_procs - 1) ] msgBuffers_secondparts = [ np.zeros(n_features) for i in range(n_procs - 1) ] g = np.zeros(n_features) betaset = np.zeros((rounds, n_features)) timeset = np.zeros(rounds) worker_timeset = np.zeros((rounds, n_procs - 1)) request_set = [] recv_reqs = [] send_set = [] cnt_groups = 0 cnt_firstpart = 0 completed_groups = np.ndarray(n_groups, dtype=bool) completed_workers = np.ndarray(n_workers, dtype=bool) completed_firstparts = np.ndarray(n_workers, dtype=bool) status = MPI.Status() eta0 = params[2] # ----- learning rate alpha = params[1] # --- coefficient of l2 regularization utemp = np.zeros(n_features) # for accelerated gradient descent # Posting all Irecv requests for master and workers if (rank): for i in range(rounds): req = comm.Irecv([beta, MPI.DOUBLE], source=0, tag=i) recv_reqs.append(req) else: for i in range(rounds): recv_reqs = [] for j in range(1, n_procs): req1 = comm.Irecv([msgBuffers_firstparts[j - 1], MPI.DOUBLE], source=j, tag=2 * rounds + i) recv_reqs.append(req1) req2 = comm.Irecv([msgBuffers_secondparts[j - 1], MPI.DOUBLE], source=j, tag=i) recv_reqs.append(req2) request_set.append(recv_reqs) ####################################################################################################################### comm.Barrier() if rank == 0: print("---- Starting Partial Replication Iterations for " + str(n_stragglers) + " stragglers ----") orig_start_time = time.time() for i in range(rounds): if rank == 0: if (i % 10 == 0): print("\t >>> At Iteration %d" % (i)) send_set[:] = [] g[:] = 0 cnt_firstpart = 0 completed_firstparts[:] = False completed_groups[:] = False cnt_groups = 0 completed_workers[:] = False start_time = time.time() for l in range(1, n_procs): sreq = comm.Isend([beta, MPI.DOUBLE], dest=l, tag=i) send_set.append(sreq) while cnt_groups < n_groups or cnt_firstpart < n_workers: req_done = MPI.Request.Waitany(request_set[i], status) src = status.Get_source() tag = status.Get_tag() worker_timeset[i, src - 1] = time.time() - start_time request_set[i].pop(req_done) if tag == i: completed_workers[src - 1] = True groupid = (src - 1) / (n_stragglers + 1) if not completed_groups[groupid]: completed_groups[groupid] = True g += msgBuffers_secondparts[src - 1] cnt_groups += 1 elif tag == 2 * rounds + i: g += msgBuffers_firstparts[src - 1] completed_firstparts[src - 1] = True cnt_firstpart += 1 grad_multiplier = eta0[i] / n_samples # ---- update step for gradient descent # np.subtract((1-2*alpha*eta0[i])*beta , grad_multiplier*g, out=beta) # ---- updates for accelerated gradient descent theta = 2.0 / (i + 2.0) ytemp = (1 - theta) * beta + theta * utemp betatemp = ytemp - grad_multiplier * g - (2 * alpha * eta0[i]) * beta utemp = beta + (betatemp - beta) * (1 / theta) beta[:] = betatemp timeset[i] = time.time() - start_time betaset[i, :] = beta ind_set = [ l for l in range(1, n_procs) if not completed_workers[l - 1] ] for l in ind_set: worker_timeset[i, l - 1] = -1 else: recv_reqs[i].Wait() sendTestBuf = send_req1.test() if not sendTestBuf[0]: send_req1.Cancel() sendTestBuf = send_req2.test() if not sendTestBuf[0]: send_req2.Cancel() predy = X_current[0:sep_lim, :].dot(beta) g_firstpart = X_current[0:sep_lim, :].T.dot( np.divide(y_current[0:sep_lim], np.exp(np.multiply(predy, y_current[0:sep_lim])) + 1)) g_firstpart *= -1 send_req1 = comm.Isend([g_firstpart, MPI.DOUBLE], dest=0, tag=2 * rounds + i) predy = X_current[sep_lim:, :].dot(beta) g_secondpart = X_current[sep_lim:, :].T.dot( np.divide(y_current[sep_lim:], np.exp(np.multiply(predy, y_current[sep_lim:])) + 1)) g_secondpart *= -1 send_req2 = comm.Isend([g_secondpart, MPI.DOUBLE], dest=0, tag=i) ###################################################################################################################### comm.Barrier() if rank == 0: elapsed_time = time.time() - orig_start_time print("Total Time Elapsed: %.3f" % (elapsed_time)) # Load all training data if not is_real_data: X_train = load_data(input_dir + "1.dat") for j in range(2, n_procs - 1): X_temp = load_data(input_dir + str(j) + ".dat") X_train = np.vstack((X_train, X_temp)) else: X_train = load_sparse_csr(input_dir + "1") for j in range(2, n_procs - 1): X_temp = load_sparse_csr(input_dir + str(j)) X_train = sps.vstack((X_train, X_temp)) y_train = load_data(input_dir + "label.dat") y_train = y_train[0:X_train.shape[0]] # Load all testing data y_test = load_data(input_dir + "label_test.dat") if not is_real_data: X_test = load_data(input_dir + "test_data.dat") else: X_test = load_sparse_csr(input_dir + "test_data") n_train = X_train.shape[0] n_test = X_test.shape[0] training_loss = np.zeros(rounds) testing_loss = np.zeros(rounds) auc_loss = np.zeros(rounds) from sklearn.metrics import roc_curve, auc for i in range(rounds): beta = np.squeeze(betaset[i, :]) predy_train = X_train.dot(beta) predy_test = X_test.dot(beta) training_loss[i] = calculate_loss(y_train, predy_train, n_train) testing_loss[i] = calculate_loss(y_test, predy_test, n_test) fpr, tpr, thresholds = roc_curve(y_test, predy_test, pos_label=1) auc_loss[i] = auc(fpr, tpr) print( "Iteration %d: Train Loss = %5.3f, Test Loss = %5.3f, AUC = %5.3f, Total time taken =%5.3f" % (i, training_loss[i], testing_loss[i], auc_loss[i], timeset[i])) output_dir = input_dir + "results/" if not os.path.exists(output_dir): os.makedirs(output_dir) save_vector( training_loss, output_dir + "partialreplication_%d_%d_training_loss.dat" % (n_stragglers, n_partitions)) save_vector( testing_loss, output_dir + "partialreplication_%d_%d_testing_loss.dat" % (n_stragglers, n_partitions)) save_vector( auc_loss, output_dir + "partialreplication_%d_%d_auc.dat" % (n_stragglers, n_partitions)) save_vector( timeset, output_dir + "partialreplication_%d_%d_timeset.dat" % (n_stragglers, n_partitions)) save_matrix( worker_timeset, output_dir + "partialreplication_%d_%d_worker_timeset.dat" % (n_stragglers, n_partitions)) print(">>> Done") comm.Barrier()