def __enter__(self): args = [] if self.profile is not None: args.append("--profile=" + self.profile) if self.cluster_id is not None: args.append("--cluster-id=" + self.cluster_id) if self.num_engines is not None: args.append("--n=" + str(self.num_engines)) if self.ipython_dir is not None: args.append("--ipython-dir=" + self.ipython_dir) cmd = " ".join(["ipcluster start --daemonize"] + args) self.logger.info('Staring IPython cluster with "' + cmd + '"') os.system(cmd) num_engines, timeout = self.num_engines, self.timeout time.sleep(self.min_wait) waited = self.min_wait client = None while client is None: try: client = Client(profile=self.profile, cluster_id=self.cluster_id) except (IOError, TimeoutError): if waited >= self.timeout: raise IOError("Could not connect to IPython cluster controller") if waited % 10 == 0: self.logger.info("Waiting for controller to start ...") time.sleep(1) waited += 1 if num_engines is None: while len(client) == 0 and waited < timeout: if waited % 10 == 0: self.logger.info("Waiting for engines to start ...") time.sleep(1) waited += 1 if len(client) == 0: raise IOError("IPython cluster engines failed to start") wait = min(waited, timeout - waited) if wait > 0: self.logger.info("Waiting {} more seconds for engines to start ...".format(wait)) time.sleep(wait) else: running = len(client) while running < num_engines and waited < timeout: if waited % 10 == 0: self.logger.info( "Waiting for {} of {} engines to start ...".format(num_engines - running, num_engines) ) time.sleep(1) waited += 1 running = len(client) running = len(client) if running < num_engines: raise IOError( "{} of {} IPython cluster engines failed to start".format(num_engines - running, num_engines) ) client.close() self.pool = IPythonPool(profile=self.profile, cluster_id=self.cluster_id) return self.pool
class ParallelPool( object ): def __init__(self): #Load configuration self.c = Configuration.Configuration( ) #Now instance the pool of batch workers according #to the technology selected in the configuration file if self.c.parallel.technology=='ipython': self.IPYc = Client( profile=self.c.parallel.ipython.profile ) self.pool = self.IPYc[:] elif self.c.parallel.technology=='python': if self.c.parallel.python.number_of_processes==0: n_cpus = multiprocessing.cpu_count() else: n_cpus = self.c.parallel.python.number_of_processes self.pool = multiprocessing.Pool( n_cpus ) else: raise ValueError("Unknown technology %s in configuration file" %(self.c.parallel.technology)) #The following methods simply forward the requests to the #batch worker technology def map( self, *args, **kwargs ): if self.c.parallel.technology=='ipython': return self.pool.map( *args, **kwargs ).get() else: return self.pool.map( *args, **kwargs ) def imap( self, *args, **kwargs ): return self.pool.imap( *args, **kwargs ) def close( self ): if self.c.parallel.technology=='ipython': self.IPYc.close() else: self.pool.close() self.pool.join()
def add_engines(n=1, profile='iptest', total=False): """add a number of engines to a given profile. If total is True, then already running engines are counted, and only the additional engines necessary (if any) are started. """ rc = Client(profile=profile) base = len(rc) if total: n = max(n - base, 0) eps = [] for i in range(n): ep = TestProcessLauncher() ep.cmd_and_args = ipengine_cmd_argv + [ '--profile=%s' % profile, '--InteractiveShell.colors=nocolor' ] ep.start() launchers.append(ep) eps.append(ep) tic = time.time() while len(rc) < base + n: if any([ep.poll() is not None for ep in eps]): raise RuntimeError("A test engine failed to start.") elif time.time() - tic > 15: raise RuntimeError("Timeout waiting for engines to connect.") time.sleep(.1) rc.close() return eps
def add_engines(n=1, profile='iptest', total=False): """add a number of engines to a given profile. If total is True, then already running engines are counted, and only the additional engines necessary (if any) are started. """ rc = Client(profile=profile) base = len(rc) if total: n = max(n - base, 0) eps = [] for i in range(n): ep = TestProcessLauncher() ep.cmd_and_args = ipengine_cmd_argv + [ '--profile=%s' % profile, '--InteractiveShell.colors=nocolor' ] ep.start() launchers.append(ep) eps.append(ep) tic = time.time() while len(rc) < base+n: if any([ ep.poll() is not None for ep in eps ]): raise RuntimeError("A test engine failed to start.") elif time.time()-tic > 15: raise RuntimeError("Timeout waiting for engines to connect.") time.sleep(.1) rc.close() return eps
class ParallelPool(object): def __init__(self): #Load configuration self.c = Configuration.Configuration() #Now instance the pool of batch workers according #to the technology selected in the configuration file if self.c.parallel.technology == 'ipython': self.IPYc = Client(profile=self.c.parallel.ipython.profile) self.pool = self.IPYc[:] elif self.c.parallel.technology == 'python': if self.c.parallel.python.number_of_processes == 0: n_cpus = multiprocessing.cpu_count() else: n_cpus = self.c.parallel.python.number_of_processes self.pool = multiprocessing.Pool(n_cpus) else: raise ValueError("Unknown technology %s in configuration file" % (self.c.parallel.technology)) #The following methods simply forward the requests to the #batch worker technology def map(self, *args, **kwargs): if self.c.parallel.technology == 'ipython': return self.pool.map(*args, **kwargs).get() else: return self.pool.map(*args, **kwargs) def imap(self, *args, **kwargs): return self.pool.imap(*args, **kwargs) def close(self): if self.c.parallel.technology == 'ipython': self.IPYc.close() else: self.pool.close() self.pool.join()
def __enter__(self): args = [] if self.profile is not None: args.append('--profile=' + self.profile) if self.cluster_id is not None: args.append('--cluster-id=' + self.cluster_id) if self.num_engines is not None: args.append('--n=' + str(self.num_engines)) if self.ipython_dir is not None: args.append('--ipython-dir=' + self.ipython_dir) cmd = ' '.join(['ipcluster start --daemonize'] + args) self.logger.info('Staring IPython cluster with "' + cmd + '"') os.system(cmd) num_engines, timeout = self.num_engines, self.timeout time.sleep(self.min_wait) waited = self.min_wait client = None while client is None: try: client = Client(profile=self.profile, cluster_id=self.cluster_id) except (IOError, TimeoutError): if waited >= self.timeout: raise IOError('Could not connect to IPython cluster controller') if waited % 10 == 0: self.logger.info('Waiting for controller to start ...') time.sleep(1) waited += 1 if num_engines is None: while len(client) == 0 and waited < timeout: if waited % 10 == 0: self.logger.info('Waiting for engines to start ...') time.sleep(1) waited += 1 if len(client) == 0: raise IOError('IPython cluster engines failed to start') wait = min(waited, timeout - waited) if wait > 0: self.logger.info(f'Waiting {wait} more seconds for engines to start ...') time.sleep(wait) else: running = len(client) while running < num_engines and waited < timeout: if waited % 10 == 0: self.logger.info(f'Waiting for {num_engines-running} of {num_engines} engines to start ...') time.sleep(1) waited += 1 running = len(client) running = len(client) if running < num_engines: raise IOError(f'{num_engines-running} of {num_engines} IPython cluster engines failed to start') # make sure all (potential) engines are in the same cwd, so they can import the same code client[:].apply_sync(os.chdir, os.getcwd()) client.close() self.pool = IPythonPool(profile=self.profile, cluster_id=self.cluster_id) return self.pool
def motion_correct_parallel(file_names,fr,template=None,margins_out=0,max_shift_w=5, max_shift_h=5,remove_blanks=False,apply_smooth=True,backend='single_thread'): """motion correct many movies usingthe ipyparallel cluster Parameters ---------- file_names: list of strings names of he files to be motion corrected fr: double fr parameters for calcblitz movie margins_out: int number of pixels to remove from the borders Return ------ base file names of the motion corrected files """ args_in=[]; for f in file_names: args_in.append((f,fr,margins_out,template,max_shift_w, max_shift_h,remove_blanks,apply_smooth)) try: if backend is 'ipyparallel': c = Client() dview=c[:] file_res = dview.map_sync(process_movie_parallel, args_in) dview.results.clear() c.purge_results('all') c.purge_everything() c.close() elif backend is 'single_thread': file_res = map(process_movie_parallel, args_in) else: raise Exception('Unknown backend') except : try: if backend is 'ipyparallel': dview.results.clear() c.purge_results('all') c.purge_everything() c.close() except UnboundLocalError as uberr: print 'could not close client' raise return file_res
def start_server(slurm_script=None, ipcluster="ipcluster", ncpus=None): """ programmatically start the ipyparallel server Parameters: ---------- ncpus: int number of processors ipcluster : str ipcluster binary file name; requires 4 path separators on Windows. ipcluster="C:\\\\Anaconda2\\\\Scripts\\\\ipcluster.exe" Default: "ipcluster" """ logger.info("Starting cluster...") if ncpus is None: ncpus = psutil.cpu_count() if slurm_script is None: if ipcluster == "ipcluster": subprocess.Popen( "ipcluster start -n {0} -- --location=localhost".format( ncpus ), # localhost needed b/c default listen only on 127.0.0.1 not external ips shell=True, close_fds=(os.name != 'nt')) else: subprocess.Popen(shlex.split("{0} start -n {1}".format( ipcluster, ncpus)), shell=True, close_fds=(os.name != 'nt')) # Check that all processes have started client = ipyparallel.Client() while len( client ) < ncpus: # client len will get longer as workers start and connect to hub time.sleep(0.5) logger.debug('Making sure everything is up and running') ar = client.direct_view().apply_async(lambda x: x + 1, 10) ar.get(timeout=10) # when done on all, we're set to go. 10s timeout client.close() logger.debug('Cluster started successfully.') else: shell_source(slurm_script) pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] print([pdir, profile]) c = Client(ipython_dir=pdir, profile=profile) ee = c[:] ne = len(ee) print(('Running on %d engines.' % (ne))) c.close() sys.stdout.write(" done\n")
def cluster_status(): try: rcl = Client() nworkers = len(rcl[:]) qstat = rcl.queue_status() queued = qstat[u'unassigned'] working = sum([qstat[w][u'tasks'] for w in rcl.ids]) idle = nworkers - working rcl.close() except: nworkers, queued, working, idle = 0, 0, 0, 0 return nworkers, queued, working, idle
def start_server(slurm_script=None, ipcluster="ipcluster", ncpus=None): ''' programmatically start the ipyparallel server Parameters ---------- ncpus: int number of processors ipcluster : str ipcluster binary file name; requires 4 path separators on Windows. ipcluster="C:\\\\Anaconda2\\\\Scripts\\\\ipcluster.exe" Default: "ipcluster" ''' sys.stdout.write("Starting cluster...") sys.stdout.flush() if ncpus is None: ncpus = psutil.cpu_count() if slurm_script is None: if ipcluster == "ipcluster": p1 = subprocess.Popen("ipcluster start -n {0}".format(ncpus), shell=True, close_fds=(os.name != 'nt')) else: p1 = subprocess.Popen(shlex.split("{0} start -n {1}".format( ipcluster, ncpus)), shell=True, close_fds=(os.name != 'nt')) # while True: try: c = ipyparallel.Client() if len(c) < ncpus: sys.stdout.write(".") sys.stdout.flush() raise ipyparallel.error.TimeoutError c.close() break except (IOError, ipyparallel.error.TimeoutError): sys.stdout.write(".") sys.stdout.flush() time.sleep(1) else: shell_source(slurm_script) pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] c = Client(ipython_dir=pdir, profile=profile) ee = c[:] ne = len(ee) print(('Running on %d engines.' % (ne))) c.close() sys.stdout.write(" done\n")
def start_server(slurm_script: str = None, ipcluster: str = "ipcluster", ncpus: int = None) -> None: """ programmatically start the ipyparallel server Args: ncpus: int number of processors ipcluster : str ipcluster binary file name; requires 4 path separators on Windows. ipcluster="C:\\\\Anaconda3\\\\Scripts\\\\ipcluster.exe" Default: "ipcluster" """ logger.info("Starting cluster...") if ncpus is None: ncpus = psutil.cpu_count() if slurm_script is None: if ipcluster == "ipcluster": subprocess.Popen("ipcluster start -n {0}".format(ncpus), shell=True, close_fds=(os.name != 'nt')) else: subprocess.Popen(shlex.split("{0} start -n {1}".format( ipcluster, ncpus)), shell=True, close_fds=(os.name != 'nt')) time.sleep(1.5) # Check that all processes have started client = ipyparallel.Client() time.sleep(1.5) while len(client) < ncpus: sys.stdout.write( ".") # Give some visual feedback of things starting sys.stdout.flush() # (de-buffered) time.sleep(0.5) logger.debug('Making sure everything is up and running') client.direct_view().execute( '__a=1', block=True) # when done on all, we're set to go else: shell_source(slurm_script) pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] logger.debug([pdir, profile]) c = Client(ipython_dir=pdir, profile=profile) ee = c[:] ne = len(ee) logger.info(('Running on %d engines.' % (ne))) c.close() sys.stdout.write("start_server: done\n")
def start_server(slurm_script=None, ipcluster="ipcluster", ncpus=None): """ programmatically start the ipyparallel server Parameters: ---------- ncpus: int number of processors ipcluster : str ipcluster binary file name; requires 4 path separators on Windows. ipcluster="C:\\\\Anaconda2\\\\Scripts\\\\ipcluster.exe" Default: "ipcluster" """ sys.stdout.write("Starting cluster...") sys.stdout.flush() if ncpus is None: ncpus = psutil.cpu_count() if slurm_script is None: if ipcluster == "ipcluster": subprocess.Popen( "ipcluster start -n {0}".format(ncpus), shell=True, close_fds=(os.name != 'nt')) else: subprocess.Popen(shlex.split( "{0} start -n {1}".format(ipcluster, ncpus)), shell=True, close_fds=(os.name != 'nt')) # Check that all processes have started time.sleep(1) client = ipyparallel.Client() while len(client) < ncpus: sys.stdout.write(".") sys.stdout.flush() client.close() time.sleep(1) client = ipyparallel.Client() print('Making Sure everything is up and running') time.sleep(10) client.close() else: shell_source(slurm_script) pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] print([pdir,profile]) c = Client(ipython_dir=pdir, profile=profile) ee = c[:] ne = len(ee) print(('Running on %d engines.' % (ne))) c.close() sys.stdout.write(" done\n")
def _nengines_up(url_file): "return the number of engines up" client = None try: client = Client(url_file, timeout=60) up = len(client.ids) client.close() # the controller isn't up yet except iperror.TimeoutError: return 0 # the JSON file is not available to parse except IOError: return 0 else: return up
def start_server(slurm_script=None, ipcluster="ipcluster", ncpus=None): """ programmatically start the ipyparallel server Parameters: ---------- ncpus: int number of processors ipcluster : str ipcluster binary file name; requires 4 path separators on Windows. ipcluster="C:\\\\Anaconda2\\\\Scripts\\\\ipcluster.exe" Default: "ipcluster" """ logger.info("Starting cluster...") if ncpus is None: ncpus = psutil.cpu_count() if slurm_script is None: if ipcluster == "ipcluster": subprocess.Popen( "ipcluster start -n {0}".format(ncpus), shell=True, close_fds=(os.name != 'nt')) else: subprocess.Popen(shlex.split( "{0} start -n {1}".format(ipcluster, ncpus)), shell=True, close_fds=(os.name != 'nt')) # Check that all processes have started time.sleep(1) client = ipyparallel.Client() while len(client) < ncpus: client.close() time.sleep(1) client = ipyparallel.Client() time.sleep(10) logger.debug('Making sure everything is up and running') client.close() else: shell_source(slurm_script) pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] print([pdir,profile]) c = Client(ipython_dir=pdir, profile=profile) ee = c[:] ne = len(ee) print(('Running on %d engines.' % (ne))) c.close() sys.stdout.write(" done\n")
def test_hubresult_timestamps(self): self.minimum_engines(4) v = self.client[:] ar = v.apply_async(time.sleep, 0.25) ar.get(2) rc2 = Client(profile='iptest') # must have try/finally to close second Client, otherwise # will have dangling sockets causing problems try: time.sleep(0.25) hr = rc2.get_result(ar.msg_ids) self.assertTrue(hr.elapsed > 0., "got bad elapsed: %s" % hr.elapsed) hr.get(1) self.assertTrue(hr.wall_time < ar.wall_time + 0.2, "got bad wall_time: %s > %s" % (hr.wall_time, ar.wall_time)) self.assertEqual(hr.serial_time, ar.serial_time) finally: rc2.close()
def test_hubresult_timestamps(self): self.minimum_engines(4) v = self.client[:] ar = v.apply_async(time.sleep, 0.25) ar.get(2) rc2 = Client(profile='iptest') # must have try/finally to close second Client, otherwise # will have dangling sockets causing problems try: time.sleep(0.25) hr = rc2.get_result(ar.msg_ids) self.assertTrue(hr.elapsed > 0., "got bad elapsed: %s" % hr.elapsed) hr.get(1) self.assertTrue(hr.wall_time < ar.wall_time + 0.2, "got bad wall_time: %s > %s" % (hr.wall_time, ar.wall_time)) self.assertEqual(hr.serial_time, ar.serial_time) finally: rc2.close()
def map(self, parallel_task, args): from ipyparallel import Client, TimeoutError chunksize = 1 if self.max_tasks > 0 and len(args) > self.max_tasks: chunksize = len(args) // self.max_tasks if chunksize * self.max_tasks < len(args): chunksize += 1 client = None try: client = Client() except TimeoutError: raise RuntimeError( 'Cannot connect to the ipyparallel client. Is it running?') ar = None try: client[:].use_cloudpickle() lbv = client.load_balanced_view() ar = lbv.map_async(IppFunctionWrapper(parallel_task, self.timeout), args, chunksize=chunksize) try: r = [] for k, z in enumerate( tqdm(ar, desc="(IPYPARALLEL)", total=len(args))): if z[0] == -1: logger.error(z[1]) engine = ar.engine_id[k] client.abort(ar) client.close() raise RuntimeError( 'remote failure (task %d of %d on engine %d)' % (k + 1, len(ar), engine)) elif z[0] == 0: r.append(z[1]) except KeyboardInterrupt: client.abort(ar) raise finally: # always close the client to release resources if ar: client.abort(ar) if client: client.close() return r
def extract_rois_patch(file_name,d1,d2,rf=5,stride = 5): idx_flat,idx_2d=extract_patch_coordinates(d1, d2, rf=rf,stride = stride) perctl=95 n_components=2 tol=1e-6 max_iter=5000 args_in=[] for id_f,id_2d in zip(idx_flat,idx_2d): args_in.append((file_name, id_f,id_2d[0].shape, perctl,n_components,tol,max_iter)) st=time.time() print len(idx_flat) try: if 1: c = Client() dview=c[:] file_res = dview.map_sync(nmf_patches, args_in) else: file_res = map(nmf_patches, args_in) finally: dview.results.clear() c.purge_results('all') c.purge_everything() c.close() print time.time()-st A1=lil_matrix((d1*d2,len(file_res))) C1=[] A2=lil_matrix((d1*d2,len(file_res))) C2=[] for count,f in enumerate(file_res): idx_,flt,ca,d=f #flt,ca,_=cse.order_components(coo_matrix(flt),ca) A1[idx_,count]=flt[:,0][:,np.newaxis] A2[idx_,count]=flt[:,1][:,np.newaxis] C1.append(ca[0,:]) C2.append(ca[1,:]) # pl.imshow(np.reshape(flt[:,0],d,order='F'),vmax=10) # pl.pause(.1) return A1,A2,C1,C2
def extract_rois_patch(file_name, d1, d2, rf=5, stride=5): idx_flat, idx_2d = extract_patch_coordinates(d1, d2, rf=rf, stride=stride) perctl = 95 n_components = 2 tol = 1e-6 max_iter = 5000 args_in = [] for id_f, id_2d in zip(idx_flat, idx_2d): args_in.append((file_name, id_f, id_2d[0].shape, perctl, n_components, tol, max_iter)) st = time.time() print len(idx_flat) try: if 1: c = Client() dview = c[:] file_res = dview.map_sync(nmf_patches, args_in) else: file_res = map(nmf_patches, args_in) finally: dview.results.clear() c.purge_results('all') c.purge_everything() c.close() print time.time() - st A1 = lil_matrix((d1 * d2, len(file_res))) C1 = [] A2 = lil_matrix((d1 * d2, len(file_res))) C2 = [] for count, f in enumerate(file_res): idx_, flt, ca, d = f #flt,ca,_=cse.order_components(coo_matrix(flt),ca) A1[idx_, count] = flt[:, 0][:, np.newaxis] A2[idx_, count] = flt[:, 1][:, np.newaxis] C1.append(ca[0, :]) C2.append(ca[1, :]) # pl.imshow(np.reshape(flt[:,0],d,order='F'),vmax=10) # pl.pause(.1) return A1, A2, C1, C2
def start_server(ncpus,slurm_script=None): ''' programmatically start the ipyparallel server Parameters ---------- ncpus: int number of processors ''' sys.stdout.write("Starting cluster...") sys.stdout.flush() if slurm_script is None: subprocess.Popen(["ipcluster start -n {0}".format(ncpus)], shell=True) while True: try: c = ipyparallel.Client() if len(c) < ncpus: sys.stdout.write(".") sys.stdout.flush() raise ipyparallel.error.TimeoutError c.close() break except (IOError, ipyparallel.error.TimeoutError): sys.stdout.write(".") sys.stdout.flush() time.sleep(1) else: shell_source(slurm_script) from ipyparallel import Client pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] c = Client(ipython_dir=pdir, profile=profile) ee = c[:] ne = len(ee) print 'Running on %d engines.'%(ne) c.close() sys.stdout.write(" done\n")
def start_server(ncpus, slurm_script=None): ''' programmatically start the ipyparallel server Parameters ---------- ncpus: int number of processors ''' sys.stdout.write("Starting cluster...") sys.stdout.flush() if slurm_script is None: subprocess.Popen(["ipcluster start -n {0}".format(ncpus)], shell=True) while True: try: c = ipyparallel.Client() if len(c) < ncpus: sys.stdout.write(".") sys.stdout.flush() raise ipyparallel.error.TimeoutError c.close() break except (IOError, ipyparallel.error.TimeoutError): sys.stdout.write(".") sys.stdout.flush() time.sleep(1) else: shell_source(slurm_script) from ipyparallel import Client pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] c = Client(ipython_dir=pdir, profile=profile) ee = c[:] ne = len(ee) print 'Running on %d engines.' % (ne) c.close() sys.stdout.write(" done\n")
t1 = time() file_res=cb.motion_correct_parallel(fnames,30,template=template,margins_out=0,max_shift_w=45, max_shift_h=45,dview=client_[::2],remove_blanks=False) t2=time()-t1 print(t2) #%% fnames=[] for file in glob.glob(base_folder+'k31_20160107_MMP_150um_65mW_zoom2p2_000*[0-9].hdf5'): fnames.append(file) fnames.sort() print(fnames) #%% file_res=cb.utils.pre_preprocess_movie_labeling(client_[::2], fnames, median_filter_size=(2,1,1), resize_factors=[.2,.1666666666],diameter_bilateral_blur=4) #%% client_.close() cse.utilities.stop_server(is_slurm=True) #%% #%% fold=os.path.split(os.path.split(fnames[0])[-2])[-1] os.mkdir(fold) #%% files=glob.glob(fnames[0][:-20]+'*BL_compress_.tif') files.sort() print(files) #%% m=cb.load_movie_chain(files,fr=3) m.play(backend='opencv',gain=10,fr=40) #%%
class ClusterLab(epyc.Lab): """A :class:`Lab` running on an ``pyparallel`` compute cluster. Experiments are submitted to engines in the cluster for execution in parallel, with the experiments being performed asynchronously to allow for disconnection and subsequent retrieval of results. Combined with a persistent :class:`LabNotebook`, this allows for fully decoupled access to an on-going computational experiment with piecewise retrieval of results. This class requires a cluster to already be set up and running, configured for persistent access, with access to the necessary code and libraries, and with appropriate security information available to the client. """ # Tuning parameters WaitingTime = 30 #: Waiting time for checking for job completion. Lower values increase network traffic. def __init__(self, notebook=None, url_file=None, profile=None, profile_dir=None, ipython_dir=None, context=None, debug=False, sshserver=None, sshkey=None, password=None, paramiko=None, timeout=10, cluster_id=None, use_dill=False, **extra_args): """Create an empty lab attached to the given cluster. Most of the arguments are as expected by the ``pyparallel.Client`` class, and are used to create the underlying connection to the cluster. The connection is opened immediately, meaning the cluster must be up and accessible when creating a lab to use it. :param notebook: the notebook used to results (defaults to an empty :class:`LabNotebook`) :param url_file: file containing connection information for accessing cluster :param profile: name of the IPython profile to use :param profile_dir: directory containing the profile's connection information :param ipython_dir: directory containing profile directories :param context: ZMQ context :param debug: whether to issue debugging information (defaults to False) :param sshserver: username and machine for ssh connections :param sshkey: file containing ssh key :param password: ssh password :param paramiko: True to use paramiko for ssh (defaults to False) :param timeout: timeout in seconds for ssh connection (defaults to 10s) :param cluster_id: string added to runtime files to prevent collisions :param use_dill: whether to use Dill as pickler (defaults to False)""" super(epyc.ClusterLab, self).__init__(notebook) # record all the connection arguments for later self._arguments = dict(url_file=url_file, profile=profile, profile_dir=profile_dir, ipython_dir=ipython_dir, context=context, debug=debug, sshserver=sshserver, sshkey=sshkey, password=password, paramiko=paramiko, timeout=timeout, cluster_id=cluster_id, use_dill=use_dill, **extra_args) self._client = None # connect to the cluster self.open() # use Dill if requested if use_dill: self.use_dill() # ---------- Protocol ---------- def open(self): """Connect to the cluster.""" if self._client is None: self._client = Client(**self._arguments) def close(self): """Close down the connection to the cluster.""" if self._client is not None: self._client.close() self._client = None def recreate(self): '''Save the arguments needed to re-connect to the cluster we use. :returns: a (classname, args) pair''' (cn, args) = super(ClusterLab, self).recreate() nargs = args.copy() nargs.update(self._arguments) return (classname, nargs) # ---------- Remote control of the compute engines ---------- def numberOfEngines(self): """Return the number of engines available to this lab. :returns: the number of engines""" return len(self.engines()) def engines(self): """Return a list of the available engines. :returns: a list of engines""" self.open() return self._client[:] def use_dill(self): """Make the cluster use Dill as pickler for transferring results. This isn't generally needed, but is sometimes useful for particularly complex experiments such as those involving closures. (Or, to put it another way, if you find yourself tempted to use this method, consider re-structuring your experiment code.)""" self.open() with self.sync_imports(quiet=True): import dill self._client.direct_view().use_dill() def sync_imports(self, quiet=False): """Return a context manager to control imports onto all the engines in the underlying cluster. This method is used within a ``with`` statement. Any imports should be done with no experiments running, otherwise the method will block until the cluster is quiet. Generally imports will be one of the first things done when connecting to a cluster. (But be careful not to accidentally try to re-import if re-connecting to a running cluster.) :param quiet: if True, suppresses messages (defaults to False) :returns: a context manager""" self.open() return self._client[:].sync_imports(quiet=quiet) # ---------- Running experiments ---------- def _mixup(self, ps): """Private method to mix up a list of values in-place using a Fisher-Yates shuffle (see https://en.wikipedia.org/wiki/Fisher-Yates_shuffle). :param ps: the array :returns: the array, shuffled in-place""" for i in range(len(ps) - 1, 0, -1): j = int(numpy.random.random() * i) temp = ps[i] ps[i] = ps[j] ps[j] = temp return ps def runExperiment(self, e): """Run the experiment across the parameter space in parallel using all the engines in the cluster. This method returns immediately. The experiments are run asynchronously, with the points in the parameter space being explored randomly so that intermediate retrievals of results are more representative of the overall result. Put another way, for a lot of experiments the results available will converge towards a final answer, so we can plot them and see the answer emerge. :param e: the experiment""" # create the parameter space space = self.parameterSpace() # only proceed if there's work to do if len(space) > 0: nb = self.notebook() # randomise the order of the parameter space so that we evaluate across # the space as we go along to try to make intermediate (incomplete) result # sets more representative of the overall result set ps = self._mixup(space) try: # connect to the cluster self.open() # submit an experiment at each point in the parameter space to the cluster view = self._client.load_balanced_view() jobs = [] for p in ps: jobs.extend((view.apply_async((lambda p: e.set(p).run()), p)).msg_ids) # there seems to be a race condition in submitting jobs, # whereby jobs get dropped if they're submitted too quickly time.sleep(0.01) # record the mesage ids of all the jobs as submitted but not yet completed psjs = zip(ps, jobs) for (p, j) in psjs: nb.addPendingResult(p, j) finally: # commit our pending results in the notebook nb.commit() self.close() def updateResults(self): """Update our results within any pending results that have completed since we last retrieved results from the cluster. :returns: the number of pending results completed at this call""" # we do all the tests for pending results against the notebook directly, # as the corresponding methods on self call this method themselves nb = self.notebook() # look for pending results if we're waiting for any n = 0 if nb.numberOfPendingResults() > 0: # we have results to get self.open() for j in set(nb.pendingResults()): # query the status of a job status = self._client.result_status(j, status_only=False) # add all completed jobs to the notebook if j in status['completed']: r = status[j] # update the result in the notebook, cancelling # the pending result as well # values come back from Client.result_status() in # varying degrees of list-nesting, which LabNotebook.addResult() # handles itself nb.addResult(r, j) # commit changes to the notebook nb.commit() # purge the completed job from the cluster self._client.purge_hub_results(j) # record that we retrieved the results for the given job n = n + 1 return n # ---------- Accessing results ---------- def numberOfResults(self): """Return the number of results we have available at the moment. :returns: the number of results""" self.updateResults() return self.notebook().numberOfResults() def numberOfPendingResults(self): """Return the number of resultswe are waiting for. :returns: the number of pending results""" self.updateResults() return self.notebook().numberOfPendingResults() def _availableResultsFraction(self): """Private method to return the fraction of results available, as a real number between 0 and 1. This does not update the results fetched from the cluster. :returns: the fraction of available results""" tr = self.notebook().numberOfResults() + self.notebook( ).numberOfPendingResults() if tr == 0: return 0 else: return (self.notebook().numberOfResults() + 0.0) / tr def readyFraction(self): """Test what fraction of results are available. This will change over time as the results come in. :returns: the fraction from 0 to 1""" self.updateResults() return self._availableResultsFraction() def ready(self): """Test whether all the results are available. This will change over time as the results come in. :returns: True if all the results are available""" return (self.readyFraction() == 1) def wait(self, timeout=-1): """Wait for all pending results to be finished. If timeout is set, return after this many seconds regardless. :param timeout: timeout period in seconds (defaults to forever) :returns: True if all the results completed""" # we can't use pyparallel.Client.wait() for this, because that # method only works for cases where the Client object is the one that # submitted the jobs to the cluster hub -- and therefore has the # necessary data structures to perform synchronisation. This isn't the # case for us, as one of the main goals of epyc is to support disconnected # operation, which implies a different Client object retrieving results # than the one that submitted the jobs in the first place. This is # unfortunate, but understandable given the typical use cases for # Client objects in pyparallel. # # Instead. we have to code around a little busily. The ClusterLab.WaitingTime # global sets the latency for waiting, and we repeatedly wait for this amount # of time before updating the results. The latency value essentially controls # how busy this process is: given that most simulations are expected to # be long, a latency in the tens of seconds feels about right as a default if self.numberOfPendingResults() > 0: # we've got pending results, wait for them timeWaited = 0 while (timeout < 0) or (timeWaited < timeout): if self.numberOfPendingResults() == 0: # no pending jobs left, we're complete return True else: # not done yet, calculate the waiting period if timeout == -1: # wait for the default waiting period dt = self.WaitingTime else: # wait for the default waiting period or until the end of the timeout. # whichever comes first if (timeout - timeWaited) < self.WaitingTime: dt = timeout - timeWaited else: dt = self.WaitingTime # sleep for a while time.sleep(dt) timeWaited = timeWaited + dt # if we get here, the timeout expired, so do a final check # and then exit return (self.numberOfPendingResults() == 0) else: # no results, so we got them all return True # ---------- Managing pending results ---------- def pendingResults(self): """Return the list of job iods for any pending results. :returns: a list of job ids""" return self.notebook().pendingResults() def pendingResultsFor(self, params): """Return a list of job ids for any results pending for experiments at the given point in the parameter space. :param params: the experimental parameters :returns: a list of job ids""" return self.notebook().pendingResultsFor(params) def _abortJobs(self, js): """Private method to abort a set of jobs. :param js: the job ids to be aborted""" self.open() self._client.abort(jobs=js) self.close() def cancelPendingResultsFor(self, params): """Cancel any results pending for experiments at the given point in the parameter space. :param params: the experimental parameters""" # grab the result job ids jobs = self.pendingResultsFor(params) if len(jobs) > 0: # abort in the cluster self._abortJobs(jobs) # cancel in the notebook self.notebook().cancelPendingResultsFor(params) def cancelAllPendingResults(self): """Cancel all pending results.""" # grab all the pending job ids jobs = self.pendingResults() if len(jobs) > 0: # abort in the cluster self._abortJobs(jobs) # cancel in the notebook self.notebook().cancelAllPendingResults()
def dask_executor(): from distributed import LocalCluster, Client client = Client(n_workers=1) yield client client.close()
def run_CNMF_patches(file_name, shape, options, rf=16, stride=4, n_processes=2, backend='single_thread', memory_fact=1): """Function that runs CNMF in patches, either in parallel or sequentiually, and return the result for each. It requires that ipyparallel is running Parameters ---------- file_name: string full path to an npy file (2D, pixels x time) containing the movie shape: tuple of thre elements dimensions of the original movie across y, x, and time options: dictionary containing all the parameters for the various algorithms rf: int half-size of the square patch in pixel stride: int amount of overlap between patches backend: string 'ipyparallel' or 'single_thread' n_processes: int nuber of cores to be used (should be less than the number of cores started with ipyparallel) memory_fact: double unitless number accounting how much memory should be used. It represents the fration of patch processed in a single thread. You will need to try different values to see which one would work Returns ------- A_tot: matrix containing all the componenents from all the patches C_tot: matrix containing the calcium traces corresponding to A_tot sn_tot: per pixel noise estimate optional_outputs: set of outputs related to the result of CNMF ALGORITHM ON EACH patch """ (d1, d2, T) = shape d = d1 * d2 K = options['init_params']['K'] options['preprocess_params']['backend'] = 'single_thread' options['preprocess_params']['n_pixels_per_process'] = np.int( (rf * rf) / memory_fact) options['spatial_params']['n_pixels_per_process'] = np.int( (rf * rf) / memory_fact) options['temporal_params']['n_pixels_per_process'] = np.int( (rf * rf) / memory_fact) options['spatial_params']['backend'] = 'single_thread' options['temporal_params']['backend'] = 'single_thread' idx_flat, idx_2d = extract_patch_coordinates(d1, d2, rf=rf, stride=stride) # import pdb # pdb.set_trace() args_in = [] for id_f, id_2d in zip(idx_flat[:], idx_2d[:]): args_in.append((file_name, id_f, id_2d[0].shape, options)) print len(idx_flat) st = time.time() if backend is 'ipyparallel': try: c = Client() dview = c[:n_processes] file_res = dview.map_sync(cnmf_patches, args_in) dview.results.clear() c.purge_results('all') c.purge_everything() c.close() except: print('Something went wrong') raise finally: print('You may think that it went well but reality is harsh') elif backend is 'single_thread': file_res = map(cnmf_patches, args_in) else: raise Exception('Backend unknown') print time.time() - st # extract the values from the output of mapped computation num_patches = len(file_res) A_tot = scipy.sparse.csc_matrix((d, K * num_patches)) B_tot = scipy.sparse.csc_matrix((d, num_patches)) C_tot = np.zeros((K * num_patches, T)) F_tot = np.zeros((num_patches, T)) mask = np.zeros(d) sn_tot = np.zeros((d1 * d2)) b_tot = [] f_tot = [] bl_tot = [] c1_tot = [] neurons_sn_tot = [] g_tot = [] idx_tot = [] shapes_tot = [] id_patch_tot = [] count = 0 patch_id = 0 print 'Transforming patches into full matrix' for idx_, shapes, A, b, C, f, S, bl, c1, neurons_sn, g, sn, _ in file_res: sn_tot[idx_] = sn b_tot.append(b) f_tot.append(f) bl_tot.append(bl) c1_tot.append(c1) neurons_sn_tot.append(neurons_sn) g_tot.append(g) idx_tot.append(idx_) shapes_tot.append(shapes) mask[idx_] += 1 F_tot[patch_id, :] = f B_tot[idx_, patch_id] = b for ii in range(np.shape(A)[-1]): new_comp = A.tocsc()[:, ii] / np.sqrt( np.sum(np.array(A.tocsc()[:, ii].todense())**2)) if new_comp.sum() > 0: A_tot[idx_, count] = new_comp C_tot[count, :] = C[ii, :] id_patch_tot.append(patch_id) count += 1 patch_id += 1 A_tot = A_tot[:, :count] C_tot = C_tot[:count, :] optional_outputs = dict() optional_outputs['b_tot'] = b_tot optional_outputs['f_tot'] = f_tot optional_outputs['bl_tot'] = bl_tot optional_outputs['c1_tot'] = c1_tot optional_outputs['neurons_sn_tot'] = neurons_sn_tot optional_outputs['g_tot'] = g_tot optional_outputs['idx_tot'] = idx_tot optional_outputs['shapes_tot'] = shapes_tot optional_outputs['id_patch_tot'] = id_patch_tot optional_outputs['B'] = B_tot optional_outputs['F'] = F_tot optional_outputs['mask'] = mask Im = scipy.sparse.csr_matrix((1. / mask, (np.arange(d), np.arange(d)))) Bm = Im.dot(B_tot) A_tot = Im.dot(A_tot) f = np.mean(F_tot, axis=0) for iter in range(10): b = Bm.dot(F_tot.dot(f)) / np.sum(f**2) f = np.dot((Bm.T.dot(b)).T, F_tot) / np.sum(b**2) return A_tot, C_tot, b, f, sn_tot, optional_outputs
idx_components=np.union1d(idx_components_r,idx_components_raw) idx_components=np.union1d(idx_components,idx_components_delta) idx_blobs=np.intersect1d(idx_components,idx_blobs) idx_components_bad=np.setdiff1d(list(range(len(traces))),idx_components) print(' ***** ') print((len(traces))) print((len(idx_components))) print((len(idx_blobs))) #%% visualize components #pl.figure(); pl.subplot(1,3,1) crd = plot_contours(A2.tocsc()[:,idx_components],Cn,thr=0.9) pl.subplot(1,3,2) crd = plot_contours(A2.tocsc()[:,idx_blobs],Cn,thr=0.9) pl.subplot(1,3,3) crd = plot_contours(A2.tocsc()[:,idx_components_bad],Cn,thr=0.9) #%% view_patches_bar(Yr,scipy.sparse.coo_matrix(A2.tocsc()[:,idx_components]),C2[idx_components,:],b2,f2, dims[0],dims[1], YrA=YrA[idx_components,:],img=Cn) #%% view_patches_bar(Yr,scipy.sparse.coo_matrix(A2.tocsc()[:,idx_components_bad]),C2[idx_components_bad,:],b2,f2, dims[0],dims[1], YrA=YrA[idx_components_bad,:],img=Cn) #%% STOP CLUSTER pl.close() if not single_thread: c.close() cm.stop_server()
def update_temporal_components_parallel(Y, A, b, Cin, fin, bl = None, c1 = None, g = None, sn = None, ITER=2, method_foopsi='constrained_foopsi', n_processes=1, backend='single_thread',memory_efficient=False, **kwargs): """Update temporal components and background given spatial components using a block coordinate descent approach. Parameters ----------- Y: np.ndarray (2D) input data with time in the last axis (d x T) A: sparse matrix (crc format) matrix of temporal components (d x K) b: ndarray (dx1) current estimate of background component Cin: np.ndarray current estimate of temporal components (K x T) fin: np.ndarray current estimate of temporal background (vector of length T) g: np.ndarray Global time constant (not used) bl: np.ndarray baseline for fluorescence trace for each column in A c1: np.ndarray initial concentration for each column in A g: np.ndarray discrete time constant for each column in A sn: np.ndarray noise level for each column in A ITER: positive integer Maximum number of block coordinate descent loops. method_foopsi: string Method of deconvolution of neural activity. constrained_foopsi is the only method supported at the moment. n_processes: int number of processes to use for parallel computation. Should be less than the number of processes started with ipcluster. backend: 'str' single_thread no parallelization ipyparallel, parallelization using the ipyparallel cluster. You should start the cluster (install ipyparallel and then type ipcluster -n 6, where 6 is the number of processes). memory_efficient: Bool whether or not to optimize for memory usage (longer running times). nevessary with very large datasets **kwargs: dict all parameters passed to constrained_foopsi except bl,c1,g,sn (see documentation). Some useful parameters are p: int order of the autoregression model method: [optional] string solution method for basis projection pursuit cvx or spgl1 or debug for fast but possibly imprecise temporal components Returns -------- C: np.matrix matrix of temporal components (K x T) f: np.array vector of temporal background (length T) Y_res: np.ndarray matrix with current residual (d x T) S: np.ndarray matrix of merged deconvolved activity (spikes) (K x T) bl: float same as input c1: float same as input g: float same as input sn: float same as input """ if not kwargs.has_key('p') or kwargs['p'] is None: raise Exception("You have to provide a value for p") d,T = np.shape(Y); nr = np.shape(A)[-1] if bl is None: bl=np.repeat(None,nr) if c1 is None: c1=np.repeat(None,nr) if g is None: g=np.repeat(None,nr) if sn is None: sn=np.repeat(None,nr) A = scipy.sparse.hstack((A,coo_matrix(b))) S = np.zeros(np.shape(Cin)); Cin = np.vstack((Cin,fin)); C = Cin; #% nA = np.squeeze(np.array(np.sum(np.square(A.todense()),axis=0))) Sp = np.zeros((nr,T)) #YrA = Y.T*A - Cin.T*(A.T*A); # Y=np.matrix(Y) # C=np.matrix(C) # Cin=np.matrix(Cin) # YrA2 = Y.T*A - Cin.T*(A.T*A); Cin=coo_matrix(Cin) YrA = (A.T.dot(Y)).T-Cin.T.dot(A.T.dot(A)) if backend == 'ipyparallel': try: # if server is not running and raise exception if not installed or not started from ipyparallel import Client c = Client() except: print "this backend requires the installation of the ipyparallel (pip install ipyparallel) package and starting a cluster (type ipcluster start -n 6) where 6 is the number of nodes" raise if len(c) < n_processes: print len(c) raise Exception("the number of nodes in the cluster are less than the required processes: decrease the n_processes parameter to a suitable value") dview=c[:n_processes] # use the number of processes Cin=np.array(Cin.todense()) for iter in range(ITER): O,lo = update_order(A.tocsc()[:,:nr]) P_=[]; for count,jo_ in enumerate(O): jo=np.array(list(jo_)) Ytemp = YrA[:,jo.flatten()] + (np.dot(np.diag(nA[jo]),Cin[jo,:])).T Ctemp = np.zeros((np.size(jo),T)) Stemp = np.zeros((np.size(jo),T)) btemp = np.zeros((np.size(jo),1)) sntemp = btemp.copy() c1temp = btemp.copy() gtemp = np.zeros((np.size(jo),kwargs['p'])); nT = nA[jo] # args_in=[(np.squeeze(np.array(Ytemp[:,jj])), nT[jj], jj, bl[jo[jj]], c1[jo[jj]], g[jo[jj]], sn[jo[jj]], kwargs) for jj in range(len(jo))] args_in=[(np.squeeze(np.array(Ytemp[:,jj])), nT[jj], jj, None, None, None, None, kwargs) for jj in range(len(jo))] if backend == 'ipyparallel': results = dview.map_sync(constrained_foopsi_parallel,args_in) elif backend == 'single_thread': results = map(constrained_foopsi_parallel,args_in) else: raise Exception('Backend not defined. Use either single_thread or ipyparallel') for chunk in results: #pars=dict(kwargs) C_,Sp_,Ytemp_,cb_,c1_,sn_,gn_,jj_=chunk Ctemp[jj_,:] = C_[None,:] Stemp[jj_,:] = Sp_ Ytemp[:,jj_] = Ytemp_[:,None] btemp[jj_] = cb_ c1temp[jj_] = c1_ sntemp[jj_] = sn_ gtemp[jj_,:] = gn_.T bl[jo[jj_]] = cb_ c1[jo[jj_]] = c1_ sn[jo[jj_]] = sn_ g[jo[jj_]] = gtemp[jj,:]#[jj_,np.abs(gtemp[jj,:])>0] #pars['b'] = cb_ # pars['c1'] = c1_ # pars['neuron_sn'] = sn_ # pars['gn'] = gtemp[jj_,np.abs(gtemp[jj,:])>0] # ## for jj = 1:length(O{jo}) ## P.gn(O{jo}(jj)) = {gtemp(jj,abs(gtemp(jj,:))>0)'}; ## end # pars['neuron_id'] = jo[jj_] # P_.append(pars) YrA[:,jo] = Ytemp C[jo,:] = Ctemp S[jo,:] = Stemp # if (np.sum(lo[:jo])+1)%1 == 0: print str(np.sum(lo[:count])) + ' out of total ' + str(nr) + ' temporal components updated \n' ii=nr YrA[:,ii] = YrA[:,ii] + nA[ii]*np.atleast_2d(Cin[ii,:]).T cc = np.maximum(YrA[:,ii]/nA[ii],0) C[ii,:] = cc[:].T YrA[:,ii] = YrA[:,ii] - nA[ii]*np.atleast_2d(C[ii,:]).T if backend == 'ipyparallel': dview.results.clear() c.purge_results('all') c.purge_everything() if scipy.linalg.norm(Cin - C,'fro')/scipy.linalg.norm(C,'fro') <= 1e-3: # stop if the overall temporal component does not change by much print "stopping: overall temporal component not changing significantly" break else: Cin = C Y_res = Y - A*C # this includes the baseline term f = C[nr:,:] C = C[:nr,:] P_ = sorted(P_, key=lambda k: k['neuron_id']) if backend == 'ipyparallel': c.close() return C,f,Y_res,S,bl,c1,sn,g
def run_CNMF_patches(file_name, shape, options, rf=16, stride = 4, n_processes=2, backend='single_thread',memory_fact=1): """Function that runs CNMF in patches, either in parallel or sequentiually, and return the result for each. It requires that ipyparallel is running Parameters ---------- file_name: string full path to an npy file (2D, pixels x time) containing the movie shape: tuple of thre elements dimensions of the original movie across y, x, and time options: dictionary containing all the parameters for the various algorithms rf: int half-size of the square patch in pixel stride: int amount of overlap between patches backend: string 'ipyparallel' or 'single_thread' n_processes: int nuber of cores to be used (should be less than the number of cores started with ipyparallel) memory_fact: double unitless number accounting how much memory should be used. It represents the fration of patch processed in a single thread. You will need to try different values to see which one would work Returns ------- A_tot: matrix containing all the componenents from all the patches C_tot: matrix containing the calcium traces corresponding to A_tot sn_tot: per pixel noise estimate optional_outputs: set of outputs related to the result of CNMF ALGORITHM ON EACH patch """ (d1,d2,T)=shape d=d1*d2 K=options['init_params']['K'] options['preprocess_params']['backend']='single_thread' options['preprocess_params']['n_pixels_per_process']=np.int((rf*rf)/memory_fact) options['spatial_params']['n_pixels_per_process']=np.int((rf*rf)/memory_fact) options['temporal_params']['n_pixels_per_process']=np.int((rf*rf)/memory_fact) options['spatial_params']['backend']='single_thread' options['temporal_params']['backend']='single_thread' idx_flat,idx_2d=extract_patch_coordinates(d1, d2, rf=rf, stride = stride) # import pdb # pdb.set_trace() args_in=[] for id_f,id_2d in zip(idx_flat[:],idx_2d[:]): args_in.append((file_name, id_f,id_2d[0].shape, options)) print len(idx_flat) st=time.time() if backend is 'ipyparallel': try: c = Client() dview=c[:n_processes] file_res = dview.map_sync(cnmf_patches, args_in) dview.results.clear() c.purge_results('all') c.purge_everything() c.close() except: print('Something went wrong') raise finally: print('You may think that it went well but reality is harsh') elif backend is 'single_thread': file_res = map(cnmf_patches, args_in) else: raise Exception('Backend unknown') print time.time()-st # extract the values from the output of mapped computation num_patches=len(file_res) A_tot=scipy.sparse.csc_matrix((d,K*num_patches)) B_tot=scipy.sparse.csc_matrix((d,num_patches)) C_tot=np.zeros((K*num_patches,T)) F_tot=np.zeros((num_patches,T)) mask=np.zeros(d) sn_tot=np.zeros((d1*d2)) b_tot=[] f_tot=[] bl_tot=[] c1_tot=[] neurons_sn_tot=[] g_tot=[] idx_tot=[]; shapes_tot=[] id_patch_tot=[] count=0 patch_id=0 print 'Transforming patches into full matrix' for idx_,shapes,A,b,C,f,S,bl,c1,neurons_sn,g,sn,_ in file_res: sn_tot[idx_]=sn b_tot.append(b) f_tot.append(f) bl_tot.append(bl) c1_tot.append(c1) neurons_sn_tot.append(neurons_sn) g_tot.append(g) idx_tot.append(idx_) shapes_tot.append(shapes) mask[idx_] += 1 F_tot[patch_id,:]=f B_tot[idx_,patch_id]=b for ii in range(np.shape(A)[-1]): new_comp=A.tocsc()[:,ii]/np.sqrt(np.sum(np.array(A.tocsc()[:,ii].todense())**2)) if new_comp.sum()>0: A_tot[idx_,count]=new_comp C_tot[count,:]=C[ii,:] id_patch_tot.append(patch_id) count+=1 patch_id+=1 A_tot=A_tot[:,:count] C_tot=C_tot[:count,:] optional_outputs=dict() optional_outputs['b_tot']=b_tot optional_outputs['f_tot']=f_tot optional_outputs['bl_tot']=bl_tot optional_outputs['c1_tot']=c1_tot optional_outputs['neurons_sn_tot']=neurons_sn_tot optional_outputs['g_tot']=g_tot optional_outputs['idx_tot']=idx_tot optional_outputs['shapes_tot']=shapes_tot optional_outputs['id_patch_tot']= id_patch_tot optional_outputs['B'] = B_tot optional_outputs['F'] = F_tot optional_outputs['mask'] = mask Im = scipy.sparse.csr_matrix((1./mask,(np.arange(d),np.arange(d)))) Bm = Im.dot(B_tot) A_tot = Im.dot(A_tot) f = np.mean(F_tot,axis=0) for iter in range(10): b = Bm.dot(F_tot.dot(f))/np.sum(f**2) f = np.dot((Bm.T.dot(b)).T,F_tot)/np.sum(b**2) return A_tot,C_tot,b,f,sn_tot, optional_outputs
def extract_rois_patch(file_name,d1,d2,rf=5,stride = 2): not_completed, in_progress rf=6 stride = 2 idx_flat,idx_2d=extract_patch_coordinates(d1, d2, rf=rf,stride = stride) perctl=95 n_components=2 tol=1e-6 max_iter=5000 args_in=[] for id_f,id_2d in zip(idx_flat,idx_2d): args_in.append((file_name, id_f,id_2d[0].shape, perctl,n_components,tol,max_iter)) st=time.time() try: if 1: c = Client() dview=c[:] file_res = dview.map_sync(nmf_patches, args_in) else: file_res = map(nmf_patches, args_in) finally: dview.results.clear() c.purge_results('all') c.purge_everything() c.close() print time.time()-st A1=lil_matrix((d1*d2,len(file_res))) C1=[] A2=lil_matrix((d1*d2,len(file_res))) C2=[] A_tot=lil_matrix((d1*d2,n_components*len(file_res))) C_tot=[]; count_out=0 for count,f in enumerate(file_res): idx_,flt,ca,d=f print count_out #flt,ca,_=cse.order_components(coo_matrix(flt),ca) # A1[idx_,count]=flt[:,0][:,np.newaxis]/np.sqrt(np.sum(flt[:,0]**2)) # A2[idx_,count]=flt[:,1][:,np.newaxis] /np.sqrt(np.sum(flt[:,1]**2)) # C1.append(ca[0,:]) # C2.append(ca[1,:]) for ccc in range(n_components): A_tot[idx_,count_out]=flt[:,ccc][:,np.newaxis]/np.sqrt(np.sum(flt[:,ccc]**2)) C_tot.append(ca[ccc,:]) count_out+=1 # pl.imshow(np.reshape(flt[:,0],d,order='F'),vmax=10) # pl.pause(.1) correlations=np.corrcoef(np.array(C_tot)) centers=cse.com(A_tot.todense(),d1,d2) distances=sklearn.metrics.pairwise.euclidean_distances(centers) pl.imshow((correlations>0.8) & (distances<10)) Yr=np.load('Yr.npy',mmap_mode='r') [d,T]=Yr.shape Y=np.reshape(Yr,(d1,d2,T),order='F') options=cse.utilities.CNMFSetParms(Y,p=0) res_merge=cse.merge_components(Yr,A_tot,[],np.array(C_tot),[],np.array(C_tot),[],options['temporal_params'],options['spatial_params'],thr=0.8) A_m,C_m,nr_m,merged_ROIs,S_m,bl_m,c1_m,sn_m,g_m=res_merge A_norm=np.array([A_m[:,rr].toarray()/np.sqrt(np.sum(A_m[:,rr].toarray()**2)) for rr in range(A_m.shape[-1])]).T options=cse.utilities.CNMFSetParms(Y,p=2,K=np.shape(A_m)[-1]) Yr,sn,g=cse.pre_processing.preprocess_data(Yr,**options['preprocess_params']) epsilon=1e-2 pixels_bckgrnd=np.nonzero(A_norm.sum(axis=-1)<epsilon)[0] f=np.sum(Yr[pixels_bckgrnd,:],axis=0) A2,b2,C2 = cse.spatial.update_spatial_components(Yr, C_m, f, A_m, sn=sn, **options['spatial_params']) A_or2, C_or2, srt2 = cse.utilities.order_components(A2,C2) A_norm2=np.array([A_or2[:,rr]/np.sqrt(np.sum(A_or2[:,rr]**2)) for rr in range(A_or2.shape[-1])]).T options['temporal_params']['p'] = 2 # set it back to original value to perform full deconvolution C2,f2,S2,bl2,c12,neurons_sn2,g21,YrA = cse.temporal.update_temporal_components(Yr,A2,b2,C2,f,bl=None,c1=None,sn=None,g=None,**options['temporal_params']) A_or, C_or, srt = cse.utilities.order_components(A2,C2) return A1,A2,C1
fnames = [] for file in glob.glob(base_folder + 'k31_20160107_MMP_150um_65mW_zoom2p2_000*[0-9].hdf5'): fnames.append(file) fnames.sort() print(fnames) #%% file_res = cb.utils.pre_preprocess_movie_labeling( client_[::2], fnames, median_filter_size=(2, 1, 1), resize_factors=[.2, .1666666666], diameter_bilateral_blur=4) #%% client_.close() cse.utilities.stop_server(is_slurm=True) #%% #%% fold = os.path.split(os.path.split(fnames[0])[-2])[-1] os.mkdir(fold) #%% files = glob.glob(fnames[0][:-20] + '*BL_compress_.tif') files.sort() print(files) #%% m = cb.load_movie_chain(files, fr=3) m.play(backend='opencv', gain=10, fr=40) #%%
def update_temporal_components(Y, A, b, Cin, fin, bl=None, c1=None, g=None, sn=None, ITER=2, method_foopsi='constrained_foopsi', n_processes=1, backend='single_thread', memory_efficient=False, debug=False, **kwargs): """Update temporal components and background given spatial components using a block coordinate descent approach. Parameters ----------- Y: np.ndarray (2D) input data with time in the last axis (d x T) A: sparse matrix (crc format) matrix of temporal components (d x K) b: ndarray (dx1) current estimate of background component Cin: np.ndarray current estimate of temporal components (K x T) fin: np.ndarray current estimate of temporal background (vector of length T) g: np.ndarray Global time constant (not used) bl: np.ndarray baseline for fluorescence trace for each column in A c1: np.ndarray initial concentration for each column in A g: np.ndarray discrete time constant for each column in A sn: np.ndarray noise level for each column in A ITER: positive integer Maximum number of block coordinate descent loops. method_foopsi: string Method of deconvolution of neural activity. constrained_foopsi is the only method supported at the moment. n_processes: int number of processes to use for parallel computation. Should be less than the number of processes started with ipcluster. backend: 'str' single_thread no parallelization ipyparallel, parallelization using the ipyparallel cluster. You should start the cluster (install ipyparallel and then type ipcluster -n 6, where 6 is the number of processes). memory_efficient: Bool whether or not to optimize for memory usage (longer running times). nevessary with very large datasets **kwargs: dict all parameters passed to constrained_foopsi except bl,c1,g,sn (see documentation). Some useful parameters are p: int order of the autoregression model method: [optional] string solution method for constrained foopsi. Choices are 'cvx': using cvxopt and picos (slow especially without the MOSEK solver) 'cvxpy': using cvxopt and cvxpy with the ECOS solver (faster, default) solvers: list string primary and secondary (if problem unfeasible for approx solution) solvers to be used with cvxpy, default is ['ECOS','SCS'] Note -------- The temporal components are updated in parallel by default by forming of sequence of vertex covers. Returns -------- C: np.ndarray matrix of temporal components (K x T) f: np.array vector of temporal background (length T) S: np.ndarray matrix of merged deconvolved activity (spikes) (K x T) bl: float same as input c1: float same as input g: float same as input sn: float same as input YrA: np.ndarray matrix of spatial component filtered raw data, after all contributions have been removed. YrA corresponds to the residual trace for each component and is used for faster plotting (K x T) """ if not kwargs.has_key('p') or kwargs['p'] is None: raise Exception("You have to provide a value for p") d, T = np.shape(Y) nr = np.shape(A)[-1] if bl is None: bl = np.repeat(None, nr) if c1 is None: c1 = np.repeat(None, nr) if g is None: g = np.repeat(None, nr) if sn is None: sn = np.repeat(None, nr) A = scipy.sparse.hstack((A, coo_matrix(b))) S = np.zeros(np.shape(Cin)) Cin = np.vstack((Cin, fin)) C = Cin nA = np.squeeze(np.array(np.sum(np.square(A.todense()), axis=0))) #import pdb #pdb.set_trace() Cin = coo_matrix(Cin) #YrA = ((A.T.dot(Y)).T-Cin.T.dot(A.T.dot(A))) YA = (A.T.dot(Y).T) * spdiags(1. / nA, 0, nr + 1, nr + 1) AA = ((A.T.dot(A)) * spdiags(1. / nA, 0, nr + 1, nr + 1)).tocsr() YrA = YA - Cin.T.dot(AA) #YrA = ((A.T.dot(Y)).T-Cin.T.dot(A.T.dot(A)))*spdiags(1./nA,0,nr+1,nr+1) if backend == 'ipyparallel': try: # if server is not running and raise exception if not installed or not started from ipyparallel import Client c = Client() except: print "this backend requires the installation of the ipyparallel (pip install ipyparallel) package and starting a cluster (type ipcluster start -n 6) where 6 is the number of nodes" raise if len(c) < n_processes: print len(c) raise Exception( "the number of nodes in the cluster are less than the required processes: decrease the n_processes parameter to a suitable value" ) dview = c[:n_processes] # use the number of processes Cin = np.array(Cin.todense()) for iter in range(ITER): O, lo = update_order(A.tocsc()[:, :nr]) P_ = [] for count, jo_ in enumerate(O): jo = np.array(list(jo_)) #Ytemp = YrA[:,jo.flatten()] + (np.dot(np.diag(nA[jo]),Cin[jo,:])).T Ytemp = YrA[:, jo.flatten()] + Cin[jo, :].T Ctemp = np.zeros((np.size(jo), T)) Stemp = np.zeros((np.size(jo), T)) btemp = np.zeros((np.size(jo), 1)) sntemp = btemp.copy() c1temp = btemp.copy() gtemp = np.zeros((np.size(jo), kwargs['p'])) nT = nA[jo] # args_in=[(np.squeeze(np.array(Ytemp[:,jj])), nT[jj], jj, bl[jo[jj]], c1[jo[jj]], g[jo[jj]], sn[jo[jj]], kwargs) for jj in range(len(jo))] args_in = [(np.squeeze(np.array(Ytemp[:, jj])), nT[jj], jj, None, None, None, None, kwargs) for jj in range(len(jo))] # import pdb # pdb.set_trace() if backend == 'ipyparallel': # if debug: results = dview.map_async(constrained_foopsi_parallel, args_in) results.get() for outp in results.stdout: print outp[:-1] sys.stdout.flush() for outp in results.stderr: print outp[:-1] sys.stderr.flush() else: results = dview.map_sync(constrained_foopsi_parallel, args_in) elif backend == 'single_thread': results = map(constrained_foopsi_parallel, args_in) else: raise Exception( 'Backend not defined. Use either single_thread or ipyparallel' ) for chunk in results: pars = dict() C_, Sp_, Ytemp_, cb_, c1_, sn_, gn_, jj_ = chunk Ctemp[jj_, :] = C_[None, :] Stemp[jj_, :] = Sp_ Ytemp[:, jj_] = Ytemp_[:, None] btemp[jj_] = cb_ c1temp[jj_] = c1_ sntemp[jj_] = sn_ gtemp[jj_, :] = gn_.T bl[jo[jj_]] = cb_ c1[jo[jj_]] = c1_ sn[jo[jj_]] = sn_ g[jo[jj_]] = gn_.T if kwargs['p'] > 0 else [] #gtemp[jj,:] pars['b'] = cb_ pars['c1'] = c1_ pars['neuron_sn'] = sn_ pars['gn'] = gtemp[jj_, np.abs(gtemp[jj, :]) > 0] pars['neuron_id'] = jo[jj_] P_.append(pars) YrA -= (Ctemp - C[jo, :]).T * AA[jo, :] #YrA[:,jo] = Ytemp C[jo, :] = Ctemp.copy() S[jo, :] = Stemp # if (np.sum(lo[:jo])+1)%1 == 0: print str(np.sum(lo[:count + 1])) + ' out of total ' + str( nr) + ' temporal components updated' ii = nr #YrA[:,ii] = YrA[:,ii] + np.atleast_2d(Cin[ii,:]).T #cc = np.maximum(YrA[:,ii],0) cc = np.maximum(YrA[:, ii] + np.atleast_2d(Cin[ii, :]).T, 0) YrA -= (cc - np.atleast_2d(Cin[ii, :]).T) * AA[ii, :] C[ii, :] = cc.T #YrA = YA - C.T.dot(AA) #YrA[:,ii] = YrA[:,ii] - np.atleast_2d(C[ii,:]).T if backend == 'ipyparallel': dview.results.clear() c.purge_results('all') c.purge_everything() if scipy.linalg.norm(Cin - C, 'fro') / scipy.linalg.norm( C, 'fro') <= 1e-3: # stop if the overall temporal component does not change by much print "stopping: overall temporal component not changing significantly" break else: Cin = C f = C[nr:, :] C = C[:nr, :] YrA = np.array(YrA[:, :nr]).T P_ = sorted(P_, key=lambda k: k['neuron_id']) if backend == 'ipyparallel': c.close() return C, f, S, bl, c1, sn, g, YrA #,P_
def update_spatial_components_parallel(Y,C,f,A_in,sn=None, d1=None,d2=None,min_size=3,max_size=8, dist=3, method = 'ellipse', expandCore = None,backend='single_thread',n_processes=4,n_pixels_per_process=128, memory_efficient=False): """update spatial footprints and background through Basis Pursuit Denoising for each pixel i solve the problem [A(i,:),b(i)] = argmin sum(A(i,:)) subject to || Y(i,:) - A(i,:)*C + b(i)*f || <= sn(i)*sqrt(T); for each pixel the search is limited to a few spatial components Parameters ---------- Y: np.ndarray (2D) movie, raw data in 2D (pixels x time). C: np.ndarray calcium activity of each neuron. f: np.ndarray temporal profile of background activity. Ain: np.ndarray spatial profile of background activity. d1: [optional] int x movie dimension d2: [optional] int y movie dimension min_size: [optional] int max_size: [optional] int dist: [optional] int sn: [optional] float noise associated with each pixel if known n_processes: [optional] int number of threads to use when the backend is multiprocessing,threading, or ipyparallel backend [optional] str 'multiprocessing', 'threading', 'ipyparallel', 'single_thread' single_thread:no parallelization. It shoul dbe used in most cases. multiprocessing or threading: use the corresponding python threading package. It has known issues on mac OS. Not to be used in most situations. ipyparallel: starts an ipython cluster and then send jobs to each of them n_pixels_per_process: [optional] int number of pixels to be processed by each thread memory_efficient [bool] whether or not to reduce memory usage (at the expense of increased computational time) method: [optional] string method used to expand the search for pixels 'ellipse' or 'dilate' expandCore: [optional] scipy.ndimage.morphology if method is dilate this represents the kernel used for expansion Returns -------- A: np.ndarray new estimate of spatial footprints b: np.ndarray new estimate of spatial background C: np.ndarray temporal components (updated only when spatial components are completely removed) """ if expandCore is None: expandCore=iterate_structure(generate_binary_structure(2,1), 2).astype(int) if d1 is None or d2 is None: raise Exception('You need to define the input dimensions') Y=np.atleast_2d(Y) if Y.shape[1]==1: raise Exception('Dimension of Matrix Y must be pixels x time') C=np.atleast_2d(C) if C.shape[1]==1: raise Exception('Dimension of Matrix C must be neurons x time') f=np.atleast_2d(f) if f.shape[1]==1: raise Exception('Dimension of Matrix f must be neurons x time ') if len(A_in.shape)==1: A_in=np.atleast_2d(A_in).T if A_in.shape[0]==1: raise Exception('Dimension of Matrix A must be pixels x neurons ') start_time = time.time() Cf = np.vstack((C,f)) # create matrix that include background components [d,T] = np.shape(Y) if n_pixels_per_process > d: raise Exception('The number of pixels per process (n_pixels_per_process) is larger than the total number of pixels!! Decrease suitably.') nr,_ = np.shape(C) # number of neurons IND = determine_search_location(A_in,d1,d2,method = method, min_size = min_size, max_size = max_size, dist = dist, expandCore = expandCore) print " find search location" ind2_ =[ np.hstack( (np.where(iid_)[0] , nr+np.arange(f.shape[0])) ) if np.size(np.where(iid_)[0])>0 else [] for iid_ in IND] folder = tempfile.mkdtemp() if backend == 'multiprocessing' or backend == 'threading': A_name = os.path.join(folder, 'A_temp') # Pre-allocate a writeable shared memory map as a container for the # results of the parallel computation print "Create Matrix for dumping data from matrix A and C for parallel computation...." A_ = np.memmap(A_name, dtype=A_in.dtype,shape=(d,nr+np.size(f,0)), mode='w+') pixels_name = os.path.join(folder, 'pixels') C_name = os.path.join(folder, 'C_temp') # Dump the input data to disk to free the memory dump(Y, pixels_name) dump(Cf, C_name) # use mempry mapped versions of C and Y Y = load(pixels_name, mmap_mode='r') Cf = load(C_name, mmap_mode='r') pixel_groups=[range(i,i+n_pixels_per_process) for i in range(0,Y.shape[0]-n_pixels_per_process+1,n_pixels_per_process)] # Fork the worker processes to perform computation concurrently print "start parallel pool..." sys.stdout.flush() Parallel(n_jobs=n_processes, backend=backend,verbose=100,max_nbytes=None)(delayed(lars_regression_noise_parallel)(Y,Cf,A_,sn,i,ind2_) for i in pixel_groups) # if n_pixels_per_process is not a multiple of Y.shape[0] run on remaining pixels pixels_remaining= Y.shape[0] % n_pixels_per_process if pixels_remaining>0: print "Running deconvolution for remaining pixels:" + str(pixels_remaining) lars_regression_noise_parallel(Y,Cf,A_,sn,range(Y.shape[0]-pixels_remaining,Y.shape[0]),ind2_,positive=1) A_=np.array(A_) elif backend == 'ipyparallel': # use the ipyparallel package, you need to start a cluster server (ipcluster command) in order to use it C_name = os.path.join(folder, 'C_temp.npy') np.save(C_name,Cf) if type(Y) is np.core.memmap: # if input file is already memory mapped then find the filename Y_name=Y.filename else: # if not create a memory mapped version (necessary for parallelization) Y_name = os.path.join(folder, 'Y_temp.npy') np.save(Y_name,Y) Y=np.load(Y_name,mmap_mode='r') # create arguments to be passed to the function. Here we are grouping bunch of pixels to be processed by each thread pixel_groups=[(Y_name,C_name,sn,ind2_,range(i,i+n_pixels_per_process)) for i in range(0,d1*d2-n_pixels_per_process+1,n_pixels_per_process)] A_ = np.zeros((d,nr+np.size(f,0))) try: # if server is not running and raise exception if not installed or not started from ipyparallel import Client c = Client() except: print "this backend requires the installation of the ipyparallel (pip install ipyparallel) package and starting a cluster (type ipcluster start -n 6) where 6 is the number of nodes" raise if len(c) < n_processes: print len(c) raise Exception("the number of nodes in the cluster are less than the required processes: decrease the n_processes parameter to a suitable value") dview=c[:n_processes] # use the number of processes #serial_result = map(lars_regression_noise_ipyparallel, pixel_groups) parallel_result = dview.map_sync(lars_regression_noise_ipyparallel, pixel_groups) for chunk in parallel_result: for pars in chunk: px,idxs_,a=pars A_[px,idxs_]=a #clean up dview.results.clear() c.purge_results('all') c.purge_everything() c.close() elif backend=='single_thread': Cf_=[Cf[idx_,:] for idx_ in ind2_] #% LARS regression A_ = np.hstack((np.zeros((d,nr)),np.zeros((d,np.size(f,0))))) for c,y,s,id2_,px in zip(Cf_,Y,sn,ind2_,range(d)): if px%1000==0: print px if np.size(c)>0: _, _, a, _ , _= lars_regression_noise(y, np.array(c.T), 1, sn[px]**2*T) if np.isscalar(a): A_[px,id2_]=a else: A_[px,id2_]=a.T else: raise Exception('Unknown backend specified: use single_thread, threading, multiprocessing or ipyparallel') #% print 'Updated Spatial Components' A_=threshold_components(A_, d1, d2) print "threshold" ff = np.where(np.sum(A_,axis=0)==0); # remove empty components if np.size(ff)>0: ff = ff[0] warn('eliminating empty components!!') nr = nr - len(ff) A_ = np.delete(A_,list(ff),1) C = np.delete(C,list(ff),0) A_ = A_[:,:nr] A_=coo_matrix(A_) if memory_efficient: print "Using memory efficient computation (slow but memory preserving)" A__=coo_matrix(A_,dtype=np.float32) C__=coo_matrix(C[:nr,:],dtype=np.float32) Y_res_name = os.path.join(folder, 'Y_res_temp.npy') Y_res = np.memmap(Y_res_name, dtype=np.float32, mode='w+', shape=Y.shape) Y_res = np.memmap(Y_res_name, dtype=np.float32, mode='r+', shape=Y.shape) print "computing residuals" Y_res[:] = -A__.dot(C__).todense()[:] Y_res[:]+=Y else: print "Using memory trade-off computation (good use of memory if input is memmaped)" Y_res = Y - A_.dot(coo_matrix(C[:nr,:])) print "Computing A_bas" A_bas = np.fmax(np.dot(Y_res,f.T)/scipy.linalg.norm(f)**2,0) # update baseline based on residual Y_res[:]=1 b = A_bas print("--- %s seconds ---" % (time.time() - start_time)) try: #clean up # remove temporary file created print "Remove temporary file created" shutil.rmtree(folder) except: raise Exception("Failed to delete: " + folder) return A_,b,C
def update_temporal_components(Y, A, b, Cin, fin, bl = None, c1 = None, g = None, sn = None, ITER=2, method_foopsi='constrained_foopsi', n_processes=1, backend='single_thread',memory_efficient=False, debug=False, **kwargs): """Update temporal components and background given spatial components using a block coordinate descent approach. Parameters ----------- Y: np.ndarray (2D) input data with time in the last axis (d x T) A: sparse matrix (crc format) matrix of temporal components (d x K) b: ndarray (dx1) current estimate of background component Cin: np.ndarray current estimate of temporal components (K x T) fin: np.ndarray current estimate of temporal background (vector of length T) g: np.ndarray Global time constant (not used) bl: np.ndarray baseline for fluorescence trace for each column in A c1: np.ndarray initial concentration for each column in A g: np.ndarray discrete time constant for each column in A sn: np.ndarray noise level for each column in A ITER: positive integer Maximum number of block coordinate descent loops. method_foopsi: string Method of deconvolution of neural activity. constrained_foopsi is the only method supported at the moment. n_processes: int number of processes to use for parallel computation. Should be less than the number of processes started with ipcluster. backend: 'str' single_thread no parallelization ipyparallel, parallelization using the ipyparallel cluster. You should start the cluster (install ipyparallel and then type ipcluster -n 6, where 6 is the number of processes). memory_efficient: Bool whether or not to optimize for memory usage (longer running times). nevessary with very large datasets **kwargs: dict all parameters passed to constrained_foopsi except bl,c1,g,sn (see documentation). Some useful parameters are p: int order of the autoregression model method: [optional] string solution method for constrained foopsi. Choices are 'cvx': using cvxopt and picos (slow especially without the MOSEK solver) 'cvxpy': using cvxopt and cvxpy with the ECOS solver (faster, default) 'spgl1': using the spgl1 package 'debug': using spgl1 without spike non-negativity constraints (just for debugging purposes) solvers: list string primary and secondary (if problem unfeasible for approx solution) solvers to be used with cvxpy, default is ['ECOS','SCS'] Note -------- The temporal components are updated in parallel by default by forming of sequence of vertex covers. Returns -------- C: np.ndarray matrix of temporal components (K x T) f: np.array vector of temporal background (length T) S: np.ndarray matrix of merged deconvolved activity (spikes) (K x T) bl: float same as input c1: float same as input g: float same as input sn: float same as input YrA: np.ndarray matrix of spatial component filtered raw data, after all contributions have been removed. YrA corresponds to the residual trace for each component and is used for faster plotting (K x T) """ if not kwargs.has_key('p') or kwargs['p'] is None: raise Exception("You have to provide a value for p") d,T = np.shape(Y); nr = np.shape(A)[-1] if bl is None: bl=np.repeat(None,nr) if c1 is None: c1=np.repeat(None,nr) if g is None: g=np.repeat(None,nr) if sn is None: sn=np.repeat(None,nr) A = scipy.sparse.hstack((A,coo_matrix(b))) S = np.zeros(np.shape(Cin)); Cin = np.vstack((Cin,fin)); C = Cin; nA = np.squeeze(np.array(np.sum(np.square(A.todense()),axis=0))) #import pdb #pdb.set_trace() Cin=coo_matrix(Cin) #YrA = ((A.T.dot(Y)).T-Cin.T.dot(A.T.dot(A))) YA = (A.T.dot(Y).T)*spdiags(1./nA,0,nr+1,nr+1) AA = ((A.T.dot(A))*spdiags(1./nA,0,nr+1,nr+1)).tocsr() YrA = YA - Cin.T.dot(AA) #YrA = ((A.T.dot(Y)).T-Cin.T.dot(A.T.dot(A)))*spdiags(1./nA,0,nr+1,nr+1) if backend == 'ipyparallel': try: # if server is not running and raise exception if not installed or not started from ipyparallel import Client c = Client() except: print "this backend requires the installation of the ipyparallel (pip install ipyparallel) package and starting a cluster (type ipcluster start -n 6) where 6 is the number of nodes" raise if len(c) < n_processes: print len(c) raise Exception("the number of nodes in the cluster are less than the required processes: decrease the n_processes parameter to a suitable value") dview=c[:n_processes] # use the number of processes Cin=np.array(Cin.todense()) for iter in range(ITER): O,lo = update_order(A.tocsc()[:,:nr]) P_=[]; for count,jo_ in enumerate(O): jo=np.array(list(jo_)) #Ytemp = YrA[:,jo.flatten()] + (np.dot(np.diag(nA[jo]),Cin[jo,:])).T Ytemp = YrA[:,jo.flatten()] + Cin[jo,:].T Ctemp = np.zeros((np.size(jo),T)) Stemp = np.zeros((np.size(jo),T)) btemp = np.zeros((np.size(jo),1)) sntemp = btemp.copy() c1temp = btemp.copy() gtemp = np.zeros((np.size(jo),kwargs['p'])); nT = nA[jo] # args_in=[(np.squeeze(np.array(Ytemp[:,jj])), nT[jj], jj, bl[jo[jj]], c1[jo[jj]], g[jo[jj]], sn[jo[jj]], kwargs) for jj in range(len(jo))] args_in=[(np.squeeze(np.array(Ytemp[:,jj])), nT[jj], jj, None, None, None, None, kwargs) for jj in range(len(jo))] # import pdb # pdb.set_trace() if backend == 'ipyparallel': # if debug: results = dview.map_async(constrained_foopsi_parallel,args_in) results.get() for outp in results.stdout: print outp[:-1] sys.stdout.flush() for outp in results.stderr: print outp[:-1] sys.stderr.flush() else: results = dview.map_sync(constrained_foopsi_parallel,args_in) elif backend == 'single_thread': results = map(constrained_foopsi_parallel,args_in) else: raise Exception('Backend not defined. Use either single_thread or ipyparallel') for chunk in results: pars=dict() C_,Sp_,Ytemp_,cb_,c1_,sn_,gn_,jj_=chunk Ctemp[jj_,:] = C_[None,:] Stemp[jj_,:] = Sp_ Ytemp[:,jj_] = Ytemp_[:,None] btemp[jj_] = cb_ c1temp[jj_] = c1_ sntemp[jj_] = sn_ gtemp[jj_,:] = gn_.T bl[jo[jj_]] = cb_ c1[jo[jj_]] = c1_ sn[jo[jj_]] = sn_ g[jo[jj_]] = gn_.T if kwargs['p'] > 0 else [] #gtemp[jj,:] pars['b'] = cb_ pars['c1'] = c1_ pars['neuron_sn'] = sn_ pars['gn'] = gtemp[jj_,np.abs(gtemp[jj,:])>0] pars['neuron_id'] = jo[jj_] P_.append(pars) YrA -= (Ctemp-C[jo,:]).T*AA[jo,:] #YrA[:,jo] = Ytemp C[jo,:] = Ctemp.copy() S[jo,:] = Stemp # if (np.sum(lo[:jo])+1)%1 == 0: print str(np.sum(lo[:count+1])) + ' out of total ' + str(nr) + ' temporal components updated' ii=nr #YrA[:,ii] = YrA[:,ii] + np.atleast_2d(Cin[ii,:]).T #cc = np.maximum(YrA[:,ii],0) cc = np.maximum(YrA[:,ii] + np.atleast_2d(Cin[ii,:]).T,0) YrA -= (cc-np.atleast_2d(Cin[ii,:]).T)*AA[ii,:] C[ii,:] = cc.T #YrA = YA - C.T.dot(AA) #YrA[:,ii] = YrA[:,ii] - np.atleast_2d(C[ii,:]).T if backend == 'ipyparallel': dview.results.clear() c.purge_results('all') c.purge_everything() if scipy.linalg.norm(Cin - C,'fro')/scipy.linalg.norm(C,'fro') <= 1e-3: # stop if the overall temporal component does not change by much print "stopping: overall temporal component not changing significantly" break else: Cin = C f = C[nr:,:] C = C[:nr,:] YrA = np.array(YrA[:,:nr]).T P_ = sorted(P_, key=lambda k: k['neuron_id']) if backend == 'ipyparallel': c.close() return C,f,S,bl,c1,sn,g,YrA #,P_
def update_spatial_components_parallel(Y, C, f, A_in, sn=None, d1=None, d2=None, min_size=3, max_size=8, dist=3, method='ellipse', expandCore=None, backend='single_thread', n_processes=4, n_pixels_per_process=128, memory_efficient=False): """update spatial footprints and background through Basis Pursuit Denoising for each pixel i solve the problem [A(i,:),b(i)] = argmin sum(A(i,:)) subject to || Y(i,:) - A(i,:)*C + b(i)*f || <= sn(i)*sqrt(T); for each pixel the search is limited to a few spatial components Parameters ---------- Y: np.ndarray (2D) movie, raw data in 2D (pixels x time). C: np.ndarray calcium activity of each neuron. f: np.ndarray temporal profile of background activity. Ain: np.ndarray spatial profile of background activity. d1: [optional] int x movie dimension d2: [optional] int y movie dimension min_size: [optional] int max_size: [optional] int dist: [optional] int sn: [optional] float noise associated with each pixel if known n_processes: [optional] int number of threads to use when the backend is multiprocessing,threading, or ipyparallel backend [optional] str 'multiprocessing', 'threading', 'ipyparallel', 'single_thread' single_thread:no parallelization. It shoul dbe used in most cases. multiprocessing or threading: use the corresponding python threading package. It has known issues on mac OS. Not to be used in most situations. ipyparallel: starts an ipython cluster and then send jobs to each of them n_pixels_per_process: [optional] int number of pixels to be processed by each thread memory_efficient [bool] whether or not to reduce memory usage (at the expense of increased computational time) method: [optional] string method used to expand the search for pixels 'ellipse' or 'dilate' expandCore: [optional] scipy.ndimage.morphology if method is dilate this represents the kernel used for expansion Returns -------- A: np.ndarray new estimate of spatial footprints b: np.ndarray new estimate of spatial background C: np.ndarray temporal components (updated only when spatial components are completely removed) """ if expandCore is None: expandCore = iterate_structure(generate_binary_structure(2, 1), 2).astype(int) if d1 is None or d2 is None: raise Exception('You need to define the input dimensions') Y = np.atleast_2d(Y) if Y.shape[1] == 1: raise Exception('Dimension of Matrix Y must be pixels x time') C = np.atleast_2d(C) if C.shape[1] == 1: raise Exception('Dimension of Matrix C must be neurons x time') f = np.atleast_2d(f) if f.shape[1] == 1: raise Exception('Dimension of Matrix f must be neurons x time ') if len(A_in.shape) == 1: A_in = np.atleast_2d(A_in).T if A_in.shape[0] == 1: raise Exception('Dimension of Matrix A must be pixels x neurons ') start_time = time.time() Cf = np.vstack((C, f)) # create matrix that include background components [d, T] = np.shape(Y) if n_pixels_per_process > d: raise Exception( 'The number of pixels per process (n_pixels_per_process) is larger than the total number of pixels!! Decrease suitably.' ) nr, _ = np.shape(C) # number of neurons IND = determine_search_location(A_in, d1, d2, method=method, min_size=min_size, max_size=max_size, dist=dist, expandCore=expandCore) print " find search location" ind2_ = [ np.hstack( (np.where(iid_)[0], nr + np.arange(f.shape[0]))) if np.size(np.where(iid_)[0]) > 0 else [] for iid_ in IND ] folder = tempfile.mkdtemp() if backend == 'multiprocessing' or backend == 'threading': A_name = os.path.join(folder, 'A_temp') # Pre-allocate a writeable shared memory map as a container for the # results of the parallel computation print "Create Matrix for dumping data from matrix A and C for parallel computation...." A_ = np.memmap(A_name, dtype=A_in.dtype, shape=(d, nr + np.size(f, 0)), mode='w+') pixels_name = os.path.join(folder, 'pixels') C_name = os.path.join(folder, 'C_temp') # Dump the input data to disk to free the memory dump(Y, pixels_name) dump(Cf, C_name) # use mempry mapped versions of C and Y Y = load(pixels_name, mmap_mode='r') Cf = load(C_name, mmap_mode='r') pixel_groups = [ range(i, i + n_pixels_per_process) for i in range(0, Y.shape[0] - n_pixels_per_process + 1, n_pixels_per_process) ] # Fork the worker processes to perform computation concurrently print "start parallel pool..." sys.stdout.flush() Parallel(n_jobs=n_processes, backend=backend, verbose=100, max_nbytes=None)(delayed(lars_regression_noise_parallel)( Y, Cf, A_, sn, i, ind2_) for i in pixel_groups) # if n_pixels_per_process is not a multiple of Y.shape[0] run on remaining pixels pixels_remaining = Y.shape[0] % n_pixels_per_process if pixels_remaining > 0: print "Running deconvolution for remaining pixels:" + str( pixels_remaining) lars_regression_noise_parallel(Y, Cf, A_, sn, range(Y.shape[0] - pixels_remaining, Y.shape[0]), ind2_, positive=1) A_ = np.array(A_) elif backend == 'ipyparallel': # use the ipyparallel package, you need to start a cluster server (ipcluster command) in order to use it C_name = os.path.join(folder, 'C_temp.npy') np.save(C_name, Cf) if type( Y ) is np.core.memmap: # if input file is already memory mapped then find the filename Y_name = Y.filename else: # if not create a memory mapped version (necessary for parallelization) Y_name = os.path.join(folder, 'Y_temp.npy') np.save(Y_name, Y) Y = np.load(Y_name, mmap_mode='r') # create arguments to be passed to the function. Here we are grouping bunch of pixels to be processed by each thread pixel_groups = [(Y_name, C_name, sn, ind2_, range(i, i + n_pixels_per_process)) for i in range(0, d1 * d2 - n_pixels_per_process + 1, n_pixels_per_process)] A_ = np.zeros((d, nr + np.size(f, 0))) try: # if server is not running and raise exception if not installed or not started from ipyparallel import Client c = Client() except: print "this backend requires the installation of the ipyparallel (pip install ipyparallel) package and starting a cluster (type ipcluster start -n 6) where 6 is the number of nodes" raise if len(c) < n_processes: print len(c) raise Exception( "the number of nodes in the cluster are less than the required processes: decrease the n_processes parameter to a suitable value" ) dview = c[:n_processes] # use the number of processes #serial_result = map(lars_regression_noise_ipyparallel, pixel_groups) parallel_result = dview.map_sync(lars_regression_noise_ipyparallel, pixel_groups) for chunk in parallel_result: for pars in chunk: px, idxs_, a = pars A_[px, idxs_] = a #clean up dview.results.clear() c.purge_results('all') c.purge_everything() c.close() elif backend == 'single_thread': Cf_ = [Cf[idx_, :] for idx_ in ind2_] #% LARS regression A_ = np.hstack((np.zeros((d, nr)), np.zeros((d, np.size(f, 0))))) for c, y, s, id2_, px in zip(Cf_, Y, sn, ind2_, range(d)): if px % 1000 == 0: print px if np.size(c) > 0: _, _, a, _, _ = lars_regression_noise(y, np.array(c.T), 1, sn[px]**2 * T) if np.isscalar(a): A_[px, id2_] = a else: A_[px, id2_] = a.T else: raise Exception( 'Unknown backend specified: use single_thread, threading, multiprocessing or ipyparallel' ) #% print 'Updated Spatial Components' A_ = threshold_components(A_, d1, d2) print "threshold" ff = np.where(np.sum(A_, axis=0) == 0) # remove empty components if np.size(ff) > 0: ff = ff[0] warn('eliminating empty components!!') nr = nr - len(ff) A_ = np.delete(A_, list(ff), 1) C = np.delete(C, list(ff), 0) A_ = A_[:, :nr] A_ = coo_matrix(A_) if memory_efficient: print "Using memory efficient computation (slow but memory preserving)" A__ = coo_matrix(A_, dtype=np.float32) C__ = coo_matrix(C[:nr, :], dtype=np.float32) Y_res_name = os.path.join(folder, 'Y_res_temp.npy') Y_res = np.memmap(Y_res_name, dtype=np.float32, mode='w+', shape=Y.shape) Y_res = np.memmap(Y_res_name, dtype=np.float32, mode='r+', shape=Y.shape) print "computing residuals" Y_res[:] = -A__.dot(C__).todense()[:] Y_res[:] += Y else: print "Using memory trade-off computation (good use of memory if input is memmaped)" Y_res = Y - A_.dot(coo_matrix(C[:nr, :])) print "Computing A_bas" A_bas = np.fmax(np.dot(Y_res, f.T) / scipy.linalg.norm(f)**2, 0) # update baseline based on residual Y_res[:] = 1 b = A_bas print("--- %s seconds ---" % (time.time() - start_time)) try: #clean up # remove temporary file created print "Remove temporary file created" shutil.rmtree(folder) except: raise Exception("Failed to delete: " + folder) return A_, b, C
def update_spatial_components(Y, C, f, A_in, sn=None, d1=None, d2=None, min_size=3, max_size=8, dist=3, method='ellipse', expandCore=None, backend='single_thread', n_processes=4, n_pixels_per_process=128): """update spatial footprints and background through Basis Pursuit Denoising for each pixel i solve the problem [A(i,:),b(i)] = argmin sum(A(i,:)) subject to || Y(i,:) - A(i,:)*C + b(i)*f || <= sn(i)*sqrt(T); for each pixel the search is limited to a few spatial components Parameters ---------- Y: np.ndarray (2D) movie, raw data in 2D (pixels x time). C: np.ndarray calcium activity of each neuron. f: np.ndarray temporal profile of background activity. Ain: np.ndarray spatial profile of background activity. d1: [optional] int x movie dimension d2: [optional] int y movie dimension min_size: [optional] int max_size: [optional] int dist: [optional] int sn: [optional] float noise associated with each pixel if known n_processes: [optional] int number of threads to use when the backend is multiprocessing,threading, or ipyparallel backend [optional] str 'ipyparallel', 'single_thread' single_thread:no parallelization. It can be used with small datasets. ipyparallel: uses ipython clusters and then send jobs to each of them n_pixels_per_process: [optional] int number of pixels to be processed by each thread method: [optional] string method used to expand the search for pixels 'ellipse' or 'dilate' expandCore: [optional] scipy.ndimage.morphology if method is dilate this represents the kernel used for expansion Returns -------- A: np.ndarray new estimate of spatial footprints b: np.ndarray new estimate of spatial background C: np.ndarray temporal components (updated only when spatial components are completely removed) """ if expandCore is None: expandCore = iterate_structure(generate_binary_structure(2, 1), 2).astype(int) if d1 is None or d2 is None: raise Exception('You need to define the input dimensions') if Y.ndim < 2 and not type(Y) is str: Y = np.atleast_2d(Y) if Y.shape[1] == 1: raise Exception('Dimension of Matrix Y must be pixels x time') C = np.atleast_2d(C) if C.shape[1] == 1: raise Exception('Dimension of Matrix C must be neurons x time') f = np.atleast_2d(f) if f.shape[1] == 1: raise Exception('Dimension of Matrix f must be neurons x time ') if len(A_in.shape) == 1: A_in = np.atleast_2d(A_in).T if A_in.shape[0] == 1: raise Exception('Dimension of Matrix A must be pixels x neurons ') start_time = time.time() Cf = np.vstack((C, f)) # create matrix that include background components [d, T] = np.shape(Y) if n_pixels_per_process > d: raise Exception( 'The number of pixels per process (n_pixels_per_process) is larger than the total number of pixels!! Decrease suitably.' ) nr, _ = np.shape(C) # number of neurons IND = determine_search_location(A_in, d1, d2, method=method, min_size=min_size, max_size=max_size, dist=dist, expandCore=expandCore) print " find search location" ind2_ = [ np.hstack( (np.where(iid_)[0], nr + np.arange(f.shape[0]))) if np.size(np.where(iid_)[0]) > 0 else [] for iid_ in IND ] folder = tempfile.mkdtemp() # use the ipyparallel package, you need to start a cluster server # (ipcluster command) in order to use it if backend == 'ipyparallel': C_name = os.path.join(folder, 'C_temp.npy') np.save(C_name, Cf) if type( Y ) is np.core.memmap: # if input file is already memory mapped then find the filename Y_name = Y.filename # if not create a memory mapped version (necessary for parallelization) elif type(Y) is str: Y_name = Y else: Y_name = os.path.join(folder, 'Y_temp.npy') np.save(Y_name, Y) Y, _, _, _ = load_memmap(Y_name) # create arguments to be passed to the function. Here we are grouping # bunch of pixels to be processed by each thread pixel_groups = [(Y_name, C_name, sn, ind2_, range(i, i + n_pixels_per_process)) for i in range(0, d1 * d2 - n_pixels_per_process + 1, n_pixels_per_process)] A_ = np.zeros((d, nr + np.size(f, 0))) try: # if server is not running and raise exception if not installed or not started from ipyparallel import Client c = Client() except: print "this backend requires the installation of the ipyparallel (pip install ipyparallel) package and starting a cluster (type ipcluster start -n 6) where 6 is the number of nodes" raise if len(c) < n_processes: print len(c) raise Exception( "the number of nodes in the cluster are less than the required processes: decrease the n_processes parameter to a suitable value" ) dview = c[:n_processes] # use the number of processes #serial_result = map(lars_regression_noise_ipyparallel, pixel_groups) parallel_result = dview.map_sync(lars_regression_noise_ipyparallel, pixel_groups) # clean up for chunk in parallel_result: for pars in chunk: px, idxs_, a = pars A_[px, idxs_] = a dview.results.clear() c.purge_results('all') c.purge_everything() c.close() elif backend == 'single_thread': Cf_ = [Cf[idx_, :] for idx_ in ind2_] #% LARS regression A_ = np.hstack((np.zeros((d, nr)), np.zeros((d, np.size(f, 0))))) for c, y, s, id2_, px in zip(Cf_, Y, sn, ind2_, range(d)): if px % 1000 == 0: print px if np.size(c) > 0: _, _, a, _, _ = lars_regression_noise(y, np.array(c.T), 1, sn[px]**2 * T) if np.isscalar(a): A_[px, id2_] = a else: A_[px, id2_] = a.T else: raise Exception( 'Unknown backend specified: use single_thread, threading, multiprocessing or ipyparallel' ) #% print 'Updated Spatial Components' A_ = threshold_components(A_, d1, d2) print "threshold" ff = np.where(np.sum(A_, axis=0) == 0) # remove empty components if np.size(ff) > 0: ff = ff[0] print('eliminating empty components!!') nr = nr - len(ff) A_ = np.delete(A_, list(ff), 1) C = np.delete(C, list(ff), 0) A_ = A_[:, :nr] A_ = coo_matrix(A_) # import pdb # pdb.set_trace() Y_resf = np.dot(Y, f.T) - A_.dot(coo_matrix(C[:nr, :]).dot(f.T)) print "Computing A_bas" A_bas = np.fmax(Y_resf / scipy.linalg.norm(f)**2, 0) # update baseline based on residual # A_bas = np.fmax(np.dot(Y_res,f.T)/scipy.linalg.norm(f)**2,0) # update # baseline based on residual b = A_bas print("--- %s seconds ---" % (time.time() - start_time)) try: # clean up # remove temporary file created print "Remove temporary file created" shutil.rmtree(folder) except: raise Exception("Failed to delete: " + folder) return A_, b, C
def stop_server(ipcluster='ipcluster', pdir=None, profile=None, dview=None): """ programmatically stops the ipyparallel server Parameters: ---------- ipcluster : str ipcluster binary file name; requires 4 path separators on Windows Default: "ipcluster" """ if 'multiprocessing' in str(type(dview)): dview.terminate() else: sys.stdout.write("Stopping cluster...\n") sys.stdout.flush() try: pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] is_slurm = True except: print('NOT SLURM') is_slurm = False if is_slurm: if pdir is None and profile is None: pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] c = Client(ipython_dir=pdir, profile=profile) ee = c[:] ne = len(ee) print(('Shutting down %d engines.' % (ne))) c.close() c.shutdown(hub=True) shutil.rmtree('profile_' + str(profile)) try: shutil.rmtree('./log/') except: print('creating log folder') files = glob.glob('*.log') os.mkdir('./log') for fl in files: shutil.move(fl, './log/') else: if ipcluster == "ipcluster": proc = subprocess.Popen("ipcluster stop", shell=True, stderr=subprocess.PIPE, close_fds=(os.name != 'nt')) else: proc = subprocess.Popen(shlex.split(ipcluster + " stop"), shell=True, stderr=subprocess.PIPE, close_fds=(os.name != 'nt')) line_out = proc.stderr.readline() if b'CRITICAL' in line_out: sys.stdout.write("No cluster to stop...") sys.stdout.flush() elif b'Stopping' in line_out: st = time.time() sys.stdout.write('Waiting for cluster to stop...') while (time.time() - st) < 4: sys.stdout.write('.') sys.stdout.flush() time.sleep(1) else: print(line_out) print( '**** Unrecognized Syntax in ipcluster output, waiting for server to stop anyways ****' ) proc.stderr.close() sys.stdout.write(" done\n")
def slurm_map(fnc, iterables, resource_spec, env='root', job_name=None, output_path=None, n_retries=5, patience=30): """ Args: fnc iterables resource_spec env: virtual env to launch engines in job_name: name of job to use. Derived from fnc name if not specified output_path: location to direct output to. If unspecified output is sent to a file (based on job name and timestamp) in ~/logs/slurm n_retries: number of times to retry connecting to client if less than the requested number of workers are available. patience: seconds to wait after failed attempt to connect to client """ resource_spec = process_resource_spec(resource_spec) if not profile_installed(PROFILE_NAME): print("No profile found for {}, installing".format(PROFILE_NAME)) install_profile(PROFILE_NAME) submission_time = time.strftime("%Y%m%d-%H%M%S") cluster_id = '{}_{}'.format(fnc.__name__, submission_time) print("Using cluster id: {}".format(cluster_id)) # break down by line: # run in bash # activate the specified environment # launch controller with desired settings controller_cmd_template = ("exec bash -c '" "source activate {env};" " ipcontroller --profile={profile} --sqlitedb --location={hostname} --ip=\'*\' --cluster-id={cluster_id}'") controller_cmd = controller_cmd_template.format( env=env, profile=PROFILE_NAME, hostname=socket.gethostname(), cluster_id=cluster_id ) print("Starting controller with: {} \n".format(controller_cmd)) # runs in the background if executed this way subprocess.Popen(controller_cmd, shell=True) print("Sleeping for 10") time.sleep(10) engine_cmd_template_path = package_path() + '/templates/slurm_template.sh' with open(engine_cmd_template_path, 'r') as engine_cmd_template_file: engine_command_template = engine_cmd_template_file.read() # prepare engine commands if job_name is None: job_name = fnc.__name__ + '_slurm_map' else: assert isinstance(job_name, str) if output_path is None: output_dir = os.path.expanduser('~/logs/slurm') output_path = '{}/{}_{}'.format (output_dir, job_name, submission_time) if not os.path.exists(output_dir): os.makedirs(output_path) else: assert isinstance(output_path, str) assert os.path.exists(output_path) # find path to engine based on specified environment if env == 'root': engine_path = 'bin/ipengine' else: engine_path = 'envs/{}/bin/ipengine'.format(env) full_engine_path = os.path.expanduser('~/anaconda3/{}'.format(engine_path)) assert os.path.exists(full_engine_path) engine_command = engine_command_template.format( job_name=job_name, output_path=output_path, n_tasks=resource_spec['max_workers'], mem_mb=resource_spec['worker_mem_mb'], n_cpus=resource_spec['worker_n_cpus'], n_gpus=resource_spec['worker_n_gpus'], engine_path=engine_path, profile=PROFILE_NAME, controller_hostname=socket.gethostname(), cluster_id=cluster_id, comment=job_name ) sbatch_file_path = '/tmp/slurm_map_sbatch_{}.sh'.format(cluster_id) with open(sbatch_file_path, 'w') as sbatch_file: sbatch_file.write(engine_command) # wrap command to execute in bash sbatch_command = "exec bash -c 'sbatch {}'".format(sbatch_file_path) print("Starting engines") # runs in the background if executed this way subprocess.Popen(sbatch_command, shell=True) print("Sleeping for {}".format(patience)) time.sleep(patience) # TODO: shut down unused engines connected = False for attempt_idx in range(n_retries): print("Attempt {} to connect to cluster".format(attempt_idx)) try: client = Client(profile=PROFILE_NAME, cluster_id=cluster_id) if resource_spec['min_workers'] <= len(client.ids) <= resource_spec['max_workers']: connected = True print('Succesfully connected to cluster with {} engines out of {} requested'.format( len(client.ids), resource_spec['max_workers'])) if len(client.ids) < resource_spec['max_workers']: warn("{} slurm jobs submitted but only {} are being used.".format( resource_spec['max_workers'], len(client.ids))) break else: print("{} available engines less than minimum requested of {}".format( len(client.ids), resource_spec['min_workers'])) print("Retrying after {}".format(patience)) client.close() time.sleep(patience) except OSError as os_err: print("Caught OSError while attempting to connect to {}: {}.".format(PROFILE_NAME, os_err)) print("Retrying after {}".format(patience)) time.sleep(patience) except TimeoutError as timeout_err: print("Caught TimeoutError while attempting to connect to {}: {}".format(PROFILE_NAME, timeout_err)) print("Retrying after {}".format(patience)) time.sleep(patience) if not connected: raise TimeoutError("Failed to connect to client after {} retries".format(n_retries)) # run tasks print("Submitting tasks") start_time = time.time() client[:].use_cloudpickle() lb_view = client.load_balanced_view() result = lb_view.map(fnc, iterables, block=True) print("Tasks finished after {} seconds".format(time.time() - start_time)) print("Shutting down cluster") client.shutdown(hub=True) print("Relinquishing slurm nodes") shutdown_cmd = 'scancel -n={job_name}'.format(job_name=job_name) shutdown_cmd = "exec bash -c '{}'".format(shutdown_cmd) # runs in the background if executed this way subprocess.Popen(shutdown_cmd, shell=True) print("Removing sbatch script") os.remove(sbatch_file_path) return result
def stop_server(ipcluster='ipcluster', pdir=None, profile=None, dview=None): """ programmatically stops the ipyparallel server Parameters: ---------- ipcluster : str ipcluster binary file name; requires 4 path separators on Windows Default: "ipcluster" """ if 'multiprocessing' in str(type(dview)): dview.terminate() else: logger.info("Stopping cluster...") try: pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] is_slurm = True except: logger.debug('stop_server: not a slurm cluster') is_slurm = False if is_slurm: if pdir is None and profile is None: pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] c = Client(ipython_dir=pdir, profile=profile) ee = c[:] ne = len(ee) print(('Shutting down %d engines.' % (ne))) c.close() c.shutdown(hub=True) shutil.rmtree('profile_' + str(profile)) try: shutil.rmtree('./log/') except: print('creating log folder') files = glob.glob('*.log') os.mkdir('./log') for fl in files: shutil.move(fl, './log/') else: if ipcluster == "ipcluster": proc = subprocess.Popen( "ipcluster stop", shell=True, stderr=subprocess.PIPE, close_fds=(os.name != 'nt')) else: proc = subprocess.Popen(shlex.split(ipcluster + " stop"), shell=True, stderr=subprocess.PIPE, close_fds=(os.name != 'nt')) line_out = proc.stderr.readline() if b'CRITICAL' in line_out: logger.info("No cluster to stop...") elif b'Stopping' in line_out: st = time.time() logger.debug('Waiting for cluster to stop...') while (time.time() - st) < 4: sys.stdout.write('.') sys.stdout.flush() time.sleep(1) else: print(line_out) print( '**** Unrecognized syntax in ipcluster output, waiting for server to stop anyways ****') proc.stderr.close() logger.info("stop_cluster(): done")
def update_spatial_components(Y, C, f, A_in, sn=None, d1=None, d2=None, min_size=3, max_size=8, dist=3, method='ellipse', expandCore=None, backend='single_thread', n_processes=4, n_pixels_per_process=128 ): """update spatial footprints and background through Basis Pursuit Denoising for each pixel i solve the problem [A(i,:),b(i)] = argmin sum(A(i,:)) subject to || Y(i,:) - A(i,:)*C + b(i)*f || <= sn(i)*sqrt(T); for each pixel the search is limited to a few spatial components Parameters ---------- Y: np.ndarray (2D) movie, raw data in 2D (pixels x time). C: np.ndarray calcium activity of each neuron. f: np.ndarray temporal profile of background activity. Ain: np.ndarray spatial profile of background activity. d1: [optional] int x movie dimension d2: [optional] int y movie dimension min_size: [optional] int max_size: [optional] int dist: [optional] int sn: [optional] float noise associated with each pixel if known n_processes: [optional] int number of threads to use when the backend is multiprocessing,threading, or ipyparallel backend [optional] str 'ipyparallel', 'single_thread' single_thread:no parallelization. It can be used with small datasets. ipyparallel: uses ipython clusters and then send jobs to each of them n_pixels_per_process: [optional] int number of pixels to be processed by each thread method: [optional] string method used to expand the search for pixels 'ellipse' or 'dilate' expandCore: [optional] scipy.ndimage.morphology if method is dilate this represents the kernel used for expansion Returns -------- A: np.ndarray new estimate of spatial footprints b: np.ndarray new estimate of spatial background C: np.ndarray temporal components (updated only when spatial components are completely removed) """ if expandCore is None: expandCore = iterate_structure(generate_binary_structure(2, 1), 2).astype(int) if d1 is None or d2 is None: raise Exception('You need to define the input dimensions') if Y.ndim<2 and not type(Y) is str: Y = np.atleast_2d(Y) if Y.shape[1] == 1: raise Exception('Dimension of Matrix Y must be pixels x time') C = np.atleast_2d(C) if C.shape[1] == 1: raise Exception('Dimension of Matrix C must be neurons x time') f = np.atleast_2d(f) if f.shape[1] == 1: raise Exception('Dimension of Matrix f must be neurons x time ') if len(A_in.shape) == 1: A_in = np.atleast_2d(A_in).T if A_in.shape[0] == 1: raise Exception('Dimension of Matrix A must be pixels x neurons ') start_time = time.time() Cf = np.vstack((C, f)) # create matrix that include background components [d, T] = np.shape(Y) if n_pixels_per_process > d: raise Exception( 'The number of pixels per process (n_pixels_per_process) is larger than the total number of pixels!! Decrease suitably.') nr, _ = np.shape(C) # number of neurons IND = determine_search_location( A_in, d1, d2, method=method, min_size=min_size, max_size=max_size, dist=dist, expandCore=expandCore) print " find search location" ind2_ = [np.hstack((np.where(iid_)[0], nr + np.arange(f.shape[0]))) if np.size(np.where(iid_)[0]) > 0 else [] for iid_ in IND] folder = tempfile.mkdtemp() # use the ipyparallel package, you need to start a cluster server # (ipcluster command) in order to use it if backend == 'ipyparallel': C_name = os.path.join(folder, 'C_temp.npy') np.save(C_name, Cf) if type(Y) is np.core.memmap: # if input file is already memory mapped then find the filename Y_name = Y.filename # if not create a memory mapped version (necessary for parallelization) elif type(Y) is str: Y_name = Y else: Y_name = os.path.join(folder, 'Y_temp.npy') np.save(Y_name, Y) Y,_,_,_=load_memmap(Y_name) # create arguments to be passed to the function. Here we are grouping # bunch of pixels to be processed by each thread pixel_groups = [(Y_name, C_name, sn, ind2_, range(i, i + n_pixels_per_process)) for i in range(0, d1 * d2 - n_pixels_per_process + 1, n_pixels_per_process)] A_ = np.zeros((d, nr + np.size(f, 0))) try: # if server is not running and raise exception if not installed or not started from ipyparallel import Client c = Client() except: print "this backend requires the installation of the ipyparallel (pip install ipyparallel) package and starting a cluster (type ipcluster start -n 6) where 6 is the number of nodes" raise if len(c) < n_processes: print len(c) raise Exception( "the number of nodes in the cluster are less than the required processes: decrease the n_processes parameter to a suitable value") dview = c[:n_processes] # use the number of processes #serial_result = map(lars_regression_noise_ipyparallel, pixel_groups) parallel_result = dview.map_sync(lars_regression_noise_ipyparallel, pixel_groups) # clean up for chunk in parallel_result: for pars in chunk: px, idxs_, a = pars A_[px, idxs_] = a dview.results.clear() c.purge_results('all') c.purge_everything() c.close() elif backend == 'single_thread': Cf_ = [Cf[idx_, :] for idx_ in ind2_] #% LARS regression A_ = np.hstack((np.zeros((d, nr)), np.zeros((d, np.size(f, 0))))) for c, y, s, id2_, px in zip(Cf_, Y, sn, ind2_, range(d)): if px % 1000 == 0: print px if np.size(c) > 0: _, _, a, _, _ = lars_regression_noise(y, np.array(c.T), 1, sn[px]**2 * T) if np.isscalar(a): A_[px, id2_] = a else: A_[px, id2_] = a.T else: raise Exception( 'Unknown backend specified: use single_thread, threading, multiprocessing or ipyparallel') #% print 'Updated Spatial Components' A_ = threshold_components(A_, d1, d2) print "threshold" ff = np.where(np.sum(A_, axis=0) == 0) # remove empty components if np.size(ff) > 0: ff = ff[0] print('eliminating empty components!!') nr = nr - len(ff) A_ = np.delete(A_, list(ff), 1) C = np.delete(C, list(ff), 0) A_ = A_[:, :nr] A_ = coo_matrix(A_) # import pdb # pdb.set_trace() Y_resf = np.dot(Y, f.T) - A_.dot(coo_matrix(C[:nr, :]).dot(f.T)) print "Computing A_bas" A_bas = np.fmax(Y_resf / scipy.linalg.norm(f)**2, 0) # update baseline based on residual # A_bas = np.fmax(np.dot(Y_res,f.T)/scipy.linalg.norm(f)**2,0) # update # baseline based on residual b = A_bas print("--- %s seconds ---" % (time.time() - start_time)) try: # clean up # remove temporary file created print "Remove temporary file created" shutil.rmtree(folder) except: raise Exception("Failed to delete: " + folder) return A_, b, C
import numpy as np import ca_source_extraction as cse from scipy.sparse import coo_matrix import scipy import pylab as pl import calblitz as cb import glob with np.load('results_analysis.npz') as ld: locals().update(ld) fname_new = glob.glob('Yr_*.mmap')[0] Yr, (d1, d2), T = cse.utilities.load_memmap(fname_new) d, T = np.shape(Yr) Y = np.reshape(Yr, (d1, d2, T), order='F') # 3D version of the movie A = scipy.sparse.coo_matrix(A) dims = (d1, d2) #%% STOP CLUSTER pl.close() if not single_thread: c.close() cse.utilities.stop_server() log_files = glob.glob('Yr*_LOG_*') for log_file in log_files: os.remove(log_file) #%% cse.utilities.stop_server(is_slurm=(backend == 'SLURM'))
class ClusterLab(epyc.Lab): """A :class:`Lab` running on an ``ipyparallel`` compute cluster. Experiments are submitted to engines in the cluster for execution in parallel, with the experiments being performed asynchronously to allow for disconnection and subsequent retrieval of results. Combined with a persistent :class:`LabNotebook`, this allows for fully decoupled access to an on-going computational experiment with piecewise retrieval of results. This class requires a cluster to already be set up and running, configured for persistent access, with access to the necessary code and libraries, and with appropriate security information available to the client. """ # Tuning parameters WaitingTime = 30 #: Waiting time for checking for job completion. Lower values increase network traffic. def __init__( self, notebook = None, url_file = None, profile = None, profile_dir = None, ipython_dir = None, context = None, debug = False, sshserver = None, sshkey = None, password = None, paramiko = None, timeout = 10, cluster_id = None, use_dill = False, **extra_args ): """Create an empty lab attached to the given cluster. Most of the arguments are as expected by the ``ipyparallel.Client`` class, and are used to create the underlying connection to the cluster. The connection is opened immediately, meaning the cluster must be up and accessible when creating a lab to use it. :param notebook: the notebook used to results (defaults to an empty :class:`LabNotebook`) :param url_file: file containing connection information for accessing cluster :param profile: name of the IPython profile to use :param profile_dir: directory containing the profile's connection information :param ipython_dir: directory containing profile directories :param context: ZMQ context :param debug: whether to issue debugging information (defaults to False) :param sshserver: username and machine for ssh connections :param sshkey: file containing ssh key :param password: ssh password :param paramiko: True to use paramiko for ssh (defaults to False) :param timeout: timeout in seconds for ssh connection (defaults to 10s) :param cluster_id: string added to runtime files to prevent collisions :param use_dill: whether to use Dill as pickler (defaults to False)""" super(epyc.ClusterLab, self).__init__(notebook) # record all the connection arguments for later self._arguments = dict(url_file = url_file, profile = profile, profile_dir = profile_dir, ipython_dir = ipython_dir, context = context, debug = debug, sshserver = sshserver, sshkey = sshkey, password = password, paramiko = paramiko, timeout = timeout, cluster_id = cluster_id, **extra_args) self._client = None # connect to the cluster self.open() # use Dill if requested if use_dill: self.use_dill() def open( self ): """Connect to the cluster.""" if self._client is None: self._client = Client(**self._arguments) def close( self ): """Close down the connection to the cluster.""" if self._client is not None: self._client.close() self._client = None def numberOfEngines( self ): """Return the number of engines available to this lab. :returns: the number of engines""" return len(self.engines()) def engines( self ): """Return a list of the available engines. :returns: a list of engines""" self.open() return self._client[:] def use_dill( self ): """Make the cluster use Dill as pickler for transferring results. This isn't generally needed, but is sometimes useful for particularly complex experiments such as those involving closures. (Or, to put it another way, if you find yourself tempted to use this method, consider re-structuring your experiment code.)""" self.open() with self.sync_imports(quiet = True): import dill self._client.direct_view().use_dill() def sync_imports( self, quiet = False ): """Return a context manager to control imports onto all the engines in the underlying cluster. This method is used within a ``with`` statement. Any imports should be done with no experiments running, otherwise the method will block until the cluster is quiet. Generally imports will be one of the first things done when connecting to a cluster. (But be careful not to accidentally try to re-import if re-connecting to a running cluster.) :param quiet: if True, suppresses messages (defaults to False) :returns: a context manager""" self.open() return self._client[:].sync_imports(quiet = quiet) def _mixup( self, ps ): """Private method to mix up a list of values in-place using a Fisher-Yates shuffle (see https://en.wikipedia.org/wiki/Fisher-Yates_shuffle). :param ps: the array :returns: the array, shuffled in-place""" for i in range(len(ps) - 1, 0, -1): j = int(numpy.random.random() * i) temp = ps[i] ps[i] = ps[j] ps[j] = temp return ps def runExperiment( self, e ): """Run the experiment across the parameter space in parallel using all the engines in the cluster. This method returns immediately. The experiments are run asynchronously, with the points in the parameter space being explored randomly so that intermediate retrievals of results are more representative of the overall result. Put another way, for a lot of experiments the results available will converge towards a final answer, so we can plot them and see the answer emerge. :param e: the experiment""" # create the parameter space space = self.parameterSpace() # only proceed if there's work to do if len(space) > 0: nb = self.notebook() # randomise the order of the parameter space so that we evaluate across # the space as we go along to try to make intermediate (incomplete) result # sets more representative of the overall result set ps = self._mixup(space) try: # connect to the cluster self.open() # submit an experiment at each point in the parameter space to the cluster view = self._client.load_balanced_view() jobs = [] for p in ps: jobs.extend((view.apply_async((lambda p: e.set(p).run()), p)).msg_ids) # there seems to be a race condition in submitting jobs, # whereby jobs get dropped if they're submitted too quickly time.sleep(0.01) # record the mesage ids of all the jobs as submitted but not yet completed psjs = zip(ps, jobs) for (p, j) in psjs: nb.addPendingResult(p, j) finally: # commit our pending results in the notebook nb.commit() self.close() def updateResults( self ): """Update our results within any pending results that have completed since we last retrieved results from the cluster. :returns: the number of pending results completed at this call""" # we do all the tests for pending results against the notebook directly, # as the corresponding methods on self call this method themselves nb = self.notebook() # look for pending results if we're waiting for any n = 0 if nb.numberOfPendingResults() > 0: # we have results to get self.open() for j in set(nb.pendingResults()): # query the status of a job status = self._client.result_status(j, status_only = False) # add all completed jobs to the notebook if j in status['completed']: r = status[j] # update the result in the notebook, cancelling # the pending result as well # values come back from Client.result_status() in # varying degrees of list-nesting, which LabNotebook.addResult() # handles itself nb.addResult(r, j) # commit changes to the notebook nb.commit() # purge the completed job from the cluster self._client.purge_hub_results(j) # record that we retrieved the results for the given job n = n + 1 return n def numberOfResults( self ): """Return the number of results we have available at the moment. :returns: the number of results""" self.updateResults() return self.notebook().numberOfResults() def numberOfPendingResults( self ): """Return the number of resultswe are waiting for. :returns: the number of pending results""" self.updateResults() return self.notebook().numberOfPendingResults() def _availableResultsFraction( self ): """Private method to return the fraction of results available, as a real number between 0 and 1. This does not update the results fetched from the cluster. :returns: the fraction of available results""" tr = self.notebook().numberOfResults() + self.notebook().numberOfPendingResults() if tr == 0: return 0 else: return (self.notebook().numberOfResults() + 0.0) / tr def readyFraction( self ): """Test what fraction of results are available. This will change over time as the results come in. :returns: the fraction from 0 to 1""" self.updateResults() return self._availableResultsFraction() def ready( self ): """Test whether all the results are available. This will change over time as the results come in. :returns: True if all the results are available""" return (self.readyFraction() == 1) def wait( self, timeout = -1 ): """Wait for all pending results to be finished. If timeout is set, return after this many seconds regardless. :param timeout: timeout period in seconds (defaults to forever) :returns: True if all the results completed""" # we can't use ipyparallel.Client.wait() for this, because that # method only works for cases where the Client object is the one that # submitted the jobs to the cluster hub -- and therefore has the # necessary data structures to perform synchronisation. This isn't the # case for us, as one of the main goals of epyc is to support disconnected # operation, which implies a different Client object retrieving results # than the one that submitted the jobs in the first place. This is # unfortunate, but understandable given the typical use cases for # Client objects. # # Instead. we have to code around a little busily. The ClusterLab.WaitingTime # global sets the latency for waiting, and we repeatedly wait for this amount # of time before updating the results. The latency value essentially controls # how busy this process is: given that most simulations are expected to # be long, a latency in the tens of seconds feels about right as a default if self.numberOfPendingResults() > 0: # we've got pending results, wait for them timeWaited = 0 while (timeout < 0) or (timeWaited < timeout): if self.numberOfPendingResults() == 0: # no pending jobs left, we're complete return True else: # not done yet, calculate the waiting period if timeout == -1: # wait for the default waiting period dt = self.WaitingTime else: # wait for the default waiting period or until the end of the timeout. # whichever comes first if (timeout - timeWaited) < self.WaitingTime: dt = timeout - timeWaited else: dt = self.WaitingTime # sleep for a while time.sleep(dt) timeWaited = timeWaited + dt # if we get here, the timeout expired, so do a final check # and then exit return (self.numberOfPendingResults() == 0) else: # no results, so we got them all return True def pendingResults( self ): """Return the list of job iods for any pending results. :returns: a list of job ids""" return self.notebook().pendingResults() def pendingResultsFor( self, params ): """Return a list of job ids for any results pending for experiments at the given point in the parameter space. :param params: the experimental parameters :returns: a list of job ids""" return self.notebook().pendingResultsFor(params) def _abortJobs( self, js ): """Private method to abort a set of jobs. :param js: the job ids to be aborted""" self.open() self._client.abort(jobs = js) self.close() def cancelPendingResultsFor( self, params ): """Cancel any results pending for experiments at the given point in the parameter space. :param params: the experimental parameters""" # grab the result job ids jobs = self.pendingResultsFor(params) if len(jobs) > 0: # abort in the cluster self._abortJobs(jobs) # cancel in the notebook self.notebook().cancelPendingResultsFor(params) def cancelAllPendingResults( self ): """Cancel all pending results.""" # grab all the pending job ids jobs = self.pendingResults() if len(jobs) > 0: # abort in the cluster self._abortJobs(jobs) # cancel in the notebook self.notebook().cancelAllPendingResults()
class AdvancedIppController(ParallelController): ''' A wrapper class to deal with monitoring and logging parallel jobs activity. Note that functions should -at least for now- accept only one parameter, (which could be a structure anyway). ''' NOT_STARTED = None FAILED = -1 RUNNING = 0 SUCCESS = 1 def __init__(self, name='ParallelController', serial_fun=None, args=None, const_vars=None, chunksize=1, logfile=None, loglevel=None, poll_interval=60, max_batch=None, max_exec_time=None, dbfile=None, fresh_run=False, retry=3, commit_timeout=30): self.name = name self._serial_fun = serial_fun # serial function to be mapped self._args = args # arguments to be mapped self.const_vars = const_vars # consts for the function self.chunksize = chunksize self.logfile = logfile self.loglevel = loglevel self.poll_interval = poll_interval # poll for status every # poll_interval seconds self.results = [] self._status = [] self._to_process = [] self._ok = False # flag for successfull completion self.chunksize self._client = None # ipyparallel client self._view = None # ipyparallel load balanced view self._logger = None # logger self._ar = None # async results #self._fwrapper = None # function wrapper self._max_batch = max_batch self.max_exec_time = max_exec_time self.retry = retry self._queue = multiprocessing.Queue() self.commit_timeout = commit_timeout if self._args is None: self._args = list() if self.const_vars is None: self.const_vars = dict() if dbfile is None: self.dbfile = '%s.db' % name else: self.dbfile = dbfile if fresh_run: if os.path.isfile(self.dbfile): os.remove(self.dbfile) def get_status(self): return self._status def set_function(self, value): self._serial_fun = value def get_function(self): return self._serial_fun def set_const(self, name, val): self.const_vars[name] = val def set_args(self, args): self._args = args self._status = [AdvancedIppController.NOT_STARTED] * len(args) self.results = [None] * len(self._args) def get_args(self): return self._args def _setup_logger(self, batch_no=None): # setup logger if batch_no is None: self._logger = logging.getLogger(self.name) else: self._logger = logging.getLogger(self.name + '/batch%d' % batch_no) # keep only stream handlers fnames = [fh.baseFilename for fh in self._logger.handlers] if self.logfile is not None and self.logfile not in fnames: fh = logging.FileHandler(self.logfile) fh.setFormatter(default_log_formatter) self._logger.addHandler(fh) self._logger.setLevel(self.loglevel) # prepare the remote function #self._fwrapper = FunctionWrapper(self._serial_fun, self.const_vars) def _setup_ipp(self): # get client and view instances, and use cloudpickle from ipyparallel import Client self._client = Client(context=zmq.Context()) self._ids = self._client.ids self._dview = self._client[self._ids] self._dview.use_cloudpickle() self._view = self._client.load_balanced_view(targets=self._ids) def _cleanup(self): if self._client: self._client.close() def _handle_errors(self): failed = [ i for i, x in enumerate(self._status) if x == AdvancedIppController.FAILED ] n_failed = len(failed) self._logger.error('%d tasks have failed.', n_failed) print('%s: %d tasks have failed.' % (self.name, n_failed), file=sys.stderr) for cnt, i in enumerate(failed): if cnt > 2: self._logger.error('... %d more errors ...', n_failed - 3) print('... %d more errors ...' % (n_failed - 3), file=sys.stderr) break self._logger.error('JOB# %d:\n %s \n' + '-' * 40, i, self.results[i]) print('JOB# %d:\n %s' % (i, self.results[i]), file=sys.stderr) print('-' * 40, file=sys.stderr) return n_failed def _split_batches(self): if self._max_batch is None: return [(0, len(self._to_process))] else: num_batch = len(self._to_process) // self._max_batch if len(self._to_process) % self._max_batch != 0: num_batch += 1 return [(b * self._max_batch, min(len(self._to_process), (b + 1) * self._max_batch)) for b in range(num_batch)] def _check_db(self): create_table = False if os.path.isfile(self.dbfile) else True with sqlite3.connect(self.dbfile) as conn: if create_table: conn.execute('CREATE TABLE completed (jid INT, ' 'completed_time INT, run_time REAL)') query = 'SELECT jid FROM completed' c = conn.execute(query) completed = {x[0] for x in c.fetchall()} not_completed = set(self._args) - completed self._to_process = list(not_completed) for i in completed: self.results[i] = 0 self._status[i] = AdvancedIppController.SUCCESS def _run_all(self): self._setup_logger() self.results = [None] * len(self._args) self._status = [AdvancedIppController.RUNNING] * len(self._args) self._check_db() tot_time = 0 trial = 0 error_count = 0 last_commit = time.time() self._start_time = time.time() with sqlite3.connect(self.dbfile) as conn: while trial < self.retry and len(self._to_process): now_completed = [] error_count = 0 self.job_batches = self._split_batches() self._logger.info( 'Starting %s - %d jobs, divided in %d batches (trial %d)', self.name, len(self._to_process), len(self.job_batches), trial) for batch_no, (batch_start, batch_end) in enumerate(self.job_batches): p = multiprocessing.Process(target=self._run_batch, args=(batch_no, )) p.start() while True: # keep the db file updated, so we can read the situation from outside if time.time() - last_commit > self.commit_timeout: conn.commit() last_commit = time.time() i, result = self._queue.get() if i >= 0: self.results[i] = result if result[0] == -1: error_count += 1 self.results[i] = result[1] self._status[i] = AdvancedIppController.FAILED elif result[0] == 0: self._status[i] = AdvancedIppController.SUCCESS self.results[i] = result[1] etime = result[2] conn.execute( 'INSERT INTO completed VALUES (?, ?, ?)', (i, int(time.time()), etime)) now_completed.append(i) elif i == -2: p.join() raise RuntimeError('Process raised error', result) elif i == -3: # batch finished signal tot_time += result break p.join() for i in now_completed: self._to_process.remove(i) if error_count: self._logger.warning( 'Got %d errors during the execution, retrying...', error_count) trial += 1 # handle errors if any occurred if error_count: n_failed = self._handle_errors() raise RuntimeError('%d jobs failed. Log file: %s' % (n_failed, self.logfile)) else: self._logger.info('Done. Time elapsed: %s', pretty_tdelta(tot_time)) self._ok = True def _run_batch(self, batch_no): self._setup_logger(batch_no) batch_start, batch_end = self.job_batches[batch_no] self._logger.info('Starting batch %d of %d: %d tasks', batch_no + 1, len(self.job_batches), batch_end - batch_start) self._setup_ipp() self._logger.info('Working on %d worker engines', len(self._ids)) # maps asyncronously on workers fwrapper = IppFunctionWrapper(self._serial_fun, self.const_vars, self.max_exec_time) self._ar = self._view.map_async( fwrapper, self._to_process[batch_start:batch_end], chunksize=self.chunksize) # start a thread to monitor progress self._monitor_flag = True monitor_thread = threading.Thread(target=self._monitor) monitor_thread.start() try: # collect results for i, r in enumerate(self._ar): self._queue.put((self._to_process[i + batch_start], r)) except: self._monitor_flag = False exc_type, exc_value, exc_traceback = sys.exc_info() tb_str = ''.join( traceback.format_exception(exc_type, exc_value, exc_traceback)) self._queue.put((-2, tb_str)) self._queue.put((-3, self._ar.elapsed)) # close the monitor thread and print details self._logger.info('Batch completed. Time elapsed: %s', pretty_tdelta(self._ar.elapsed)) monitor_thread.join() def _monitor(self): while not self._ar.ready() and self._monitor_flag: n_tasks = len(self._ar) if self._ar.progress > 0: time_per_task = float(self._ar.elapsed) / self._ar.progress eta = (n_tasks - self._ar.progress) * time_per_task etastr = pretty_tdelta(eta) else: etastr = 'N/A' self._logger.info( 'Completed %d of %d tasks. Time elapsed: %s Remaining: %s', self._ar.progress, n_tasks, pretty_tdelta(self._ar.elapsed), etastr) elapsed = 0 while elapsed < self.poll_interval: if not self._monitor_flag: break self._ar.wait(1) elapsed += 1 def submit(self): if not self._serial_fun: raise RuntimeError('AdvancedIppController.serial_fun not set') if not self._args: raise RuntimeError('AdvancedIppController.args not set') try: self._run_all() finally: self._cleanup() def success(self): return self._ok def map(self, parallel_task, args): self.serial_fun = parallel_task self.args = args self.submit() return self.results status = property(get_status, None, None) serial_fun = property(get_function, set_function, None) args = property(get_args, set_args, None)
def get_noise_fft_parallel(Y, n_processes=4,n_pixels_per_process=100, backend='ipyparallel', **kwargs): """parallel version of get_noise_fft. Params: ------- Y: ndarray input movie (n_pixels x Time). Can be also memory mapped file. n_processes: [optional] int number of processes/threads to use concurrently n_pixels_per_process: [optional] int number of pixels to be simultaneously processed by each process backend: [optional] string the type of concurrency to be employed. only 'multithreading' for the moment **kwargs: [optional] dict all the parameters passed to get_noise_fft Returns: -------- sn: ndarray(double) noise associated to each pixel """ folder = tempfile.mkdtemp() # Pre-allocate a writeable shared memory map as a container for the # results of the parallel computation pixel_groups=range(0,Y.shape[0]-n_pixels_per_process+1,n_pixels_per_process) # if backend=="threading": # case joblib # sn_name = os.path.join(folder, 'sn_s') # print "using threading" # # sn_s = np.memmap(sn_name, dtype=np.float32,shape=Y.shape[0], mode='w+') # # Fork the worker processes to perform computation concurrently # Parallel(n_jobs=n_processes, backend=backend)(delayed(fft_psd_parallel)(Y, sn_s, i, n_pixels_per_process, **kwargs) # for i in pixel_groups) if backend=='multithreading': pool = ThreadPool(n_processes) argsin=[(Y, i, n_pixels_per_process, kwargs) for i in pixel_groups] results = pool.map(fft_psd_multithreading, argsin) _,_,psx_= results[0] sn_s=np.zeros(Y.shape[0]) psx_s=np.zeros((Y.shape[0],psx_.shape[-1])) for idx,sn,psx_ in results: sn_s[idx]=sn psx_s[idx,:]=psx_ elif backend=='ipyparallel' or backend=='SLURM': if type(Y) is np.core.memmap: # if input file is already memory mapped then find the filename Y_name = Y.filename else: raise Exception('ipyparallel backend only works with memory mapped files') try: if backend=='SLURM': if 'IPPPDIR' in os.environ and 'IPPPROFILE' in os.environ: pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE'] else: raise Exception('envirnomment variables not found, please source slurmAlloc.rc') c = Client(ipython_dir=pdir, profile=profile) else: c = Client() dview = c[:] ne = len(dview) print 'Running on %d engines.'%(ne) argsin=[(Y_name, i, n_pixels_per_process, kwargs) for i in pixel_groups] if backend=='SLURM': results = dview.map(fft_psd_multithreading, argsin) else: results = dview.map_sync(fft_psd_multithreading, argsin) _,_,psx_= results[0] psx_s=np.zeros((Y.shape[0],psx_.shape[-1])) sn_s=np.zeros(Y.shape[0]) for idx,sn, psx_ in results: sn_s[idx]=sn psx_s[idx,:]=psx_ finally: try: c.close() except: print 'closing c failed' elif backend=='single_thread': # pool = ThreadPool(n_processes) argsin=[(Y, i, n_pixels_per_process, kwargs) for i in pixel_groups] results = map(fft_psd_multithreading, argsin) _,_,psx_= results[0] sn_s=np.zeros(Y.shape[0]) psx_s=np.zeros((Y.shape[0],psx_.shape[-1])) for idx,sn,psx_ in results: sn_s[idx]=sn psx_s[idx,:]=psx_ else: raise Exception('Unknown method') # if n_pixels_per_process is not a multiple of Y.shape[0] run on remaining pixels pixels_remaining= Y.shape[0] % n_pixels_per_process if pixels_remaining>0: print "Running fft for remaining pixels:" + str(pixels_remaining) if type(Y) is np.core.memmap: # if input file is already memory mapped then find the filename Y_name = Y.filename elif type(Y) is str: Y_name=Y else: raise Exception('ipyparallel backend only works with memory mapped files') idx,sn, psx_=fft_psd_multithreading((Y_name,Y.shape[0]-pixels_remaining, pixels_remaining, kwargs)) sn_s[idx]=sn psx_s[idx,:]=psx_ sn_s=np.array(sn_s) psx_s=np.array(psx_s) try: shutil.rmtree(folder) except: print("Failed to delete: " + folder) raise return sn_s,psx_s
def run_CNMF_patches(file_name, shape, options, rf=16, stride = 4, n_processes=2, backend='single_thread'): """ Function that runs CNMF in patches, either in parallel or sequentiually, and return the result for each. It requires that ipyparallel is running Parameters ---------- file_name: string full path to an npy file (2D, pixels x time) containing the movie shape: tuple of thre elements dimensions of the original movie across y, x, and time options: dictionary containing all the parameters for the various algorithms rf: int half-size of the square patch in pixel stride: int amount of overlap between patches backend: string 'ipyparallel' or 'single_thread' Returns ------- A_tot: C_tot: sn_tot: optional_outputs: """ (d1,d2,T)=shape d=d1*d2 K=options['init_params']['K'] idx_flat,idx_2d=extract_patch_coordinates(d1, d2, rf=rf, stride = stride) args_in=[] for id_f,id_2d in zip(idx_flat[:],idx_2d[:]): args_in.append((file_name, id_f,id_2d[0].shape, options)) print len(idx_flat) st=time.time() if backend is 'ipyparallel': try: c = Client() dview=c[:n_processes] file_res = dview.map_sync(cnmf_patches, args_in) finally: dview.results.clear() c.purge_results('all') c.purge_everything() c.close() elif backend is 'single_thread': file_res = map(cnmf_patches, args_in) else: raise Exception('Backend unknown') print time.time()-st # extract the values from the output of mapped computation num_patches=len(file_res) A_tot=scipy.sparse.csc_matrix((d,K*num_patches)) C_tot=np.zeros((K*num_patches,T)) sn_tot=np.zeros((d1*d2)) b_tot=[] f_tot=[] bl_tot=[] c1_tot=[] neurons_sn_tot=[] g_tot=[] idx_tot=[]; shapes_tot=[] id_patch_tot=[] count=0 patch_id=0 print 'Transforming patches into full matrix' for idx_,shapes,A,b,C,f,S,bl,c1,neurons_sn,g,sn,_ in file_res: sn_tot[idx_]=sn b_tot.append(b) f_tot.append(f) bl_tot.append(bl) c1_tot.append(c1) neurons_sn_tot.append(neurons_sn) g_tot.append(g) idx_tot.append(idx_) shapes_tot.append(shapes) for ii in range(np.shape(A)[-1]): new_comp=A.tocsc()[:,ii]/np.sqrt(np.sum(np.array(A.tocsc()[:,ii].todense())**2)) if new_comp.sum()>0: A_tot[idx_,count]=new_comp C_tot[count,:]=C[ii,:] id_patch_tot.append(patch_id) count+=1 patch_id+=1 A_tot=A_tot[:,:count] C_tot=C_tot[:count,:] optional_outputs=dict() optional_outputs['b_tot']=b_tot optional_outputs['f_tot']=f_tot optional_outputs['bl_tot']=bl_tot optional_outputs['c1_tot']=c1_tot optional_outputs['neurons_sn_tot']=neurons_sn_tot optional_outputs['g_tot']=g_tot optional_outputs['idx_tot']=idx_tot optional_outputs['shapes_tot']=shapes_tot optional_outputs['id_patch_tot']= id_patch_tot return A_tot,C_tot,sn_tot, optional_outputs
def power_of_test(data1, data2, rvs_func='rvs_pairs', tests=['chi2_2samp'], rvs_key={}, test_key={}, parallel=None, sync=True): """Compute the corresponding p-values for each histrogram pairs from the random variates of the given 2 samples/frequencies for size_times. Parameters ---------- data1, data2 : sequence of 1-D ndarrays Input data. Observed samples or frequencies. rvs_func : [callable|str], optional, default : "rvs_pairs" The random variates function. The rvs_func can be either a callable or one of the following strings:: String Description "rvs_pairs" Compute the histogram pairs from the random variates of the given 2 samples/frequencies for size_times. tests : ([callable|str],...), optional, default : ["chi2_2samp"] A list of *test* statistical functions. The *test* can be either a callable or one of the following strings:: String Description "chi2_2samp" Read TS.chi2_2samp for further information. "BDM_2samp" Read TS.BDM_2samp for further information. "likelihoodratio_ksamp" Read TS.likelihoodratio_ksamp for further information. "likelihoodvalue_ksamp" Read TS.likelihoodvalue_ksamp for further information. "ks_2samp" Read TS.ks_2samp for further information. "anderson_ksamp" Read TS.anderson_ksamp for further information. "CVM_2samp" Read TS.CVM_2samp for further information. rvs_key : dict, optional, default : {} Keyword arguments for the rvs function, rvs_func. test_key : dict, optional Keyword arguments for the test statistical function, test. parallel : bool, optional, default : None If True, import IPyParallel package to do the parallel computation. If parallel is None, the global variable PARALLEL will be used instead. sync : bool, optional, default : True When sync is False, an IPyParallel AsyncResult Object will be returned instead. Onyl affect when parallel is True. Returns ------- [p1, p2, ...] : 1-D array The corresponding p-values for each histogram pairs. """ if parallel == None: parallel = PARALLEL if parallel: try: global client client = Client(**ipp_profile) size = rvs_key['size'] N = len(client) jobs = [] for i in range(N): rvs_key['size'] = (size // N + 1) if (i < size % N) else size // N jobs.append(client[client.ids[i]].apply_async( power_of_test, data1, data2, rvs_func, test, rvs_key, test_key, False)) ars = client._asyncresult_from_jobs(jobs) if sync: ars.wait_interactive() ret = {} for key, val in ars.get(): ret.setdefault(key, []).extend(val) else: return ars finally: client.close() return ret if type(rvs_func) == str: rvs_func = globals()[rvs_func] if type(tests) not in (list, tuple): tests = [tests] tests = [(t, getattr(TS, t)) if type(t) == str else (str(t), t) for t in tests] ret = {} for rvs1, rvs2 in rvs_func(data1, data2, **rvs_key): for tname, test in tests: ret.setdefault(tname, []).append( test(rvs1, rvs2, binned=True, **test_key).pvalue) return ret