示例#1
0
 def __init__(self,file_list, modification_pipeline=None, use_menpo_type=False):
     videos=OrderedDict()
     file_list_filtered=[]
     
     video_getter_pool=ThreadPool(nodes=8)
     mpio_obj_list = video_getter_pool.map(menpo_import_video_verbose,file_list)
     
     for vp,mpio_obj in zip(file_list,mpio_obj_list):
         if not mpio_obj is None:
             videos[vp]=mpio_obj
             file_list_filtered.append(vp)
     
     #for vp in file_list:
         #print('menpo.io.import_video importing %s' % vp)
         #try:
             #mpio_obj=menpo.io.import_video(vp, exact_frame_count=False, normalise=False)
             #videos[vp]=mpio_obj
         #except Exception as err:
             #print('menpo.io.import_video could not import %s' % vp)
             #print(err)
             
     self.file_list = file_list_filtered    
     self.videos=videos
     if not modification_pipeline is None:
         self.modification_pipeline=modification_pipeline
     self.fps=mpio_obj.fps
     self.use_menpo_type=use_menpo_type
示例#2
0
 def __init__(self, 
              image_generators=[],
              fps=None,
              modification_pipelines=None,
              structured_modification_pipelines=None,
              use_menpo_type=False,
              opts={}):
     self.video_lists=[]
     self.file_lists=[]
     print('GroupedVideoGenerator3()')
     for i in range(0,len(image_generators)):
         if(isinstance(image_generators[i], VideoGenerator3)):
             assert(isinstance(image_generators[i].videos, OrderedDict))
             self.file_lists.append(image_generators[i].file_list);
             self.video_lists.append(image_generators[i].videos)
             fps=image_generators[i].fps
         elif(isinstance(image_generators[i],list)):
             video_getter_pool=ThreadPool(nodes=8)                
             tmp_mpio_obj_list=video_getter_pool.map(menpo_import_video_verbose, image_generators[i])
             tmp_mpio_obj_list = [x for x in tmp_mpio_obj_list if x is not None]
             safe_idxs = [idx for idx,x in enumerate(tmp_mpio_obj_list) if x is not None]
             tmp_file_list=image_generators[i]
             self.file_lists.append([tmp_file_list[safe_idx] for safe_idx in safe_idxs])                
             self.video_lists.append(tmp_mpio_obj_list)
         else:
             raise TypeError('You can only make a GroupedImageGenerator2 '+\
                             'from a list of string-lists or ImageGenerator2s')
         
         self.file_lists[i] = [tmp for tmp in self.file_lists[i] if not tmp is None]
     
     self.fps=fps
     self.modification_pipelines= \
         modification_pipelines if not modification_pipelines is None else {}
     self.structured_modification_pipelines= \
         structured_modification_pipelines if not structured_modification_pipelines is None else {}
     self.io_pool = None
     self.enable_caching = 0
     self.enable_pp_caching = 0
     self.img_pp_cache = {}
     self.img_pp_oo_cache = {}
     self.img_cache = {}
     self.opts = opts
     self.use_menpo_type = use_menpo_type
示例#3
0
 def init_thread_pool(self):
     self.io_pool=ThreadPool(nodes=4)
示例#4
0
class GroupedVideoGenerator2:


    def __init__(self, image_generators,fps=None,modification_pipelines=None,structured_modification_pipelines=None):
        self.video_lists=[]
        self.file_lists=[]
        for i in range(0,len(image_generators)):
            if(isinstance(image_generators[i], VideoGenerator)):
                assert(isinstance(image_generators[i].videos, OrderedDict))
                self.file_lists.append(image_generators[i].file_list);
                self.video_lists.append(image_generators[i].videos)
                fps=image_generators[i].fps
            elif(isinstance(image_generators[i],list)):
                self.file_lists.append(image_generators[i]);
                tmp_mpio_obj_list=map(lambda vp:menpo.io.import_video(vp, exact_frame_count=False, normalise=False), image_generators[i])
                self.video_lists.append(tmp_mpio_obj_list)
            else:
                raise TypeError('You can only make a GroupedImageGenerator2 from a list of string-lists or ImageGenerator2s')
            
            self.file_lists[i] = filter(None,self.file_lists[i]);
        
        self.fps=fps
            

        self.modification_pipelines=modification_pipelines if not modification_pipelines is None else {}
        self.structured_modification_pipelines=structured_modification_pipelines if not structured_modification_pipelines is None else {}
        self.io_pool = None
        self.enable_caching=0;
        self.enable_pp_caching=0;
        self.img_pp_cache={}
        self.img_pp_oo_cache={}
        self.img_cache={}
        self.opts={}

    def group_lists(self,get_key_callback):
        
        new_file_lists=sbpy_utils.core.sets.group_by(self.file_lists,get_key_callback);
        self.file_lists=new_file_lists;
        
        reordered_video_lists=[]
        old_video_lists=self.video_lists
        
        for view_idx,file_list in enumerate(new_file_lists):
            reordered_video_list=OrderedDict()
            old_video_list=old_video_lists[view_idx]
            for g in file_list:
                reordered_video_list[g]=old_video_list[g]
                
    def dynamic_func(self,key,*args):
        fhandle = self.opts[key];
        out = fhandle(*args);
        return out    
                       
    def get_sample_dims(self, pipeline_mask=[], invert_pipeline_mask=True):
        modified_img,_=self.get(0,0,pipeline_mask, invert_pipeline_mask)
        all_dims=[]
        for view in modified_img:
            dims = view[0].shape;
            all_dims.append(dims)
        return all_dims
        
    def structured_get(self, idx, time_idxs, pipeline_mask=[], invert_pipeline_mask=True): 
        return self._get(idx, time_idxs, pipeline_mask, invert_pipeline_mask,collect_pipeline_multi_outputs=True)
    
    def get(self, idx, time_idxs, pipeline_mask=[], invert_pipeline_mask=True):
        return self._get(idx, time_idxs, pipeline_mask, invert_pipeline_mask,collect_pipeline_multi_outputs=False)
    
    def get_num_samples(self):
        return len(self.file_lists[0]);
    

    def _get(self, idx, time_idxs, pipeline_mask=[], invert_pipeline_mask=True,collect_pipeline_multi_outputs=False):
        segments=[]
        metadatas=[]
        all_oos=[]
        
        if isinstance(time_idxs, int):
            time_idxs=[time_idxs]        
        
        for i in range(0,len(self.file_lists)):

            vid_path = self.video_lists[i].keys()[idx]
            video_obj = self.video_lists[i][vid_path]
    
    
            safe_time_idxs=[safe_idx for safe_idx in time_idxs if safe_idx <len(video_obj)]
            sliced_frame_list=video_obj[safe_time_idxs]
            
            if self.io_pool is None:
                
                processed_imgs=[]
                pipeline_oos=[]
                for frame in sliced_frame_list:
                    if collect_pipeline_multi_outputs:
                        processed_img,oo=apply_func_pipeline_masked_wrapped(menpo_grayscale_to_rgb(frame), self.structured_modification_pipelines[i],pipeline_mask, invert_pipeline_mask)
                        pipeline_oos.append(oo)
                        processed_imgs.append(processed_img)
                    else:
                        processed_img = apply_func_pipeline_masked(menpo_grayscale_to_rgb(frame), self.modification_pipelines[i],pipeline_mask, invert_pipeline_mask)
                        processed_imgs.append(processed_img)
            else:
                if collect_pipeline_multi_outputs:
                    frame_proc = lambda frame: apply_func_pipeline_masked_wrapped(menpo_grayscale_to_rgb(frame), self.structured_modification_pipelines[i],pipeline_mask, invert_pipeline_mask)     
                    processed_imgs_oos = self.io_pool.map(frame_proc,sliced_frame_list)
                    processed_imgs,pipeline_oos=zip(*processed_imgs_oos)
                else:
                    frame_proc = lambda frame: apply_func_pipeline_masked(menpo_grayscale_to_rgb(frame), self.modification_pipelines[i],pipeline_mask, invert_pipeline_mask)
                    processed_imgs = self.io_pool.map(frame_proc,sliced_frame_list)
                
            metadata={'video_path':vid_path,'time_idxs':time_idxs}
            
            
            
            #if(self.enable_caching and (mij_path in self.img_cache)):
                #img = self.img_cache[mij_path];
                ##print(mij_path+'(using cache) ; ')
            #elif(self.enable_caching and (not mij_path in self.img_cache)):
                #img = self.imreader_callback(mij_path);
                #self.img_cache[mij_path]=img
                ##print(mij_path + '(caching) ; ');
            #else:
                #img = self.imreader_callback(mij_path);
                ##print(mij_path+' ; ');
            
           
            #pp_cache_key=stringman.sanitize_string(mij_path+str(pipeline_mask)+str(invert_pipeline_mask));
            #if(self.enable_pp_caching and kv_haskey(pp_cache_key,self.img_pp_cache)):
                #modified_img = self.img_pp_cache[pp_cache_key];
                ##print(mij_path+'(using img_pp_cache) ; ');
            #elif(self.enable_pp_caching and not (pp_cache_key in self.img_pp_cache)):
                #modified_img = apply_func_pipeline_masked(img, self.modification_pipelines[i], pipeline_mask, invert_pipeline_mask);
                #self.img_pp_cache = kv_set(pp_cache_key,modified_img,self.img_pp_cache);
                ##print([mij_path,'(caching to img_pp_cache) ; ']);
            #else:
                #modified_img = apply_func_pipeline_masked(img, self.modification_pipelines[i], pipeline_mask, invert_pipeline_mask);
                ##print([mij_path,' ; ']);
            
            segments.append(processed_imgs);
            metadatas.append(metadata);
            all_oos.append(pipeline_oos)
        
        ret_val=(segments, metadatas)
        if collect_pipeline_multi_outputs:
            ret_val+=(all_oos,)
        
        
        return ret_val
    
    
    def start_fill(self,queuesize=10,pipeline_mask=[],invert_pipeline_mask=True,min_length=3,max_length=5,alpha=12,beta=0.5):
        
        self.read_queue = Queue(maxsize=queuesize)
        self.worker_threads_events=[]
        self.worker_threads=[]
        def structured_get_loop(queue_obj,event_obj):
            while True:
                print('fetching sequence')
                data_obj,metadatas,data_obj_oos = self.get_random_segment_group(pipeline_mask, invert_pipeline_mask,min_length,max_length,alpha,beta)
                queue_obj.put( {'data_obj':data_obj,
                                'data_paths':metadatas,
                                'data_obj_oos':data_obj_oos,
                                'metadatas':metadatas} )
                event_is_set = event_obj.wait()
 
        num_threads=1
        for i in range(num_threads):
            worker_event = threading.Event()
            worker_event.set()
            worker = threading.Thread(target=structured_get_loop, args=(self.read_queue,worker_event))
            worker.setDaemon(True)
            self.worker_threads.append(worker)
            self.worker_threads_events.append(worker_event)
            worker.start()
        
    def get_random_segment_group_from_q(self,num_items=1):
        return self.structured_gets_from_q(num_items)
        
    def structured_gets_from_q(self,num_items=1):
        items=[]
        for i in range(0,num_items):
            res=self.read_queue.get()
            item_consumed_handle = self.read_queue.task_done
            items.append((res,item_consumed_handle))
        return items

    def structured_get_from_q(self):
        return self.structured_gets_from_q()
    
    def pause_fill(self):
        for event_obj in self.worker_threads_events:
            event_obj.clear()
        with self.read_queue.mutex:
            self.read_queue.queue.clear()        
    
    def resume_fill(self):
        for event_obj in self.worker_threads_events:
            event_obj.set()    

    def gets(self, idxs, time_idxs_s, io_pool, pipeline_mask=[], invert_pipeline_mask=True):
        f = lambda idx,time_idxs: self.get(idx, time_idxs,pipeline_mask, invert_pipeline_mask)
        data_objs = io_pool.map(f,idxs,time_idxs_s)
        return data_objs
    
    def structured_gets(self, idxs, time_idxs_s, io_pool, pipeline_mask=[], invert_pipeline_mask=True):
        f = lambda idx, time_idx: self.structured_get(idx, time_idxs, pipeline_mask, invert_pipeline_mask)
        data_objs_tmp = io_pool.map(f,idxs,time_idxs_s)
        data_obj_tmp,data_paths_tmp,data_obj_oos_tmp = [list(c) for c in zip(*data_objs_tmp)]
        data_obj,data_paths,data_obj_oos=[list(x) for x in zip(*data_obj_tmp)],[list(x) for x in zip(*data_paths_tmp)],[list(x) for x in zip(*data_obj_oos_tmp)]
        return (data_obj,data_paths,data_obj_oos)
    
    def get_random_segment_group(self,pipeline_mask=[], invert_pipeline_mask=True,min_length=3,max_length=5,alpha=12,beta=0.5):
        file_group_idx=random.randint(0,len(self.video_lists[0].keys())-1)
        vid_paths=[vlist[file_group_idx] for vlist in self.file_lists]
        
        vid_lengths=map(lambda vlist,path: len(vlist[path]), self.video_lists, vid_paths)
        vid_length=min(vid_lengths)
        start_idx=random.randint(0, vid_length-min_length)
        segment_length=min(max_length,max(random.gammavariate(alpha,beta),min_length))*self.fps
        end_idx=int(np.round(start_idx+segment_length))
        time_idxs=range(start_idx,end_idx)
        processed_segment = self.structured_get(file_group_idx,time_idxs,pipeline_mask,invert_pipeline_mask)
        return processed_segment
    
               
    def set_image_modifiers(self,modification_pipelines):
        self.modification_pipelines=modification_pipelines;
    
    def set_structured_image_modifiers(self,structured_modification_pipelines):
        self.structured_modification_pipelines=structured_modification_pipelines;
    
    def get_func_pipeline(self):
        return copy.deepcopy(self.modification_pipelines);
    
    def get_structured_func_pipeline(self):
        return copy.deepcopy(self.structured_modification_pipelines)
    
    def get_num_groups(self):
        return len(self.file_lists);

    def init_thread_pool(self):
        self.io_pool=ThreadPool(nodes=4)

    def close_thread_pool(self): 
        self.io_pool.close()
示例#5
0
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2019 The Uncertainty Quantification Foundation.
# License: 3-clause BSD.  The full license text is available at:
#  - https://github.com/uqfoundation/pathos/blob/master/LICENSE

def host(id):
    import socket
    return "Rank: %d -- %s" % (id, socket.gethostname())


if __name__ == '__main__':
    from pathos.pools import ThreadPool as TPool
    tpool = TPool()

    print("Evaluate 10 items on 1 thread")
    tpool.nthreads = 1
    res3 = tpool.map(host, range(10))
    print(tpool)
    print('\n'.join(res3))
    print('')

    print("Evaluate 10 items on 2 threads")
    tpool.nthreads = 2
    res5 = tpool.map(host, range(10))
    print(tpool)
    print('\n'.join(res5))
    print('')
示例#6
0
# build from inner function
add_me = adder(5)

# build from lambda functions
squ = lambda x:x**2


if __name__ is '__main__':
    from pathos.helpers import freeze_support
    freeze_support()

    from pathos.pools import ProcessPool as Pool
    from pathos.pools import ThreadPool as TPool
    pool = Pool()
    tpool = TPool()

    # test 'dilled' multiprocessing for inner
    print("Evaluate 10 items on 2 proc:")
    pool.ncpus = 2
    print(pool)
    print(pool.map(add_me, range(10)))
    print('')

    # test 'dilled' multiprocessing for lambda
    print("Evaluate 10 items on 4 proc:")
    pool.ncpus = 4
    print(pool)
    print(pool.map(squ, range(10)))
    print('')
示例#7
0
    xp = np.arange(N * nodes, dtype=np.float64)[::-1]
    print("Input: %s\n" % x)

    # map sin_diff to the workers, then print to screen
    print("Running serial python ...")
    y = map(sin_diff, x, xp)
    print("Output: %s\n" % np.asarray(y))

    if HAS_PYINA:
        # map sin_diff to the workers, then print to screen
        print("Running mpi4py on %d cores..." % nodes)
        y = MpiPool(nodes).map(sin_diff, x, xp)
        print("Output: %s\n" % np.asarray(y))

    # map sin_diff to the workers, then print to screen
    print("Running multiprocesing on %d processors..." % nodes)
    y = ProcessPool(nodes).map(sin_diff, x, xp)
    print("Output: %s\n" % np.asarray(y))

    # map sin_diff to the workers, then print to screen
    print("Running multiprocesing on %d threads..." % nodes)
    y = ThreadPool(nodes).map(sins_diff, x, xp)
    print("Output: %s\n" % np.asarray(y))

    # map sin_diff to the workers, then print to screen
    print("Running parallelpython on %d cpus..." % nodes)
    y = ParallelPool(nodes).map(sin_diff, x, xp)
    print("Output: %s\n" % np.asarray(y))

# EOF
示例#8
0
    print("Running serial python ...")
    y = map(sin2, x)
    print("Output: %s\n" % np.asarray(y))


    if HAS_PYINA:
        # map sin2 to the workers, then print to screen
        print("Running mpi4py on %d cores..." % nodes)
        y = MpiPool(nodes).map(sin2, x)
        print("Output: %s\n" % np.asarray(y))


    # map sin2 to the workers, then print to screen
    print("Running multiprocesing on %d processors..." % nodes)
    y = ProcessPool(nodes).map(sin2, x)
    print("Output: %s\n" % np.asarray(y))


    # map sin2 to the workers, then print to screen
    print("Running multiprocesing on %d threads..." % nodes)
    y = ThreadPool(nodes).map(sin2, x)
    print("Output: %s\n" % np.asarray(y))


    # map sin2 to the workers, then print to screen
    print("Running parallelpython on %d cpus..." % nodes)
    y = ParallelPool(nodes).map(sin2, x)
    print("Output: %s\n" % np.asarray(y))

# EOF
示例#9
0

def f(x, y):
    return x * y


x = range(10)
y = range(5)

if __name__ == '__main__':
    from pathos.helpers import freeze_support, shutdown
    freeze_support()

    from pathos.pools import ProcessPool, ThreadPool
    amap = ProcessPool().amap
    tmap = ThreadPool().map

    print(amap(f, [h(x), h(x), h(x), h(x), h(x)], y).get())

    def _f(m, g, x, y):
        return sum(m(g, x)) * y

    print(amap(_f, [tmap] * len(y), [g] * len(y), [x] * len(y), y).get())

    from math import sin, cos

    print(amap(tmap, [sin, cos], [x, x]).get())

    shutdown()

# EOF
示例#10
0
 def __init__(self):
     self.num_partitions = self.num_cores
     self.pool = ThreadPool(self.num_cores)
示例#11
0
 def threadcompute(self, xs):
     pool = ThreadPool(4)
     results = pool.map(self.compute, xs)
     return results
示例#12
0
def orm_extract(args):
    """
        Function for the ORMExtractParser
        :param args: Namespace
        :return: nothing
        """

    # Load database
    Base = databaseManage.WebsiteBase(args.database[0])
    Base.create_tables()

    if type(args.thread) is list:
        args.thread = args.thread[0]

    # Load data
    URLs = list(importData.csv_to_list(args.path[0])[1].keys())

    # ---------------------
    #  Filter the results already in database
    # ---------------------
    alreadyIn = []
    for url in Base.session.query(Base.__getattribute__(args.table[0])).all():
        alreadyIn.append(url.url)

    for url in URLs:
        if "http://" in url[:7]:
            URLs[URLs.index(url)] = url[7:]
        elif "https://" in url[:8]:
            URLs[URLs.index(url)] = url[8:]

    URLs = set(URLs)

    for url in alreadyIn:
        try:
            URLs.remove(url)
        except KeyError:
            pass
    logger.info("{} websites will be added to the database".format(len(URLs)))
    itera = iter(URLs)
    URLs = zip(*[itera] * args.thread)

    # ---------------------
    #  Add to the database
    # --------------------
    dBase = databaseManage.NormalizationBase("DB/norm.db")
    normDict = {}
    for norm in dBase.session.query(dBase.Normalization).all():
        normDict[norm.feature] = {"data": norm.data, "normalizer": norm.normalizer, "scaler": norm.scaler}

    i = 1
    for url in URLs:
        logger.debug(str(i))
        logger.info("Add : {}".format(url))
        i += args.thread

        # Create URL object
        result1 = ThreadPool().map(Website.website, url)
        result2 = []
        tmp = []
        for web in result1:
            if web.html is None:
                result2.append(web)
                # result1.remove(web)
            else:
                tmp.append(web)
        if args.extraction:
            # Extract features
            fct = partial(Website.website.features_extraction, normDict=normDict)
            ThreadPool().map(fct, tmp)
            result2 += tmp
            for web in result2:
                print(web)
                # Add in database
                Base.adding(web, args.table[0])
        else:
            for web in result1:
                # Add in database
                Base.adding(web, args.table[0])

        if i % ((50 // args.thread) * args.thread) == 1 and i != 1:
            # Get new identity with tor
            with Controller.from_port(port=9051) as controller:
                controller.authenticate()
                controller.signal(Signal.NEWNYM)
示例#13
0
class GroupedVideoGenerator3:


    def __init__(self, 
                 image_generators=[],
                 fps=None,
                 modification_pipelines=None,
                 structured_modification_pipelines=None,
                 use_menpo_type=False,
                 opts={}):
        self.video_lists=[]
        self.file_lists=[]
        print('GroupedVideoGenerator3()')
        for i in range(0,len(image_generators)):
            if(isinstance(image_generators[i], VideoGenerator3)):
                assert(isinstance(image_generators[i].videos, OrderedDict))
                self.file_lists.append(image_generators[i].file_list);
                self.video_lists.append(image_generators[i].videos)
                fps=image_generators[i].fps
            elif(isinstance(image_generators[i],list)):
                video_getter_pool=ThreadPool(nodes=8)                
                tmp_mpio_obj_list=video_getter_pool.map(menpo_import_video_verbose, image_generators[i])
                tmp_mpio_obj_list = [x for x in tmp_mpio_obj_list if x is not None]
                safe_idxs = [idx for idx,x in enumerate(tmp_mpio_obj_list) if x is not None]
                tmp_file_list=image_generators[i]
                self.file_lists.append([tmp_file_list[safe_idx] for safe_idx in safe_idxs])                
                self.video_lists.append(tmp_mpio_obj_list)
            else:
                raise TypeError('You can only make a GroupedImageGenerator2 '+\
                                'from a list of string-lists or ImageGenerator2s')
            
            self.file_lists[i] = [tmp for tmp in self.file_lists[i] if not tmp is None]
        
        self.fps=fps
        self.modification_pipelines= \
            modification_pipelines if not modification_pipelines is None else {}
        self.structured_modification_pipelines= \
            structured_modification_pipelines if not structured_modification_pipelines is None else {}
        self.io_pool = None
        self.enable_caching = 0
        self.enable_pp_caching = 0
        self.img_pp_cache = {}
        self.img_pp_oo_cache = {}
        self.img_cache = {}
        self.opts = opts
        self.use_menpo_type = use_menpo_type
        
    def copy3(self):
        ret_val=GroupedVideoGenerator3([],
                                       fps=self.fps,
                                       modification_pipelines=self.modification_pipelines,
                                       structured_modification_pipelines=self.structured_modification_pipelines,
                                       use_menpo_type=self.use_menpo_type,
                                       opts=self.opts)
        ret_val.file_lists = copy.copy(self.file_lists)
        ret_val.video_lists = copy.copy(self.video_lists)
        return ret_val
        
    def group_lists(self,get_key_callback):
        
        new_file_lists=sbpy_utils.core.sets.group_by(self.file_lists,get_key_callback);
        self.file_lists=new_file_lists;
        
        reordered_video_lists=[]
        old_video_lists=self.video_lists
        
        for view_idx,file_list in enumerate(new_file_lists):
            reordered_video_list=OrderedDict()
            old_video_list=old_video_lists[view_idx]
            for g in file_list:
                reordered_video_list[g]=old_video_list[g]
                
    def dynamic_func(self,key,*args):
        fhandle = self.opts[key];
        out = fhandle(*args);
        return out    
                       
    def get_sample_dims(self, pipeline_mask=[], invert_pipeline_mask=True):
        vidgroup_obj=self.get(0,0,pipeline_mask, invert_pipeline_mask)
        return vidgroup_obj.shape()
        
    def structured_get(self, idx, time_idxs, pipeline_mask=[], invert_pipeline_mask=True): 
        return self._get(idx, time_idxs, pipeline_mask, invert_pipeline_mask,collect_pipeline_multi_outputs=True)
    
    def get(self, idx, time_idxs, pipeline_mask=[], invert_pipeline_mask=True):
        return self._get(idx, time_idxs, pipeline_mask, invert_pipeline_mask,collect_pipeline_multi_outputs=False)
    
    def get_num_samples(self):
        return len(self.file_lists[0]);
    

    def _get(self, idx, time_idxs, pipeline_mask=[], invert_pipeline_mask=True,collect_pipeline_multi_outputs=False):
        segments=[]
        metadatas=[]
        all_oos=[]
        
        if isinstance(time_idxs, int):
            time_idxs=[time_idxs]        
        
        img_type_converter = (lambda x:x) if self.use_menpo_type else menpo_to_rgb
        
        for i in range(0,len(self.file_lists)):

            vid_path = self.video_lists[i].keys()[idx]
            video_obj = self.video_lists[i][vid_path]
    
    
            safe_time_idxs=[safe_idx for safe_idx in time_idxs if safe_idx <len(video_obj)]
            sliced_frame_list=video_obj[safe_time_idxs]
            
            if self.io_pool is None:
                
                processed_imgs=[]
                pipeline_oos=[]
                for frame in sliced_frame_list:
                    if collect_pipeline_multi_outputs:
                        processed_img,oo=apply_func_pipeline_masked_wrapped(img_type_converter(frame), self.structured_modification_pipelines[i],pipeline_mask, invert_pipeline_mask)
                        pipeline_oos.append(oo)
                        processed_imgs.append(processed_img)
                    else:
                        processed_img = apply_func_pipeline_masked(img_type_converter(frame), self.modification_pipelines[i],pipeline_mask, invert_pipeline_mask)
                        processed_imgs.append(processed_img)
            else:
                if collect_pipeline_multi_outputs:
                    frame_proc = lambda frame: apply_func_pipeline_masked_wrapped(img_type_converter(frame), self.structured_modification_pipelines[i],pipeline_mask, invert_pipeline_mask)     
                    processed_imgs_oos = self.io_pool.map(frame_proc,sliced_frame_list)
                    processed_imgs,pipeline_oos=zip(*processed_imgs_oos)
                else:
                    frame_proc = lambda frame: apply_func_pipeline_masked(img_type_converter(frame), self.modification_pipelines[i],pipeline_mask, invert_pipeline_mask)
                    processed_imgs = self.io_pool.map(frame_proc,sliced_frame_list)
                
            #if(self.enable_caching and (mij_path in self.img_cache)):
                #img = self.img_cache[mij_path];
                ##print(mij_path+'(using cache) ; ')
            #elif(self.enable_caching and (not mij_path in self.img_cache)):
                #img = self.imreader_callback(mij_path);
                #self.img_cache[mij_path]=img
                ##print(mij_path + '(caching) ; ');
            #else:
                #img = self.imreader_callback(mij_path);
                ##print(mij_path+' ; ');
            
           
            #pp_cache_key=stringman.sanitize_string(mij_path+str(pipeline_mask)+str(invert_pipeline_mask));
            #if(self.enable_pp_caching and kv_haskey(pp_cache_key,self.img_pp_cache)):
                #modified_img = self.img_pp_cache[pp_cache_key];
                ##print(mij_path+'(using img_pp_cache) ; ');
            #elif(self.enable_pp_caching and not (pp_cache_key in self.img_pp_cache)):
                #modified_img = apply_func_pipeline_masked(img, self.modification_pipelines[i], pipeline_mask, invert_pipeline_mask);
                #self.img_pp_cache = kv_set(pp_cache_key,modified_img,self.img_pp_cache);
                ##print([mij_path,'(caching to img_pp_cache) ; ']);
            #else:
                #modified_img = apply_func_pipeline_masked(img, self.modification_pipelines[i], pipeline_mask, invert_pipeline_mask);
                ##print([mij_path,' ; ']);
            if collect_pipeline_multi_outputs:
                per_frame_extra_data=pipeline_oos
            else:
                per_frame_extra_data=None
                
            new_vid_obj=Video(processed_imgs, video_path=vid_path, frame_idxs=safe_time_idxs, per_frame_extra_data=per_frame_extra_data)
            segments.append(new_vid_obj);
        
        return VideoGroup(segments)
    
    def start_fill(self,queuesize=10,pipeline_mask=[],invert_pipeline_mask=True,min_length=3,max_length=5,alpha=12,beta=0.5):
        
        self.read_queue = Queue(maxsize=queuesize)
        self.worker_threads_events=[]
        self.worker_threads=[]
        def structured_get_loop(queue_obj,event_obj):
            while True:
                grouped_video_lists = self.get_random_segment_group(pipeline_mask, invert_pipeline_mask,min_length,max_length,alpha,beta)
                queue_obj.put( grouped_video_lists )
                event_is_set = event_obj.wait()
 
        num_threads=1
        for i in range(num_threads):
            worker_event = threading.Event()
            worker_event.set()
            worker = threading.Thread(target=structured_get_loop, args=(self.read_queue,worker_event))
            worker.setDaemon(True)
            self.worker_threads.append(worker)
            self.worker_threads_events.append(worker_event)
            worker.start()
        
    def get_random_segment_group_from_q(self,num_items=1):
        return self.structured_gets_from_q(num_items)
        
    def structured_gets_from_q(self,num_items=1):
        video_groups=[]
        for i in range(0,num_items):
            video_group=self.read_queue.get()
            video_groups.append(video_group)
        
        return (VideoGroupList(video_groups).as_GroupedVideoLists(),[self.read_queue.task_done]*num_items)

    def structured_get_from_q(self):
        return self.structured_gets_from_q()
    
    def pause_fill(self):
        for event_obj in self.worker_threads_events:
            event_obj.clear()
        with self.read_queue.mutex:
            self.read_queue.queue.clear()        
    
    def resume_fill(self):
        for event_obj in self.worker_threads_events:
            event_obj.set()    

    def gets(self, idxs, time_idxs_s, io_pool, pipeline_mask=[], invert_pipeline_mask=True):
        f = lambda idx,time_idxs: self.get(idx, time_idxs,pipeline_mask, invert_pipeline_mask)
        data_objs = io_pool.map(f,idxs,time_idxs_s)
        return data_objs
    
    def structured_gets(self, idxs, time_idxs_s, io_pool, pipeline_mask=[], invert_pipeline_mask=True):
        f = lambda idx, time_idx: self.structured_get(idx, time_idxs, pipeline_mask, invert_pipeline_mask)
        video_group_list = io_pool.map(f,idxs,time_idxs_s)
        return video_group_list.as_GroupedVideoLists()
    
    def get_random_segment_group(self,pipeline_mask=[], invert_pipeline_mask=True,min_length=3,max_length=5,alpha=12,beta=0.5):
        file_group_idx=random.randint(0,len(self.video_lists[0].keys())-1)
        vid_paths=[vlist[file_group_idx] for vlist in self.file_lists]
        
        vid_lengths=map(lambda vlist,path: len(vlist[path]), self.video_lists, vid_paths)
        vid_length=min(vid_lengths)
        start_idx=random.randint(0, vid_length-min_length)
        segment_length=min(max_length,max(random.gammavariate(alpha,beta),min_length))*self.fps
        end_idx=int(np.round(start_idx+segment_length))
        time_idxs=range(start_idx,end_idx)
        #print('intended_segment length: '+str(len(time_idxs)))
        processed_segment = self.structured_get(file_group_idx,time_idxs,pipeline_mask,invert_pipeline_mask)
        print('processed_segment length: '+str(len(processed_segment)))
        return processed_segment
    
               
    def set_image_modifiers(self,modification_pipelines):
        self.modification_pipelines=modification_pipelines;
    
    def set_structured_image_modifiers(self,structured_modification_pipelines):
        self.structured_modification_pipelines=structured_modification_pipelines;
    
    def get_func_pipeline(self):
        return copy.deepcopy(self.modification_pipelines);
    
    def get_structured_func_pipeline(self):
        return copy.deepcopy(self.structured_modification_pipelines)
    
    def get_num_groups(self):
        return len(self.file_lists);

    def init_thread_pool(self):
        self.io_pool=ThreadPool(nodes=4)

    def close_thread_pool(self): 
        self.io_pool.close()
示例#14
0
 def threadcompute(self, xs):
     pool = ThreadPool(4)
     results = pool.map(self.compute, xs)
     return results
示例#15
0

# build from inner function
add_me = adder(5)

# build from lambda functions
squ = lambda x: x**2

if __name__ == '__main__':
    from pathos.helpers import freeze_support, shutdown
    freeze_support()

    from pathos.pools import ProcessPool as Pool
    from pathos.pools import ThreadPool as TPool
    pool = Pool()
    tpool = TPool()

    # test 'dilled' multiprocessing for inner
    print("Evaluate 10 items on 2 proc:")
    pool.ncpus = 2
    print(pool)
    print(pool.map(add_me, range(10)))
    print('')

    # test 'dilled' multiprocessing for lambda
    print("Evaluate 10 items on 4 proc:")
    pool.ncpus = 4
    print(pool)
    print(pool.map(squ, range(10)))
    print('')
示例#16
0
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2015 California Institute of Technology.
# License: 3-clause BSD.  The full license text is available at:
#  - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pathos/LICENSE

def host(id):
    import socket
    return "Rank: %d -- %s" % (id, socket.gethostname())


if __name__ == '__main__':
    from pathos.pools import ThreadPool as TPool
    tpool = TPool()

    print "Evaluate 10 items on 1 thread"
    tpool.nthreads = 1
    res3 = tpool.map(host, range(10))
    print tpool
    print '\n'.join(res3)
    print ''

    print "Evaluate 10 items on 2 threads"
    tpool.nthreads = 2
    res5 = tpool.map(host, range(10))
    print tpool
    print '\n'.join(res5)
    print ''

    print "Evaluate 10 items on ? threads"
示例#17
0
def df_apply(df, f, pool=None, n_cpus=None, return_df=True):
    """Apply the function `f` to each row in `df` in a parallel fashion.
    """
    if pool is None:
        if n_cpus is None:
            n_cpus = cpu_count()
        pool = ThreadPool(n_cpus)

    class RecordProxy:
        """A proxy object to wrap a `DataFrame.iat[row_i, col_i]` access model and
        provide a dictionary style interface.
        """

        __df = df
        __field_names = list(df.columns)

        @classmethod
        def _field_i(cls, name):
            try:
                return cls.__field_names.index(name)
            except ValueError as e:
                raise KeyError(
                    f"key '{name}' not found on record. Available keys are: {cls.__field_names}"
                )

        @classmethod
        def wrap_map_func(cls, f):
            """Wraps the given function to be passed to a map() style function.
            Returns a function that expects to be called with an index value and it will call
            the given function passing it an object with a python dictionary style interface to the row.
            """
            return lambda row_i: f(cls(row_i))

        @property
        def index(self):
            return self.__row_i

        def __init__(self, row_i):
            self.__row_i = row_i

        def __getitem__(self, key):
            i = self._field_i(key)
            return self.__df.iat[self.__row_i, i]

        def __setitem__(self, key, value):
            i = self._field_i(key)
            self.__df.iat[self.__row_i, i] = value

        def get(self, key, value=None):
            try:
                i = self._field_i(key)
                return self.__df.iat[self.__row_i, i]
            except KeyError:
                return value

        def __str__(self):
            parts = ["Record({"]
            fields_repr = []
            for field_name in self.__field_names:
                field_repr = self.__getitem__(field_name).__repr__()
                fields_repr.append(f"'{field_name}': {field_repr}")
            parts.extend(",".join(fields_repr))
            parts.append("})")
            return "".join(parts)

        def dict(self, keys=None):
            if keys is None:
                keys = self.__field_names

            return {
                key: self.__df.iat[self.__row_i, i]
                for i, key in enumerate(self.__field_names) if key in keys
            }

        def __iter__(self):
            return (self.__df.iat[self.__row_i, i]
                    for i in range(len(self.__field_names)))

    results = pool.map(RecordProxy.wrap_map_func(f), range(df.shape[0]))

    if return_df:
        return df
    else:
        return results