Example #1
0
    def fit_loop(self, loop_num=1000):
        parameter = {}
        #########################
        #
        # use parallel computing
        #
        #>>>>>>>>>>>>>>>>>>>>>>>>
        pool = mp.Pool(mp.cpu_count())
        print('CPU number :  ', mp.cpu_count())
        print('loop start ..............')
        for i in range(loop_num):
            print('\n Fit experiment in loop  >>>>>  ', i)
            seed = np.random.randint(500) * i + np.random.randint(500)
            parameter[i] = pool.apply_async(self.fit, args=(seed,)).get()
            if i % 100 == 98:
                self.autosave(parameter)
        pool.close()
        pool.join()
        print('loop finished')
        self.params = np.array([parameter[i] for i in range(loop_num)])

        np.savetxt(os.path.join(os.getcwd() + '/result/'+self.subfolder+ 'parameters_' + self.name + '.csv'), self.params, delimiter=',',
                   header='N,  R,  delss, ' * len(self.shells) + 'e, LL')

        return _plot_(self.params, self.shells, self.name)
Example #2
0
def simulate_noise(n1,n2, sigma, size, I_op, transform, lvl, Npar = np.int(mtp.cpu_count()/2)):
    ##DESCRIPTION:
    ##  Simulates noise levels in source plane from lensing operator and convolution operator.
    ##
    ##INPUTS:
    ##  -n1,n2: the shape of the images for which to simulate noise maps.
    ##  -size: scaling factor for the shape of the source.
    ##  -Fkappa: Projection operator between lens and source plane.
    ##  -lensed: mapping of an all at one image to source plane.
    ##  -PSFconj: the conjugate of the PSF
    ##
    ##OPTIONS:
    ##  -n: size of the median filter. Default is 3.
    ## 
    ##OUTPUTS:
    ##  -S: the source light profile.
    ##  -FS: the lensed version of the estimated source light profile
    n = 500
    if Npar>mtp.cpu_count():
        Npar = mtp.cpu_count()
    ns1,ns2 = n1*size, n2*size
#    lvl = np.int(np.log2(ns1))
    w_levels = np.zeros((lvl,ns1,ns2))

    p = Pool(Npar)
    storage = mk_simu(n1,n2,lvl,size, sigma, I_op, transform,n)

    w_levels = np.std(storage, axis = 3)
#    w_levels[0,:,:] = w_levels[0,:,:]*6/5

    return w_levels
def _n_jobs_actual(n_jobs):
    """Actual number of jobs to execute in parallel"""
    try:
        result = mp.cpu_count() if n_jobs == -1 else min(
            n_jobs, mp.cpu_count())
    except:
        result = 1
    return n_jobs
Example #4
0
 def resolve_pods_multiprocessing(self):
     n_core = mp.cpu_count()  # set to the number of cores you want to use
     try:
         with mp.Pool(n_core) as pool:
             self.pods = pool.map(worker, self.pods)
     except TimeoutError:
         "We lacked patience and got a multiprocessing.TimeoutError"
 def test_Lattice_Stability(self,
                            ringTuningBounds,
                            injectorTuningBounds,
                            numEdgePoints=30,
                            parallel=False):
     assert len(ringTuningBounds) == 2 and len(injectorTuningBounds) == 2
     ringKnob1Arr = np.linspace(ringTuningBounds[0][0],
                                ringTuningBounds[0][1], numEdgePoints)
     ringKnob2Arr = np.linspace(ringTuningBounds[1][0],
                                ringTuningBounds[1][1], numEdgePoints)
     injectorKnob1Arr_Constant = ringTuningBounds[0][1] * np.ones(
         numEdgePoints**2)
     injectorKnobA2rr_Constant = ringTuningBounds[1][1] * np.ones(
         numEdgePoints**2)
     testCoords = np.asarray(np.meshgrid(ringKnob1Arr,
                                         ringKnob2Arr)).T.reshape(-1, 2)
     if self.whichKnobs == 'all':
         testCoords = np.column_stack(
             (testCoords, injectorKnob1Arr_Constant,
              injectorKnobA2rr_Constant))
     if parallel == False:
         stabilityList = [self.is_Stable(coords) for coords in testCoords]
     else:
         with mp.Pool(mp.cpu_count()) as pool:
             stabilityList = pool.map(self.is_Stable, testCoords)
     assert len(stabilityList) == numEdgePoints**2
     if sum(stabilityList) == 0:
         return False
     else:
         return True
Example #6
0
 def calculate(self):
     pool = Pool(processes=min(cpu_count(), 8))
     results = [pool.apply(self.square, (i, )) for i in self.getNumbers()]
     pool.close()
     pool.join()
     for result in results:
         print result
Example #7
0
    def _train_batch_parallelize(self, trees, n_incorrect_answers):
        """Parallelizes training for a list of trees.
        Uses the number of threads given by multiprocessing.cpu_count()

        Updates model parameters directly, and returns batch error.
        """
        # Defaults to using cpu_count() threads
        pool = Pool()
        
        def get_subbatch_deltas(_trees):
            return self._train_batch(_trees, n_incorrect_answers, 
                                     apply_learning=False)

        subbatches = utils.split(trees, n_slices=cpu_count())

        # result will be a list of tuples (error, deltas)
        result = pool.map(get_subbatch_deltas, subbatches)

        # no more processes accepted by this pool
        pool.close()   
        # Wait until mapping is completed
        pool.join()

        error = sum([r[0] for r in result])
        deltas = [r[1] for r in result]
        for (delta_Wv, delta_b, delta_We, delta_Wr) in deltas:
            self.Wv -= delta_Wv
            self.b -=  delta_b
            self.We -= delta_We
            self.Wr -= delta_Wr

        return error
Example #8
0
def run_parallel_async(graph, nprocs=None, sleep=0.2):
    if nprocs == 1:
        return run_async(graph)

    nprocs = nprocs or mp.cpu_count() // 2

    with mp.Manager() as manager:
        graph = tgraph.create_parallel_compatible_graph(graph, manager)

        ioq = mp.Queue(len(graph.funcs.keys()))
        cpuq = mp.Queue(len(graph.funcs.keys()))

        for _ in range(nprocs):
            proc = mp.Process(target=run_scheduler,
                              args=(graph, sleep, ioq, cpuq))
            proc.start()

        while not tgraph.all_done(graph):
            for task in tgraph.get_ready_tasks(graph):
                graph = tgraph.mark_as_in_progress(graph, task)
                mlog(graph).info('pid {}: queueing task {}'.format(
                    os.getpid(), task))
                if task in graph.io_bound:
                    ioq.put(task)
                else:
                    cpuq.put(task)

            time.sleep(sleep)

        return tgraph.recover_values_from_manager(graph)
Example #9
0
 def __init__(self, loglikelihood,
              prior_transform, ndim,
              sample='auto', bound='multi',
              n_cpu=None, n_thread=None):
     
     if n_cpu is None:
         n_cpu = mp.cpu_count()
         
     if n_thread is not None:
         n_thread = max(n_thread, n_cpu-1)
     
     if n_cpu > 1:
         self.open_pool(n_cpu)
         self.use_pool = {'update_bound': False}
     else:
         self.pool = None
         self.use_pool = None
             
     self.prior_tf = prior_transform
     self.loglike = loglikelihood
     self.ndim = ndim
     
     dsampler = dynesty.DynamicNestedSampler(self.loglike,
                                             self.prior_tf,
                                             self.ndim,
                                             sample=sample,
                                             bound=bound,
                                             pool=self.pool,
                                             queue_size=n_thread,
                                             use_pool=self.use_pool)
     self.dsampler = dsampler
Example #10
0
def pool_map(
        worker: typing.Callable,
        worker_args: typing.Union[list, dict],
        constant_args=None,
        cpu_ratio=0.75,
        single_thread=False
) -> list:

    worker_args = _group_args(worker_args)

    if constant_args is not None:
        try:
            [args.update(copy.deepcopy(constant_args)) for args in worker_args]
        except AttributeError:
            [args.extend(copy.deepcopy(constant_args)) for args in worker_args]

    worker = pool_map_more_than_one_arg(worker)

    if single_thread:
        resp = map(worker, worker_args)
    else:
        count_cpu_workers = int(
            (cpu_ratio * multiprocess.cpu_count())
            if cpu_ratio <= 1.0 else cpu_ratio
        )
        pool = multiprocess.Pool(count_cpu_workers)
        resp = pool.imap(worker, worker_args)
        pool.close()
        pool.join()

    return list(resp)
Example #11
0
    def save_topics(cls, topic_words_path, start_date, end_date, year_interval):
        topic_words = []
        cls.cursor.execute(f"DROP TABLE IF EXISTS {cls.table}_topics")
        cls.cursor.execute(
            f"CREATE TABLE {cls.table}_topics(topic_id INTEGER, word_distribution JSONB, topic_evolution JSONB, frequency FLOAT, docs JSONB)"
        )
        with tqdm(total=cls.model.nb_topics, leave=False, desc="Generating topic stats") as pbar:
            with Pool(cpu_count() - 1) as pool:
                for (
                    topic_id,
                    word_distribution,
                    topic_evolution,
                    frequency,
                    docs,
                    description,
                ) in pool.imap_unordered(
                    cls.compute_topic,
                    zip(range(cls.model.nb_topics), repeat(start_date), repeat(end_date), repeat(year_interval),),
                ):
                    cls.cursor.execute(
                        f"INSERT INTO {cls.table}_topics (topic_id, word_distribution, topic_evolution, frequency, docs) VALUES (%s, %s, %s, %s, %s)",
                        (topic_id, word_distribution, topic_evolution, frequency, docs),
                    )
                    topic_words.append(
                        {"name": topic_id, "frequency": frequency, "description": ", ".join(description),}
                    )
                    pbar.update()

        topic_words.sort(key=lambda x: x["name"])
        with open(topic_words_path, "w") as out_file:
            json.dump(topic_words, out_file)

        cls.cursor.execute(f"CREATE INDEX {cls.table}_topic_id_index on {cls.table}_topics USING HASH(topic_id)")
        cls.db.commit()
Example #12
0
    def test_init_cpus_max(self):
        runmodel = RunModel(model=TestingModel1d(),
                            parameters=self.parameters,
                            logger_level="error",
                            CPUs="max")

        self.assertEqual(runmodel.CPUs, mp.cpu_count())
Example #13
0
 def get_new_tickets(self, from_time=utils.pre_day_to_string(1)):
     search_conditions = {
         "skip": 0,
         "query": {
             "ctimeGte": "{}T21:00:00.000Z".format(from_time)
         }
     }
     pool_size = multiprocess.cpu_count()
     pool_volume = 10 * pool_size
     index = 0
     tickets_num = self._get_number_of_tickets(from_time, to_time)
     req_num = utils.ceil_division(tickets_num, 1000)
     pool = Pool(pool_size)
     for req_count in range(req_num):
         search_tickets = self.search_tickets(search_conditions)
         while True:
             tickets = pool.map(
                 self.add_attr_to_ticket,
                 itertools.islice(search_tickets, pool_volume))
             if tickets:
                 print('Downloaded {}/{} tickets'.format(
                     index, tickets_num),
                       end='\r')
                 index += pool_volume
                 yield tickets
             else:
                 break
         search_conditions['skip'] += 1000
Example #14
0
def run_parallel_async(graph, nprocs=None, sleep=0.2, raise_errors=False):
    if nprocs == 1:
        return run_async(graph, sleep=sleep, raise_errors=raise_errors)

    nprocs = nprocs or mp.cpu_count() // 2

    with mp.Manager() as manager:
        graph = tgraph.create_parallel_compatible_graph(graph, manager)

        ioq = mp.Queue(len(graph.funcs.keys()))
        cpuq = mp.Queue(len(graph.funcs.keys()))

        procs = [mp.Process(target=run_scheduler,
                            args=(graph, sleep, ioq, cpuq, raise_errors))
                 for _ in range(nprocs)]
        for proc in procs:
            proc.start()

        while not tgraph.all_done(graph):
            for task in tgraph.get_ready_tasks(graph):
                graph = tgraph.mark_as_in_progress(graph, task)
                mlog(graph).info(
                    'pid {}: queueing task {}'.format(os.getpid(), task))
                if task in graph.io_bound:
                    ioq.put(task)
                else:
                    cpuq.put(task)

            time.sleep(sleep)

            if raise_errors and sum(not p.exitcode for p in procs):
                raise RuntimeError('An async task has failed. Please check your logs')

        return tgraph.recover_values_from_manager(graph)
Example #15
0
def pplinearFold(listOfSeqs) :

    pool = mp.Pool(mp.cpu_count())
    result = numpy.array(pool.map(fold_with_LinearFold, listOfSeqs))
    pool.close()

    return list(result[:,0]),list(result[:,1])
def getInputTargetPaths(path):
    """
    From a parent path, get all paths for all inputs and all targets

    Params:
        - path : str
            Parent folder

    Outputs:
        - x_paths : list
            Set of input paths
        - y_paths : list
            Set of fluorescence paths

    """
    folders = sorted(os.listdir(path))
    parallel_out = Parallel(multiprocess.cpu_count())(
        delayed(ParallelPathAccess)(path, folder) for folder in folders)

    # Parallel out is a list of parallelized tuples: [(x1,y1),(x2,y2),...,(xN,y)]
    parallel_out = np.array(parallel_out)

    x_paths = parallel_out[:, 0]
    y_paths = parallel_out[:, 1]

    return x_paths, y_paths
Example #17
0
def run_parallel(graph, nprocs=None, sleep=0.2, raise_errors=False):
    nprocs = nprocs or mp.cpu_count() - 1
    with mp.Manager() as manager:
        graph = tgraph.create_parallel_compatible_graph(graph, manager)
        with mp.Pool(nprocs) as pool:

            exception_q = mp.Queue(10)

            def error_callback(exception):
                exception_q.put_nowait(exception)
                pool.terminate()

            while not tgraph.all_done(graph):
                for task in tgraph.get_ready_tasks(graph, reverse=False):
                    graph = tgraph.mark_as_in_progress(graph, task)
                    mlog(graph).info('pid {}: assigning task {}'.format(
                        os.getpid(), task))
                    pool.apply_async(run_task,
                                     args=(graph, task, raise_errors),
                                     error_callback=error_callback)
                time.sleep(sleep)

                if not exception_q.empty():
                    raise exception_q.get()

        return tgraph.recover_values_from_manager(graph)
def check_ncpus(arg_value):
    arg_value = int(arg_value)

    if arg_value <= 0:
        raise click.BadParameter("n_cpus must be >= 1")
    else:
        return min(arg_value, mp.cpu_count())
Example #19
0
 def _super_Fast_Trace(self,swarm,trace_Particle):
     # use trick of accessing only the important class variabels and passing those through to reduce pickle time
     def fastFunc(compactDict):
         particle = Particle()
         for key, val in compactDict.items():
             setattr(particle, key, val)
         particle = trace_Particle(particle)
         compactDictTraced = {}
         for key, val in vars(particle).items():
             if val is not None:
                 compactDictTraced[key] = val
         return compactDictTraced
     compactDictList = []
     for particle in swarm:
         compactDict = {}
         for key, val in vars(particle).items():
             if val is not None:
                 if not (isinstance(val, Iterable) and len(val) == 0):
                     compactDict[key] = val
         compactDictList.append(compactDict)
     with mp.Pool(mp.cpu_count()) as Pool:
         compactDictTracedList = Pool.map(fastFunc, compactDictList)
     for particle, compactDict in zip(swarm.particles, compactDictTracedList):
         for key, val in compactDict.items():
             setattr(particle, key, val)
     return swarm
def pool_map(worker,
             worker_args,
             constant_args=None,
             cpu_ratio=0.75,
             single_thread=False):

    worker_args = _group_args(worker_args)

    if constant_args is not None:
        try:
            [args.update(copy.deepcopy(constant_args)) for args in worker_args]
        except AttributeError:
            [args.extend(copy.deepcopy(constant_args)) for args in worker_args]

    worker = pool_map_more_than_one_arg(worker)

    if single_thread:
        resp = map(worker, worker_args)
    else:
        count_cpu_workers = int((
            cpu_ratio *
            multiprocess.cpu_count()) if cpu_ratio <= 1.0 else cpu_ratio)
        pool = multiprocess.Pool(count_cpu_workers)
        resp = pool.imap(worker, worker_args)
        pool.close()
        pool.join()

    return list(resp)
def compute_distance(points):
    all_combinations = itertools.product(points, points)
    pool = mp.Pool(math.ceil(mp.cpu_count() * .75))
    calculations = pool.imap_unordered(point_distance,
                                       all_combinations,
                                       chunksize=1000000)
    return np.max(list(tqdm(calculations, total=len(points)**2)))
Example #22
0
def check_ncpus(arg_value):
    arg_value = int(arg_value)

    if arg_value <= 0:
        raise argparse.ArgumentTypeError("n_cpus must be >= 1")
    else:
        return min(arg_value, mp.cpu_count())
Example #23
0
def main():

    nucleotides = ["A", "U", "G", "C"]
    pop_size = 100
    target = '..((((((.........))))))..((((.....))))'
    rate = None
    length = len(target)
    ldscape = Landscape.Landscape(target)
    time = 50

    print("Starting RNA evolution with target {} of length {}".format(
        target, len(target)))
    db, init_pop = init(pop_size, length, ['A'], ldscape)
    params = {
        'c': rate,
        "pop_size": pop_size,
        'init_pop': init_pop,
        'landscape': ldscape,
        'nucleotides': nucleotides,
        'time': time,
        'db': db,
        "pos": get_bp_position(target)
    }
    print(init_pop)
    data_min = []
    data_mean = []

    for c in np.arange(1, 2, 0.5):
        params['c'] = c
        print("Start evolution with c = {}......".format(params['c']))
        pool = mp.Pool(mp.cpu_count())
        results = pool.map(evolution, [params] * 3)
        print(results)
    """
def start_train(resume):

    urllib.request.urlretrieve(const.url + '/network_file',
                               'deliverables/network.py')
    urllib.request.urlretrieve(const.url + '/config_file',
                               'deliverables/input_params.py')
    urllib.request.urlretrieve(const.url + '/observation_file',
                               'deliverables/observation.py')
    urllib.request.urlretrieve(const.url + '/curriculum_file',
                               'deliverables/curriculum.py')

    num_workers = mp.cpu_count() - 1
    should_stop = mp.Value(c_bool, False)

    while True:
        worker_processes = []

        # create_worker(0, should_stop)

        # Start process 1 - n, running in other processes
        for w_num in range(0, num_workers):
            process = mp.Process(target=create_worker,
                                 args=(w_num, should_stop))
            process.start()
            sleep(0.5)
            worker_processes.append(process)

        try:
            for p in worker_processes:
                p.join()
        except KeyboardInterrupt:
            should_stop.value = True

    print("Looks like we're done")
Example #25
0
def MultiNMF(V, rank, nrun, cores, return_best=True):
    '''Run NMF for a target matrix multiple times'''
    available_cores = multiprocess.cpu_count()
    if available_cores < cores:
        print(
            "Only %d cores available but %d cores requested, reset it to available number."
            % (available_cores, cores))
        cores = available_cores
    xs = range(1, nrun + 1)
    with multiprocess.Pool(processes=cores) as p:
        v = p.starmap(NMF2, zip(xs, repeat(V), repeat(rank)))

    if return_best:
        print("Compare KLD and return the best.")
        l = len(v)
        kld = v[0]["KLD"]
        idx = 0
        for i in range(1, l):
            k = v[i]["KLD"]
            print("KLD to compare: %f, KLD to be compared: %f" % (k, kld))
            if kld > k:
                print("Current KLD value updated to %f" % k)
                kld = k
                idx = i
        print("Return the run #%d" % (idx + 1))
        v = v[idx]

    return v
Example #26
0
def dataAccess(path, size, output_folder, key="max"):
    """
    Provide Maximum Intensity Projection (MIP) of given images in folders, plus
    provide the corresponding flourescent target images
    
    Params:
        path : str
            Root folder from where to start examining
        size : int or list of int
            Size of images to be obtained
        output_folder : str
            Folder where to save MIP file as .npy (if "save" flag is True)
        key : str
            Type of projection to provide ("max" for maximum intensity projection, "mean" for mean intensity projection,
            "median" for median intensity projection)
            Default: "max"
            
    Outputs:
        mips : list of np.array
            List with MIP results [MIP_20x, MIP_40x, MIP_60x]
        targets : list of np.array
            List with fluorescent target images [ch01_images, ch02_images, ch03_images] --> ch01_images = [ch01_20x, ch01_40x, ch01_60x]
            
    """
    init = time.time()

    if not (os.path.exists(path)):
        print("Folder {} does not exist".format(path))

    folders = sorted(os.listdir(path))  # Read all folders in the given path
    Parallel(multiprocess.cpu_count())(delayed(parallelAccess)(os.path.join(
        path, folder), size, output_folder, key) for folder in folders)

    print("Preprocessing completed! Time ellapsed: {} seconds".format(
        round(time.time() - init, 2)))
Example #27
0
    def energy_basins(self, X, multicore=True):
        """Energy basin for each given state.
        
        Parameters
        ----------
        X : ndarray

        Returns
        -------
        energybasin : ndarray
            Each row is the energy basin for the given row of X.
        """
        def f(v):
            return self._find_energy_basin(v, self.multipliers)

        if multicore:
            pool = mp.Pool(mp.cpu_count())
            energybasin = np.vstack(pool.map(f, X))
            pool.close()
        else:
            energybasin = []
            for v in X:
                energybasin.append(f(v))
            energybasin = np.vstack(energybasin)
        return energybasin
Example #28
0
def insert_to_prod(stage_table, prod_table, sql_execute):
    file_location = raw_input(
        'Please input the location of the CSV file to ingest\n')
    insert_data = ImportCsv(stage_table)
    batch_size = 5000
    chunk_gen = insert_data.get_chunks(batch_size, file_location)
    pool = mp.Pool(mp.cpu_count() - 1)
    results = pool.map(insert_data.worker, chunk_gen)
    pool.close()
    pool.join()
    logging.info(
        'Insert to Stage Complete complete. Insert to Prod Table Start')
    result = sql_execute.execute_query(
        "SELECT COUNT(1) FROM {0}".format(prod_table))
    count = result.fetchone()[0]
    if count == 0:
        insert_query = """INSERT INTO {1} SELECT name,sku,description FROM {0}""".format(
            stage_table, prod_table)
        logging.info(
            'Bulk Insert from stage to prod table (One time Full Load)')
        sql_execute.execute_query(insert_query)
    else:
        logging.info('Inserting New Records to Prod')
        new_records = """INSERT INTO {1} SELECT ps.name, ps.sku, ps.description FROM {0} ps left join {1} pp on ps.name = pp.name and ps.sku = pp.sku and ps.description = pp.description WHERE pp.sku is NULL""".format(
            stage_table, prod_table)
        sql_execute.execute_query(new_records)

    logging.info('Prod Table insertion Complete. Truncating Stage Table')
    sql_execute.execute_query("TRUNCATE TABLE {0}".format(stage_table))
    logging.info(
        'File Ingestion Succeeded. Please query {0} Table to check the results.'
        .format(prod_table))
Example #29
0
    def __init__(self, mapping, output_directory=None):
        self._logger = logging.getLogger(__name__)

        # FIXME: not raising errors due to schema validation for now
        mapping = validate(mapping, silent=True)

        _processes = mapping['resources']['processes']
        mapping['resources']['processes'] = (
            multiprocess.cpu_count() if _processes == 'max' else _processes)

        self._frozenjson = FrozenJSON(mapping)

        if output_directory is not None:
            if path.isabs(output_directory):
                self._path_to_output_directory = output_directory
            else:
                _ = Path(self.data.root_folder, output_directory)
                self._path_to_output_directory = str(_)
        else:
            self._path_to_output_directory = None

        # init the rest of the parameters, these parameters are used
        # througout the pipeline so we compute them once to avoid redudant
        # computations

        # GEOMETRY PARAMETERS
        path_to_geom = path.join(self.data.root_folder, self.data.geometry)

        self._set_param('geom',
                        geom.parse(path_to_geom, self.recordings.n_channels))

        # check dimensions of the geometry file
        n_channels_geom, _ = self.geom.shape

        if self.recordings.n_channels != n_channels_geom:
            raise ValueError('Channels in the geometry file ({}) does not '
                             'value in the configuration file ({})'.format(
                                 n_channels_geom, self.recordings.n_channels))

        neigh_channels = geom.find_channel_neighbors(
            self.geom, self.recordings.spatial_radius)
        self._set_param('neigh_channels', neigh_channels)

        channel_groups = geom.make_channel_groups(self.recordings.n_channels,
                                                  self.neigh_channels,
                                                  self.geom)
        self._set_param('channel_groups', channel_groups)

        self._set_param(
            'spike_size',
            int(
                np.round(self.recordings.spike_size_ms *
                         self.recordings.sampling_rate / (2 * 1000))))

        channel_index = geom.make_channel_index(self.neigh_channels,
                                                self.geom,
                                                steps=2)

        self._set_param('channel_index', channel_index)
Example #30
0
def main():
    with open("config.json", "r") as f:
        conf = json.load(f)
    args = argue(conf)
    dest = conf["host"] + conf["endpoint"]
    head = {
        "api_key": args.apikey,
        "api_secret": args.apisecret,
        "Content-type": "application/json"
    }

    if args.group:
        data = grouper(importer(args.document))
    else:
        data = importer(args.document)

    if int(args.n) != 0:
        data = data[0:int(args.n)]

    pts = [Patient(x["metadata"], x["data"], args.group) for x in data]

    if args.group and args.abstractions:
        csv_header = [
            "patient_id", "focus", "chunk", "encoding", "abstractions"
        ]
    elif args.group:
        csv_header = ["patient_id", "focus", "chunk", "encoding"]
    elif args.abstractions:
        csv_header = [
            "patient_id", "document_id", "surname", "forename", "dob",
            "gender", "visit_id", "observation_datetime", "project", "author",
            "focus", "chunk", "encoding", "abstractions"
        ]
    else:
        csv_header = [
            "patient_id", "document_id", "surname", "forename", "dob",
            "gender", "visit_id", "observation_datetime", "project", "author",
            "focus", "chunk", "encoding"
        ]

    header_str = "\t".join(csv_header)
    args.outfile.write(header_str + "\n")

    pool = mp.Pool(mp.cpu_count())

    def process(pt):
        pt.request(args.resource, dest, head, args.cacert)

        csv_str = []
        for line in pt.format(args.abstractions):
            csv_str.append("\t".join([str(x).replace("\n", " ")
                                      for x in line]))
        return "\n".join(csv_str)

    result_async = [pool.apply_async(process, (pt, )) for pt in pts]
    results = [r.get() for r in result_async]
    for line in results:
        if line is not None and line != "":
            args.outfile.write(line + "\n")
Example #31
0
    def train(self, trees, n_incorrect_answers=100, 
              n_epochs=30, n_batches=None):
        """Trains the QANTA model on the sentence trees.

        trees is a list of DependencyTree
        n_incorrect_answers is |Z| in the paper's eq. 5. It determines
            how many incorrect answers are sampled from the training data
            to be used in calculating sentence error.
        n_epochs is the number of times the model trains on the input data
        """
        # Grab new answers, and enforce uniqueness
        for t in trees:
            if not t.answer in self.answers:
                self.answers.append(t.answer)

        answers = self.answers # A tiny bit shorter
        
        # Make sure we can sample n_incorrect_answers different answers.
        if len(answers) - 1 < n_incorrect_answers:
            n_incorrect_answers = len(answers) - 1

        # QANTA original code says 'ideally 25 minibatches per epoch'
        n_batches = n_batches or min(25, len(trees))
        batch_size = len(trees) / n_batches

        if cpu_count() < batch_size:
            trainer = self._train_batch_parallelize
        else:
            # No reason to parallelize
            trainer = self._train_batch

        for epoch in xrange(n_epochs):
            epoch_error = 0
            epoch_start = time()
            for batch in xrange(n_batches):
                batch_start = time()

                # Index range for this batch
                lo = batch * batch_size
                hi = lo + batch_size

                batch_trees = trees[lo:hi]

                # Parallel training
                batch_error = trainer(batch_trees, n_incorrect_answers)
                # Serial training
                # batch_error = self._train_batch(batch_trees, n_incorrect_answers)

                # Only print batch stats if it takes more than 5 seconds
                if time() - batch_start > 5:
                    print ("Training error epoch {}, batch {}: {} "
                           "({:.2f} seconds)").format(epoch, batch, 
                           batch_error, time() - batch_start)
                epoch_error += batch_error

            if time() - epoch_start > 5:
                print ("Total training error for epoch {}: {} "
                       "({:.2f} seconds)").format(epoch, epoch_error, 
                        time() - epoch_start)
Example #32
0
    def __init__(self, processes=None, initializer=None, initargs=(),
                 maxtasksperchild=None):
        self._setup_queues()
        self._taskqueue = Queue.Queue()
        self._cache = {}
        self._state = RUN
        self._maxtasksperchild = maxtasksperchild
        self._initializer = initializer
        self._initargs = initargs

        if processes is None:
            try:
                processes = cpu_count()
            except NotImplementedError:
                processes = 1
        if processes < 1:
            raise ValueError("Number of processes must be at least 1")

        if initializer is not None and not hasattr(initializer, '__call__'):
            raise TypeError('initializer must be a callable')

        self._processes = processes
        self._pool = []
        self._repopulate_pool()

        self._worker_handler = threading.Thread(
            target=Pool._handle_workers,
            args=(self, )
            )
        self._worker_handler.daemon = True
        self._worker_handler._state = RUN
        self._worker_handler.start()


        self._task_handler = threading.Thread(
            target=Pool._handle_tasks,
            args=(self._taskqueue, self._quick_put, self._outqueue,
                  self._pool, self._cache)
            )
        self._task_handler.daemon = True
        self._task_handler._state = RUN
        self._task_handler.start()

        self._result_handler = threading.Thread(
            target=Pool._handle_results,
            args=(self._outqueue, self._quick_get, self._cache)
            )
        self._result_handler.daemon = True
        self._result_handler._state = RUN
        self._result_handler.start()

        self._terminate = Finalize(
            self, self._terminate_pool,
            args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
                  self._worker_handler, self._task_handler,
                  self._result_handler, self._cache),
            exitpriority=15
            )
Example #33
0
    def __init__(self, processes=None, initializer=None, initargs=()):
        self._setup_queues()
        self._taskqueue = queue.Queue()
        self._cache = {}
        self._state = RUN

        if processes is None:
            try:
                processes = cpu_count()
            except NotImplementedError:
                processes = 1

        if initializer is not None and not hasattr(initializer, '__call__'):
            raise TypeError('initializer must be a callable')

        self._pool = []
        for i in range(processes):
            w = self.Process(
                target=worker,
                args=(self._inqueue, self._outqueue, initializer, initargs)
                )
            self._pool.append(w)
            w.name = w.name.replace('Process', 'PoolWorker')
            w.daemon = True
            w.start()

        self._task_handler = threading.Thread(
            target=Pool._handle_tasks,
            args=(self._taskqueue, self._quick_put, self._outqueue, self._pool)
            )
        self._task_handler.daemon = True
        self._task_handler._state = RUN
        self._task_handler.start()

        self._result_handler = threading.Thread(
            target=Pool._handle_results,
            args=(self._outqueue, self._quick_get, self._cache)
            )
        self._result_handler.daemon = True
        self._result_handler._state = RUN
        self._result_handler.start()

        self._terminate = Finalize(
            self, self._terminate_pool,
            args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
                  self._task_handler, self._result_handler, self._cache),
            exitpriority=15
            )
Example #34
0
    def eval_EFG(self,x,num_procs=None,info=False):

        from multiprocess import Pool,cpu_count

        if not num_procs:
            num_procs = cpu_count()
        num_samples = self.parameters['num_samples']
        pool = Pool(num_procs)
        num = int(np.ceil(float(num_samples)/float(num_procs)))
        results = list(zip(*pool.map(lambda i: self.eval_EFG_sequential(x,num,i,info),range(num_procs),chunksize=1)))
        pool.terminate()
        pool.join()
        if not info:
            assert(len(results) == 4)
        else:
            assert(len(results) == 5)
        assert(all([len(vals) == num_procs for vals in results]))
        return [sum(vals)/float(num_procs) for vals in results]
Example #35
0
    def eval_EQ(self,p,num_procs=None,quiet=True):
        """
        Evaluates E[Q(p,r)] and its gradient in parallel. 

        Parameters
        ----------
        p : generator powers
        num_procs : number of parallel processes
        quiet : flag
        """
       
        from multiprocess import Pool,cpu_count
 
        if not num_procs:
            num_procs = cpu_count()
        num_samples = self.parameters['num_samples']
        pool = Pool(num_procs)
        num = int(np.ceil(float(num_samples)/float(num_procs)))
        results = list(zip(*pool.map(lambda i: self.eval_EQ_sequential(p,num,i,quiet),range(num_procs),chunksize=1)))
        pool.terminate()
        pool.join()
        assert(len(results) == 2)
        assert(all([len(vals) == num_procs for vals in results]))
        return [sum(vals)/float(num_procs) for vals in results]
Example #36
0
 def get_new_tickets(self, from_time=utils.pre_day_to_string(1)):
     search_conditions = {
         "skip": 0,
         "query": {
             "ctimeGte": "{}T21:00:00.000Z".format(from_time)
          }
     }
     pool_size = multiprocess.cpu_count()
     pool_volume = 10 * pool_size
     index = 0
     tickets_num = self._get_number_of_tickets(from_time, to_time)
     req_num = utils.ceil_division(tickets_num, 1000)
     pool = Pool(pool_size)
     for req_count in range(req_num):
         search_tickets = self.search_tickets(search_conditions)
         while True:
             tickets = pool.map(self.add_attr_to_ticket, itertools.islice(search_tickets, pool_volume))
             if tickets:
                 print('Downloaded {}/{} tickets'.format(index, tickets_num), end='\r')
                 index += pool_volume
                 yield tickets
             else:
                 break
         search_conditions['skip'] += 1000
Example #37
0
Future = namedtuple('Future', ['function', 'args', 'kwargs'])

def launch(pool, fs):
    results = []
    for f in fs:
        with redirected("%s.out" % f.function.__name__):
            results.append(pool.apply_async(f.function, f.args, f.kwargs))
    return results
            

def _f(x, y=0):
    return [x * 42 for i in range(1000)]

def run_tasks(futures):
    futures = [future for futuregroup in futures for future in futuregroup]
    pool    = Pool(processes=cpu_count())
    results = launch(pool, futures)
    results = [result.get() for result in results]
    return results

if __name__ == "__main__":
    print("Creating pool with %s CPUs" % cpu_count())
    pool = Pool(processes=cpu_count())
    futures = [Future(_f, [10], {'y':0})] 
    print(futures)
    results = launch(pool, futures)
    for result in results:
        result.get()
    

Example #38
0
            perc_sum, count_sum = 0, 0
            for i in (self.one, self.three, self.seven, self.nine):
                count_sum += i
                perc_sum += self.rounder(i)
            print "\n" + "one" + "\t" + "= %.2f" % self.rounder(self.one) + "%" + "\t" + "(%i)" % self.one
            print "three" + "\t" + "= %.2f" % self.rounder(self.three) + "%" + "\t" + "(%i)" % self.three
            print "seven" + "\t" + "= %.2f" % self.rounder(self.seven) + "%" + "\t" + "(%i)" % self.seven
            print "nine" + "\t" + "= %.2f" % self.rounder(self.nine) + "%" + "\t" + "(%i)" % self.nine
            print "\n" + "Sum" + "\t" + "= %.2f" % perc_sum + "%" + "\t" + "(%i)" % count_sum
        except ZeroDivisionError:
            print "\nNo primes found!"


if __name__ == '__main__':

    cores = cpu_count()


    def screen_clear():  # Small function for clearing the screen on Unix or Windows
        if os.name == 'nt':
            return os.system('cls')
        else:
            return os.system('clear')


    def usage():  # any incorrect input will display the usage instructions
        screen_clear()
        print """Prime_Perc.py
    Version 0.1 - Written by Halsandr

    =======
Example #39
0
# routines to call a function in parallel
from __future__ import print_function
import scipy.linalg as lg
from . import algebra
try:
  from multiprocess import Pool
  import multiprocess
  maxcpu = multiprocess.cpu_count()
except:
    print("Multiprocess not working")
    def Pool(n=1): # workaround
            class mpool():
                def map(self,f,xs):
                  return [f(x) for x in xs]
                def terminate(self): return None # dummy function
            return mpool()

cores = 1 # call in a single by default


def set_cores(n=1):
    global cores
    cores = n


#mainpool = None

#def initialize(): 
#  global mainpool
#  if cores>1:
#    mainpool = Pool(cores) # create pool
Example #40
0
#!/usr/bin/python
# coding: utf-8

import threading
import multiprocess

def loop():
    x = 0
    while True:
        x = x ^ 1


if __name__ == "__main__":
    for i in range(multiprocess.cpu_count()):
        t = threading.Thread(target=loop)
        t.start()

        
Example #41
0
from scipy import interpolate

#import multiprocessing
import multiprocess # BETTER THAN MULTIPROCESSING
import datetime
from tempfile import TemporaryFile
import os.path
#import matplotlib
#from matplotlib import pyplot as plt
import vegas

from extras import set_active, cartesian, interp_list, set_shift

#import psutil # to see how many cores are allocated
#ncores =  len(psutil.Process().cpu_affinity()) # for cluster
ncores =  multiprocess.cpu_count() # for local

############################
#    numerical parameters  #
############################
E = np.exp(1.) # no longer needed?
qmin = 0.0001
qmax = 10. #default
n=1.; # default consider every n*kf for the bispectrum . for the power specutrm it computes every kf.
ni = 2; #number iterations
ne = 5000; #number of evaluations #3500 normal, 5000 for high prec
chunksize = 500 # divide the triangle list in chunks of size "chunksize". carefull not to change the chunksize between runs!

#############################
#  cosmological parameters  #
#############################
Example #42
0
from server_json_wrapper import JSONServerWrapper

REPEATS_PER_CONFIG = 300
DIMS_LEN = 6
NUM_DIMS = 2
SEEDS_SEED = 7
MINES_MIN = 1
MINES_MAX = (DIMS_LEN ** NUM_DIMS) // 2

# Reduce this when using very slow clients
POOL_MAX_CHUNKSIZE = 10

SERVER = PythonInternalServer

if hasattr(multiprocessing, "cpu_count"):
	no_cores = multiprocessing.cpu_count()
else:
	no_cores = 1

plot_clients = [
	("blue", ReactiveClient),
	("green", ReactiveClientCheckShared),
	#("red", ReactiveClientGuess),
	#("cyan", ReactiveClientGuessAny),
	("orange", ReactiveClientAvgEmptiesBalanced),
	#("purple", ReactiveClientExhaustiveTest),
	#("pink", ReactiveClientExhaustiveSplit),
]

# Chunksize for pools as calculated in multiprocessing module, but with the
# addition of a specified cap. Allows for more frequent progress counter updates
Example #43
0
    def _parallel_solve(self,
                        solver=None,
                        ignore_dcp=False,
                        warm_start=False,
                        verbose=False, **kwargs):
        """Solves a DCP compliant optimization problem in parallel.

        Saves the values of primal and dual variables in the variable
        and constraint objects, respectively.

        Parameters
        ----------
        solver : str, optional
            The solver to use. Defaults to ECOS.
        ignore_dcp : bool, optional
            Overrides the default of raising an exception if the problem is not
            DCP.
        warm_start : bool, optional
            Should the previous solver result be used to warm start?
        verbose : bool, optional
            Overrides the default of hiding solver output.
        kwargs : dict, optional
            A dict of options that will be passed to the specific solver.
            In general, these options will override any default settings
            imposed by cvxpy.

        Returns
        -------
        float
            The optimal value for the problem, or a string indicating
            why the problem could not be solved.
        """
        def _solve_problem(problem):
            """Solve a problem and then return the optimal value, status,
            primal values, and dual values.
            """
            opt_value = problem.solve(solver=solver,
                                      ignore_dcp=ignore_dcp,
                                      warm_start=warm_start,
                                      verbose=verbose,
                                      parallel=False, **kwargs)
            status = problem.status
            primal_values = [var.value for var in problem.variables()]
            dual_values = [constr.dual_value for constr in problem.constraints]
            return SolveResult(opt_value, status, primal_values, dual_values)

        pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
        solve_results = pool.map(_solve_problem, self._separable_problems)
        pool.close()
        pool.join()
        statuses = {solve_result.status for solve_result in solve_results}
        # Check if at least one subproblem is infeasible or inaccurate
        for status in s.INF_OR_UNB:
            if status in statuses:
                self._handle_no_solution(status)
                break
        else:
            for subproblem, solve_result in zip(self._separable_problems,
                                                solve_results):
                for var, primal_value in zip(subproblem.variables(),
                                             solve_result.primal_values):
                    var.save_value(primal_value)
                for constr, dual_value in zip(subproblem.constraints,
                                              solve_result.dual_values):
                    constr.save_value(dual_value)
            self._value = sum(solve_result.opt_value
                              for solve_result in solve_results)
            if s.OPTIMAL_INACCURATE in statuses:
                self._status = s.OPTIMAL_INACCURATE
            else:
                self._status = s.OPTIMAL
        return self._value
Example #44
0
from Queue import Empty
from multiprocess import JoinableQueue, Process, cpu_count, Queue, Pool
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings()

re_youtube = re.compile(r'((?<=watch\?v=)[-\w]+\
        |(?<=youtube.com/embed/)[-\w]+)', re.I)

re_images = re.compile(r'(?<=")[^"]+jpg(?=")', re.I)
re_splitter = re.compile(r"[^\w@#]+", re.UNICODE)

HOME = os.path.join(os.path.expanduser("~"), ".config/anchorbot")
HERE = os.path.realpath(os.path.dirname(__file__))
CONFIGFILE = os.path.join(HOME, "config")
NUM_THREADS = max(1, cpu_count() - 1)

class Config(dict):
    def __init__(self, configfile):
        dict.__init__(self)

        self["redis_keys"] = {}
        self["abos"] = []
        self.configfile = configfile

        if not os.path.exists(HOME):
            os.mkdir(HOME)
        if os.path.exists(configfile):
            if os.path.getsize(configfile) > 0:
                with open(configfile, "r") as f:
                    content = json.load(f)
Example #45
0
def run_tasks(futures):
    futures = [future for futuregroup in futures for future in futuregroup]
    pool    = Pool(processes=cpu_count())
    results = launch(pool, futures)
    results = [result.get() for result in results]
    return results