def fit_loop(self, loop_num=1000):
parameter = {}
#########################
#
# use parallel computing
#
#>>>>>>>>>>>>>>>>>>>>>>>>
pool = mp.Pool(mp.cpu_count())
print('CPU number : ', mp.cpu_count())
print('loop start ..............')
for i in range(loop_num):
print('\n Fit experiment in loop >>>>> ', i)
seed = np.random.randint(500) * i + np.random.randint(500)
parameter[i] = pool.apply_async(self.fit, args=(seed,)).get()
if i % 100 == 98:
self.autosave(parameter)
pool.close()
pool.join()
print('loop finished')
self.params = np.array([parameter[i] for i in range(loop_num)])
np.savetxt(os.path.join(os.getcwd() + '/result/'+self.subfolder+ 'parameters_' + self.name + '.csv'), self.params, delimiter=',',
header='N, R, delss, ' * len(self.shells) + 'e, LL')
return _plot_(self.params, self.shells, self.name)
def simulate_noise(n1,n2, sigma, size, I_op, transform, lvl, Npar = np.int(mtp.cpu_count()/2)):
##DESCRIPTION:
## Simulates noise levels in source plane from lensing operator and convolution operator.
##
##INPUTS:
## -n1,n2: the shape of the images for which to simulate noise maps.
## -size: scaling factor for the shape of the source.
## -Fkappa: Projection operator between lens and source plane.
## -lensed: mapping of an all at one image to source plane.
## -PSFconj: the conjugate of the PSF
##
##OPTIONS:
## -n: size of the median filter. Default is 3.
##
##OUTPUTS:
## -S: the source light profile.
## -FS: the lensed version of the estimated source light profile
n = 500
if Npar>mtp.cpu_count():
Npar = mtp.cpu_count()
ns1,ns2 = n1*size, n2*size
# lvl = np.int(np.log2(ns1))
w_levels = np.zeros((lvl,ns1,ns2))
p = Pool(Npar)
storage = mk_simu(n1,n2,lvl,size, sigma, I_op, transform,n)
w_levels = np.std(storage, axis = 3)
# w_levels[0,:,:] = w_levels[0,:,:]*6/5
return w_levels
def _n_jobs_actual(n_jobs):
"""Actual number of jobs to execute in parallel"""
try:
result = mp.cpu_count() if n_jobs == -1 else min(
n_jobs, mp.cpu_count())
except:
result = 1
return n_jobs
def resolve_pods_multiprocessing(self):
n_core = mp.cpu_count() # set to the number of cores you want to use
try:
with mp.Pool(n_core) as pool:
self.pods = pool.map(worker, self.pods)
except TimeoutError:
"We lacked patience and got a multiprocessing.TimeoutError"
def test_Lattice_Stability(self,
ringTuningBounds,
injectorTuningBounds,
numEdgePoints=30,
parallel=False):
assert len(ringTuningBounds) == 2 and len(injectorTuningBounds) == 2
ringKnob1Arr = np.linspace(ringTuningBounds[0][0],
ringTuningBounds[0][1], numEdgePoints)
ringKnob2Arr = np.linspace(ringTuningBounds[1][0],
ringTuningBounds[1][1], numEdgePoints)
injectorKnob1Arr_Constant = ringTuningBounds[0][1] * np.ones(
numEdgePoints**2)
injectorKnobA2rr_Constant = ringTuningBounds[1][1] * np.ones(
numEdgePoints**2)
testCoords = np.asarray(np.meshgrid(ringKnob1Arr,
ringKnob2Arr)).T.reshape(-1, 2)
if self.whichKnobs == 'all':
testCoords = np.column_stack(
(testCoords, injectorKnob1Arr_Constant,
injectorKnobA2rr_Constant))
if parallel == False:
stabilityList = [self.is_Stable(coords) for coords in testCoords]
else:
with mp.Pool(mp.cpu_count()) as pool:
stabilityList = pool.map(self.is_Stable, testCoords)
assert len(stabilityList) == numEdgePoints**2
if sum(stabilityList) == 0:
return False
else:
return True
def calculate(self):
pool = Pool(processes=min(cpu_count(), 8))
results = [pool.apply(self.square, (i, )) for i in self.getNumbers()]
pool.close()
pool.join()
for result in results:
print result
def _train_batch_parallelize(self, trees, n_incorrect_answers):
"""Parallelizes training for a list of trees.
Uses the number of threads given by multiprocessing.cpu_count()
Updates model parameters directly, and returns batch error.
"""
# Defaults to using cpu_count() threads
pool = Pool()
def get_subbatch_deltas(_trees):
return self._train_batch(_trees, n_incorrect_answers,
apply_learning=False)
subbatches = utils.split(trees, n_slices=cpu_count())
# result will be a list of tuples (error, deltas)
result = pool.map(get_subbatch_deltas, subbatches)
# no more processes accepted by this pool
pool.close()
# Wait until mapping is completed
pool.join()
error = sum([r[0] for r in result])
deltas = [r[1] for r in result]
for (delta_Wv, delta_b, delta_We, delta_Wr) in deltas:
self.Wv -= delta_Wv
self.b -= delta_b
self.We -= delta_We
self.Wr -= delta_Wr
return error
def run_parallel_async(graph, nprocs=None, sleep=0.2):
if nprocs == 1:
return run_async(graph)
nprocs = nprocs or mp.cpu_count() // 2
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
ioq = mp.Queue(len(graph.funcs.keys()))
cpuq = mp.Queue(len(graph.funcs.keys()))
for _ in range(nprocs):
proc = mp.Process(target=run_scheduler,
args=(graph, sleep, ioq, cpuq))
proc.start()
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info('pid {}: queueing task {}'.format(
os.getpid(), task))
if task in graph.io_bound:
ioq.put(task)
else:
cpuq.put(task)
time.sleep(sleep)
return tgraph.recover_values_from_manager(graph)
def __init__(self, loglikelihood,
prior_transform, ndim,
sample='auto', bound='multi',
n_cpu=None, n_thread=None):
if n_cpu is None:
n_cpu = mp.cpu_count()
if n_thread is not None:
n_thread = max(n_thread, n_cpu-1)
if n_cpu > 1:
self.open_pool(n_cpu)
self.use_pool = {'update_bound': False}
else:
self.pool = None
self.use_pool = None
self.prior_tf = prior_transform
self.loglike = loglikelihood
self.ndim = ndim
dsampler = dynesty.DynamicNestedSampler(self.loglike,
self.prior_tf,
self.ndim,
sample=sample,
bound=bound,
pool=self.pool,
queue_size=n_thread,
use_pool=self.use_pool)
self.dsampler = dsampler
def pool_map(
worker: typing.Callable,
worker_args: typing.Union[list, dict],
constant_args=None,
cpu_ratio=0.75,
single_thread=False
) -> list:
worker_args = _group_args(worker_args)
if constant_args is not None:
try:
[args.update(copy.deepcopy(constant_args)) for args in worker_args]
except AttributeError:
[args.extend(copy.deepcopy(constant_args)) for args in worker_args]
worker = pool_map_more_than_one_arg(worker)
if single_thread:
resp = map(worker, worker_args)
else:
count_cpu_workers = int(
(cpu_ratio * multiprocess.cpu_count())
if cpu_ratio <= 1.0 else cpu_ratio
)
pool = multiprocess.Pool(count_cpu_workers)
resp = pool.imap(worker, worker_args)
pool.close()
pool.join()
return list(resp)
def save_topics(cls, topic_words_path, start_date, end_date, year_interval):
topic_words = []
cls.cursor.execute(f"DROP TABLE IF EXISTS {cls.table}_topics")
cls.cursor.execute(
f"CREATE TABLE {cls.table}_topics(topic_id INTEGER, word_distribution JSONB, topic_evolution JSONB, frequency FLOAT, docs JSONB)"
)
with tqdm(total=cls.model.nb_topics, leave=False, desc="Generating topic stats") as pbar:
with Pool(cpu_count() - 1) as pool:
for (
topic_id,
word_distribution,
topic_evolution,
frequency,
docs,
description,
) in pool.imap_unordered(
cls.compute_topic,
zip(range(cls.model.nb_topics), repeat(start_date), repeat(end_date), repeat(year_interval),),
):
cls.cursor.execute(
f"INSERT INTO {cls.table}_topics (topic_id, word_distribution, topic_evolution, frequency, docs) VALUES (%s, %s, %s, %s, %s)",
(topic_id, word_distribution, topic_evolution, frequency, docs),
)
topic_words.append(
{"name": topic_id, "frequency": frequency, "description": ", ".join(description),}
)
pbar.update()
topic_words.sort(key=lambda x: x["name"])
with open(topic_words_path, "w") as out_file:
json.dump(topic_words, out_file)
cls.cursor.execute(f"CREATE INDEX {cls.table}_topic_id_index on {cls.table}_topics USING HASH(topic_id)")
cls.db.commit()
def test_init_cpus_max(self):
runmodel = RunModel(model=TestingModel1d(),
parameters=self.parameters,
logger_level="error",
CPUs="max")
self.assertEqual(runmodel.CPUs, mp.cpu_count())
def get_new_tickets(self, from_time=utils.pre_day_to_string(1)):
search_conditions = {
"skip": 0,
"query": {
"ctimeGte": "{}T21:00:00.000Z".format(from_time)
}
}
pool_size = multiprocess.cpu_count()
pool_volume = 10 * pool_size
index = 0
tickets_num = self._get_number_of_tickets(from_time, to_time)
req_num = utils.ceil_division(tickets_num, 1000)
pool = Pool(pool_size)
for req_count in range(req_num):
search_tickets = self.search_tickets(search_conditions)
while True:
tickets = pool.map(
self.add_attr_to_ticket,
itertools.islice(search_tickets, pool_volume))
if tickets:
print('Downloaded {}/{} tickets'.format(
index, tickets_num),
end='\r')
index += pool_volume
yield tickets
else:
break
search_conditions['skip'] += 1000
def run_parallel_async(graph, nprocs=None, sleep=0.2, raise_errors=False):
if nprocs == 1:
return run_async(graph, sleep=sleep, raise_errors=raise_errors)
nprocs = nprocs or mp.cpu_count() // 2
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
ioq = mp.Queue(len(graph.funcs.keys()))
cpuq = mp.Queue(len(graph.funcs.keys()))
procs = [mp.Process(target=run_scheduler,
args=(graph, sleep, ioq, cpuq, raise_errors))
for _ in range(nprocs)]
for proc in procs:
proc.start()
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info(
'pid {}: queueing task {}'.format(os.getpid(), task))
if task in graph.io_bound:
ioq.put(task)
else:
cpuq.put(task)
time.sleep(sleep)
if raise_errors and sum(not p.exitcode for p in procs):
raise RuntimeError('An async task has failed. Please check your logs')
return tgraph.recover_values_from_manager(graph)
def pplinearFold(listOfSeqs) :
pool = mp.Pool(mp.cpu_count())
result = numpy.array(pool.map(fold_with_LinearFold, listOfSeqs))
pool.close()
return list(result[:,0]),list(result[:,1])
def getInputTargetPaths(path):
"""
From a parent path, get all paths for all inputs and all targets
Params:
- path : str
Parent folder
Outputs:
- x_paths : list
Set of input paths
- y_paths : list
Set of fluorescence paths
"""
folders = sorted(os.listdir(path))
parallel_out = Parallel(multiprocess.cpu_count())(
delayed(ParallelPathAccess)(path, folder) for folder in folders)
# Parallel out is a list of parallelized tuples: [(x1,y1),(x2,y2),...,(xN,y)]
parallel_out = np.array(parallel_out)
x_paths = parallel_out[:, 0]
y_paths = parallel_out[:, 1]
return x_paths, y_paths
def run_parallel(graph, nprocs=None, sleep=0.2, raise_errors=False):
nprocs = nprocs or mp.cpu_count() - 1
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
with mp.Pool(nprocs) as pool:
exception_q = mp.Queue(10)
def error_callback(exception):
exception_q.put_nowait(exception)
pool.terminate()
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph, reverse=False):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info('pid {}: assigning task {}'.format(
os.getpid(), task))
pool.apply_async(run_task,
args=(graph, task, raise_errors),
error_callback=error_callback)
time.sleep(sleep)
if not exception_q.empty():
raise exception_q.get()
return tgraph.recover_values_from_manager(graph)
def check_ncpus(arg_value):
arg_value = int(arg_value)
if arg_value <= 0:
raise click.BadParameter("n_cpus must be >= 1")
else:
return min(arg_value, mp.cpu_count())
def _super_Fast_Trace(self,swarm,trace_Particle):
# use trick of accessing only the important class variabels and passing those through to reduce pickle time
def fastFunc(compactDict):
particle = Particle()
for key, val in compactDict.items():
setattr(particle, key, val)
particle = trace_Particle(particle)
compactDictTraced = {}
for key, val in vars(particle).items():
if val is not None:
compactDictTraced[key] = val
return compactDictTraced
compactDictList = []
for particle in swarm:
compactDict = {}
for key, val in vars(particle).items():
if val is not None:
if not (isinstance(val, Iterable) and len(val) == 0):
compactDict[key] = val
compactDictList.append(compactDict)
with mp.Pool(mp.cpu_count()) as Pool:
compactDictTracedList = Pool.map(fastFunc, compactDictList)
for particle, compactDict in zip(swarm.particles, compactDictTracedList):
for key, val in compactDict.items():
setattr(particle, key, val)
return swarm
def pool_map(worker,
worker_args,
constant_args=None,
cpu_ratio=0.75,
single_thread=False):
worker_args = _group_args(worker_args)
if constant_args is not None:
try:
[args.update(copy.deepcopy(constant_args)) for args in worker_args]
except AttributeError:
[args.extend(copy.deepcopy(constant_args)) for args in worker_args]
worker = pool_map_more_than_one_arg(worker)
if single_thread:
resp = map(worker, worker_args)
else:
count_cpu_workers = int((
cpu_ratio *
multiprocess.cpu_count()) if cpu_ratio <= 1.0 else cpu_ratio)
pool = multiprocess.Pool(count_cpu_workers)
resp = pool.imap(worker, worker_args)
pool.close()
pool.join()
return list(resp)
def compute_distance(points):
all_combinations = itertools.product(points, points)
pool = mp.Pool(math.ceil(mp.cpu_count() * .75))
calculations = pool.imap_unordered(point_distance,
all_combinations,
chunksize=1000000)
return np.max(list(tqdm(calculations, total=len(points)**2)))
def check_ncpus(arg_value):
arg_value = int(arg_value)
if arg_value <= 0:
raise argparse.ArgumentTypeError("n_cpus must be >= 1")
else:
return min(arg_value, mp.cpu_count())
def main():
nucleotides = ["A", "U", "G", "C"]
pop_size = 100
target = '..((((((.........))))))..((((.....))))'
rate = None
length = len(target)
ldscape = Landscape.Landscape(target)
time = 50
print("Starting RNA evolution with target {} of length {}".format(
target, len(target)))
db, init_pop = init(pop_size, length, ['A'], ldscape)
params = {
'c': rate,
"pop_size": pop_size,
'init_pop': init_pop,
'landscape': ldscape,
'nucleotides': nucleotides,
'time': time,
'db': db,
"pos": get_bp_position(target)
}
print(init_pop)
data_min = []
data_mean = []
for c in np.arange(1, 2, 0.5):
params['c'] = c
print("Start evolution with c = {}......".format(params['c']))
pool = mp.Pool(mp.cpu_count())
results = pool.map(evolution, [params] * 3)
print(results)
"""
def start_train(resume):
urllib.request.urlretrieve(const.url + '/network_file',
'deliverables/network.py')
urllib.request.urlretrieve(const.url + '/config_file',
'deliverables/input_params.py')
urllib.request.urlretrieve(const.url + '/observation_file',
'deliverables/observation.py')
urllib.request.urlretrieve(const.url + '/curriculum_file',
'deliverables/curriculum.py')
num_workers = mp.cpu_count() - 1
should_stop = mp.Value(c_bool, False)
while True:
worker_processes = []
# create_worker(0, should_stop)
# Start process 1 - n, running in other processes
for w_num in range(0, num_workers):
process = mp.Process(target=create_worker,
args=(w_num, should_stop))
process.start()
sleep(0.5)
worker_processes.append(process)
try:
for p in worker_processes:
p.join()
except KeyboardInterrupt:
should_stop.value = True
print("Looks like we're done")
def MultiNMF(V, rank, nrun, cores, return_best=True):
'''Run NMF for a target matrix multiple times'''
available_cores = multiprocess.cpu_count()
if available_cores < cores:
print(
"Only %d cores available but %d cores requested, reset it to available number."
% (available_cores, cores))
cores = available_cores
xs = range(1, nrun + 1)
with multiprocess.Pool(processes=cores) as p:
v = p.starmap(NMF2, zip(xs, repeat(V), repeat(rank)))
if return_best:
print("Compare KLD and return the best.")
l = len(v)
kld = v[0]["KLD"]
idx = 0
for i in range(1, l):
k = v[i]["KLD"]
print("KLD to compare: %f, KLD to be compared: %f" % (k, kld))
if kld > k:
print("Current KLD value updated to %f" % k)
kld = k
idx = i
print("Return the run #%d" % (idx + 1))
v = v[idx]
return v
def dataAccess(path, size, output_folder, key="max"):
"""
Provide Maximum Intensity Projection (MIP) of given images in folders, plus
provide the corresponding flourescent target images
Params:
path : str
Root folder from where to start examining
size : int or list of int
Size of images to be obtained
output_folder : str
Folder where to save MIP file as .npy (if "save" flag is True)
key : str
Type of projection to provide ("max" for maximum intensity projection, "mean" for mean intensity projection,
"median" for median intensity projection)
Default: "max"
Outputs:
mips : list of np.array
List with MIP results [MIP_20x, MIP_40x, MIP_60x]
targets : list of np.array
List with fluorescent target images [ch01_images, ch02_images, ch03_images] --> ch01_images = [ch01_20x, ch01_40x, ch01_60x]
"""
init = time.time()
if not (os.path.exists(path)):
print("Folder {} does not exist".format(path))
folders = sorted(os.listdir(path)) # Read all folders in the given path
Parallel(multiprocess.cpu_count())(delayed(parallelAccess)(os.path.join(
path, folder), size, output_folder, key) for folder in folders)
print("Preprocessing completed! Time ellapsed: {} seconds".format(
round(time.time() - init, 2)))
def energy_basins(self, X, multicore=True):
"""Energy basin for each given state.
Parameters
----------
X : ndarray
Returns
-------
energybasin : ndarray
Each row is the energy basin for the given row of X.
"""
def f(v):
return self._find_energy_basin(v, self.multipliers)
if multicore:
pool = mp.Pool(mp.cpu_count())
energybasin = np.vstack(pool.map(f, X))
pool.close()
else:
energybasin = []
for v in X:
energybasin.append(f(v))
energybasin = np.vstack(energybasin)
return energybasin
def insert_to_prod(stage_table, prod_table, sql_execute):
file_location = raw_input(
'Please input the location of the CSV file to ingest\n')
insert_data = ImportCsv(stage_table)
batch_size = 5000
chunk_gen = insert_data.get_chunks(batch_size, file_location)
pool = mp.Pool(mp.cpu_count() - 1)
results = pool.map(insert_data.worker, chunk_gen)
pool.close()
pool.join()
logging.info(
'Insert to Stage Complete complete. Insert to Prod Table Start')
result = sql_execute.execute_query(
"SELECT COUNT(1) FROM {0}".format(prod_table))
count = result.fetchone()[0]
if count == 0:
insert_query = """INSERT INTO {1} SELECT name,sku,description FROM {0}""".format(
stage_table, prod_table)
logging.info(
'Bulk Insert from stage to prod table (One time Full Load)')
sql_execute.execute_query(insert_query)
else:
logging.info('Inserting New Records to Prod')
new_records = """INSERT INTO {1} SELECT ps.name, ps.sku, ps.description FROM {0} ps left join {1} pp on ps.name = pp.name and ps.sku = pp.sku and ps.description = pp.description WHERE pp.sku is NULL""".format(
stage_table, prod_table)
sql_execute.execute_query(new_records)
logging.info('Prod Table insertion Complete. Truncating Stage Table')
sql_execute.execute_query("TRUNCATE TABLE {0}".format(stage_table))
logging.info(
'File Ingestion Succeeded. Please query {0} Table to check the results.'
.format(prod_table))
def __init__(self, mapping, output_directory=None):
self._logger = logging.getLogger(__name__)
# FIXME: not raising errors due to schema validation for now
mapping = validate(mapping, silent=True)
_processes = mapping['resources']['processes']
mapping['resources']['processes'] = (
multiprocess.cpu_count() if _processes == 'max' else _processes)
self._frozenjson = FrozenJSON(mapping)
if output_directory is not None:
if path.isabs(output_directory):
self._path_to_output_directory = output_directory
else:
_ = Path(self.data.root_folder, output_directory)
self._path_to_output_directory = str(_)
else:
self._path_to_output_directory = None
# init the rest of the parameters, these parameters are used
# througout the pipeline so we compute them once to avoid redudant
# computations
# GEOMETRY PARAMETERS
path_to_geom = path.join(self.data.root_folder, self.data.geometry)
self._set_param('geom',
geom.parse(path_to_geom, self.recordings.n_channels))
# check dimensions of the geometry file
n_channels_geom, _ = self.geom.shape
if self.recordings.n_channels != n_channels_geom:
raise ValueError('Channels in the geometry file ({}) does not '
'value in the configuration file ({})'.format(
n_channels_geom, self.recordings.n_channels))
neigh_channels = geom.find_channel_neighbors(
self.geom, self.recordings.spatial_radius)
self._set_param('neigh_channels', neigh_channels)
channel_groups = geom.make_channel_groups(self.recordings.n_channels,
self.neigh_channels,
self.geom)
self._set_param('channel_groups', channel_groups)
self._set_param(
'spike_size',
int(
np.round(self.recordings.spike_size_ms *
self.recordings.sampling_rate / (2 * 1000))))
channel_index = geom.make_channel_index(self.neigh_channels,
self.geom,
steps=2)
self._set_param('channel_index', channel_index)
def main():
with open("config.json", "r") as f:
conf = json.load(f)
args = argue(conf)
dest = conf["host"] + conf["endpoint"]
head = {
"api_key": args.apikey,
"api_secret": args.apisecret,
"Content-type": "application/json"
}
if args.group:
data = grouper(importer(args.document))
else:
data = importer(args.document)
if int(args.n) != 0:
data = data[0:int(args.n)]
pts = [Patient(x["metadata"], x["data"], args.group) for x in data]
if args.group and args.abstractions:
csv_header = [
"patient_id", "focus", "chunk", "encoding", "abstractions"
]
elif args.group:
csv_header = ["patient_id", "focus", "chunk", "encoding"]
elif args.abstractions:
csv_header = [
"patient_id", "document_id", "surname", "forename", "dob",
"gender", "visit_id", "observation_datetime", "project", "author",
"focus", "chunk", "encoding", "abstractions"
]
else:
csv_header = [
"patient_id", "document_id", "surname", "forename", "dob",
"gender", "visit_id", "observation_datetime", "project", "author",
"focus", "chunk", "encoding"
]
header_str = "\t".join(csv_header)
args.outfile.write(header_str + "\n")
pool = mp.Pool(mp.cpu_count())
def process(pt):
pt.request(args.resource, dest, head, args.cacert)
csv_str = []
for line in pt.format(args.abstractions):
csv_str.append("\t".join([str(x).replace("\n", " ")
for x in line]))
return "\n".join(csv_str)
result_async = [pool.apply_async(process, (pt, )) for pt in pts]
results = [r.get() for r in result_async]
for line in results:
if line is not None and line != "":
args.outfile.write(line + "\n")
def train(self, trees, n_incorrect_answers=100,
n_epochs=30, n_batches=None):
"""Trains the QANTA model on the sentence trees.
trees is a list of DependencyTree
n_incorrect_answers is |Z| in the paper's eq. 5. It determines
how many incorrect answers are sampled from the training data
to be used in calculating sentence error.
n_epochs is the number of times the model trains on the input data
"""
# Grab new answers, and enforce uniqueness
for t in trees:
if not t.answer in self.answers:
self.answers.append(t.answer)
answers = self.answers # A tiny bit shorter
# Make sure we can sample n_incorrect_answers different answers.
if len(answers) - 1 < n_incorrect_answers:
n_incorrect_answers = len(answers) - 1
# QANTA original code says 'ideally 25 minibatches per epoch'
n_batches = n_batches or min(25, len(trees))
batch_size = len(trees) / n_batches
if cpu_count() < batch_size:
trainer = self._train_batch_parallelize
else:
# No reason to parallelize
trainer = self._train_batch
for epoch in xrange(n_epochs):
epoch_error = 0
epoch_start = time()
for batch in xrange(n_batches):
batch_start = time()
# Index range for this batch
lo = batch * batch_size
hi = lo + batch_size
batch_trees = trees[lo:hi]
# Parallel training
batch_error = trainer(batch_trees, n_incorrect_answers)
# Serial training
# batch_error = self._train_batch(batch_trees, n_incorrect_answers)
# Only print batch stats if it takes more than 5 seconds
if time() - batch_start > 5:
print ("Training error epoch {}, batch {}: {} "
"({:.2f} seconds)").format(epoch, batch,
batch_error, time() - batch_start)
epoch_error += batch_error
if time() - epoch_start > 5:
print ("Total training error for epoch {}: {} "
"({:.2f} seconds)").format(epoch, epoch_error,
time() - epoch_start)
def __init__(self, processes=None, initializer=None, initargs=(),
maxtasksperchild=None):
self._setup_queues()
self._taskqueue = Queue.Queue()
self._cache = {}
self._state = RUN
self._maxtasksperchild = maxtasksperchild
self._initializer = initializer
self._initargs = initargs
if processes is None:
try:
processes = cpu_count()
except NotImplementedError:
processes = 1
if processes < 1:
raise ValueError("Number of processes must be at least 1")
if initializer is not None and not hasattr(initializer, '__call__'):
raise TypeError('initializer must be a callable')
self._processes = processes
self._pool = []
self._repopulate_pool()
self._worker_handler = threading.Thread(
target=Pool._handle_workers,
args=(self, )
)
self._worker_handler.daemon = True
self._worker_handler._state = RUN
self._worker_handler.start()
self._task_handler = threading.Thread(
target=Pool._handle_tasks,
args=(self._taskqueue, self._quick_put, self._outqueue,
self._pool, self._cache)
)
self._task_handler.daemon = True
self._task_handler._state = RUN
self._task_handler.start()
self._result_handler = threading.Thread(
target=Pool._handle_results,
args=(self._outqueue, self._quick_get, self._cache)
)
self._result_handler.daemon = True
self._result_handler._state = RUN
self._result_handler.start()
self._terminate = Finalize(
self, self._terminate_pool,
args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
self._worker_handler, self._task_handler,
self._result_handler, self._cache),
exitpriority=15
)
def __init__(self, processes=None, initializer=None, initargs=()):
self._setup_queues()
self._taskqueue = queue.Queue()
self._cache = {}
self._state = RUN
if processes is None:
try:
processes = cpu_count()
except NotImplementedError:
processes = 1
if initializer is not None and not hasattr(initializer, '__call__'):
raise TypeError('initializer must be a callable')
self._pool = []
for i in range(processes):
w = self.Process(
target=worker,
args=(self._inqueue, self._outqueue, initializer, initargs)
)
self._pool.append(w)
w.name = w.name.replace('Process', 'PoolWorker')
w.daemon = True
w.start()
self._task_handler = threading.Thread(
target=Pool._handle_tasks,
args=(self._taskqueue, self._quick_put, self._outqueue, self._pool)
)
self._task_handler.daemon = True
self._task_handler._state = RUN
self._task_handler.start()
self._result_handler = threading.Thread(
target=Pool._handle_results,
args=(self._outqueue, self._quick_get, self._cache)
)
self._result_handler.daemon = True
self._result_handler._state = RUN
self._result_handler.start()
self._terminate = Finalize(
self, self._terminate_pool,
args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
self._task_handler, self._result_handler, self._cache),
exitpriority=15
)
def eval_EFG(self,x,num_procs=None,info=False):
from multiprocess import Pool,cpu_count
if not num_procs:
num_procs = cpu_count()
num_samples = self.parameters['num_samples']
pool = Pool(num_procs)
num = int(np.ceil(float(num_samples)/float(num_procs)))
results = list(zip(*pool.map(lambda i: self.eval_EFG_sequential(x,num,i,info),range(num_procs),chunksize=1)))
pool.terminate()
pool.join()
if not info:
assert(len(results) == 4)
else:
assert(len(results) == 5)
assert(all([len(vals) == num_procs for vals in results]))
return [sum(vals)/float(num_procs) for vals in results]
def eval_EQ(self,p,num_procs=None,quiet=True):
"""
Evaluates E[Q(p,r)] and its gradient in parallel.
Parameters
----------
p : generator powers
num_procs : number of parallel processes
quiet : flag
"""
from multiprocess import Pool,cpu_count
if not num_procs:
num_procs = cpu_count()
num_samples = self.parameters['num_samples']
pool = Pool(num_procs)
num = int(np.ceil(float(num_samples)/float(num_procs)))
results = list(zip(*pool.map(lambda i: self.eval_EQ_sequential(p,num,i,quiet),range(num_procs),chunksize=1)))
pool.terminate()
pool.join()
assert(len(results) == 2)
assert(all([len(vals) == num_procs for vals in results]))
return [sum(vals)/float(num_procs) for vals in results]
def get_new_tickets(self, from_time=utils.pre_day_to_string(1)):
search_conditions = {
"skip": 0,
"query": {
"ctimeGte": "{}T21:00:00.000Z".format(from_time)
}
}
pool_size = multiprocess.cpu_count()
pool_volume = 10 * pool_size
index = 0
tickets_num = self._get_number_of_tickets(from_time, to_time)
req_num = utils.ceil_division(tickets_num, 1000)
pool = Pool(pool_size)
for req_count in range(req_num):
search_tickets = self.search_tickets(search_conditions)
while True:
tickets = pool.map(self.add_attr_to_ticket, itertools.islice(search_tickets, pool_volume))
if tickets:
print('Downloaded {}/{} tickets'.format(index, tickets_num), end='\r')
index += pool_volume
yield tickets
else:
break
search_conditions['skip'] += 1000
Future = namedtuple('Future', ['function', 'args', 'kwargs'])
def launch(pool, fs):
results = []
for f in fs:
with redirected("%s.out" % f.function.__name__):
results.append(pool.apply_async(f.function, f.args, f.kwargs))
return results
def _f(x, y=0):
return [x * 42 for i in range(1000)]
def run_tasks(futures):
futures = [future for futuregroup in futures for future in futuregroup]
pool = Pool(processes=cpu_count())
results = launch(pool, futures)
results = [result.get() for result in results]
return results
if __name__ == "__main__":
print("Creating pool with %s CPUs" % cpu_count())
pool = Pool(processes=cpu_count())
futures = [Future(_f, [10], {'y':0})]
print(futures)
results = launch(pool, futures)
for result in results:
result.get()
perc_sum, count_sum = 0, 0
for i in (self.one, self.three, self.seven, self.nine):
count_sum += i
perc_sum += self.rounder(i)
print "\n" + "one" + "\t" + "= %.2f" % self.rounder(self.one) + "%" + "\t" + "(%i)" % self.one
print "three" + "\t" + "= %.2f" % self.rounder(self.three) + "%" + "\t" + "(%i)" % self.three
print "seven" + "\t" + "= %.2f" % self.rounder(self.seven) + "%" + "\t" + "(%i)" % self.seven
print "nine" + "\t" + "= %.2f" % self.rounder(self.nine) + "%" + "\t" + "(%i)" % self.nine
print "\n" + "Sum" + "\t" + "= %.2f" % perc_sum + "%" + "\t" + "(%i)" % count_sum
except ZeroDivisionError:
print "\nNo primes found!"
if __name__ == '__main__':
cores = cpu_count()
def screen_clear(): # Small function for clearing the screen on Unix or Windows
if os.name == 'nt':
return os.system('cls')
else:
return os.system('clear')
def usage(): # any incorrect input will display the usage instructions
screen_clear()
print """Prime_Perc.py
Version 0.1 - Written by Halsandr
=======
# routines to call a function in parallel
from __future__ import print_function
import scipy.linalg as lg
from . import algebra
try:
from multiprocess import Pool
import multiprocess
maxcpu = multiprocess.cpu_count()
except:
print("Multiprocess not working")
def Pool(n=1): # workaround
class mpool():
def map(self,f,xs):
return [f(x) for x in xs]
def terminate(self): return None # dummy function
return mpool()
cores = 1 # call in a single by default
def set_cores(n=1):
global cores
cores = n
#mainpool = None
#def initialize():
# global mainpool
# if cores>1:
# mainpool = Pool(cores) # create pool
#!/usr/bin/python
# coding: utf-8
import threading
import multiprocess
def loop():
x = 0
while True:
x = x ^ 1
if __name__ == "__main__":
for i in range(multiprocess.cpu_count()):
t = threading.Thread(target=loop)
t.start()
from scipy import interpolate #import multiprocessing import multiprocess # BETTER THAN MULTIPROCESSING import datetime from tempfile import TemporaryFile import os.path #import matplotlib #from matplotlib import pyplot as plt import vegas from extras import set_active, cartesian, interp_list, set_shift #import psutil # to see how many cores are allocated #ncores = len(psutil.Process().cpu_affinity()) # for cluster ncores = multiprocess.cpu_count() # for local ############################ # numerical parameters # ############################ E = np.exp(1.) # no longer needed? qmin = 0.0001 qmax = 10. #default n=1.; # default consider every n*kf for the bispectrum . for the power specutrm it computes every kf. ni = 2; #number iterations ne = 5000; #number of evaluations #3500 normal, 5000 for high prec chunksize = 500 # divide the triangle list in chunks of size "chunksize". carefull not to change the chunksize between runs! ############################# # cosmological parameters # #############################
from server_json_wrapper import JSONServerWrapper
REPEATS_PER_CONFIG = 300
DIMS_LEN = 6
NUM_DIMS = 2
SEEDS_SEED = 7
MINES_MIN = 1
MINES_MAX = (DIMS_LEN ** NUM_DIMS) // 2
# Reduce this when using very slow clients
POOL_MAX_CHUNKSIZE = 10
SERVER = PythonInternalServer
if hasattr(multiprocessing, "cpu_count"):
no_cores = multiprocessing.cpu_count()
else:
no_cores = 1
plot_clients = [
("blue", ReactiveClient),
("green", ReactiveClientCheckShared),
#("red", ReactiveClientGuess),
#("cyan", ReactiveClientGuessAny),
("orange", ReactiveClientAvgEmptiesBalanced),
#("purple", ReactiveClientExhaustiveTest),
#("pink", ReactiveClientExhaustiveSplit),
]
# Chunksize for pools as calculated in multiprocessing module, but with the
# addition of a specified cap. Allows for more frequent progress counter updates
def _parallel_solve(self,
solver=None,
ignore_dcp=False,
warm_start=False,
verbose=False, **kwargs):
"""Solves a DCP compliant optimization problem in parallel.
Saves the values of primal and dual variables in the variable
and constraint objects, respectively.
Parameters
----------
solver : str, optional
The solver to use. Defaults to ECOS.
ignore_dcp : bool, optional
Overrides the default of raising an exception if the problem is not
DCP.
warm_start : bool, optional
Should the previous solver result be used to warm start?
verbose : bool, optional
Overrides the default of hiding solver output.
kwargs : dict, optional
A dict of options that will be passed to the specific solver.
In general, these options will override any default settings
imposed by cvxpy.
Returns
-------
float
The optimal value for the problem, or a string indicating
why the problem could not be solved.
"""
def _solve_problem(problem):
"""Solve a problem and then return the optimal value, status,
primal values, and dual values.
"""
opt_value = problem.solve(solver=solver,
ignore_dcp=ignore_dcp,
warm_start=warm_start,
verbose=verbose,
parallel=False, **kwargs)
status = problem.status
primal_values = [var.value for var in problem.variables()]
dual_values = [constr.dual_value for constr in problem.constraints]
return SolveResult(opt_value, status, primal_values, dual_values)
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
solve_results = pool.map(_solve_problem, self._separable_problems)
pool.close()
pool.join()
statuses = {solve_result.status for solve_result in solve_results}
# Check if at least one subproblem is infeasible or inaccurate
for status in s.INF_OR_UNB:
if status in statuses:
self._handle_no_solution(status)
break
else:
for subproblem, solve_result in zip(self._separable_problems,
solve_results):
for var, primal_value in zip(subproblem.variables(),
solve_result.primal_values):
var.save_value(primal_value)
for constr, dual_value in zip(subproblem.constraints,
solve_result.dual_values):
constr.save_value(dual_value)
self._value = sum(solve_result.opt_value
for solve_result in solve_results)
if s.OPTIMAL_INACCURATE in statuses:
self._status = s.OPTIMAL_INACCURATE
else:
self._status = s.OPTIMAL
return self._value
from Queue import Empty
from multiprocess import JoinableQueue, Process, cpu_count, Queue, Pool
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings()
re_youtube = re.compile(r'((?<=watch\?v=)[-\w]+\
|(?<=youtube.com/embed/)[-\w]+)', re.I)
re_images = re.compile(r'(?<=")[^"]+jpg(?=")', re.I)
re_splitter = re.compile(r"[^\w@#]+", re.UNICODE)
HOME = os.path.join(os.path.expanduser("~"), ".config/anchorbot")
HERE = os.path.realpath(os.path.dirname(__file__))
CONFIGFILE = os.path.join(HOME, "config")
NUM_THREADS = max(1, cpu_count() - 1)
class Config(dict):
def __init__(self, configfile):
dict.__init__(self)
self["redis_keys"] = {}
self["abos"] = []
self.configfile = configfile
if not os.path.exists(HOME):
os.mkdir(HOME)
if os.path.exists(configfile):
if os.path.getsize(configfile) > 0:
with open(configfile, "r") as f:
content = json.load(f)
def run_tasks(futures):
futures = [future for futuregroup in futures for future in futuregroup]
pool = Pool(processes=cpu_count())
results = launch(pool, futures)
results = [result.get() for result in results]
return results
