def CreateMovie(saveFrame, nFrames, fps, test='Ronchi', fixedLight=True, fringe=stepFringe, nX=1001):
'''Generate all the frames, create the video,
then delete the individual frames.
'''
# file name for the final animation
if fixedLight:
name = test + "_fixedlight"
else:
name = test + "_samelightgrating"
print("Generate all frames")
f = lambda iFrame: saveFrame(iFrame, './figures/_tmp%05d.jpg'%iFrame, test=test, fixedLight=fixedLight)
pool = ProcessPool(nodes=3)
pool.map(f, range(nFrames))
print("Resize images")
# resize the images to have even pixel sizes on both dimensions, important for ffmpeg
# this commands preserves the aspect ratio, rescales the image to fill HD as much as possible,
# without cropping, then pads the rest with white
for iFrame in range(nFrames):
fname = './figures/_tmp%05d.jpg'%iFrame
#os.system("convert "+fname+" -resize 1280x720 -gravity center -extent 1280x720 -background white "+fname)
os.system("convert "+fname+" -resize 1000x1000 -gravity center -extent 1000x1000 -background white "+fname)
# delete old animation
os.system("rm ./figures/"+name+".mp4")
print("Create new animation")
#os.system("ffmpeg -r "+str(fps)+" -i ./figures/_tmp%05d.jpg -s 1280x720 -vcodec libx264 -pix_fmt yuv420p ./figures/ronchi.mp4")
os.system("ffmpeg -r "+str(fps)+" -i ./figures/_tmp%05d.jpg -s 1000x1000 -vcodec libx264 -pix_fmt yuv420p ./figures/"+name+".mp4")
# delete images
os.system("rm ./figures/_tmp*.jpg")
def make_predictions_by_t_local_map(model_name, chunk_size=(300, 150, 150), allowed_gpus=list(range(8)), timepoints=None):
""" Make predictions for all timepoints, using all local gpus
"""
from division_detection.vol_preprocessing import VOL_DIR_H5
# fetch the number of timepoints
num_vols = len(os.listdir(VOL_DIR_H5))
if timepoints is None:
timepoints = timepoints or np.arange(3, num_vols - 4)
n_gpus = len(allowed_gpus)
split_timepoints = np.array_split(timepoints, n_gpus)
devices = ['/gpu:{}'.format(idx) for idx in allowed_gpus]
starred_args = list(zip(split_timepoints,
[model_name] * n_gpus,
[chunk_size] * n_gpus,
devices))
def _star_helper(args):
return _predict_local_helper(*args)
print("Creating pool")
pool = Pool(n_gpus)
print("Dispatching jobs")
pool.map(_star_helper, starred_args)
def make_predictions_by_t_local_map_general(model_name, in_dir, timepoints,
allowed_gpus=[0],
chunk_size=(200, 150, 150)):
""" Make predictions for all timepoints, using all local gpus
Args:
model_name: name of model to use, will be looked up
in_dir: absolute path to data dir
timepoints: list of timepoints to process
chunk_size: size of chunks to proces volume in
allowed_gpus: CUDA ids of GPUs to use
job will be parallelized across GPUs
"""
n_gpus = len(allowed_gpus)
split_timepoints = np.array_split(timepoints, n_gpus)
devices = ['/gpu:{}'.format(idx) for idx in allowed_gpus]
starred_args = list(zip(split_timepoints,
[in_dir] * n_gpus,
[model_name] * n_gpus,
[chunk_size] * n_gpus,
devices))
def _star_helper(args):
return _local_predict_helper_general(*args)
print("Creating pool")
pool = Pool(n_gpus)
print("Dispatching jobs")
pool.map(_star_helper, starred_args)
def get_grid_fill_objs(self, size):
max_n_threads = 40
n_rows, n_cols = size
n_cameras = n_rows * n_cols
# print('n_cameras',n_cameras)
n_threads = min(n_cameras, max_n_threads)
cam_path = join(self.sample_dir, 'camera.txt')
cams = np.loadtxt(cam_path)
def f(camera_i):
row_i = camera_i // n_cols
col_i = camera_i % n_cols
y = row_i / (n_rows - 1)
x = col_i / (n_cols - 1)
camera_pos = (1 - x) * (1 - y) * cams[3] + x * (
1 - y) * cams[2] + (1 - x) * y * cams[1] + x * y * cams[0]
self.write_meshlab_camera(
join(self.keys_dir, 'meshlab_camera_{}.txt'.format(camera_i)),
camera_pos, np.eye(3))
self.get_fill_obj(camera_pos,
postfix=str(camera_i),
frame=camera_i,
check=False)
pool = ProcessPool(nodes=n_threads)
pool.map(f, range(n_cameras))
def get_circle_fill_objs(self, start_camera_pos, n_cameras):
max_n_threads = 40
n_threads = min(n_cameras, max_n_threads)
cx, cy = start_camera_pos[0], start_camera_pos[1]
cz = self.center[2]
r = np.abs(start_camera_pos[2] - cz)
def f(camera_i):
theta = camera_i / n_cameras * (2 * np.pi)
x = r * np.sin(theta) + cx
z = r * np.cos(theta) + cz
camera_pos = np.array([x, cy, z])
deg = theta / np.pi * 180
deg_str = '{}'.format(int(deg))
meshlab_R = self.get_meshlab_R(camera_pos, np.array([cx, cy, cz]))
self.write_meshlab_camera(
join(self.frames_dir, 'meshlab_camera_{}.txt'.format(deg_str)),
camera_pos, meshlab_R)
self.get_fill_obj(camera_pos,
postfix=deg_str,
frame=camera_i,
check=False)
pool = ProcessPool(nodes=n_threads)
pool.map(f, range(n_cameras))
def step_generation(self):
'''
evaluate fitness of pop, and create new pop after elitist_selection and mutation
'''
global __evolsearch_process_pool
# estimate fitness using multiprocessing pool
if __evolsearch_process_pool:
# pool exists
__evolsearch_process_pool = ProcessPool(self.num_processes)
self.fitness = np.asarray(
__evolsearch_process_pool.map(self.evaluate_fitness,
np.arange(self.pop_size)))
else:
# re-create pool
__evolsearch_process_pool = ProcessPool(self.num_processes)
self.fitness = np.asarray(
__evolsearch_process_pool.map(self.evaluate_fitness,
np.arange(self.pop_size)))
# elitist_selection
self.elitist_selection()
# mutation
self.mutation()
def run_vt(self):
cams=self.load_cam()
def f(cam_i):
cam_pos=cams[cam_i]
self.get_fill_obj(cam_pos,'{}'.format(cam_i),cam_i,check=False)
n_cams=len(cams)
n_threads=n_cams
pool=ProcessPool(nodes=n_threads)
pool.map(f,range(n_cams))
def main(trackingnet_dir="TrackingNet",
csv_dir=".",
overwrite=False,
chunks=[],
data=["ANNO", "ZIPS"]):
for chunk_folder in chunks:
chunk_folder = chunk_folder.upper()
#if ("TEST" in chunk_folder):
# my_data = ["ZIPS"]
#else:
my_data = data
for datum in my_data:
data_folder = os.path.join(trackingnet_dir, chunk_folder,
datum.lower())
if (not os.path.exists(data_folder)):
os.makedirs(data_folder)
csv_file = os.path.join(
csv_dir, chunk_folder + "_" + datum.upper() + ".csv")
df = pd.read_csv(csv_file)
# for index, row in tqdm(df.iterrows(), desc="_".join([chunk_folder,datum]), total=len(df.index)):
# Google_drive_file_id = row["link"]
# Google_drive_file_name = row["name"]
# destination_path = os.path.join(trackingnet_dir, chunk_folder, datum.lower(), Google_drive_file_name)
# if (not os.path.exists(destination_path)):
# downloader.download(url='https://drive.google.com/uc?id={id}'.format(id=Google_drive_file_id),
# output=destination_path,
# quiet=False,
# )
pool = ProcessPool(nodes=6)
file_attributes_list = [{
'Google_drive_file_id': Google_drive_file_id,
'Google_drive_file_name': Google_drive_file_name,
'chunk_folder': chunk_folder,
'datum': datum.lower(),
'trackingnet_dir': trackingnet_dir
} for Google_drive_file_id, Google_drive_file_name in zip(
df['link'], df['name'])]
pool.map(get_file, file_attributes_list)
return True
def main(args):
setup = experiment_setups.parse(args.setup)
# use_base_dir = False
dirname = fileutil.run_dir(setup.dest_dir, setup.name,
setup.max_quantifier_length, setup.model_size,
setup.pareto_name)
file_util = FileUtil(dirname)
expressions = language_loader.load_all_evaluated_expressions(file_util)
languages_0 = language_generator.generate_sampled(
expressions, args.lang_size, int(args.sample_size / args.lang_size))
universe = generator.generate_simplified_models(setup.model_size)
measure_complexity = SumComplexityMeasurer(args.lang_size, 1)
measure_informativeness = SimMaxInformativenessMeasurer(universe)
pool = ProcessPool(nodes=setup.processes)
languages = languages_0 #lanuages will be iteratively updated in subsequent loop
for gen in range(args.generations):
print('GENERATION {0}'.format(gen))
print('measuring')
complexity = pool.map(measure_complexity, languages)
informativeness = pool.map(measure_informativeness, languages)
measurements = [(1 - inf, comp)
for inf, comp in zip(informativeness, complexity)]
print('calculating dominating')
dominating_indices = pygmo.non_dominated_front_2d(measurements)
dominating_languages = [languages[i] for i in dominating_indices]
print('mutating')
languages = sample_mutated(dominating_languages, args.sample_size,
expressions)
language_indices = [[e.index for e in lang]
for lang in dominating_languages]
dominating_complexity = [complexity[i] for i in dominating_indices]
dominating_informativeness = [
informativeness[i] for i in dominating_indices
]
file_util.dump_dill(dominating_complexity,
'complexity_wordcomplexity.dill')
file_util.dump_dill(dominating_informativeness,
'informativeness_simmax.dill')
file_util.dump_dill(language_indices, 'language_indices.dill')
file_util.save_stringlist([list(map(str, lang)) for lang in languages],
'languages.txt')
print("generate_evolutionary.py finished.")
def write_para_record(addrs, labels, split='train', n_shards=10):
train_filenames = [
expanduser("~") +
'/domain_adaptation/data/syn_data_4_april_2018_norm/{}_{:0>3}_{:0>3}.tfrecords'
.format(split, i, n_shards - 1) for i in range(n_shards)
]
addrs_split, labels_split = np.array_split(np.array(addrs),
n_shards), np.array_split(
np.array(labels), n_shards)
# print(type(addrs_split), len(addrs_split), addrs_split[0].shape)
p = Pool(n_shards)
p.map(write_record, train_filenames, addrs_split, labels_split,
[split for i in range(n_shards)])
# write_record(train_filenames[0], addrs_split, labels_split, split)
sys.stdout.flush()
def cluster_search(self):
_log('CLUSTER SEARCH')
target_kwargs = dict(self.cfg)
def target(infile):
return _process_clusters(infile, **target_kwargs)
if self.cfg.ncpu > 1:
pool = ProcessPool(nodes=min(self.cfg.ncpu, len(self.infiles)))
all_out_arrays = pool.map(target, self.infiles)
_log('CLUSTER REDUCTION')
all_out_arrays = reduce(lambda a, b: a + b, all_out_arrays)
_log('CLUSTER OUTPUT')
with h5py.File(self.cfg.outfile, 'a', libver=libver) as f:
for out_array in all_out_arrays:
self._write_cluster(f, out_array)
else:
for infile in self.infiles:
all_out_arrays = target(infile)
_log('CLUSTER OUTPUT (PARTIAL)')
with h5py.File(self.cfg.outfile, 'a', libver=libver) as f:
for out_array in all_out_arrays:
self._write_cluster(f, out_array)
return
def start_cache(self):
pool = Pool(self.WORKERS)
for status in pool.map(self.parallel_cache, self.nodes):
if "ERROR" in status:
STREAM.error(status)
else:
STREAM.success(status)
def fit(self, X, y):
"""Fit estimator.
Args:
X : array-like
The data to fit.
y : array-like
The target variable.
"""
self.X = np.asarray(X)
self.y = np.asarray(y).reshape(y.shape[0], )
self._base_score = cross_val_score(self.clf,
self.X,
y=self.y,
scoring=self.metric,
cv=self.cv).mean()
print("Base score: {}\n".format(self._base_score))
self._best_score = self._base_score
population = self._create_population()
gen = 0
total_time = 0
for i in trange(self.max_iter, desc='Generation', leave=False):
self.generation_plot.append(population.tolist())
p = ProcessPool(nodes=multiprocessing.cpu_count())
start = timer()
self._individuals = p.map(self._score_ind, population)
total_time = total_time + timer() - start
self._Generations.append(self._individuals)
best = sorted(self._individuals,
key=lambda tup: tup.score,
reverse=True)[0]
self._best_individuals.append(
self._BestIndividual(gen, best.params, best.score))
if (gen == 0):
self._best_score = self._best_individuals[gen]
if (best.score > self._best_score.score):
self._best_score = self._best_individuals[gen]
else:
pass
self._gen_score.append(
self._Generation(
gen,
sum([tup[1] for tup in self._individuals]) /
len(self._individuals), self._best_individuals[gen]))
population = self._create_next_generation(self._individuals)
self._individuals = []
gen += 1
else:
print('gen: {}'.format(gen))
print('avg time per gen: {0:0.1f}'.format(total_time / gen))
def interloper_search(self):
_log('INTERLOPER SEARCH')
# must not put 'self' in the function, so copy the dict
target_kwargs = dict(self.cfg)
def target(infile):
return _process_interlopers(infile, **target_kwargs)
if self.cfg.ncpu > 1:
pool = ProcessPool(ncpus=min(self.cfg.ncpu, len(self.infiles)))
all_out_arrays = pool.map(target, self.infiles)
else:
all_out_arrays = list()
for infile in self.infiles:
all_out_arrays.append(target(infile))
_log('INTERLOPER REDUCTION')
all_out_arrays = np.vstack(all_out_arrays)
unique_keys = np.unique(all_out_arrays['is_near'])
_log('INTERLOPER OUTPUT')
with h5py.File(self.cfg.outfile, 'a', libver=libver) as f:
for ik, cluster_id in enumerate(unique_keys):
if ik % 1000 == 0:
_log(' ', ik, '/', unique_keys.size)
interlopers = all_out_arrays[all_out_arrays['is_near'] ==
cluster_id]
self._write_interlopers(f, cluster_id, interlopers)
return
def pairs_construction(seqs: List[List[Union[str, int]]], window_size: int = 2,
drop_duplicates: bool = True,
n_jobs: int = 4, **kwargs):
"""
Helper function to make pairs from sequences in parallel
Parameters
----------
seqs : input sequences of nodes
window_size : int, default is 2
drop_duplicates : bool, default if True
Delete pairs where both elements are the same
n_jobs : int, default is 4
Number of workers to be created in parallel pool
Returns
-------
List of pairs of nodes as <cur_vertex, context_vertex>
"""
set_new_config(window_size=window_size, **kwargs)
local_logger = logging.getLogger(f"{__name__}")
max_processes = max(n_jobs, os.cpu_count())
pairs_pool = ProcessPool(nodes=max_processes)
pairs_pool.terminate()
pairs_pool.restart()
local_logger.info("Started making pairs from the sequences.")
pairs = pairs_pool.map(_make_pairs, seqs)
local_logger.info(f"Total number of raw sampled pairs is {len(pairs)}")
if drop_duplicates:
pairs = [item for sublist in pairs for item in sublist if item[0] != item[1]]
else:
pairs = [item for sublist in pairs for item in sublist]
pairs = [item for item in pairs if (item[0] != -3) & (item[1] != -3)]
pairs_pool.terminate()
pairs_pool.restart()
return pairs
def map_list_in_chunks(l, f, extra_data):
'''
A wrapper around ProcessPool.uimap that processes a list in chunks.
Differs from `map_list_as_chunks` in that this method calls `f` once for each item in `l`.
uimap already chunks but if you have extra data to pass in it will pickle
it for every item. This function passes in the extra data to each chunk
which significantly saves on pickling.
https://stackoverflow.com/questions/53604048/iterating-the-results-of-a-multiprocessing-list-is-consuming-large-amounts-of-me
Parameters
----------
l : list
the list
f : function
the function to process each item
takes two parameters: item, extra_data
extra_data : object
the extra data to pass to each f
'''
cpus = cpu_count()
chunk_length = max(1, int(len(l) / cpus))
chunks = [l[x:x + chunk_length] for x in range(0, len(l), chunk_length)]
pool = Pool(nodes=cpus)
f_dumps = cloudpickle.dumps(f)
tuples = [(chunk, f_dumps, extra_data) for chunk in chunks]
mapped_chunks = pool.map(_process_chunk, tuples)
return (item for chunk in mapped_chunks for item in chunk)
def pareto(callback, budgets : List[float], heuristic, runtime, verbose=True, **kwargs):
def safe_callback(rt):
result = 'pass'
t = time.time()
try:
callback(rt)
except MemoryError:
result = 'fail (OOM)'
except RematExceededError:
result = 'fail (thrashed)'
except:
import traceback
traceback.print_exc()
print(flush=True)
raise
total_time = time.time() - t
rt.meta['total_time'] = total_time
if verbose:
print(' budget {} finished in {} seconds: {}'.format(
rt.budget, total_time, result
), flush=True)
rt._prepickle()
return rt
if verbose:
print('running pareto trial for budgets: {}'.format(budgets), flush=True)
p = Pool()
runtimes = list(map(lambda b: runtime(b, heuristic, **kwargs), budgets))
runtimes = p.map(safe_callback, runtimes)
return runtimes
def _calculate_powder(self):
"""
Calculates powder data (a_tensors, b_tensors according to aCLIMAX manual).
"""
# define container for powder data
powder = AbinsModules.PowderData(num_atoms=self._num_atoms)
k_indices = sorted(self._frequencies.keys(
)) # make sure dictionary keys are in the same order on each machine
b_tensors = {}
a_tensors = {}
if PATHOS_FOUND:
threads = AbinsModules.AbinsParameters.threads
p_local = ProcessPool(nodes=threads)
tensors = p_local.map(self._calculate_powder_k, k_indices)
else:
tensors = [self._calculate_powder_k(k=k) for k in k_indices]
for indx, k in enumerate(k_indices):
a_tensors[k] = tensors[indx][0]
b_tensors[k] = tensors[indx][1]
# fill powder object with powder data
powder.set(dict(b_tensors=b_tensors, a_tensors=a_tensors))
return powder
def build_coarse_model_voronoi(B, num_samples_per_bin, num_nodes, num_steps):
n_bins = B.length()
pool = ProcessPool(nodes = num_nodes)
num_steps = [num_steps for x in range(num_samples_per_bin)]
seed_data = [x for x in range(num_samples_per_bin)]
Transitions = []
particle = AlanineDipeptideSimulation()
particle.Bins = B
particle.temperature = 1000
T = np.zeros((n_bins,n_bins))
for j in range(n_bins):
particle.positions = B.Ω[j]
Transitions.append(pool.map(particle.sample_voronoi, num_steps, seed_data))
for j in range(len(Transitions)):
T[Transitions[j][0], Transitions[j][1]] = T[Transitions[j][0], Transitions[j][1]] + 1
for j in range(n_bins):
if (sum(T[j,:])==0):
#print('No transitions in row', j, flush = True)
T[j,j]=1
T[j,:] = T[j,:] / sum(T[j,:])
return T
def map_list_as_chunks(l, f, extra_data, cpus=None, max_chunk_size=None):
'''
A wrapper around `pathos.multiprocessing.ProcessPool.uimap` that processes a list in chunks.
Differs from `map_list_in_chunks` in that this method calls `f` once for each chunk.
uimap already chunks but if you have extra data to pass in it will pickle
it for every item. This function passes in the extra data to each chunk
which significantly saves on pickling.
https://stackoverflow.com/questions/53604048/iterating-the-results-of-a-multiprocessing-list-is-consuming-large-amounts-of-me
Parameters
----------
l : list
the list
f : function
the function to process each item
takes two parameters: chunk, extra_data
extra_data : object
the extra data to pass to each f
cpus : int
the number of cores to use to split the chunks across
max_chunk_size : int
the maximum size for each chunk
'''
cpus = cpu_count() if cpus is None else cpus
max_chunk_size = float('inf') if max_chunk_size is None else max_chunk_size
chunk_length = min(max_chunk_size, max(1, ceil(len(l) / cpus)))
chunks = [l[x:x + chunk_length] for x in range(0, len(l), chunk_length)]
pool = Pool(nodes=cpus)
f_dumps = cloudpickle.dumps(f)
tuples = [(chunk, f_dumps, extra_data) for chunk in chunks]
return pool.map(_process_whole_chunk, tuples)
def correction_factor(p, number_of_runs, method, X, y, n_jobs=None):
# Setup parallel job
if n_jobs == -1:
n_jobs = cpu_count()
elif n_jobs == None:
n_jobs = 1
pool = Pool(n_jobs, maxtasksperchild=1000)
def run(_):
# Artificially falsify
y_f = falsify(y, p, random_state=_)
# Correct labels
y_corrected = method.fit_transform(X, y_f)
N = X.shape[0]
return ((y == y_corrected).sum() - (1 - p) * N) / (p * N)
factor = np.array(pool.map(run, range(number_of_runs)))
# Close the pool again
pool.close()
pool.join()
pool.clear()
return np.mean(factor), np.std(factor)
def _calculate_powder(self):
"""
Calculates powder data (a_tensors, b_tensors according to aCLIMAX manual).
"""
# define container for powder data
powder = AbinsModules.PowderData(num_atoms=self._num_atoms)
k_indices = sorted(self._frequencies.keys()) # make sure dictionary keys are in the same order on each machine
b_tensors = {}
a_tensors = {}
if PATHOS_FOUND:
threads = AbinsModules.AbinsParameters.threads
p_local = ProcessPool(nodes=threads)
tensors = p_local.map(self._calculate_powder_k, k_indices)
else:
tensors = [self._calculate_powder_k(k=k) for k in k_indices]
for indx, k in enumerate(k_indices):
a_tensors[k] = tensors[indx][0]
b_tensors[k] = tensors[indx][1]
# fill powder object with powder data
powder.set(dict(b_tensors=b_tensors, a_tensors=a_tensors))
return powder
def AllocateParticlesInBins(B, num_nodes, step_data):
#seeds not being used at the moment
magnitude = 1000.0
particle_data = []
phi_data = []
psi_data = []
target_ids = []
B_copies = [B]
magnitude_copies=[]
for j in range(B.length()):
angles = BinMidpoints(B, j)
phi_data.append(angles[0])
psi_data.append(angles[1])
target_ids.append(j)
B_copies.append(B)
magnitude_copies.append(magnitude)
pool = ProcessPool(nodes = num_nodes)
particle = AlanineDipeptideSimulation()
#positionsset = pool.map(particle.move_particle_to_bin, phi_data, psi_data, step_data, seeds, B_copies, target_ids)
positionsset = pool.map(particle.move_particle_to_bin_minenergy, phi_data, psi_data, magnitude_copies)
#positions = particle.move_particle_to_bin(1,1, step_data[0], seeds[0])
return positionsset
def sparsify_predictions(model_name, timepoints=None):
""" Computes and saves sparse representations for model predictions
Args:
model_name: name of model whose predictions you wish to sparsify
timepoints: list of timepoints to sparsify - [int]
"""
from ipp_tools.slurm import slurm_map
from division_detection.constants import NUM_TIMEPOINTS
pred_dir = '/nrs/turaga/bergera/division_detection/prediction_outbox/{}'.format(model_name)
sparse_pred_dir = '{}/sparse'.format(pred_dir)
if not os.path.exists(sparse_pred_dir):
os.mkdir(sparse_pred_dir)
existing_tps = set([int(fname[:-3]) for fname in os.listdir(sparse_pred_dir)])
if timepoints is None:
timepoints = np.arange(3, NUM_TIMEPOINTS - 4)
timepoints = [t for t in timepoints if t not in existing_tps]
def _sparsify_predictions_helper(t_idx):
""" Helper function that sparsifies a single timepoint.
"""
from division_detection.sparse_utils import save_dense_as_coo
pred_dir = '/nrs/turaga/bergera/division_detection/prediction_outbox/{}'.format(model_name)
dense_pred_dir = '{}/dense'.format(pred_dir)
sparse_pred_dir = '{}/sparse'.format(pred_dir)
if not os.path.exists('{}/{}.h5'.format(dense_pred_dir, t_idx)):
warn('You asked me to sparsify predictions for {} but none exist'.format(t_idx))
try:
with h5py.File('{}/{}.h5'.format(dense_pred_dir, t_idx), 'r') as prediction_file:
predictions = prediction_file['predictions']
print("Loading ", t_idx)
tp_preds = predictions[:]
print("Saving ", t_idx)
save_dense_as_coo(tp_preds, '{}/{}'.format(sparse_pred_dir, t_idx))
except OSError as os_err:
warn("Caught OS error while trying to read {}; continuing".format(t_idx))
pool = Pool(20)
pool.map(_sparsify_predictions_helper, timepoints)
def write_data(img_data_path):
categories = filter(lambda x: os.path.isdir(os.path.join(img_data_path,x)), os.listdir(img_data_path))
addrs = [i.strip().split(' ')[0] for i in open(img_data_path + 'image_list.txt').readlines()]
print(len(addrs))
n_shards = 16
addrs = np.array_split(np.array(addrs), n_shards)
write_data_path = expanduser("~") + '/abhinav/RelayVision/data/' + split_str + '_tfrecords/' # for Aggie's laptop
# write_data_path = '/home/arna/Project/RelayVision/' + split_str + '_tfrecords/' # for Arna's lab PC
filenames = [write_data_path+'{}_{:0>3}_{:0>3}.tfrecords'.format(split_str, i, n_shards-1) for i in range(n_shards)]
# for i in range(len(filenames)):
# i=0
# write_record(img_data_path, filenames[i], addrs[i], split_str)
p = Pool(n_shards)
p.map(write_record, [img_data_path for i in range(n_shards)], filenames, addrs, [split_str for i in range(n_shards)])
def get_square_fill_objs(self):
n_threads = 4
cam_path = join(self.sample_dir, 'camera.txt')
cams = np.loadtxt(cam_path)
def f(camera_i):
camera_pos = cams[camera_i]
self.write_meshlab_camera(
join(self.keys_dir, 'meshlab_camera_{}.txt'.format(camera_i)),
camera_pos, np.eye(3))
self.get_fill_obj(camera_pos,
postfix=str(camera_i),
frame=camera_i,
check=False)
pool = ProcessPool(nodes=n_threads)
pool.map(f, range(n_threads))
def obs_temp_p(self, dtobj):
'''
get observed temperature in amsterdam parallel
'''
self.dtobjP = dtobj
pool = Pool()
obs = pool.map(self.obs_temp, self.filelist)
self.obs = [ob for ob in obs if ob is not None]
def _prepare_eigen(self):
"""
calculate eigen values and vectors for all kpts and save.
Note that the convention 2 is used here, where the
phase factor is e^(ik.R), not e^(ik.(R+rj-ri))
"""
nkpts = len(self.kpts)
self.evals = np.zeros((nkpts, self.nbasis), dtype=float)
self.nkpts = nkpts
self.H0 = np.zeros((self.nbasis, self.nbasis), dtype=complex)
self.evecs = np.zeros((nkpts, self.nbasis, self.nbasis), dtype=complex)
H = np.zeros((nkpts, self.nbasis, self.nbasis), dtype=complex)
if not self.is_orthogonal:
self.S = np.zeros((nkpts, self.nbasis, self.nbasis), dtype=complex)
else:
self.S = None
if self.nproc == 1:
results = map(self.tbmodel.HSE_k, self.kpts)
else:
executor = ProcessPool(nodes=self.nproc)
results = executor.map(self.tbmodel.HSE_k, self.kpts,
[2] * len(self.kpts))
executor.close()
executor.join()
executor.clear()
for ik, result in enumerate(results):
if self.is_orthogonal:
H[ik], _, self.evals[ik], self.evecs[ik] = result
else:
H[ik], self.S[ik], self.evals[ik], self.evecs[ik] = result
self.H0 += H[ik] / self.nkpts
self.evals, self.evecs = self._reduce_eigens(self.evals,
self.evecs,
emin=self.efermi - 10.0,
emax=self.efermi + 10.1)
if self._use_cache:
evecs = self.evecs
self.evecs_shape = self.evecs.shape
self.evecs = np.memmap(os.path.join(self.cache_path, 'evecs.dat'),
mode='w+',
shape=self.evecs.shape,
dtype=complex)
self.evecs[:, :, :] = evecs[:, :, :]
if self.is_orthogonal:
self.S = None
else:
S = self.S
self.S = np.memmap(os.path.join(self.cache_path, 'S.dat'),
mode='w+',
shape=(nkpts, self.nbasis, self.nbasis),
dtype=complex)
self.S[:] = S[:]
del self.evecs
if not self.is_orthogonal:
del self.S
def _parallel_final_score(self, smiles: List[str]) -> FinalSummary:
molecules, valid_indices = self._smiles_to_mols(smiles)
component_smiles_pairs = [[
component, molecules, valid_indices, smiles
] for component in self.scoring_components]
pool = ProcessPool(nodes=len(self.scoring_components))
mapped_pool = pool.map(parallel_run, component_smiles_pairs)
pool.clear()
return self._score_summary(mapped_pool, smiles, valid_indices)
def para_data_allo_1(Theta, cpu_num, rng, d_struct, Data_struct):
time.sleep(1)
pub = Data_struct.pub_info[0, ]
print(" id: {} , is dealing the auction with {} bidder ".format(
threading.get_ident(), pub[2]))
JJ = d_struct["JJ"]
# number of auctions in the data || maximum length of an auction
TT, T_end = Data_struct.data_act.shape
TT = int(TT)
T_end = int(T_end)
'''
take the grid generation outsides
'''
# num of bidders in the auction
N = int(pub[2])
# setup the env info structure
info_flag = pub[3]
# setup the env info structure
Env = ENV(N, Theta)
if info_flag == 0:
para = Env.Uninform()
else:
para = Env.Info_ID()
[x_signal, w_x] = signal_DGP(para, rng, N, JJ)
results = []
func = partial(para_fun, para, info_flag, rng, T_end, int(JJ * N),
x_signal, w_x)
pool = ProcessPool(nodes=cpu_num)
# pool = ProcessPoolExecutor(max_workers=cpu_num)
start = time.time()
results = pool.map(
func,
zip(range(0, TT), Data_struct.data_act, Data_struct.data_state,
Data_struct.pub_info))
MoM = np.nanmean(list(results))
end = time.time()
print('time expenditure for the auction estimation under N = {}'.format(N))
print(end - start)
return MoM
def run_asymptotics(base,
ns,
heuristic,
bound,
runtime,
releases=True,
**kwargs):
config = {
'ns': ns,
'heuristic': str(heuristic),
'heuristic_features': list(heuristic.FEATURES),
'memory': str(bound),
'releases': releases,
'runtime': runtime.ID,
'runtime_features': list(runtime.FEATURES),
'kwargs': kwargs
}
p = Pool()
print('generating asymptotics data for config: {}...'.format(
json.dumps(config, indent=2)))
args = []
for n in ns:
args.append([n, bound(n), heuristic, runtime, releases, kwargs])
t = time.time()
rts = p.map(run, *zip(*args))
t = time.time() - t
succ_ns, succ_rts = chop_failures(ns, rts)
print(' - succeeded between n={} and n={}'.format(succ_ns[0],
succ_ns[-1]))
print(' done, took {} seconds.'.format(t))
results = {
'layers':
succ_ns,
'computes':
list(map(lambda rt: rt.telemetry.summary['remat_compute'], rts)),
'had_OOM':
ns[0] != succ_ns[0],
'had_thrash':
ns[-1] != succ_ns[-1]
}
date_str = datetime.now().strftime('%Y%m%d-%H%M%S-%f')
base_mod = ASYMPTOTICS_MOD + '/' + base
out_file = '{}-{}-{}.json'.format(date_str, heuristic.ID, bound.ID)
util.ensure_output_path(base_mod)
out_path = util.get_output_path(base_mod, out_file)
with open(out_path, 'w') as out_f:
out_f.write(
json.dumps({
'config': config,
'results': results
}, indent=2))
print('-> done, saved to "{}"'.format(out_path))
def run( self ): # from pathos.multiprocessing import Pool from pathos.multiprocessing import ProcessPool as Pool args = self._interpna_setup( ) pool = Pool( processes=self.ncpus ) out = pool.map( self._interpna, args[:400] ) pool.close() lons = self._lonpc # stack em and roll-its axis so time is dim0 dat = np.rollaxis( np.dstack( out ), -1 ) if self._rotated == True: # rotate it back dat, lons = self.rotate( dat, lons, to_pacific=False ) # place back into a new xarray.Dataset object for further processing # function to make a new xarray.Dataset object with the mdata we need? # ds = self.ds # var = ds[ self.variable ] # setattr( var, 'data', dat ) # self.ds = ds print( 'ds interpolated updated into self.ds' ) return dat
def spaceConvNumbaThreadedOuter2(self):
""" `Block` threading example """
def divider(arr_dims, coreNum=1):
""" Get a bunch of iterable ranges;
Example input: [[[0, 24], [15, 25]]]"""
if (coreNum == 1):
return arr_dims
elif (coreNum < 1):
raise ValueError(\
'partitioner expected a positive number of cores, got %d'\
% coreNum
)
elif (coreNum % 2):
raise ValueError(\
'partitioner expected an even number of cores, got %d'\
% coreNum
)
total = []
# Split each coordinate in arr_dims in _half_
for arr_dim in arr_dims:
dY = arr_dim[0][1] - arr_dim[0][0]
dX = arr_dim[1][1] - arr_dim[1][0]
if ((coreNum,)*2 > (dY, dX)):
coreNum = max(dY, dX)
coreNum -= 1 if (coreNum % 2 and coreNum > 1) else 0
new_c1, new_c2, = [], []
if (dY >= dX):
# Subimage height is greater than its width
half = dY // 2
new_c1.append([arr_dim[0][0], arr_dim[0][0] + half])
new_c1.append(arr_dim[1])
new_c2.append([arr_dim[0][0] + half, arr_dim[0][1]])
new_c2.append(arr_dim[1])
else:
# Subimage width is greater than its height
half = dX // 2
new_c1.append(arr_dim[0])
new_c1.append([arr_dim[1][0], half])
new_c2.append(arr_dim[0])
new_c2.append([arr_dim[1][0] + half, arr_dim[1][1]])
total.append(new_c1), total.append(new_c2)
# If the number of cores is 1, we get back the total; Else,
# we split each in total, etc.; it's turtles all the way down
return divider(total, coreNum // 2)
def numer(start, finish):
count = start
iteration = 0
while count < finish:
yield iteration, count
iteration += 1
count += 1
@checkarrays
@jit
def dotJit(subarray, kernel):
total = 0.0
for i in xrange(subarray.shape[0]):
for j in xrange(subarray.shape[1]):
total += subarray[i][j] * kernel[i][j]
return total
def outer(subset):
a, b, = subset
ai, bi, = map(sub, *reversed(zip(*subset)))
temp = np.zeros((ai, bi))
for ind, i in numer(*a):
for jnd, j in numer(*b):
temp[ind, jnd] = dotJit(\
self.array[i:i+self.__rangeKX_,
j:j+self.__rangeKY_]
, self.kernel
)
return temp, a, b
# ProcessPool auto-detects processors, but my function above
# only accepts an even number; I'm still working on it.
# Otherwise I wouldn't mess with cpu_count()
cores = cpu_count()
cores -= 1 if (cores % 2 == 1 and cores > 1) else 0
# Get partitioning indices and the usable number of cores
shape = [[[0, self.__rangeX_ - 1], [0, self.__rangeY_ - 1]]]
partitions = divider(shape, cores)
# Map partitions to threads and process
pool = ProcessPool(nodes=cores)
results = pool.map(outer, partitions)
#pool.close()
#pool.join()
for ind, res in enumerate(results):
X, Y, = results[ind][1:]
self.__arr_[slice(*X), slice(*Y)] += results[ind][0]
return self.__arr_
# from pathos.multiprocessing import ProcessingPool as Pool
# import pathos.multiprocessing as mp
from pathos.multiprocessing import ProcessPool as Pool
from ActiveShapeModelsBetter import ASMB, Point, Shape
import dill
if __name__ == "__main__":
asm = ASMB([0, 1], 10)
asm.addShape(Shape([Point(100, 200), Point(200, 440), Point(400, 300)]))
p = Pool()
p.map(Point.rotate, asm.allShapes, [[-1, 1], [1, -1]])