def optimize_trials_parallel_gpu(
self,
train_function,
nb_trials,
trials,
gpu_ids,
nb_workers=4,
):
"""
Runs optimization across gpus with cuda drivers
:param train_function:
:param nb_trials:
:param gpu_ids: List of strings like: ['0', '1, 3']
:param nb_workers:
:return:
"""
self.trials = trials
self.trials = [(x, train_function) for x in self.trials]
# build q of gpu ids so we can use them in each process
# this is thread safe so each process can pull out a gpu id, run its task and put it back when done
if self.pool is None:
gpu_q = Queue()
for gpu_id in gpu_ids:
gpu_q.put(gpu_id)
# init a pool with the nb of worker threads we want
self.pool = Pool(processes=nb_workers, initializer=init, initargs=(gpu_q,))
# apply parallelization
results = self.pool.map(optimize_parallel_gpu_private, self.trials)
return results
def __init__(self,
path,
seqNames,
nProcessLoader=50,
MAX_SIZE_LOADED=4000000000):
"""
Args:
- path (string): path to the training dataset
- seqNames (list): sequences to load
- nProcessLoader (int): number of processes to call when loading the
data from the disk
- MAX_SIZE_LOADED (int): target maximal size of the floating array
containing all loaded data.
"""
self.MAX_SIZE_LOADED = MAX_SIZE_LOADED
self.nProcessLoader = nProcessLoader
self.dbPath = Path(path)
self.seqNames = [self.dbPath / x for _, x in seqNames]
self.reload_pool = Pool(nProcessLoader)
self.prepare()
self.data = []
self.loadNextPack(first=True)
self.loadNextPack()
def optimize_parallel_cpu(
self,
train_function,
nb_trials,
nb_workers=4,
):
"""
Runs optimization across n cpus
:param train_function:
:param nb_trials:
:param nb_workers:
:return:
"""
self.trials = strategies.generate_trials(
strategy=self.strategy,
flat_params=self.__flatten_params(self.opt_args),
nb_trials=nb_trials
)
self.trials = [(self.__namespace_from_trial(x), train_function) for x in self.trials]
# init a pool with the nb of worker threads we want
if self.pool is None:
self.pool = Pool(processes=nb_workers)
# apply parallelization
results = self.pool.map(optimize_parallel_cpu_private, self.trials)
return results
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-modelType',
default=4,
type=int,
help='Refer train_utils.py ')
parser.add_argument('-numSpkrs',
default=7323,
type=int,
help='Number of output labels for model')
parser.add_argument('modelDirectory',
help='Directory containing the model checkpoints')
parser.add_argument(
'featDir', help='Directory containing features ready for extraction')
parser.add_argument('embeddingDir', help='Output directory')
args = parser.parse_args()
modelFile = max(glob.glob(args.modelDirectory + '/*'),
key=os.path.getctime)
# Load model definition
if args.modelType == 3:
net = simpleTDNN(args.numSpkrs, p_dropout=0)
else:
net = xvecTDNN(args.numSpkrs, p_dropout=0)
checkpoint = torch.load(modelFile, map_location=torch.device('cuda'))
new_state_dict = OrderedDict()
for k, v in checkpoint['model_state_dict'].items():
if k.startswith('module.'):
new_state_dict[k[7:]] = v # ugly fix to remove 'module' from key
else:
new_state_dict[k] = v
# load trained weights
net.load_state_dict(new_state_dict)
net = net.cuda()
net.eval()
# Parallel Processing
try:
nSplits = int(
sorted(glob.glob(args.featDir + '/split*'),
key=getSplitNum)[-1].split('/')[-1].lstrip('split'))
except:
print('Cannot find %s/splitN directory' % args.featDir)
sys.exit(1)
if not os.path.isdir(args.embeddingDir):
os.makedirs(args.embeddingDir)
nProcs = nSplits
L = [('%s/split%d/%d/feats.scp' % (args.featDir, nSplits, i),
'%s/xvector.%d.ark' % (args.embeddingDir, i),
'%s/xvector.%d.scp' % (args.embeddingDir, i), net, 'fc1')
for i in range(1, nSplits + 1)]
pool2 = Pool(processes=nProcs)
result = pool2.starmap(par_core_extractXvectors, L)
pool2.terminate()
os.system('cat %s/xvector.*.scp > %s/xvector.scp' %
(args.embeddingDir, args.embeddingDir))
def optimize_parallel_gpu(
self,
train_function,
gpu_ids,
max_nb_trials=None,
):
"""
Runs optimization across gpus with cuda drivers
:param train_function:
:param max_nb_trials:
:param gpu_ids: List of strings like: ['0', '1, 3']
:return:
"""
self.trials = strategies.generate_trials(
strategy=self.strategy,
flat_params=self.__flatten_params(self.opt_args),
nb_trials=max_nb_trials,
)
self.trials = [(self.__namespace_from_trial(x), train_function) for x in self.trials]
# build q of gpu ids so we can use them in each process
# this is thread safe so each process can pull out a gpu id, run its task and put it back when done
if self.pool is None:
gpu_q = Queue()
for gpu_id in gpu_ids:
gpu_q.put(gpu_id)
# init a pool with the nb of worker threads we want
nb_workers = len(gpu_ids)
self.pool = Pool(processes=nb_workers, initializer=init, initargs=(gpu_q,))
# apply parallelization
results = self.pool.map(optimize_parallel_gpu_private, self.trials)
return results
def setup_class(self):
"""Setup the metric class. This will spawn the pool of workers that are
used for metric testing and setup_ddp
"""
self.poolSize = NUM_PROCESSES
self.pool = Pool(processes=self.poolSize)
self.pool.starmap(setup_ddp, [(rank, self.poolSize) for rank in range(self.poolSize)])
def meta_ars(env_name,
policy,
meta_epochs,
meta_seed,
n_seeds=4,
n_top_seeds=1,
n_workers=4,
mean_lookback=10,
ars_epochs=10,
env_config=None,
step_size=.02,
n_delta=32,
n_top=16,
exp_noise=0.03):
n_children = n_seeds // n_top_seeds
np.random.seed(meta_seed)
W = torch.nn.utils.parameters_to_vector(policy.parameters())
W = torch.zeros_like(W)
torch.nn.utils.vector_to_parameters(W, policy.parameters())
pool = Pool(processes=n_seeds)
ars_partial = partial(ars, env_name, ars_epochs, env_config, step_size,
n_delta, n_top, exp_noise, n_workers)
#root = Node(meta_seed)
reward_log = []
top_policies = []
for _ in range(n_top_seeds):
top_policies.append(copy.deepcopy(policy))
for epoch in range(meta_epochs):
pols_and_seeds = []
for pol in top_policies:
for _ in range(n_children):
pols_and_seeds.append(
(pol, int(np.random.randint(0, 2**32 - 1, 1))))
results = pool.starmap(ars_partial, pols_and_seeds)
p_list = []
r_list = []
for result in results:
policy, rews = result
p_list.append(policy)
r = torch.stack(rews[-mean_lookback:])
r_list.append(r.mean())
top_idx = sorted(range(len(r_list)),
key=lambda k: r_list[k],
reverse=True)[:n_top_seeds]
for i in top_idx:
top_policies.append(p_list[i])
reward_log.append(max(r_list))
return top_policies, reward_log
def match(self):
pool = Pool(self.num_processes)
results = []
total = len(self.pairs_to_match)
counter = 0
matches_pairs_fname = os.path.join(
self.input_dir, "matches_pairs" + self.suffix + ".txt")
mp_file = open(matches_pairs_fname, 'w')
for pair in tqdm(self.pairs_to_match):
orig_name1 = self.image_names[pair[0]]
orig_name2 = self.image_names[pair[1]]
name1 = os.path.join(self.input_dir, orig_name1)
name2 = os.path.join(self.input_dir, orig_name2)
image_id1 = self.image_ids[pair[0]]
image_id2 = self.image_ids[pair[1]]
if self.cuda:
# use single process
data = self.matchPairCached(orig_name1, orig_name2, image_id1,
image_id2, self.recompute,
self.cuda)
self.db.add_matches(data[0], data[1], data[2])
mp_file.write(data[3] + " " + data[4] + "\n")
counter += 1
if counter % 50000 == 0:
counter = 0
self.db.commit()
# print("Matched", counter, "out of", total)
else:
# use multiple processes
res = pool.apply_async(self.matchPairCached,
(orig_name1, orig_name2, image_id1,
image_id2, self.recompute, self.cuda))
results.append(res)
if not self.cuda:
# if more processes were used, collect the results
counter = 0
for res in tqdm(results):
data = res.get()
if data is None:
continue
self.db.add_matches(data[0], data[1], data[2])
name1 = data[3]
name2 = data[4]
mp_file.write(name1 + " " + name2 + "\n")
counter += 1
if counter % 50000 == 0:
counter = 0
self.db.commit()
# be sure to commit everything at the end
self.db.commit()
mp_file.close()
def __init__(self,
hd_folder,
reg_folder=None,
batch_size=64,
ext_list=['.bmp', '.tif', '.png'],
img_size=128): #
super(DatasetSmall, self).__init__()
self.hd_imgs_org = []
self.reg_imgs_org = []
self.batch_size = batch_size
self.ratio_pool = np.arange(0.5, 0.75, 0.01)
img_count = 0
self.hd_filelist, self.hd_filenames = getfilelist(hd_folder,
ext_list,
with_ext=True)
self.count = 0
self.reset_count = 20000
self.img_size = img_size
self.epoch_iteration = 5000
self.pool = Pool(6)
for this_hd_path, this_hd_name in zip(self.hd_filelist,
self.hd_filenames):
this_reg_path = os.path.join(reg_folder, this_hd_name)
reg_img_orig = imread(this_reg_path).astype(np.float32)
hd_img_orig = imread(this_hd_path).astype(np.float32)
hd_row, hd_col, chn = hd_img_orig.shape
#reg_row, reg_col, chn = reg_img_orig.shape
# now pad the image to at least img_size * 1.4
pad_hd_row, pad_hd_col = max(0,
int(1.4 * img_size) - hd_row), max(
0,
int(1.4 * img_size) - hd_col)
npad3 = ((pad_hd_row // 2, pad_hd_row - pad_hd_row // 2),
(pad_hd_col // 2, pad_hd_col - pad_hd_col // 2), (0, 0))
hd_img_orig = np.pad(hd_img_orig, npad3, 'symmetric')
reg_img_orig = np.pad(reg_img_orig, npad3, 'symmetric')
hd_img = (hd_img_orig / 127.5 - 1)
reg_img = (reg_img_orig / 127.5 - 1)
self.hd_imgs_org.append(hd_img)
self.reg_imgs_org.append(reg_img)
img_count += 1
print('Generated {} images'.format(img_count), hd_img_orig.shape)
self.num_imgs = len(self.hd_imgs_org)
print('start rotate')
self.hd_imgs = []
self.reg_imgs = []
self.rotate_img()
def setup_class(self):
""" Setup the metric class. This will spawn the pool of workers that are
used for metric testing and setup_ddp
"""
try:
set_start_method('spawn')
except RuntimeError:
pass
self.poolSize = NUM_PROCESSES
self.pool = Pool(processes=self.poolSize)
self.pool.starmap(setup_ddp, [(rank, self.poolSize) for rank in range(self.poolSize)])
def sample(self, observations, num_samples, thetas=None):
assert (thetas is None or len(thetas) is self.chains)
self.sampler.reset()
if thetas is None:
inputs = self._prepare_inputs()
pool = Pool(processes=self.workers)
arguments = self._prepare_arguments(observations, inputs, num_samples)
chains = pool.map(self.sample_chain, arguments)
del pool
return chains
def __init__(self, *args, preprocessing_transform=None, preprocessing_params: dict = None,
transform=None, no_progress: bool = False, **kwargs):
"""
Args:
positional and keyword arguments for initialize(*args, **kwargs) (see class and initialize documentation)
preprocessing_params (dict): parameters for the preprocessing:
- 'cache_dir': path to the cached preprocessed data.
- 'num_workers': number of process used in parallel for preprocessing (default: number of cores)
preprocessing_transform (Callable): Called on the outputs of _get_data over the indices
from 0 to len(self) during the construction of the dataset,
the preprocessed outputs are then cached to 'cache_dir'.
transform (Callable): Called on the preprocessed data at __getitem__.
no_progress (bool): disable tqdm progress bar for preprocessing.
"""
self.initialize(*args, **kwargs)
if preprocessing_transform is not None:
desc = 'Applying preprocessing'
if preprocessing_params is None:
preprocessing_params = {}
cache_dir = preprocessing_params.get('cache_dir')
assert cache_dir is not None, 'Cache directory is not given'
self.cache_convert = helpers.Cache(
preprocessing_transform,
cache_dir=cache_dir,
cache_key=helpers._get_hash(repr(preprocessing_transform))
)
use_cuda = preprocessing_params.get('use_cuda', False)
num_workers = preprocessing_params.get('num_workers')
uncached = [idx for idx in range(len(self)) if self._get_attributes(idx)[
'name'] not in self.cache_convert.cached_ids]
if len(uncached) > 0:
if num_workers == 0:
with torch.no_grad():
for idx in tqdm(range(len(self)), desc=desc, disable=no_progress):
name = self._get_attributes(idx)['name']
data = self._get_data(idx)
self.cache_convert(name, *data)
else:
p = Pool(num_workers)
iterator = p.imap_unordered(
_preprocess_task,
[(idx, self._get_data, self._get_attributes, self.cache_convert)
for idx in uncached])
for i in tqdm(range(len(uncached)), desc=desc, disable=no_progress):
next(iterator)
else:
self.cache_convert = None
self.transform = transform
def make_dataset_XNAT(project, opt, phase):
session = xnat.connect('http://rufus.stanford.edu', user='******', password='******') #make XNAT connection
#xnat version of make_dataset_dicom()
images = []
#parsing xnat hierarchy: project -> subject -> experiment -> scan -> dicom img
subjs = session.projects[project].subjects
try:
set_start_method('spawn')
except RuntimeError:
pass
#multiprocessing code (not working) ------------
pool = Pool(os.cpu_count())
images = pool.map(process_subj, [subjs[s] for s in subjs])
images = [i for im in images for i in im]
#-----------------------------------------------
#original code (before multiproc, works fine) -------------------
'''
for s in subjs:
exps = subjs[s].experiments
for e in exps:
scans = exps[e].scans
for sc in scans:
my_file = scans[sc].resources['DICOM'].files[0]
fname = my_file.data['Name']
path = my_file.uri
with my_file.open() as f:
dicom = pydicom.read_file(f)
#save images whose filenames are in phase
if fname.endswith('.dcm'):
try:
opt.names_phase[fname]
except:
print('Error: we donot find the phase for file name' , fname, 'ignore this file')
continue
# sys.exit(3)
if opt.names_phase[fname] == phase:
Img = process_dicom(dicom)
if Img is not None:
print("x" + fname)
images.append([path, dicom]) #each image list item has filename + FileData obj
'''
#-------------------------------------------------
session.disconnect()
return images
def tune_from_args(args):
params_grid = list(
product(
torch.linspace(args.alpha_from, args.alpha_to, args.alpha_steps),
torch.linspace(args.beta_from, args.beta_to, args.beta_steps)))
LOG.info(
'Scheduling {} jobs for alphas=linspace({}, {}, {}), betas=linspace({}, {}, {})'
.format(len(params_grid), args.alpha_from, args.alpha_to,
args.alpha_steps, args.beta_from, args.beta_to,
args.beta_steps))
# start worker processes
LOG.info(
f"Using {args.num_workers} processes and {args.lm_workers} for each CTCDecoder."
)
extract_start = default_timer()
if args.unit == 'char':
vocab = CharTokenizer(args.vocab_file,
reserved_tokens={
'blank': '<blank>',
'space': args.space_token
})
else:
vocab = WordTokenizer(args.vocab_file)
p = Pool(args.num_workers, init, [
args.logits_targets_file, vocab, args.lm_path, args.lm_trie_path,
args.lm_unit, args.beam_size, args.cutoff_prob, args.cutoff_top_n
])
scores = []
best_wer = float('inf')
with tqdm.tqdm(p.imap(tune_step, params_grid),
total=len(params_grid),
desc='Grid search') as pbar:
for params in pbar:
alpha, beta, wer, cer = params
scores.append([alpha, beta, wer, cer])
if wer < best_wer:
best_wer = wer
pbar.set_postfix(alpha=alpha, beta=beta, wer=wer, cer=cer)
LOG.info(
f"Finished {len(params_grid)} processes in {default_timer() - extract_start:.1f}s"
)
df = pd.DataFrame(scores, columns=['alpha', 'beta', 'wer', 'cer'])
df.to_csv(args.output_file, index=False)
def multitrainer(trainers, keep_on_gpu=True, parallel=False, cuda_pool=(0, )):
"""Train multiple models in parallel.
Parameters
----------
trainers : sequence[ModelTrainer]
keep_on_gpu : bool, default=True
Keep all models on GPU (risk of out-of-memory)
parallel : bool or int, default=False
Train model in parallel.
If an int, only this number of models will be trained in parallel.
cuda_pool : sequence[int], default=[0]
IDs of GPUs that can be used to dispatch the models.
"""
n = len(trainers)
initial_epoch = max(min(trainer.initial_epoch for trainer in trainers), 1)
nb_epoch = max(trainer.nb_epoch for trainer in trainers)
trainers = tuple(trainers)
if parallel:
parallel = len(trainers) if parallel is True else parallel
chunksize = max(len(trainers) // (2 * parallel), 1)
pool = Pool(parallel)
cuda_pool = py.make_list(pool, parallel)
if not torch.cuda.is_available():
cuda_pool = []
if not keep_on_gpu:
for trainer in trainers:
trainer.to(device='cpu')
# --- init ---
if parallel:
args = zip(trainers, [keep_on_gpu] * n, [cuda_pool] * n)
trainers = list(pool.map(_init1, args, chunksize=chunksize))
else:
args = zip(trainers, [keep_on_gpu] * n, [cuda_pool] * n)
trainers = list(map(_init1, args))
# --- train ---
for epoch in range(initial_epoch + 1, nb_epoch + 1):
if parallel:
args = zip(trainers, [epoch] * n, [keep_on_gpu] * n,
[cuda_pool] * n)
trainers = list(pool.map(_train1, args, chunksize=chunksize))
else:
args = zip(trainers, [epoch] * n, [keep_on_gpu] * n,
[cuda_pool] * n)
trainers = list(map(_train1, args))
def async_build_examples(
self, data_type: str,
dials: List[Tuple[str, dict]]) -> Tuple[list, list]:
"""Use multiprocessing to process raw dialogue data.
Args:
data_type: train, dev or test
dials: raw dialogues data
Returns:
new examples by all processes
"""
neg_examples = Manager().list()
pos_examples = Manager().list()
dials4single_process = (len(dials) -
1) // self.config['num_processes'] + 1
print(f'Single process have {dials4single_process} dials ...')
pool = Pool(self.config['num_processes'])
for i in range(self.config['num_processes']):
pool.apply_async(func=self.iter_dials,
args=(dials[dials4single_process *
i:dials4single_process * (i + 1)],
data_type, pos_examples, neg_examples, i))
pool.close()
pool.join()
pos_examples = list(pos_examples)
neg_examples = list(neg_examples)
return neg_examples, pos_examples
def setup_pool(self, n=None):
"""Setup process pool."""
if n is None:
n = self['config', 'num_workers']
if n == 1:
self.pool = None
else:
if self['config', 'backend'].lower() == 'torch':
logging.info('Using torch multiprocessing')
from torch.multiprocessing import Pool
self.pool = Pool(n)
else:
logging.info('Using usual multiprocessing')
from multiprocessing import Pool
self.pool = Pool(n)
def thread_fn(self, data, fn, num_threads):
tmp_result = []
thread_args = self.gen_args(data, num_threads)
with Pool(processes=num_threads) as pool:
for result in pool.imap(fn, thread_args):
tmp_result.extend(result)
return tmp_result
def multiprocess(self, *args, **kwargs):
foundIter = False
if self.reference > -1:
foundIter = self.reference
length = len(args[self.reference])
else:
for i,x in enumerate(args):
if isIterable(x):
foundIter = i
length = len(x)
break
if self.n_jobs is None:
self.n_jobs = length if length > self.count else self.count
self.n_jobs = length if self.count > length else self.n_jobs
if foundIter is False:
raise Parallel_ReferenceError()
toCall = []
for i,x in enumerate(args):
if foundIter == i:
toCall.append(x)
continue
if isTensor(x):
x.share_memory_()
toCall.append([x]*length)
with Pool(processes = self.n_jobs) as pool:
output = pool.starmap(self.f, zip(*toCall) )
return output
def load_async(pool: Pool,
fn: Callable,
*args,
callback: Callable = None,
**kwargs) -> Any:
"""
Load data asynchronously and serialize data via dill
Args:
pool: multiprocessing pool to use for :func:`apply_async`
fn: function to load a single sample
*args: positional arguments to dump with dill
callback: optional callback. defaults to None.
**kwargs: keyword arguments to dump with dill
Returns:
Any: reference to obtain data with :func:`get`
"""
if not DILL_AVAILABLE:
raise RuntimeError('dill is not installed. For async loading '
'please install it')
payload = dill.dumps((fn, args, kwargs))
return pool.apply_async(dill_helper, (payload, ), callback=callback)
def load_multi_process(self, load_fn: Callable, path: Sequence) -> List:
"""
Helper function to load dataset with multiple processes
Args:
load_fn: function to load a single sample
path: a sequence of paths which should be loaded
Returns:
list: loaded data
"""
_processes = cpu_count(
) if self._num_workers is None else self._num_workers
if self._verbosity:
pbar = tqdm(total=len(path),
unit='samples',
desc="Loading Samples")
def update(*a):
pbar.update(1)
callback = update
else:
callback = None
with Pool(processes=_processes) as pool:
jobs = [
load_async(pool, load_fn, p, callback=callback) for p in path
]
_data = [j.get() for j in jobs]
return _data
def batch_clique_graph(batch, classes_dataframe, processes=None):
"""Creates a graph Data object from an image batch, to use with a semi-supervised graph learning model. The created
graph connects all batch elements with each other (clique graph) and graph vertex weights correspond to word
vector distances of class labels. Assumes data and labels are the first two parameters of each sample.
Args:
batch: data to be sent to device.
classes_dataframe: dataframe containing class names and their word vectors.
processes: number of parallel workers to be used for creating batch graphs. If `None`, then `os.cpu_count()`
will be used. (Default value = None)
Returns:
the batch clique graph.
"""
x, y, *_ = batch # unpack extra parameters into `_`
edge_index = torch.stack(
[ # create the binary adjacency matrix for the clique graph
torch.arange(x.shape[0]).repeat_interleave(
x.shape[0]), # each index repeated num_edges times
torch.arange(x.shape[0]).repeat(x.shape[0]),
]) # the index range repeated num_edges times
with Pool(processes=processes
) as pool: # create edge weights from the word vector distances
edge_classes = torch.stack(
[y.repeat_interleave(y.shape[0]),
y.repeat(y.shape[0])]).t().contiguous()
edge_attr = torch.stack(
pool.starmap(
wordvector_distance,
zip(edge_classes,
repeat(torch.tensor(classes_dataframe["distances"])))))
return Data(x=x, edge_index=edge_index, edge_attr=edge_attr, y=y)
def test(n=10000):
n_cpus = mp.cpu_count()
print("num cpus: ", n_cpus)
p = Pool(n_cpus)
import time
# s1 = time.time()
data = []
for i in range(n):
inp_val = np.random.random(size=10)
vec_val = np.random.random(size=10)
data.append((inp_val, vec_val))
#
# res = p.map(compute_hvp, data)
# e1 = time.time()
# print ("Time 1: ", (e1-s1))
s2 = time.time()
for i in range(n):
inp_val, vec_val = data[i]
inp = Variable(torch.FloatTensor([inp_val]), requires_grad=True)
v = Variable(torch.FloatTensor([vec_val]), requires_grad=False)
z = three_sin(inp)
l = F.mse_loss(z, torch.zeros_like(z))
# hvp_rop_lop = Hvp_RopLop(f, inp, v)
# print ("hvp: ", hvp_rop_lop.data)
# hvp_dbl_bp = Hvp_dbl_bp(l, inp, v)
# print ("hvp: ", hvp_dbl_bp.data)
# print ("hvp: ", hvp_rop_lop.data, hvp_dbl_bp.data)
gnvp_roplop = GNvp_RopLop(l, z, inp, v)
e2 = time.time()
print("Time 2: ", (e2 - s2))
def policy_iteration(args):
set_start_method('spawn')
agent = Agent(args)
eval_policy(agent, 500)
replay_memory = deque(maxlen=args.MAX_MEMORY_SIZE)
num_episode_per_thread = args.step_per_iteration // args.NUM_OF_PROCESSES // (
GAMEOVER_ROUND - CUT)
outer = tqdm(range(args.total_iterations), desc='Iteration', position=0)
for i in outer:
if i == 0:
num_episode = args.observation_data // args.NUM_OF_PROCESSES // (
GAMEOVER_ROUND - CUT)
thread_args = gen_args(agent, num_episode, args.NUM_OF_PROCESSES)
else:
thread_args = gen_args(agent, num_episode_per_thread,
args.NUM_OF_PROCESSES)
with Pool(processes=args.NUM_OF_PROCESSES) as pool:
for result in pool.imap(thread_thunk, thread_args):
replay_memory.extend(result)
train_data = list(replay_memory)
if i == 0:
message = "num of training data: {}".format(len(train_data))
outer.write(message)
agent.train(train_data, 2)
else:
message = "num of training data: {}".format(
args.step_per_iteration)
outer.write(message)
agent.train(train_data[-args.step_per_iteration:])
eval_policy(agent, 500)
agent.update_epsilon()
def _greedy_bayes_multiprocess(self, encoded_df, k=1):
"""Construct a Bayesian Network (BN) using greedy algorithm."""
dataset = encoded_df.astype(str, copy=False)
root_attribute = choice(dataset.columns)
V = [root_attribute]
rest_attributes = set(dataset.columns)
rest_attributes.remove(root_attribute)
bayesian_net = []
while rest_attributes:
parents_pair_list = []
mutual_info_list = []
num_parents = min(len(V), k)
tasks = [(child, V, num_parents, split, dataset) for child, split in
product(rest_attributes, range(len(V) - num_parents + 1))]
with Pool(processes=PROCESSES) as pool:
res_list = pool.map(bayes_worker, tasks)
for res in res_list:
parents_pair_list += res[0]
mutual_info_list += res[1]
idx = mutual_info_list.index(max(mutual_info_list))
bayesian_net.append(parents_pair_list[idx])
adding_attribute = parents_pair_list[idx][0]
V.append(adding_attribute)
rest_attributes.remove(adding_attribute)
return bayesian_net
def run_synthetic_confounded_cdm_experiment():
print(f'Running run_synthetic_confounded_cdm_experiment()...')
models = [Logit, CDM, MultinomialLogit, MultinomialCDM]
learning_rates = [0.01]
use_ipws = [True, False]
samples = [10000]
embedding_dims = [2]
seeds = list(range(16))
context_strengths = [1]
confounding_strengths = np.linspace(0, 8, 21)
params = list(
itertools.product(models, learning_rates, use_ipws, samples,
embedding_dims, seeds, context_strengths,
confounding_strengths))
random.shuffle(params)
results = dict()
with Pool(N_THREADS) as pool:
for args, state_dict, train_loss, test_loss, test_mrr, num_params, mcar_loss, mcar_mrr in tqdm(
pool.imap_unordered(synthetic_confounded_cdm_experiment_helper,
params),
total=len(params)):
results[
args] = state_dict, train_loss, test_loss, test_mrr, num_params, mcar_loss, mcar_mrr
fname = f'{RESULTS_DIR}/synthetic_counfounded_cdm_results.pt'
with open(fname, 'wb') as f:
torch.save(results, f)
def setup_featurizer(self, dataset, total_vars, classes, processes=20):
self.ds = dataset
self.total_vars = total_vars
self.classes = classes
self.pool = Pool(processes)
self.setup_done = True
self.specific_setup()
def train_one_round(sites: Sequence[Site],
num_epochs: int) -> Dict[str, float]:
train = partial(train_site, num_epochs=num_epochs)
with Pool(processes=len(sites)) as pool:
results = pool.map_async(train, sites)
per_site_metrics = results.get()
return synchronize_metrics(sites, per_site_metrics)
def main(data_dir: Path, save_dir: Path, encoder_path: Path, seg_len: int,
n_workers: int):
device = "cuda" if torch.cuda.is_available() else "cpu"
set_start_method("spawn")
encoder = torch.jit.load(encoder_path).eval().to(device)
meta_data = defaultdict(list)
(save_dir / "uttrs").mkdir(exist_ok=True)
(save_dir / "embed").mkdir(exist_ok=True)
spk_dirs = data_dir.iterdir()
wav2mel = Wav2Mel()
file2mel = partial(process_file, wav2mel=wav2mel)
for spk in tqdm(spk_dirs):
wav_files = list(spk.iterdir())
with Pool(n_workers) as p:
mels = p.map(file2mel, wav_files)
mels = list(filter(lambda x: x is not None, mels))
mels = [mel.to(device) for mel in mels]
embed = embed_uttrs(encoder, mels, seg_len)
rnd_paths = [f"uttrs/{uuid4().hex}.pt" for _ in range(len(mels))]
dummy = [
torch.save(mel.cpu(), save_dir / path)
for path, mel in zip(rnd_paths, mels)
]
emb_path = f"embed/{spk}.pt"
torch.save(embed.cpu(), save_dir / emb_path)
meta_data[spk] = {"embed": emb_path, "uttrs": rnd_paths}
json.dump(meta_data, (save_dir / "metadata.json").open(mode="w"))
class ParallDataWraper():
def __init__(self, loader, batch_size, thread=0):
self.loader = loader
assert loader.shuffle == False, 'Shuffle in loader should be False'
self.batch_size = batch_size
self.pool = Pool(thread)
self.create_loader(self.loader.num_data)
def create_loader(self, num):
ids = [a for a in range(num)]
random.shuffle(ids)
self.targets = self.pool.imap(self.loader.next, (id for id in ids))
def reset(self):
self.create_loader(self.loader.num_data)
def get_iter_epoch(self):
return self.loader.get_iter_epoch()
def load_batch(self):
all_outputs = []
for i in range(self.batch_size):
try:
outputs = self.targets.__next__()
except StopIteration:
self.create_loader(self.loader.num_data)
outputs = self.targets.__next__()
all_outputs.append(outputs)
return pack_data(all_outputs)
net = Darknet19()
net_utils.load_net(trained_model, net)
# net.load_from_npz(npz_fname)
# net_utils.save_net(h5_fname, net)
net.cuda()
net.eval()
print('load model succ...')
t_det = Timer()
t_total = Timer()
im_fnames = sorted((fname
for fname in os.listdir(im_path)
if os.path.splitext(fname)[-1] == '.jpg'))
im_fnames = (os.path.join(im_path, fname) for fname in im_fnames)
pool = Pool(processes=1)
for i, (image, im_data) in enumerate(pool.imap(
preprocess, im_fnames, chunksize=1)):
t_total.tic()
im_data = net_utils.np_to_variable(
im_data, is_cuda=True, volatile=True).permute(0, 3, 1, 2)
t_det.tic()
bbox_pred, iou_pred, prob_pred = net(im_data)
det_time = t_det.toc()
# to numpy
bbox_pred = bbox_pred.data.cpu().numpy()
iou_pred = iou_pred.data.cpu().numpy()
prob_pred = prob_pred.data.cpu().numpy()
# print bbox_pred.shape, iou_pred.shape, prob_pred.shape
