def create(self, numOfEpoch, *args):
self.args = args
self.data = dict()
for item in self.args:
self.data[item] = np.frombuffer(sharedmem.full(numOfEpoch,
[0] * numOfEpoch,
dtype=np.float64),
dtype=np.float64)
self.data['tried_times'] = np.frombuffer(sharedmem.full(1, [0],
dtype=np.int8),
dtype=np.int8)
def coarse_graph(graph, similarity, max_levels, method_option, reduction_factor):
#### Coarsening ####
while not graph['level'] == max_levels:
# The common neighbors measure is used to contract the network.
graph['similarity'] = getattr(Similarity(graph, graph['adjlist']), similarity)
matching = sharedmem.full(graph.vcount(), range(graph.vcount()), dtype='int')
processes = []
# Sets the method that will be used for creating the matching.
method = None
if(method_option == 0):
method = graph.greed_two_hops
elif(method_option == 1):
method = graph.greed_rand_two_hops
for layer in options.layers:
start = sum(graph['vertices'][0:layer])
end = sum(graph['vertices'][0:layer + 1])
processes.append(Process(target = method, args = (range(start, end), reduction_factor[layer], matching)))
for p in processes:
p.start()
for p in processes:
p.join()
# Coarsening the graph.
coarser = graph.coarsening(matching)
graph = coarser
return graph
def adjoint_transform(self, coeffs, do_norm=True, total_cores=None):
if total_cores is None:
total_cores = self.total_cores
# result = sm.full((self.height, self.width), 0, dtype='complex128')
result = sm.full((self.height, self.width), 0, dtype=DTYPES[1])
numexpr_flag = NUM_FFTW_THREADS != 1
with sharedmem_pool(total_cores, numexpr=numexpr_flag) as pool:
def work(i):
# local_fft = make_fft(self.width, self.height)
self.add_adjoint_part(coeffs[i], i, result, do_norm, pool)
# if do_norm:
# # temp = ((spects[i] / self._space_norms[i]) *
# # my_fft_shift(self._fft(coeffs[i], local_fft)))
# temp = ((self._spectrograms[i] / self._space_norms[i]) *
# my_fft_shift(self._fft(coeffs[i])))
# else:
# # temp = (self._spectrograms[i] *
# # my_fft_shift(self._fft(coeffs[i], local_fft)))
# temp = (self._spectrograms[i] *
# my_fft_shift(self._fft(coeffs[i])))
# with pool.critical:
# result[:] += temp
pool.map(work, list(range(len(self._spectrograms))))
# return ifft2(my_ifft_shift(result))
return self._ifft(my_ifft_shift(result))
def tf_trainAndVal(idxData, prlog, train_flag, resultDir, trainDataShared,
spcl):
# creat shared memory variable
# data_shared, dataLabel_shared, data_shared_stepInfo, data_shared_stateInfo = sharedDataCreat(batch_size, batchBlockNum, global_step_local=train_flag[0])
read_timer_batch_org = sharedmem.full(idxData.maxStep * 2,
[-1] * idxData.maxStep * 2,
dtype=np.float64)
read_timer_batch = np.frombuffer(read_timer_batch_org,
dtype=np.float64).reshape(
idxData.maxStep, 2)
# creat a processor to manage reading data simutanously
Process(target=readData,
args=(trainDataShared, idxData, spcl),
kwargs={
'mode': 'train',
'read_timer_batch': read_timer_batch
}).start()
# a process to clean cache simutanously, for performance testing
if cleanCacheFlag == 1:
Process(target=cleanMem,
args=(trainDataShared.step, idxData.maxStep, spcl)).start()
# f*****g train now
tfTrain(trainDataShared, idxData, prlog, train_flag, resultDir, spcl)
# terminate reading processors, # stop all children process
spcl.passiveFinishAll()
def generate_roadmap_parallel(samples, env, max_dist, leafsize, knn):
"""Parallelized roadmap generator """
n_sample = len(samples)
leafsize = knn
if len(samples) < leafsize: leafsize = len(samples) - 1
import sharedmem
sample_ids = np.arange(n_sample, dtype='i')
roadmap = sharedmem.full((n_sample, knn), 0)
# Start multi processing over samples
with sharedmem.MapReduce() as pool:
if n_sample % sharedmem.cpu_count() == 0:
chunksize = n_sample / sharedmem.cpu_count()
else:
chunksize = n_sample / sharedmem.cpu_count() + 1
def work(i):
skdtree = KDTree(samples, leafsize=leafsize)
sub_sample_ids = sample_ids[slice(i, i + chunksize)]
for j, sub_sample_id in enumerate(sub_sample_ids):
x = samples[sub_sample_id]
try:
inds, dists = skdtree.search(x, k=leafsize)
except:
print "skdtree search failed"
sys.exit()
edge_id = []
append = edge_id.append
for ii, (ind, dist) in enumerate(zip(inds, dists)):
if dist > max_dist: break # undirected
if len(edge_id) >= knn: break # directed?
append(ind)
# to complement fewer number of edges for vectorized valueiteration
if len(edge_id) < knn:
for ii in range(0, len(inds)):
#for ind in edge_id:
# edge_id.append(ind)
# if len(edge_id) >= knn: break
append(inds[0])
if len(edge_id) >= knn: break
assert len(
edge_id
) <= leafsize, "fewer leaves than edges {} (dists={})".format(
len(edge_id), dists[:len(edge_id)])
for k in range(len(edge_id)):
roadmap[sub_sample_id][k] = edge_id[k]
pool.map(work, range(0, n_sample, chunksize)) #, reduce=reduce)
# convert sharedmem array to list
roadmap = np.array(roadmap).astype(int)
skdtree = None #KDTree(samples, leafsize=leafsize)
return roadmap.tolist(), skdtree
def test_create():
a = sharedmem.empty(100, dtype='f8')
l = list(a)
b = sharedmem.empty_like(a)
assert b.shape == a.shape
b = sharedmem.empty_like(l)
assert b.shape == a.shape
c = sharedmem.full(100, 1.0, dtype='f8')
assert c.shape == a.shape
c = sharedmem.full_like(a, 1.0)
assert c.shape == a.shape
def test_create():
a = sharedmem.empty(100, dtype='f8')
l = list(a)
b = sharedmem.empty_like(a)
assert b.shape == a.shape
b = sharedmem.empty_like(l)
assert b.shape == a.shape
c = sharedmem.full(100, 1.0, dtype='f8')
assert c.shape == a.shape
c = sharedmem.full_like(a, 1.0)
assert c.shape == a.shape
def create(self, sampleShape_local, labelShape_local, batch_size_local,
batchPoolSize_local, batchBlockNum_local, latestStep):
datashape = [batchPoolSize_local, batch_size_local] + sampleShape_local
labelshape = [batchPoolSize_local, batch_size_local] + labelShape_local
self.data = np.frombuffer(sharedmem.empty(datashape, dtype=np.float32),
dtype=np.float32).reshape(datashape)
self.label = np.frombuffer(sharedmem.empty(labelshape, dtype=np.int64),
dtype=np.int64).reshape(labelshape)
self.step = np.frombuffer(sharedmem.full(
batchPoolSize_local, [latestStep] * batchPoolSize_local,
dtype=np.int64),
dtype=np.int64)
# 0: ready to use, 1: used, 2:full of updating, 3 updating conti.
self.state = np.frombuffer(sharedmem.full(
batchPoolSize_local * (batchBlockNum_local + 1),
[1] * batchPoolSize_local * (batchBlockNum_local + 1),
dtype=np.int8),
dtype=np.int8).reshape(
batchBlockNum_local + 1,
batchPoolSize_local)
def inverse_transform(self,
coeffs,
real=False,
do_norm=True,
total_cores=None):
if total_cores is None:
total_cores = self.total_cores
# result = sm.full((self.height, self.width), 0, dtype='complex128')
result = sm.full((self.height, self.width), 0, dtype=DTYPES[1])
with sharedmem_pool(total_cores) as pool:
def work(i):
# local_fft = make_fft(self.width, self.height)
# coeff_f = my_fft_shift(self._fft(coeffs[i], local_fft))
self.add_inverse_part(coeffs[i], i, result, do_norm, pool)
# coeff_fourier = my_fft_shift(self._fft(coeffs[i]))
# assert coeff_fourier is not None # silence pyflakes
# spec = self._spectrograms[i]
# dual_w = self.dual_frame_weight
# temp = np.zeros((self.height, self.width),
# dtype=DTYPES[1])
# if do_norm:
# norm = self._space_norms[i]
# ne.evaluate('spec * norm * coeff_fourier / dual_w',
# out=temp)
# else:
# ne.evaluate('spec * coeff_fourier / dual_w', out=temp)
# with pool.critical:
# result[:] += temp
pool.map(work, list(range(len(self._spectrograms))))
result = np.array(result)
if real:
# return np.real(ifft2(my_ifft_shift(result)))
return np.real(self._ifft(my_ifft_shift(result)))
else:
# return ifft2(my_ifft_shift(result))
return self._ifft(my_ifft_shift(result))
def readData(dataShared, idxData, spcl, **kwargs):
lock = Lock()
mode = kwargs['mode']
spcl.start(mode, -1) # the read manager will be record in the final row
print '%s read manager - %d - mother start' % (mode, os.getpid())
if mode == 'train':
readNum = readProcessorNum
elif mode == 'val' or mode == 'test':
readNum = eval_readProcessorNum
# a flag which tell the readData manager that the fill slaver is activated.
token = sharedmem.full(readNum, [0] * readNum, dtype=np.int8)
manager = [[]] * readNum
for num in range(readNum):
manager[num] = Process(target=fillData,
args=(lock, dataShared, idxData, spcl, num,
readNum, token),
kwargs=kwargs)
manager[num].start()
for num in range(readNum):
manager[num].join()
while True:
if sum(token) == readNum and spcl.evaluateChild(
os.getpid(), mode, readNum) == 'dead':
break
time.sleep(1)
spcl.activeFinish(mode, -1)
print '%s read manager - %d - mother dead' % (mode, os.getpid())
print np.concatenate(
(spcl.table[0, :, :], spcl.table[1, :, :], spcl.table[2, :, :]), 1)
def get_features_from_states(env, states, feature_fn):
import sharedmem
n_states = len(states)
feat_len = len(feature_fn(env, states[0]))
state_ids = np.arange(n_states, dtype='i')
features = sharedmem.full((n_states, feat_len), 0.)
# Start multi processing over support states
with sharedmem.MapReduce() as pool:
if n_states % sharedmem.cpu_count() == 0:
chunksize = n_states / sharedmem.cpu_count()
else:
chunksize = n_states / sharedmem.cpu_count() + 1
def work(i):
s_ids = state_ids[slice(i, i + chunksize)]
for j, s_id in enumerate(s_ids):
s = states[s_id] # state id in states
features[s_id] = feature_fn(env, s)
pool.map(work, range(0, n_states, chunksize)) #, reduce=reduce)
return np.array(features)
def __init__(self, size=1, lock=None):
super().__init__(lock)
self.size = size
self.index = sharedmem.full(1, 0, int)
def generate_graph(PB, diff_l, diff_h):
print('generating graph...')
#getcontext().prec = 6
#ttl_path=Decimal(0.0)
ttl_path = Decimal((2**(timestamp - 2) - 1))
#eps_edges=np.float64(0.0)
'''
for idx in range(2,timestamp+1):
#if is_in_bound(1,idx,idx-1,diff_l,diff_h):
ttl_path+= Decimal(find_total_path_with_edge(1, 1, idx, 1, timestamp))
'''
lock1 = mp.Lock()
lock2 = mp.Lock()
lock3 = mp.Lock()
#for i in range(num_timeseries):
# lock3.append(mp.Lock())
manager = mp.Manager()
#ttl_Prest = np.array([Decimal(0)]*num_timeseries,dtype=np.dtype(Decimal))
ttl_Prest = np.zeros(num_timeseries, dtype=np.float128)
#print('total prest type')
#print(type(ttl_Prest[0]))
#print(ttl_Prest.shape)
#'''
ttl_nodes = timestamp * timestamp
num_cores = mp.cpu_count()
if num_cores > ttl_nodes:
n_jobs = ttl_nodes
else:
n_jobs = num_cores
batch_size = int(ttl_nodes / n_jobs)
node_num = 1
start = time.time()
for batch in range(batch_size):
job = []
num_path = mp.Value('i', 0)
Prest = shm.full(num_timeseries, 0, dtype=np.float128)
#Prest = shm.full(num_timeseries,Decimal(0),dtype=np.dtype(Decimal))
#print('prest type when created:')
#print(type(Prest[0]))
for idx in range(node_num, min(node_num + n_jobs, ttl_nodes)):
node_start = int(idx / timestamp + 1)
node_end = int(idx % timestamp + 1)
if node_start >= node_end: #or not is_in_bound(node_start,node_end,node_end-node_start,diff_l,diff_h):
continue
#firstend=node_start+diff_l
#print(node_start,node_end,firstend,diff_l)
#if (node_end-firstend)<diff_l:
# continue
key = str(node_start) + '-' + str(node_end)
print(key)
p = mp.Process(target=create_graph_path,
args=(key, node_start, node_end, PB, Prest, idx,
timestamp, lock1, lock3, ttl_path))
job.append(p)
p.start()
node_num += n_jobs
_ = [p.terminate() for p in job]
_ = [p.join() for p in job]
#ttl_path+=num_path.value
ttl_Prest = np.add(ttl_Prest, Prest)
#print('process Prest:')
#print(Prest)
#'''
stop = time.time()
comp_time = stop - start
print('time: %.4f' % comp_time)
#print('Prest:')
#print(ttl_Prest)
#print('nodes')
#print(graph_nodes.keys())
return ttl_Prest, ttl_path, comp_time
def create(self, readProcessorNum, eval_readProcessorNum):
lenth = max(readProcessorNum, eval_readProcessorNum) + 1
self.table = np.frombuffer(sharedmem.full(lenth * 3 * 4,
[-1] * lenth * 3 * 4,
dtype=np.int64),
dtype=np.int64).reshape(3, lenth, 4)
# CNN-training parametes
# os.environ["CUDA_VISIBLE_DEVICES"]="1"
num_epochs = 20
SEED = 66478
num_label_classes = 2
max_keep = 5
loss_frequency = 200 # how much steps to print the loss infomation
def data_type():
return tf.float32
# early stop parametes
train_flag = sharedmem.full(1, [-1], dtype=np.int64)
initialLR = 0.01
reTryNum = 5
decentRate = 0.5
stopLR = initialLR * (decentRate**reTryNum)
def main():
for vp in range(10):
for cross in range(10):
# setup result output path
if not os.path.isdir(os.path.join(projectPath,
'View{}'.format(vp))):
os.makedirs(os.path.join(projectPath, 'View{}'.format(vp)))
def computeQ(mdp, support_states, error=1e-10, support_features=None, support_feature_state_dict=None,
cstr_fn=None, add_no_cstr=True, max_cnt=100, **kwargs):
"""Compute Q using multi-process
"""
# initialization of variables
n_support_states = len(support_states)
n_actions, n_states = mdp.n_actions, mdp.n_states
eps = np.finfo(float).eps
roadmap = mdp.roadmap
states = mdp.states
gamma = mdp.gamma
T = mdp.T
rewards = mdp.get_rewards()
#from IPython import embed; embed(); sys.exit()
if rewards is None: rewards=np.zeros(len(mdp.states))
else:
rewards = np.array(rewards)
rewards[np.where(rewards>0)]=0.
support_state_ids = np.arange(n_support_states, dtype='i')
if support_features is not None:
support_feature_ids, support_feature_values = support_features
computed_f_id = sharedmem.full(len(support_feature_values), False, dtype='b')
else:
return NotImplementedError
if cstr_fn is None:
support_q_mat = sharedmem.full((n_support_states, mdp.n_states, mdp.n_actions), 0.)
support_values = sharedmem.full((n_support_states, mdp.n_states), 0.)
support_validity = sharedmem.full((n_support_states), True)
else:
if add_no_cstr: n_cstr_fn = len(cstr_fn) + 1
else: n_cstr_fn = len(cstr_fn)
support_q_mat = sharedmem.full((n_support_states, n_cstr_fn, mdp.n_states,
mdp.n_actions), 0.)
support_values = sharedmem.full((n_support_states, n_cstr_fn, mdp.n_states), 0.)
support_validity = sharedmem.full((n_support_states, n_cstr_fn), True)
if len(cstr_fn)>0:
feat_map = kwargs['feat_map']
roadmap = kwargs['roadmap']
states = mdp.states
cstr_T = []
for i in range(len(cstr_fn)):
validity_map = cstr_fn[i](None, f=feat_map)[roadmap]
validity_map[:,0] = True
Tc = mdp.T*validity_map[:,np.newaxis,:]
Tc[:,:,0] = eps
sum_T = np.sum(Tc, axis=-1)
Tc /= sum_T[:,:,np.newaxis]
cstr_T.append(Tc)
# Start multi processing over support states
with sharedmem.MapReduce() as pool:
if n_support_states % sharedmem.cpu_count() == 0:
chunksize = n_support_states / sharedmem.cpu_count()
else:
chunksize = n_support_states / sharedmem.cpu_count() + 1
def work(i):
state_ids = support_state_ids[slice (i, i + chunksize)]
new_rewards = copy.copy(rewards)
values = np.zeros(n_states)
for j, state_id in enumerate(state_ids):
s = support_states[state_id] # state id in states
# vi agent
mdp = vi.valueIterAgent(n_actions, n_states,
roadmap, None, states,
gamma=gamma, T=T)
mdp.set_goal(s)
if support_feature_ids is None:
if new_rewards[s] >= 0.:
new_rewards[s] = 1.
else:
# find all states that gives f_g in states
f_id = support_feature_ids[state_id]
goal_state_ids = support_feature_state_dict[f_id]
if computed_f_id[f_id]:
continue
else:
computed_f_id[f_id] = True
new_rewards[goal_state_ids] = 1.
mdp.set_rewards(new_rewards)
# Store q_mat and validity mat per state
if cstr_fn is not None:
for k in range(len(cstr_fn)):
# check if the goal is isolated
if np.sum(cstr_T[k][goal_state_ids])>0.:
values, param_dict = mdp.solve_mdp(error, init_values=values,
T=cstr_T[k], max_cnt=max_cnt,
goal=s,
return_params=True)
support_q_mat[state_id][k] = param_dict['q']
support_validity[state_id][k] = cstr_fn[k](s)
support_values[state_id][k] = values
if add_no_cstr:
values, param_dict = mdp.solve_mdp(error, init_values=values,
T=T, max_cnt=max_cnt,
## goal=s,
return_params=True)
support_q_mat[state_id][-1] = param_dict['q']
support_validity[state_id][-1] = True
support_values[state_id][-1] = values
else:
values, param_dict = mdp.solve_mdp(error, init_values=values,
max_cnt=max_cnt,
return_params=True)
support_q_mat[state_id] = param_dict['q']
support_values[state_id] = values
# find all states that gives f_g in states
for gs in goal_state_ids:
k = support_states.index(gs)
if k!=state_id:
support_q_mat[k] = support_q_mat[state_id]
# reset
## new_rewards = copy.copy(rewards)
if support_feature_ids is None:
new_rewards[s] = 0.
else:
new_rewards[goal_state_ids] = 0.
pool.map(work, range(0, n_support_states, chunksize))#, reduce=reduce)
# convert sharedmem array to dict
support_q_mat_dict = {}
support_values_dict = {}
support_validity_dict = {}
for i, s in enumerate(support_states):
support_q_mat_dict[s] = np.array(support_q_mat[i])
support_values_dict[s] = np.array(support_values[i])
if cstr_fn is not None:
support_validity_dict[s] = np.array(support_validity[i])
if cstr_fn is not None:
return support_q_mat_dict, support_values_dict, support_validity_dict
else:
return support_q_mat_dict, support_values_dict
def main():
"""
Main entry point for the application when run from the command line.
"""
# Timing instanciation
timing = Timing(['Snippet', 'Time [m]', 'Time [s]'])
with timing.timeit_context_add('Pre-processing'):
# Setup parse options command line
current_path = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parser = args.setup_parser(current_path + '/args/coarsening.json')
options = parser.parse_args()
args.update_json(options)
args.check_output(options)
# Log instanciation
log = helper.initialize_logger(dir='log', output='log')
if options.input and options.vertices is None:
log.warning('Vertices are required when input is given.')
sys.exit(1)
# Create default values for optional parameters
if options.reduction_factor is None:
options.reduction_factor = [0.5] * len(options.vertices)
if options.max_levels is None:
options.max_levels = [3] * len(options.vertices)
if options.matching is None:
options.matching = ['rgmb'] * len(options.vertices)
if options.similarity is None:
options.similarity = ['weighted_common_neighbors'] * len(
options.vertices)
if options.itr is None:
options.itr = [10] * len(options.vertices)
if options.upper_bound is None:
options.upper_bound = [2.0] * len(options.vertices)
if options.global_min_vertices is None:
options.global_min_vertices = [None] * len(options.vertices)
if options.tolerance is None:
options.tolerance = [0.01] * len(options.vertices)
# Validation of list values
if len(options.reduction_factor) == 1:
options.reduction_factor = [options.reduction_factor[0]] * len(
options.vertices)
if len(options.max_levels) == 1:
options.max_levels = [options.max_levels[0]] * len(
options.vertices)
if len(options.matching) == 1:
options.matching = [options.matching[0]] * len(options.vertices)
if len(options.similarity) == 1:
options.similarity = [options.similarity[0]] * len(
options.vertices)
if len(options.itr) == 1:
options.itr = [options.itr[0]] * len(options.vertices)
if len(options.upper_bound) == 1:
options.upper_bound = [options.upper_bound[0]] * len(
options.vertices)
if len(options.global_min_vertices) == 1:
options.global_min_vertices = [options.global_min_vertices[0]
] * len(options.vertices)
if len(options.tolerance) == 1:
options.tolerance = [options.tolerance[0]] * len(options.vertices)
# Verification of the dimension of the parameters
if len(options.vertices) != len(options.reduction_factor):
log.warning('Sizes of input arguments -v and -r do not match.')
sys.exit(1)
if len(options.vertices) != len(options.max_levels):
log.warning('Sizes of input arguments -v and -m do not match.')
sys.exit(1)
if len(options.vertices) != len(options.matching):
log.warning('Sizes of input arguments -v and -c do not match.')
sys.exit(1)
if len(options.vertices) != len(options.similarity):
log.warning('Sizes of input arguments -v and -s do not match.')
sys.exit(1)
if len(options.vertices) != len(options.itr):
log.warning('Size of input arguments -v and -i do not match.')
sys.exit(1)
if len(options.vertices) != len(options.upper_bound):
log.warning('Size of input arguments -v and -ub do not match.')
sys.exit(1)
if len(options.vertices) != len(options.global_min_vertices):
log.warning('Size of input arguments -v and -gmv do not match.')
sys.exit(1)
if len(options.vertices) != len(options.tolerance):
log.warning('Size of input arguments -v and -t do not match.')
sys.exit(1)
# Validation of matching method
valid_matching = ['rgmb', 'gmb', 'mlpb', 'hem', 'lem', 'rm']
for index, matching in enumerate(options.matching):
matching = matching.lower()
if matching not in valid_matching:
log.warning('Matching ' + matching + ' method is unvalid.')
sys.exit(1)
options.matching[index] = matching
# Validation of sedd priority
valid_seed_priority = ['strength', 'degree', 'random']
for index, seed_priority in enumerate(options.seed_priority):
seed_priority = seed_priority.lower()
if seed_priority not in valid_seed_priority:
log.warning('Seed priotiry ' + seed_priority + ' is unvalid.')
sys.exit(1)
options.seed_priority[index] = seed_priority
# Validation reverse
for index, reverse in enumerate(options.reverse):
if reverse.lower() in ('yes', 'true', 't', 'y', '1'):
options.reverse[index] = True
elif reverse.lower() in ('no', 'false', 'f', 'n', '0'):
options.reverse[index] = False
else:
log.warning('Boolean value expected in -rv.')
sys.exit(1)
# isinstance(data[i][k], bool)
# Validation of similarity measure
valid_similarity = [
'common_neighbors', 'weighted_common_neighbors', 'salton',
'preferential_attachment', 'jaccard', 'weighted_jaccard',
'adamic_adar', 'resource_allocation', 'sorensen', 'hub_promoted',
'hub_depressed', 'leicht_holme_newman'
]
for index, similarity in enumerate(options.similarity):
similarity = similarity.lower()
if similarity not in valid_similarity:
log.warning('Similarity ' + similarity + ' misure is unvalid.')
sys.exit(1)
options.similarity[index] = similarity
for layer in range(len(options.vertices)):
if options.matching[layer] in ['rgmb', 'gmb', 'hem', 'lem', 'rm']:
if options.global_min_vertices[layer] is not None:
options.global_min_vertices[layer] = None
text = 'Matching method ' + options.matching[layer]
text += ' (setted in layer '
text += str(layer) + ') does not accept -gmv parameter.'
log.warning(text)
if options.reduction_factor[layer] > 0.5:
options.reduction_factor[layer] = 0.5
text = 'Matching method ' + options.matching[layer]
text += ' (setted in layer '
text += str(layer) + ') does not accept -rf > 0.5.'
log.warning(text)
# Load bipartite graph
with timing.timeit_context_add('Load'):
graph = helpermgraph.load(options.input, options.vertices)
graph['level'] = [0] * graph['layers']
source_ecount = graph.ecount()
# Coarsening
with timing.timeit_context_add('Coarsening'):
hierarchy_graphs = []
hierarchy_levels = []
running = True
while running:
running = False
membership = sharedmem.full(graph.vcount(),
range(graph.vcount()),
dtype='int')
levels = graph['level']
contract = False
processes = []
for layer in range(len(graph['vertices'])):
matching_layer = True
if (options.global_min_vertices[layer] is None):
if levels[layer] >= options.max_levels[layer]:
matching_layer = False
elif (graph['vertices'][layer] <=
options.global_min_vertices[layer]):
matching_layer = False
if matching_layer:
contract = True
running = True
levels[layer] += 1
graph['similarity'] = getattr(
Similarity(graph, graph['adjlist']),
options.similarity[layer])
start = sum(graph['vertices'][0:layer])
end = sum(graph['vertices'][0:layer + 1])
vertices = range(start, end)
param = dict(
reduction_factor=options.reduction_factor[layer])
if options.matching[layer] in ['mlpb', 'gmb', 'rgmb']:
param['vertices'] = vertices
param['reverse'] = options.reverse[layer]
if options.matching[layer] in ['mlpb', 'rgmb']:
param['seed_priority'] = options.seed_priority[layer]
if options.matching[layer] in ['mlpb']:
param['upper_bound'] = options.upper_bound[layer]
param['n'] = options.vertices[layer]
param[
'global_min_vertices'] = options.global_min_vertices[
layer]
param['tolerance'] = options.tolerance[layer]
param['itr'] = options.itr[layer]
if options.matching[layer] in ['hem', 'lem', 'rm']:
one_mode_graph = graph.weighted_one_mode_projection(
vertices)
matching_method = getattr(one_mode_graph,
options.matching[layer])
else:
matching_method = getattr(graph,
options.matching[layer])
processes.append(
Process(target=matching_method,
args=[membership],
kwargs=param))
for p in processes:
p.start()
for p in processes:
p.join()
if contract:
coarse = graph.contract(membership)
coarse['level'] = levels
if coarse.vcount() == graph.vcount():
break
graph = coarse
if options.save_hierarchy or not running:
hierarchy_graphs.append(graph)
hierarchy_levels.append(levels[:])
if not hierarchy_graphs:
hierarchy_graphs.append(graph)
hierarchy_levels.append(levels[:])
# Save
with timing.timeit_context_add('Save'):
output = options.output
for index, obj in enumerate(zip(hierarchy_levels, hierarchy_graphs)):
levels, graph = obj
index += 1
if options.save_conf or options.show_conf:
d = {}
d['source_input'] = options.input
d['source_vertices'] = [
options.vertices[0], options.vertices[1]
]
d['source_vcount'] = options.vertices[0] + options.vertices[1]
d['source_ecount'] = source_ecount
d['ecount'] = graph.ecount()
d['vcount'] = graph.vcount()
d['vertices'] = graph['vertices']
d['reduction_factor'] = options.reduction_factor
d['max_levels'] = options.max_levels
d['achieved_levels'] = graph['level']
d['similarity'] = options.similarity
d['matching'] = options.matching
d['level'] = levels
d['upper_bound'] = options.upper_bound
d['global_min_vertices'] = options.global_min_vertices
d['itr'] = options.itr
if options.save_conf:
with open(output + '-' + str(index) + '.conf', 'w+') as f:
json.dump(d, f, indent=4)
if options.show_conf:
print(json.dumps(d, indent=4))
if options.save_ncol:
graph.write(output + '-' + str(index) + '.ncol', format='ncol')
if options.save_source:
with open(output + '-' + str(index) + '.source', 'w+') as f:
for v in graph.vs():
f.write(' '.join(map(str, v['source'])) + '\n')
if options.save_membership:
membership = [0] * (options.vertices[0] + options.vertices[1])
for v in graph.vs():
for source in v['source']:
membership[source] = v.index
numpy.savetxt(output + '-' + str(index) + '.membership',
membership,
fmt='%d')
if options.save_predecessor:
with open(output + '-' + str(index) + '.predecessor',
'w+') as f:
for v in graph.vs():
f.write(' '.join(map(str, v['predecessor'])) + '\n')
if options.save_successor:
numpy.savetxt(output + '-' + str(index) + '.successor',
graph.vs['successor'],
fmt='%d')
if options.save_weight:
numpy.savetxt(output + '-' + str(index) + '.weight',
graph.vs['weight'],
fmt='%d')
if options.save_gml:
del graph['adjlist']
del graph['similarity']
graph['layers'] = str(graph['layers'])
graph['vertices'] = ','.join(map(str, graph['vertices']))
graph['level'] = ','.join(map(str, graph['level']))
graph.vs['name'] = map(str, range(0, graph.vcount()))
graph.vs['type'] = map(str, graph.vs['type'])
graph.vs['weight'] = map(str, graph.vs['weight'])
graph.vs['successor'] = map(str, graph.vs['successor'])
for v in graph.vs():
v['source'] = ','.join(map(str, v['source']))
v['predecessor'] = ','.join(map(str, v['predecessor']))
graph.write(output + '-' + str(index) + '.gml', format='gml')
if not options.save_hierarchy:
break
if options.show_timing:
timing.print_tabular()
if options.save_timing_csv:
timing.save_csv(output + '-timing.csv')
if options.save_timing_json:
timing.save_json(output + '-timing.json')
import time
import numpy as np
import sharedmem
from multiprocessing import Process, Manager
def function(counts, lock):
with lock:
if counts[0]['foo'] == 0:
time.sleep(1)
counts[0]['foo'] += 1
processes = []
manager = Manager()
counts = sharedmem.full(1, (0, ), dtype=[('foo', 'i1')])
lock = manager.Lock()
for i in range(2):
processes.append(Process(target=function, args=(counts, lock)))
processes[-1].start()
for process in processes:
process.join()
print(counts[0]['foo'])
def __init__(self,
path: str,
log1p: Optional[bool] = False,
nproc: Optional[int] = 1,
selection: Optional[list] = None,
silent: Optional[bool] = False) -> None:
"""
PyTorch Dataset wrapper to handle the GSE93421 hdf5 dataset
:param path: path to hdf5 file for e18 Mouse Data
:param nproc: number of processes to use in contructing vectors
:param ratio: ratio of the dataset to actually load
:param selection: list of cells to load data for
:param silent: whether print statements should print
"""
hdf5 = h5py.File(path, 'r', driver='core')
self.dims = len(hdf5['mm10']['genes'])
# allow a customizable selection of cells
if selection is not None:
self._len = len(selection)
else:
self._len = len(hdf5['mm10']['indptr'])
selection = range(0, self._len)
self.selection = selection
# get a list that can be shared between processes
selected_cells = sm.empty(self._len, dtype=np.int)
for i in range(0, self._len):
selected_cells[i] = self.selection[i]
#self.cells = sm.full((self._len,self.dims),0,dtype=np.int16)
# Load all of the important information into memory
#############
# Data
#############
if not silent: print("Reading data ...")
start = time()
ds = hdf5['mm10']['data']
data = sm.empty(len(ds), dtype=ds.dtype)
ds.read_direct(data)
tmp = ds.dtype
end = time()
if not silent: print("\t" + str(end - start) + "s ...")
#############
# Indices
#############
if not silent: print("Reading indices ...")
start = time()
ds = hdf5['mm10']['indices']
indx = sm.empty(len(ds), dtype=ds.dtype)
ds.read_direct(indx)
end = time()
if not silent: print("\t" + str(end - start) + "s ...")
#############
# Indptr
#############
if not silent: print("Reading indptr ...")
start = time()
ds = hdf5['mm10']['indptr']
iptr = sm.empty(len(ds), dtype=ds.dtype)
ds.read_direct(iptr)
end = time()
if not silent: print("\t" + str(end - start) + "s ...")
hdf5.close()
del ds
###########################
# Create empty cell vectors
###########################
# build the vector foreach cell
if not silent: print("Creating 0 vectors ...")
start = time()
self.data = sm.full((self._len, self.dims), 0, dtype=tmp)
#self.cells = sm.full((self._len,self.dims),0,dtype=float)
end = time()
if not silent: print("\t" + str(end - start) + "s ...")
###########################
# Multi-core loading ...
###########################
if not silent: print("Building Tensor List ...")
start = time()
with sm.MapReduce(np=nproc) as pool:
pool.map(
_build_tensor,
list(
zip([self.data] * nproc, [iptr] * nproc, [indx] * nproc,
[data] * nproc, range(0, nproc), [nproc] * nproc,
[selected_cells] * nproc, [log1p] * nproc)))
end = time()
if not silent: print("\t" + str(end - start) + "s ...")
# Some explicit cleanup to conserve memory
# Not sure if necessary, but I don't trust Python
del iptr
del indx
del data
del selected_cells
def computePolicies(mdp, goal_states, error=1e-10):
"""Compute Q using multi-process
"""
# initialization of variables
n_goal_states = len(goal_states)
n_actions, n_states = mdp.n_actions, mdp.n_states
roadmap = mdp.roadmap
states = mdp.states
gamma = mdp.gamma
T = mdp.T
rewards = mdp.get_rewards()
#from IPython import embed; embed(); sys.exit()
if rewards is None: rewards=np.zeros(len(mdp.states))
else:
rewards = np.array(rewards)
rewards[np.where(rewards>0)]=0.
goal_state_ids = np.arange(n_goal_states, dtype='i')
goal_policy_mat = sharedmem.full((n_goal_states, mdp.n_states, mdp.n_actions), 0.)
## goal_values = sharedmem.full((n_goal_states, mdp.n_states), 0.)
## goal_validity = sharedmem.full((n_goal_states), True)
# Start multi processing over goal states
with sharedmem.MapReduce() as pool:
if n_goal_states % sharedmem.cpu_count() == 0:
chunksize = n_goal_states / sharedmem.cpu_count()
else:
chunksize = n_goal_states / sharedmem.cpu_count() + 1
def work(i):
state_ids = goal_state_ids[slice (i, i + chunksize)] #0,1,2,3
new_rewards = copy.copy(rewards)
values = np.zeros(n_states)
for j, goal_state_id in enumerate(state_ids):
s = goal_states[goal_state_id] # state id in states
## s_idx = states.index(s)
# vi agent
mdp = vi.valueIterAgent(n_actions, n_states,
roadmap, None, states,
gamma=gamma, T=T)
mdp.set_goal(s)
if new_rewards[s] >= 0.:
new_rewards[s] = 1.
mdp.set_rewards(new_rewards)
# Store q_mat and validity mat per state
policy, _ = mdp.find_policy(error)
goal_policy_mat[goal_state_id] = policy
# find all states that gives f_g in states
for k, gs in enumerate(goal_states):
if s == gs:
goal_policy_mat[k] = policy
# reset
new_rewards[s] = 0.
pool.map(work, range(0, n_goal_states, chunksize))#, reduce=reduce)
# convert sharedmem array to list
policies = []
for i in range(n_goal_states):
policies.append( np.array(goal_policy_mat[i]) )
return policies
