def parallel_computation(values,
ranks,
to_compute,
output_file,
threads=1,
mode="all2all"):
"""
Create shared variables and pool for multiprocessing.
Params: logcounts and sigma numpy array, to_compute list with the index and
the number of processeurs to use.
Return: all the correlations and index
"""
# Define the corresponding shared ctype arrays
# Create ctypes object from numpy array
tmp_values = np.ctypeslib.as_ctypes(values)
shared_values = sharedctypes.Array(tmp_values._type_,
tmp_values,
lock=False)
# Returns a ctypes array allocated from shared memory.
# tmp_dhs._type_: type of the returned array elements
# tmp_dhs: initialize the array
# lock=True (default): synchronize access to the value
# lock=False: access to returned object not protected
tmp_ranks = np.ctypeslib.as_ctypes(ranks)
shared_ranks = sharedctypes.Array(tmp_ranks._type_, tmp_ranks, lock=False)
# we need to define a partial function here
temp_file = tempfile.NamedTemporaryFile(delete=False)
with open(temp_file.name, "w") as temp_fh:
with Pool(processes=threads,
initializer=_init_parallel,
initargs=(
shared_values,
shared_ranks,
)) as pool:
# controls a pool of worker processes
# processes: number of worker processes to use
# initializer is not None: each worker process will
# call initializer(*initargs)
# we split the elements to compute in batchs of size 50000
launch_para_correlations = partial(parallel_correlations,
mode=mode)
num_pass = 0
for group_to_compute in grouping(50000, to_compute):
num_pass += 1
eprint("Pass %d" % num_pass)
nested_result = pool.map(launch_para_correlations,
group_to_compute, 100)
# supports only one iterable argument,
# blocks until the result is ready
result = [
item for sublist in nested_result for item in sublist
]
dump_correlations(result, temp_fh, mode)
sort_and_write(temp_file.name, output_file, mode)
return 0
def __init__(self,
update_data_func=None,
state_transition_func=None,
thermostat=False):
"""Create variables used to send in packets to the roaster. The update
data function is called when a packet is opened. The state transistion
function is used by the timer thread to know what to do next. See wiki
for more information on packet structure and fields."""
self.update_data_func = update_data_func
self.state_transition_func = state_transition_func
self._header = sharedctypes.Array('c', b'\xAA\xAA')
self._temp_unit = sharedctypes.Array('c', b'\x61\x74')
self._flags = sharedctypes.Array('c', b'\x63')
self._current_state = sharedctypes.Array('c', b'\x02\x01')
self._footer = b'\xAA\xFA'
self._fan_speed = sharedctypes.Value('i', 1)
self._heat_setting = sharedctypes.Value('i', 0)
self._target_temp = sharedctypes.Value('i', 150)
self._current_temp = sharedctypes.Value('i', 150)
self._time_remaining = sharedctypes.Value('i', 0)
self._total_time = sharedctypes.Value('i', 0)
self._cont = sharedctypes.Value('i', 1)
if (thermostat is True):
self.thermostat_process = mp.Process(target=self.thermostat)
self.thermostat_process.start()
def main():
workers = []
task_queue = Queue()
result_queue = Queue()
# input data
n = 10000
m = 100
X_in = np.random.rand(n, m)
size_in = X_in.size
shape_in = X_in.shape
X_in.shape = size_in
X_ctypes_in = sharedctypes.RawArray(ctypes.c_double, X_in)
X_in = np.frombuffer(X_ctypes_in, dtype=np.float64, count=size_in)
X_in.shape = shape_in
# output data
X_out = np.random.rand(n, n)
size_out = X_out.size
shape_out = X_out.shape
X_out.shape = size_out
X_ctypes_out = sharedctypes.Array(ctypes.c_double, X_out)
#X_out = np.frombuffer(X_ctypes_out, dtype=np.float64, count=size_out)
#X_out.shape = shape_out
# create worker and start
concurrency = 4
unit = n / 4
for i in xrange(concurrency):
worker = Worker(task_queue, result_queue,
X_ctypes_in, shape_in,
X_ctypes_out, shape_out)
worker.start()
workers.append(worker)
# put task
for i in xrange(concurrency):
task_queue.put((i, unit*i, unit*i+unit))
pass
# get result of task
elapsed_times = []
while True:
elapsed_time = result_queue.get()
elapsed_times.append(elapsed_time)
if len(elapsed_times) == concurrency:
break
# stop worker
for i in xrange(concurrency):
workers[i].terminate()
# do math for elapsed time
print "Elapsed time {} [s]".format(np.max(elapsed_times))
def test_with_shared(N, M, complexity):
x = ones(N)
x_sct = sharedctypes.Array(ctypes.c_double, x, lock=False)
A = 1 + arange(M)
#pool = multiprocessing.Pool()
start = time.time()
processes = [
multiprocessing.Process(target=test_with_shared_func,
args=(x_sct, a, complexity)) for a in A
]
for p in processes:
p.start()
for p in processes:
p.join()
end = time.time()
t_multiprocessing = end - start
results = []
args = [(x, a, complexity) for a in A]
start = time.time()
for x, a, complexity in args:
results.append(test_with_shared_func(x, a, complexity))
end = time.time()
t_single = end - start
print 'Test with shared'
print '----------------'
print 'Single:', t_single
print 'Multiprocessing:', t_multiprocessing
print 'S/M:', t_single / t_multiprocessing
def parallel_implementation_corr(logcounts,
sigma,
ranks,
to_compute,
threads=1):
"""
Create shared variables and pool for multiprocessing.
Params: logcounts and sigma numpy array, to_compute list with the index and
the number of processeurs to use.
Return: all the correlations and index
"""
# Define the corresponding shared ctype arrays
# Create ctypes object from numpy array
tmp_logcounts = np.ctypeslib.as_ctypes(logcounts)
shared_logcounts = sharedctypes.Array(tmp_logcounts._type_,
tmp_logcounts,
lock=False)
# Returns a ctypes array allocated from shared memory.
# tmp_dhs._type_: type of the returned array elements
# tmp_dhs: initialize the array
# lock=True (default): synchronize access to the value
# lock=False: access to returned object not protected
tmp_sigma = np.ctypeslib.as_ctypes(sigma)
shared_sigma = sharedctypes.Array(tmp_sigma._type_, tmp_sigma, lock=False)
tmp_ranks = np.ctypeslib.as_ctypes(ranks)
shared_ranks = sharedctypes.Array(tmp_ranks._type_, tmp_ranks, lock=False)
pool = Pool(processes=threads,
initializer=_init_parallel,
initargs=(
shared_logcounts,
shared_sigma,
shared_ranks,
))
# controls a pool of worker processes
# processes: number of worker processes to use
# initializer is not None: each worker process will
# call initializer(*initargs)
result = pool.map(parallel_correlations, to_compute)
# supports only one iterable argument,
# blocks until the result is ready
results = [item for sublist in result for item in sublist]
return results
def np_to_c(np_array):
"""
Converts numpy array to c-type to allow for sharing across different threads
:param np_array:
:return: ctype array
"""
tmp = np.ctypeslib.as_ctypes(np_array)
c_array = sharedctypes.Array(tmp._type_, tmp, lock=False)
return c_array
def main():
"""run radio display"""
parser = argparse.ArgumentParser(description=main.__doc__)
parser.add_argument('--color', default='black')
subparsers = parser.add_subparsers(dest='subparser')
serial_parser = subparsers.add_parser('serial')
serial_parser.add_argument('--device', default='/dev/ttyS0')
serial_parser.add_argument('--timeout', default=10, type=float)
serial_parser.add_argument('--baud', default=115200, type=int)
subparsers.add_parser('keyboard')
args = parser.parse_args()
channel_models = sharedctypes.Array(models.Channel, [
('', 0, '---.-', '!----------') + 6 * (False, ),
('', 0, '---.-', '!----------') + 6 * (False, ),
],
lock=True)
width, height = 480, 320
window = pyglet.window.Window(width=width, height=height)
channel_views, bg, batch = views.build(color=args.color)
@window.event
def on_draw():
window.clear()
zipped = zip(channel_views, channel_models)
for view, model in zipped:
view(model)
batch.draw()
def update(dt):
for view in channel_views:
view.scroll()
pyglet.clock.schedule_interval(update, 1.0 / 25)
stop = threading.Event()
if args.subparser == 'keyboard':
controller = controllers.KeyboardInput(channel_models, stop)
else:
controller = controllers.SerialInput(channel_models,
stop,
port=args.device,
baudrate=args.baud,
timeout=args.timeout)
t = threading.Thread(target=controller)
t.start()
pyglet.app.run()
stop.set()
t.join()
def __init__(self, calculator, nombreThreads):
self.calculator = calculator
self.nombreThreads = nombreThreads
self.nombrePoints = self.calculator.nbPoints
self.nombreCentroides = len(self.calculator.centroides)
#définition des objets partagés entre les threads
self.sharedArrayPoints = sharedctypes.Array(
'd', self.nombrePoints *
self.nombrePoints) #la matrice, convertie en une ligne
#création du dictionnaire contenant les clusters
self.manager = Manager()
self.dictClusters = self.manager.dict()
#initialisation des clusters
for i in range(self.nombreCentroides):
self.dictClusters[i] = []
self.sharedArrayCentroides = sharedctypes.Array(
'd', self.nombreCentroides * self.nombrePoints
) #la matrice de centroides, ceonvertie en une ligne
#définition de la matrice de centroides
self.matrCentroides = np.frombuffer(
self.sharedArrayCentroides.get_obj()).reshape(
(self.nombreCentroides, self.nombrePoints))
for i in range(self.nombreCentroides):
self.matrCentroides[i] = self.calculator.centroides[i]
#définition de la matrice des Points
self.matrPoints = np.frombuffer(
self.sharedArrayPoints.get_obj()).reshape(
(self.nombrePoints, self.nombrePoints))
for i in range(self.nombrePoints):
self.matrPoints[i] = self.calculator.matrice[i]
self.nombreEntreesParThread = self.nombrePoints // self.nombreThreads
self.indexes = []
#définition des indexes pour les threads
for i in range(self.nombreThreads):
idxD = i * self.nombreEntreesParThread
if i == self.nombreThreads - 1:
idxF = self.nombrePoints - 1
else:
idxF = ((i + 1) * self.nombreEntreesParThread) - 1
self.indexes.append((idxD, idxF))
def make_common_item(v):
shape = v.shape
mapping = {
numpy.dtype(numpy.float64): ctypes.c_double,
numpy.dtype(numpy.int32): ctypes.c_int,
}
ctype = mapping.get(v.dtype, None)
if ctype is not None:
v = v.flatten()
v = sharedctypes.Array(ctype, v, lock=False)
return v, shape
def processes_start(self):
"""."""
# Create shared memory objects to be shared with worker processes.
arr = self._sofb_current_readback_ref
rbref = _shm.Array(_shm.ctypes.c_double, arr.size, lock=False)
self._sofb_current_readback_ref = _np.ndarray(
arr.shape, dtype=arr.dtype, buffer=memoryview(rbref))
ref = _shm.Array(_shm.ctypes.c_double, arr.size, lock=False)
self._sofb_current_refmon = _np.ndarray(
arr.shape, dtype=arr.dtype, buffer=memoryview(ref))
fret = _shm.Array(_shm.ctypes.c_int, arr.size, lock=False)
self._sofb_func_return = _np.ndarray(
arr.shape, dtype=_np.int32, buffer=memoryview(fret))
# Unit converter.
self.converter = UnitConverter(self._sofb_psnames)
# subdivide the pv list for the processes
nr_bbbs = len(PSSOFB.BBBNAMES)
div = nr_bbbs // self._nr_procs
rem = nr_bbbs % self._nr_procs
sub = [div*i + min(i, rem) for i in range(self._nr_procs+1)]
for i in range(self._nr_procs):
bbbnames = PSSOFB.BBBNAMES[sub[i]:sub[i+1]]
evt = _Event()
evt.set()
theirs, mine = _Pipe(duplex=False)
proc = _Process(
target=PSSOFB._run_process,
args=(self._ethbridge_cls, bbbnames, theirs, evt,
arr.shape, rbref, ref, fret),
daemon=True)
proc.start()
self._procs.append(proc)
self._doneevts.append(evt)
self._pipes.append(mine)
def create_diamond_square_map(self, low_val=0, high_val=100):
"""
Runs DiamondSquare algorithm to get a 2d grid of values.
Because of the algorithm's inherent bias towards the initial values,
expect to see each grid biasing towards the values in the corners.
:param low_val: minimum value that should appear in the resulting grid
:param high_val: maximum value that should appear in the resulting grid
:return: 2d grid of procedurally generated values
"""
mid_value = math.floor((low_val + high_val) / 2)
quarter_value = mid_value / 2
three_quart_value = mid_value + mid_value / 2
seed_options = [
mid_value, mid_value, quarter_value, three_quart_value,
three_quart_value
]
# initialize grid with corners
left = top = 0
right = bottom = self.size - 1
x_center = math.floor((left + right) / 2)
y_center = math.floor((top + bottom) / 2)
init_grid = np.zeros((self.size, self.size))
init_grid[top, left] = random.choice(seed_options)
init_grid[top, right] = random.choice(seed_options)
init_grid[bottom, left] = random.choice(seed_options)
init_grid[bottom, right] = random.choice(seed_options)
tmp = np.ctypeslib.as_ctypes(init_grid.ravel())
shared_array = sharedctypes.Array(tmp._type_, tmp, lock=False)
# I found that there's a nice speed boost if you split the first recursion between four processes
# but anything more than that gave diminishing returns.
with Pool(processes=4, initializer=mp_init,
initargs=(shared_array, )) as pool:
# do first step
self._diamond_square(left, top, right, bottom, mid_value,
shared_array)
pool.map(self._ds_recurse,
[(left, top, x_center, y_center, mid_value),
(left, y_center, x_center, bottom, mid_value),
(x_center, top, right, y_center, mid_value),
(x_center, y_center, right, bottom, mid_value)])
grid = pu.convert_to_np(shared_array, self.size)
return grid
def main():
size = int(10 * 1e6)
array = sharedctypes.Array("f", size)
max_epoch = 150
batch_size = 128
n_samples = int(10 * 10000)
iter_per_epoch = n_samples / batch_size
n_iters = max_epoch * iter_per_epoch
st = time.time()
for i in range(n_iters):
with array.get_lock():
pass
et = time.time() - st
print("ElapsedTime:{} [s]".format(et))
def make_common_item(v):
if isinstance(v, numpy.ndarray):
shape = v.shape
mapping = {
numpy.dtype(numpy.float64): ctypes.c_double,
numpy.dtype(numpy.int32): ctypes.c_int,
}
ctype = mapping.get(v.dtype, None)
if ctype is not None:
log_debug('converting numpy array to common array')
v = v.flatten()
v = sharedctypes.Array(ctype, v, lock=False)
else:
# shape = None means that v is not an array and should not be converted
# back as numpy item
shape = None
return v, shape
def __init__(self, sync_timer, event_trigger = (), event_ignore_tag = None):
"""Initialize UDPConnectionProcess
Parameters
----------
receive_queue: multiprocessing.Queue
the queue to which the received data should be put
peer_ip : string
the IP of the peer to which the connection should be established
sync_clock : Clock
the internal clock for timestamps will synchronized with this clock
event_trigger: multiprocessing.Event() (or list of..)
event trigger(s) to be set. If Udp event is received and it is not a
command to set this event (typical ofsensor recording processes).
event_ignore_tag:
udp data that start with this tag will be ignored for event triggering
""" # todo docu
super(UDPConnectionProcess, self).__init__()
self._sync_timer = sync_timer
self.receive_queue = Queue()
self.send_queue = Queue()
self.event_is_connected = Event()
self._event_stop_request = Event()
self._event_is_polling = Event()
self._ip_address = sharedctypes.Array('c', 'xxx.xxx.xxx.xxx')
self._event_ignore_tag = event_ignore_tag
if isinstance(event_trigger, type(Event) ):
event_trigger = (event_trigger)
try:
self._event_trigger = tuple(event_trigger)
except:
self._event_trigger = ()
atexit.register(self.stop)
def __init__(self, image_dir, short_edge, crop_edge, filenames, randoms,
train):
if short_edge < crop_edge:
raise ValueError
self._image_dir = image_dir
self._short_edge = short_edge
self._crop_edge = crop_edge
self._filenames = filenames
self._train = train
if train:
randoms = np.asarray(randoms, dtype=np.uint32).ravel()
self._randoms = sharedctypes.Array(ctypes.c_uint32, randoms)
self._class_seps = []
t = 0
for i, filename in enumerate(filenames):
if not filename.startswith(self.classes[t]):
t += 1
if i == 0 or not filename.startswith(self.classes[t]):
raise ValueError
self._class_seps.append(i)
if t != len(self.classes) - 1:
raise ValueError
def create_share_type(np_array):
np_carr = np.ctypeslib.as_ctypes(np_array)
shared_array = sharedctypes.Array(np_carr._type_, np_carr, lock=False)
return shared_array
def compute_similarity_wem(n_iter, n_cores, n_locks, data_type, n_samples,
data, ground_truth_file, temp_folder, output_folder,
annotation_params, classification_params, verbose,
debug, with_unique_occurrences, preclustering):
"""
Computes the simlarity matrix in the basic setting.
Args:
* ``n_iter`` (*int*): number of iterations.
* ``n_cores`` (*int*): number of cores to use.
* ``n_locks`` (*int*): number of locks to use on the full shared matrix.
* ``data_type`` (*str*): data set to use.
* ``n_samples`` (*int*): number of samples.
* ``data`` (*struct*): initial data.
* ``ground_truth_file`` (*str*): path to ground-truth.
* ``temp_folder`` (*str*): path to the temporary folder.
* ``output_folder`` (*str*): path to the output folder.
* ``annotation_params`` (*list*): parameters for synthetic annotation.
* ``classification_params`` (*list*): parameters for classification.
* ``verbose`` (*int*): sets the verbosity level.
* ``with_unique_occurrences`` (*bool*): indicates wether samples occur only once in the data set or not.
* ``with_common_label_wordform`` (*bool*): indicates wether to give the same label to samples with identical wordform or not.
Returns:
* ``n_samples_occurrences`` (*list*): number of occurrences of each sample in a test set over all iterations.
* ``synthetic_labels`` (*list*): synthetic labels repartition for each iteration.
* ``co_occ`` (*ndarray*): full similarity matrix.
"""
#Additional variables for estimating the different parameters
positive_pairs = ground_truth_pairs(data_type, ground_truth_file,
n_samples)
true_pi0 = 1.0 - float(2 * len(positive_pairs)) / (n_samples *
(n_samples - 1))
true_p1 = [0] * n_iter
true_p0 = [0] * n_iter
indep_p0 = [0] * n_iter
#----------------------------------------------------------- SHARED MEMORY
# Create shared array with ctype structure
print '> Creating Shared Arrays'
n_locks = n_samples * 2 if n_locks == -1 else n_locks
total_length = n_samples * (n_samples - 1) / 2
cell_length, rst = total_length / n_locks, total_length % n_locks
if rst > 0:
n_locks += rst / cell_length
rst = rst % cell_length
if rst > 0:
n_locks += 1
# An object of type Cell contains the sequence of iteration scores for one pair of samples
UnitcellType = n_iter * c_int
shared_coocc = [
sharedctypes.Array(UnitcellType, [
UnitcellType(*(2 for _ in xrange(n_iter)))
for _ in xrange(cell_length)
],
lock=True) for x in xrange(n_locks)
]
#----------------------------------------------------------- INITIAL WORKER PROCESSES
threads = {}
sim_queue = Queue()
iterations = range(n_iter)
n_samples_occurrences = np.zeros(n_samples)
synthetic_labels = []
n_iterations_done = 0
#Start initial processes
for c in xrange(n_cores):
t = Process(name='Worker %d' % c,
target=thread_step,
args=(iterations.pop(0), shared_coocc, cell_length,
n_samples, sim_queue, data, temp_folder, data_type,
annotation_params, classification_params, verbose,
debug, with_unique_occurrences, preclustering))
threads[t.name] = t
t.start()
#----------------------------------------------------------- THREADED ADDITIONAL COMPUTATIONS
############################################################################################
def aux_computation_WEM(incoming_queue, n_iter, shared_coocc, total_length,
positive_pairs, true_p1, true_p0, indep_p0):
"""
Additional computations for the EM similarity
"""
for k in xrange(n_iter):
(iter, id, n_steps, counts, synth_lab) = incoming_queue.get()
#Compute Indep p0
repartition = synth_lab[-1]
indep_p0[n_steps] = float(sum([x**2 for x in repartition
])) / sum(repartition)**2
print 'Computation %d done' % (iter + 1)
############################################################################################
incoming_queue = Queue_thread()
aux_thread = threading.Thread(target=aux_computation_WEM,
args=(incoming_queue, n_iter, shared_coocc,
total_length, positive_pairs, true_p1,
true_p0, indep_p0))
aux_thread.start()
# Retrieve results from queue and restat threads if needed
for k in xrange(n_iter):
#---- Retrieve and restart
(id, n_steps, counts, synth_lab, _, b) = sim_queue.get()
threads[id].join()
#Launch Compute thread
incoming_queue.put((k, id, n_steps, counts, synth_lab))
#Start next process
if len(iterations) > 0:
t = Process(name=id,
target=thread_step,
args=(iterations.pop(0), shared_coocc, cell_length,
n_samples, sim_queue, data, temp_folder,
data_type, annotation_params,
classification_params, verbose, debug,
with_unique_occurrences, preclustering))
threads[id] = t
t.start()
else:
del threads[id]
#---- Additional Computations
n_samples_occurrences += counts
synthetic_labels.append(synth_lab)
#----- Verbose outputs
if verbose >= 1:
n_iterations_done += 1
print >> sys.stderr, '\n>>>> %d/%d iterations done\n' % (
n_iterations_done, n_iter)
# Join remaining processes (normally, they are processe that crashed)
for i, t in enumerate(threads.values()):
t.join()
if verbose >= 1:
print 'joined thread %d' % i
aux_thread.join()
gc.collect()
#------------------------------------------------------------------------- POST PROCESS MATRIX
####### Reshapping the result into a true numpy matrix
print '> Reshaping co_occurence matrix'
#flat_cooc = np.hstack(shared_coocc)[:total_length]
print 'Long format'
flat_cooc = np.zeros(total_length, dtype='object')
i = 0
for x in shared_coocc:
for y in x:
if i >= total_length:
break
flat_cooc[i] = long(''.join([str(z) for z in y]))
i += 1
del shared_coocc
# True parameters
print 'Compute ground-truth paramters'
for n_steps in xrange(n_iter):
to_iter = np.vectorize(lambda x: int(str(x).zfill(n_iter)[n_steps]))
mat = to_iter(flat_cooc)
true_p1[n_steps] = float(len(
np.where(mat[positive_pairs] == 1)[0])) / len(positive_pairs)
true_p0[n_steps] = float(
len(
np.where(mat[np.setdiff1d(range(total_length), positive_pairs)]
== 1)[0])) / (total_length - len(positive_pairs))
del mat
###### EM Estimation
print 'Running EM'
z1, pi0, p0, p1 = estimate_parameters_em(flat_cooc,
n_iter,
p1i=0.9,
p0i=0.1,
pi0i=0.8,
n_iter=25,
cores=n_cores)
# Estimates
print 'Estimated parameters (pi0, p0, p1) %s \n %s \n %s' % (pi0, p0, p1)
print 'True parameters (pi0, p0, p1) %s \n %s \n %s' % (true_pi0, true_p0,
true_p1)
print 'Independant parameters p0 \n %s \n ' % (indep_p0)
a = [0 if p == 0 else np.sqrt((1 - p) / p) for p in p0]
b = [0 if p == 0 else -np.sqrt(p / (1 - p)) for p in p0]
# Similarities
if verbose >= 2:
print 'Binary sim'
to_bin_sim = np.vectorize(lambda x: float(str(long(x)).count(1)) /
float(str(long(x)).count(2)))
save_coocc(output_folder,
to_bin_sim(flat_cooc).astype(np.float),
suffix='binary_final_flat')
print 'WBIN with EM weights'
to_wbinem_sim = np.vectorize(lambda x: np.sum([
a[i] if int(y) == 1 else b[i] if int(y) == 2 else 0
for y in str(long(x))
]) / float((str(long(x)).count(2))))
save_coocc(output_folder,
to_wbinem_sim(flat_cooc).astype(np.float),
suffix='weighted_final_flat')
print 'EM similarities'
to_em_sim = np.vectorize(lambda x: z1[str(long(x)).zfill(n_iter)])
co_occ = np.zeros((n_samples, n_samples), dtype=float)
co_occ[np.triu_indices(n_samples, k=1)] = to_em_sim(flat_cooc)
return n_samples_occurrences, synthetic_labels, co_occ, None
def f(mp_arr, mp_it, mp_it2, mp_val, num, N, rr):
for (i, j, r) in itt.izip(mp_it[(num * rr):((num + 1) * rr)],
mp_it2[(num * rr):((num + 1) * rr)],
mp_val[(num * rr):((num + 1) * rr)]):
mp_arr[i * N + j] = r
if __name__ == '__main__':
N = 6000
nbModif = N * 200
nbThreads = 4
rr = nbModif / nbThreads
lock = mp.Lock()
pool = []
mp_arr = sct.Array(c.c_double, N * N, lock=False)
mp_it = sct.Array(c.c_int, nbModif, lock=False)
mp_it2 = sct.Array(c.c_int, nbModif, lock=False)
mp_val = sct.Array(c.c_double, nbModif, lock=False)
listi = np.random.permutation(np.arange(N))
listj = np.random.permutation(np.arange(N))
# just random things to assign
a = []
for num in np.arange(nbModif):
n = num % N
a.append((listi[n], listj[n], n))
mp_it[num] = listi[n]
mp_it2[num] = listj[n]
mp_val[num] = n
def balance_ratio(G,
length,
exact=False,
n_samples=100000,
accuracy=None,
sampling_func=nrsampling,
parallel=True):
"""
Returns balance ratio of "G" based on simple cycles of length upto "length". Please note
that it catches the Keyboard interruption if "exact" is False. It is helpful when one wants
to interrupt the algorithm because it is running for longer than expected or the accuracy
has reached an acceptable value.
Parameters
----------
G : numpy.ndarray
Adjacency matrix of graph
length : int
Maximum length of simple cycles
exact : bool
If True, the algorithm counts all subgraphs of G.
Otherwise, it uses a sampling technique to sample
enough number of subgraphs so that the estimate of balance
would converge.
n_samples : int
If exact is False, how many samples to take from graph. Use a float number
accuracy : float
If provided, it is used as a threshold on standard deviation of estimated ratios. In this
case, the "n_samples" parameter is ignored and algorithm runs until the desired accuracy
is reached.
sampling_func : function
A sampling function from `cycleindex.sampling` module.
parallel : bool
If True, the sampling is done in parallel.
If "exact" is True, "parallel" is ignored.
Returns
-------
numpy.ndarray
A numpy array containing balance ratios upto desired length "length".
"""
if exact:
counts = cycle_count(G, length)
else:
counts = ([], [])
batch_count = batch_count_
args = (sampling_func, True, counts)
if parallel:
global shared_G
n_cores = cpu_count()
if G.__array_interface__['strides']:
G = np.ascontiguousarray(G)
tmp = np.ctypeslib.as_ctypes(G)
shared_G = sharedctypes.Array(tmp._type_, tmp, lock=False)
G = None
pool = Pool(n_cores, init_worker_)
batch_count = batch_count_parallel_
args = (n_cores, pool, sampling_func, True, counts)
try:
if accuracy:
last_ratios = []
batch_size = 1000
deviation = accuracy
while np.any(deviation >= accuracy):
batch_count(G, length, batch_size, *args)
ratios = calc_ratio(*counts)
last_ratios.append(ratios)
if len(last_ratios) > 5:
deviation = np.std(last_ratios, axis=0)
else:
batch_count(G, length, n_samples, *args)
except KeyboardInterrupt:
if pool:
pool.terminate()
pool.join()
if parallel:
pool.close()
pool.join()
del shared_G
return calc_ratio(*counts)
def compute_similarity_basic(
n_iter, n_cores, n_locks, data_type, n_samples, data, data_occurrences,
temp_folder, output_folder, annotation_params, classification_params,
verbose, debug, with_unique_occurrences, preclustering, writing_steps,
convergence_step, convergence_criterion):
"""
Computes the simlarity matrix in the basic setting.
Args:
* ``n_iter`` (*int*): number of iterations.
* ``n_cores`` (*int*): number of cores to use.
* ``n_locks`` (*int*): number of locks to use on the full shared matrix.
* ``data_type`` (*str*): data set to use.
* ``n_samples`` (*int*): number of samples.
* ``data`` (*struct*): initial data.
* ``data_occurrences`` (*struct*): indicates where each of the samples occurs in the data base.
* ``temp_folder`` (*str*): path to the temporary folder.
* ``output_folder`` (*str*): path to the output folder.
* ``annotation_params`` (*list*): parameters for synthetic annotation.
* ``classification_params`` (*list*): parameters for classification.
* ``verbose`` (*int*): sets the verbosity level.
* ``with_unique_occurrences`` (*bool*): indicates wether samples occur only once in the data set or not.
* ``with_common_label_wordform`` (*bool*): indicates wether to give the same label to samples with identical wordform or not.
* ``writing_steps`` (*list*): steps at which to save the partial matrix.
* ``convergence_step`` (*int*): every 'convergence_step', the script checks if convergence is reached.
Returns:
* ``n_samples_occurrences`` (*list*): number of occurrences of each sample in a test set over all iterations.
* ``synthetic_labels`` (*list*): synthetic labels repartition for each iteration.
* ``co_occ`` (*ndarray*): full similarity matrix.
"""
#----------------------------------------------------------- SHARED MEMORY
# Create shared array
print '> Creating Shared Array'
n_locks = n_samples * 2 if n_locks == -1 else n_locks
total_length = n_samples * (n_samples - 1) / 2
cell_length, rst = total_length / n_locks, total_length % n_locks
if rst > 0:
n_locks += rst / cell_length
rst = rst % cell_length
if rst > 0:
n_locks += 1
shared_coocc = [
sharedctypes.Array(np.ctypeslib.ctypes.c_double,
np.zeros(cell_length),
lock=True) for x in xrange(n_locks)
]
test_occurrences = np.zeros(total_length, dtype=int)
#----------------------------------------------------------- INITIAL WORKER PROCESSES
threads = {}
sim_queue = Queue()
iterations = range(n_iter)
n_samples_occurrences = np.zeros(n_samples)
synthetic_labels = []
n_iterations_done = 0
score_penalty = 0.
convergence_values = []
# Start initial processes
for c in xrange(n_cores):
t = Process(name='Worker %d' % c,
target=thread_step,
args=(iterations.pop(0), shared_coocc, cell_length,
n_samples, sim_queue, data, temp_folder, data_type,
annotation_params, classification_params, verbose,
debug, with_unique_occurrences, preclustering))
threads[t.name] = t
t.start()
running_threads = len(threads)
# Compute entropy values
last_mean_ent = 0.
#----------------------------------------------------------- THREADED ADDITIONAL COMPUTATIONS
############################################################################################
def aux_computation(incoming_queue, convergence_check_queue, n_iter,
n_samples, cell_length, total_length, shared_coocc,
data_occurrences, test_occurrences, verbose,
convergence_step, writing_steps, last_mean_ent,
convergence_criterion, output_folder):
"""
Additional computations for the BIN and WEIGHT similarity
"""
for k in xrange(n_iter):
(iter, id, n_steps, counts, synth_lab, mat) = incoming_queue.get()
# Updating number of test occurences
if mat is not None:
if data_type == 'NER':
mat = np.sum([data_occurrences[x] for x in mat])
in_train = np.setdiff1d(range(n_samples), mat.nonzero()[1])
elif data_type in ['AQUA', 'AUDIO', 'AUDIOTINY']:
in_train = np.where(mat == 0)[0]
test_occurrences += 1
for i in in_train:
test_occurrences[i - 1 + np.cumsum(
[n_samples - np.arange(2, i + 1)])] -= 1 # Column
test_occurrences[(i + 1 + i * n_samples - (i + 1) *
(i + 2) / 2):(n_samples - 1 +
i * n_samples - (i + 1) *
(i + 2) / 2)] -= 1 # Line
# Convergence check if required
if convergence_step > 1 and k % convergence_step == 1:
print 'Checking Convergence'
entropies = np.zeros(total_length)
# Lock
for cell in shared_coocc:
cell.get_lock().acquire()
# Compute entropies
for v, cell in enumerate(shared_coocc):
start_ind = v * cell_length
if v == len(shared_coocc) - 1 and rst > 0:
sim = np.array(np.frombuffer(
cell.get_obj()))[:rst] / np.clip(
test_occurrences[start_ind:], 1, n_iter)
else:
sim = np.array(np.frombuffer(
cell.get_obj())) / np.clip(
test_occurrences[start_ind:(
start_ind + cell_length)], 1, n_iter)
entropies[start_ind:(start_ind + cell_length)] = [
-np.sum(st * np.log(st) +
(1. - st) * np.log(1. - st)) if
(st > 0 and st < 1) else 0. for st in sim
]
del sim
# Release
for cell in shared_coocc:
cell.get_lock().release()
entropies /= np.log(2)
mean_ent = np.mean(entropies)
nonz_mean_ent = np.mean(entropies[entropies != 0])
del entropies
if verbose >= 2:
print 'Mean Shannon Entropy: %.4f' % mean_ent
print '(Non-zero) Mean Shannon Entropy: %.4f' % nonz_mean_ent
# Compare to threshold
if np.abs(mean_ent -
last_mean_ent) < convergence_criterion:
print 'convergence reached for criterion %s at step %d' % (
convergence_criterion, iter + 1)
# Reached convergence: stop execution
convergence_check_queue.put(
(k, mean_ent, nonz_mean_ent, True))
return
last_mean_ent = mean_ent
convergence_check_queue.put(
(k, mean_ent, nonz_mean_ent, False))
# Write matrix if required
if verbose >= 4 and (iter + 1) in writing_steps:
print 'Saving co-occurrence matrix at step %d' % (iter + 1)
for cell in shared_coocc:
cell.get_lock().acquire()
np.save(
os.path.join(output_folder,
'sim_matrix_%d_partial' % (iter + 1)),
(np.hstack(shared_coocc)[:total_length] + score_penalty) /
np.clip(test_occurrences, 1, n_iter))
for cell in shared_coocc:
cell.get_lock().release()
if verbose >= 1:
print 'Computation %d done' % (iter + 1)
############################################################################################
incoming_queue = Queue_thread()
convergence_check_queue = Queue_thread()
aux_thread = threading.Thread(
target=aux_computation,
args=(incoming_queue, convergence_check_queue, n_iter, n_samples,
cell_length, total_length, shared_coocc, data_occurrences,
test_occurrences, verbose, convergence_step, writing_steps,
last_mean_ent, convergence_criterion, output_folder))
aux_thread.start()
waiting_for_cvg_check = []
# Retrieve results from queue and restat threads if needed
for k in xrange(n_iter):
# Retrieve and restart
(id, n_steps, counts, synth_lab, mat, b) = sim_queue.get()
threads[id].join()
running_threads -= 1
# Launch Compute thread
incoming_queue.put((k, id, n_steps, counts, synth_lab, mat))
# Wait for convergence check before modifying the cooc matrix even further
if convergence_step > 1 and k % convergence_step == 1:
print 'Waiting for convergence check'
a, b, c, d = convergence_check_queue.get()
convergence_values.append((a, b, c))
if d:
print 'Convergence reached, ending program'
n_samples_occurrences += counts
synthetic_labels.append(synth_lab)
score_penalty += b
iterations[:] = []
# Terminates other processes + get lock to avoid termination problems
for cell in shared_coocc:
cell.get_lock().acquire()
for t in threads.values():
t.terminate()
for cell in shared_coocc:
cell.get_lock().release()
break
# Start next process
if len(iterations) > 0:
t = Process(name=id,
target=thread_step,
args=(iterations.pop(0), shared_coocc, cell_length,
n_samples, sim_queue, data, temp_folder,
data_type, annotation_params,
classification_params, verbose, debug,
with_unique_occurrences, preclustering))
threads[id] = t
running_threads += 1
t.start()
else:
del threads[id]
# Additional Computations
n_samples_occurrences += counts
synthetic_labels.append(synth_lab)
score_penalty += b
# Verbose outputs
del mat
if verbose >= 1:
n_iterations_done += 1
print >> sys.stderr, '\n>>>> %d/%d iterations done\n' % (
n_iterations_done, n_iter)
# Join remaining processes (normally, they are processe that crashed)
for i, t in enumerate(threads.values()):
t.join()
if verbose >= 1:
print 'joined thread %d' % i
aux_thread.join()
gc.collect()
#------------------------------------------------------------------------- POST PROCESS MATRIX
# Reshapping the result into a true numpy matrix
print '> Reshaping co_occurence matrix'
co_occ = np.zeros((n_samples, n_samples), dtype=float)
co_occ[np.triu_indices(
n_samples,
k=1)] = np.hstack(shared_coocc)[:total_length] + score_penalty
del shared_coocc
print 'Normalizing similarities...'
test_occurrences[test_occurrences == 0] = 1.
co_occ[np.triu_indices(n_samples, k=1)] /= test_occurrences
if convergence_step > 0:
print 'Saving Convergence criterion values'
with open(os.path.join(output_folder, 'convergence_check'), 'w') as f:
f.write('Iteration\tMean\tNonZeroMean\n')
f.write('\n'.join(
['\t'.join(map(str, obj)) for obj in convergence_values]))
return n_samples_occurrences, synthetic_labels, co_occ, test_occurrences
classification_params['similarity_type'] = 'BIN'
map_save_step = 5 # save mAP values every ``step`` iteration
steps = [
1, 5, 10, 15, 20, 25, 30, 40, 50, 60, 70, 80, 90, 100, 120, 140, 160,
180, 210, 230, 250, 270
] # iterations to consider for the correlation comparisons
with_unique_occurrences = (data_type in ['NER', 'AUDIO', 'AUDIOTINY'])
with_common_label_wordform = (data_type == 'NER' or data_type == 'AQUA')
preclustering = index_to_label if (data_type in ['NER', 'AQUA']) else []
#----------------------------------------------------------- RUN THREADS
# Initialize shared array
total_length = n_samples * (n_samples - 1) / 2
co_occ = [
sharedctypes.Array(np.ctypeslib.ctypes.c_long,
np.zeros(total_length, dtype='int32'),
lock=True)
]
count_lab = np.zeros(n_samples)
synthetic_labels = [0] * n_iter
compares = {}
# Sequential loop to build the similarity
for n in xrange(n_iter):
# Split + Annotation
train, test, _ = split_data(n,
data,
data_type,
classification_params['training_size'],
classification_params['classifier_type'],
annotation_params,
# y = 2048
y = 256
num_cpus = 2
#num_cpus = 4
a_shape = (x, y)
b_shape = (y, x)
# allocate source and dest. arrays
a = numpy.random.uniform(size=a_shape)
b = numpy.random.uniform(size=b_shape)
c = numpy.empty((x, x))
# allocated shared memory
shared_a = sharedctypes.Array(ctypes.c_double, a.flat, lock=False)
shared_b = sharedctypes.Array(ctypes.c_double, b.flat, lock=False)
shared_c = sharedctypes.Array(ctypes.c_double, c.flat, lock=False)
# access the answer as a numpy array, set dimensions
nd_c = ctypeslib.as_array(shared_c).reshape((a_shape[0], b_shape[1]))
# 1 process reference
print("starting.")
t1 = time()
ans1 = numpy.dot(a, b)
print("1 CPU:", time() - t1)
# x must be a multiple of num_cpus
assert (x % num_cpus == 0)
def modifiedGreedyInsertionCPU(asdGMM, xTilde, y, nPartial=10, speedUp=False, regVal=1e-2, doOpt=True, add2Sigma=1e-3, iterMax=100, relTolLogLike=1e-3,
absTolLogLike=1e-3, relTolMSE=1e-3, absTolMSE=1e-3,mseConverged=0., convBounds=[0.5,0.3], regValStep=[0.08, 4], addKWARGS={} ):
addKWARGS.setdefault('JacSpace')
nPt = xTilde.shape[1]
nK = asdGMM.nK
weights = asdGMM._getWeightsCPU(xTilde)# TBD or based on x only; I odn't think so
indKernel = np.argmax(weights,axis=0)
if mainDoParallel_:
xL = nPartial*[x]
doOptL = nPartial*[doOpt]
add2SigmaL = nPartial*[add2Sigma]
iterMaxL = nPartial*[iterMax]
relTolL = nPartial*[relTol]
absTolL = nPartial*[absTol]
mseConvergedL = nPartial*[mseConverged]
# Now split each of the kernels
#This is somewhat dirty
bestOverall = sharedctypes.Value(ctypes.c_double); bestOverall = 1e200
bestThetaOpt = sharedctypes.Array(ctypes.c_double,varsInMat(y.shape[0])*y.shape[1])
addKWARGS.update({'bestOverall':bestOverall,'bestThetaOpt':bestThetaOpt})
resultList = []
usedClass = asdGMM.__class__
for k in range(nK):
indKernelK = indKernel == k
xTildeK = xTilde[:,indKernelK]
nPtK = xTildeK.shape[1]
if xTildeK.shape[1] < asdGMM.nVarTot*4:
warnings.warn("Skipping kernel {0} in update process due to a lack of points".format(k))
continue
if speedUp:
# Check consistency if assumption that the base of each kernel is disjoint from the others
parasitInfl = np.mean(weights[np.hstack((np.arange(0,k),np.arange(k+1,nK))),indKernelK])
if parasitInfl > 0.05:
warnings.warn("Disjoint base assumption might not be valid for kernel {0}".format(k))
del parasitInfl
# Generate random samples
randSampI = np.random.choice(nPtK,nPartial)
randSampJ = np.random.choice(nPtK,nPartial)
indReplace = randSampI == randSampJ
while np.any(indReplace):
randSampI[indReplace] = np.random.choice(nPtK,sum(indReplace))
randSampJ[indReplace] = np.random.choice(nPtK,sum(indReplace))
indReplace = randSampI == randSampJ
if speedUp:
assert 0,"TBD"
else:
if mainDoParallel_:
assert 0, "TBD cuda driver error and other"
GMMparsL = [asdGMM.toPars() for k in range(nPartial)]
xTildeKL = nPartial*[xTildeK]
# partialInsertionCPU(GMMpars,x,xK,k,i,j,doOpt=True,add2Sigma=1e-3,iterMax=100,relTol=1e-3,absTol=1e-3)
with Pool(4) as p:
newList = p.starmap(modifiedPartialInsertionCPU,
zip(asdGMMparsL,xTildeL,xTildeKL,y,nPartial*[k],randSampI,randSampJ,doOptL,regValL,add2SigmaL,
iterMaxL,relTolL,absTolL,mseConvergedL))
resultList += newList
else:
resultList += lmap(
lambda ij:modifiedPartialInsertionCPU(asdGMM.toPars(),xTilde,xTildeK,y,k,ij[0],ij[1],doOpt=doOpt,regVal=regVal,add2Sigma=add2Sigma,
iterMax=iterMax,relTolLogLike=relTolLogLike,absTolLogLike=absTolLogLike,relTolMSE=relTolMSE,absTolMSE=absTolMSE,mseConverged=mseConverged, convBounds=convBounds, regValStep=regValStep, addKWARGS=addKWARGS, usedClass=usedClass), zip(randSampI,randSampJ))
#Get the best updated model among all tested ones
#Here we care more about mse than loglike
bestVal = np.Inf
bestPars = None
for newVal, newPars in resultList:
if newVal < bestVal:
bestVal = newVal
bestPars = newPars
# Load pars
newasdGMM = aSymDynamicsGMM(parDict=bestPars)
return newasdGMM
def __init__(self,
update_data_func=None,
state_transition_func=None,
thermostat=False,
kp=0.06, ki=0.0075, kd=0.01,
heater_segments=8,
ext_sw_heater_drive=False):
"""Create variables used to send in packets to the roaster. The update
data function is called when a packet is opened. The state transistion
function is used by the timer thread to know what to do next. See wiki
for more information on packet structure and fields."""
# constants for protocol decoding
self.LOOKING_FOR_HEADER_1 = 0
self.LOOKING_FOR_HEADER_2 = 1
self.PACKET_DATA = 2
self.LOOKING_FOR_FOOTER_2 = 3
# constants for connection state monitoring
self.CS_NOT_CONNECTED = -2
self.CS_ATTEMPTING_CONNECT = -1
self.CS_CONNECTING = 0
self.CS_CONNECTED = 1
# constants for connection attempt type
self.CA_NONE = 0
self.CA_AUTO = 1
self.CA_SINGLE_SHOT = 2
self._create_update_data_system(update_data_func)
self._create_state_transition_system(state_transition_func)
self._header = sharedctypes.Array('c', b'\xAA\xAA')
self._temp_unit = sharedctypes.Array('c', b'\x61\x74')
self._flags = sharedctypes.Array('c', b'\x63')
self._current_state = sharedctypes.Array('c', b'\x02\x01')
self._footer = b'\xAA\xFA'
self._fan_speed = sharedctypes.Value('i', 1)
self._heat_setting = sharedctypes.Value('i', 0)
self._target_temp = sharedctypes.Value('i', 150)
self._current_temp = sharedctypes.Value('i', 150)
self._time_remaining = sharedctypes.Value('i', 0)
self._total_time = sharedctypes.Value('i', 0)
self._disconnect = sharedctypes.Value('i', 0)
self._teardown = sharedctypes.Value('i', 0)
self._cooling_for_pid_control = False
# for SW PWM heater setting
self._heater_level = sharedctypes.Value('i', 0)
# the following vars are not process-safe, do not access them
# from the comm or timer threads, nor from the callbacks.
self._ext_sw_heater_drive = ext_sw_heater_drive
if not self._ext_sw_heater_drive:
self._thermostat = thermostat
else:
self._thermostat = False
self._pid_kp = kp
self._pid_ki = ki
self._pid_kd = kd
self._heater_bangbang_segments = heater_segments
# initialize to 'not connected'
self._connected = sharedctypes.Value('i', 0)
self._connect_state = sharedctypes.Value('i', self.CS_NOT_CONNECTED)
# initialize to 'not trying to connect'
self._attempting_connect = sharedctypes.Value('i', self.CA_NONE)
# create comm process
self.comm_process = mp.Process(
target=self._comm,
args=(
self._thermostat,
self._pid_kp,
self._pid_ki,
self._pid_kd,
self._heater_bangbang_segments,
self._ext_sw_heater_drive,
self.update_data_event,))
self.comm_process.daemon = True
self.comm_process.start()
# create timer process that counts down time_remaining
self.time_process = mp.Process(
target=self._timer,
args=(
self.state_transition_event,))
self.time_process.daemon = True
self.time_process.start()
def get_multiprocess_numpy(dtype, shape):
tmp = np.ctypeslib.as_ctypes(np.zeros(shape, dtype=dtype))
return sharedctypes.Array(tmp._type_, tmp, lock=False)
def compute_similarity_ova(n_iter,
n_cores,
n_locks,
input_file,
ground_truth_file,
data_type,
n_samples,
data,
data_occurrences,
index_to_label,
label_to_index,
temp_folder,
output_folder,
annotation_params,
classification_params,
verbose,
debug,
with_unique_occurrences,
preclustering,
gtonly=True):
"""
Computes the simlarity matrix in the basic setting.
Args:
* ``n_iter`` (*int*): number of iterations.
* ``n_cores`` (*int*): number of cores to use.
* ``n_locks`` (*int*): number of locks to use on the full shared matrix.
* ``data_type`` (*str*): data set to use.
* ``n_samples`` (*int*): number of samples.
* ``data`` (*struct*): initial data.
* ``data_occurrences`` (*struct*): indicates where each of the samples occurs in the data base.
* ``index_to_label`` (*list*): maps a sample's index to a string representation.
* ``label_to_index`` (*dict*): reverse index_to_label mapping.
* ``temp_folder`` (*str*): path to the temporary folder.
* ``output_folder`` (*str*): path to the output folder.
* ``annotation_params`` (*list*): parameters for synthetic annotation.
* ``classification_params`` (*list*): parameters for classification.
* ``verbose`` (*int*): sets the verbosity level.
* ``with_unique_occurrences`` (*bool*): indicates wether samples occur only once in the data set or not.
* ``with_common_label_wordform`` (*bool*): indicates wether to give the ame label to samples with identical wordform or not.
* ``writing_steps`` (*list*): steps at which to save the partial matrix.
* ``gt_only`` (*boolean, optional*): if True, experiments will only be conducted from query samples from the ground-truth. Defqults to True.
Returns:
* ``n_samples_occurrences`` (*list*): number of occurrences of each sample in a test set over all iterations.
* ``synthetic_labels`` (*list*): synthetic labels repartition for each iteration.
* ``co_occ`` (*ndarray*): full similarity matrix.
"""
# Additional parameters
full_matrix = np.zeros((n_samples, n_samples))
n_locks, rst, cell_length = 1, 0, n_samples
ground_truth = parse_ground_truth(data_type, ground_truth_file,
label_to_index)
sampleindx = ground_truth_indices(
data_type, ground_truth_file,
label_to_index) if gtonly else xrange(n_samples)
shuffle(sampleindx)
#################################### LOOP OVER EACH SAMPLE
for train_sample in sampleindx:
name = index_to_label[train_sample]
print "Iteration for sample %d - %s" % (train_sample, name)
iter_annotation_params = annotation_params.copy()
iter_annotation_params['sample'] = 'B-%d' % train_sample
print 'Pre-parsing train sentences'
data_files = [
f for f in os.listdir(input_file) if f.endswith('.xml.u8')
]
with open(
os.path.join(iter_annotation_params['ova_occurrences'],
'%s.pkl' % (name)), 'rb') as f:
train_docs = pickle.load(f)
train_files = train_docs.keys()
partial_load_file = partial(
load_file,
input_file=input_file,
train_docs=train_docs,
label_to_index=label_to_index,
classification_params=classification_params)
pool = Pool(processes=n_cores)
samples = pool.map(partial_load_file, train_files)
train = ()
for tl in samples:
train = chain(train, tl)
del samples
iter_annotation_params['ova_occurrences'] = (list(train), train_files)
# Shared Array
print 'Creating Shared Array'
shared_coocc = [
sharedctypes.Array(np.ctypeslib.ctypes.c_double,
np.zeros(n_samples),
lock=True)
]
test_occurrences = np.zeros(n_samples, dtype=int)
#----------------------------------------------------------- INITIAL WORKER PROCESSES
threads = {}
sim_queue = Queue()
iterations = range(n_iter)
n_samples_occurrences = np.zeros(n_samples)
synthetic_labels = []
n_iterations_done = 0
score_penalty = 0.
# Start initial processes
for c in xrange(n_cores):
t = Process(name='Worker %d' % c,
target=thread_step,
args=(iterations.pop(0), shared_coocc, cell_length,
n_samples, sim_queue, data, temp_folder,
data_type, iter_annotation_params,
classification_params, verbose, debug,
with_unique_occurrences, preclustering))
threads[t.name] = t
t.start()
#----------------------------------------------------------- THREADED ADDITIONAL COMPUTATIONS
############################################################################################
def aux_computation_OVA(incoming_queue, n_iter, test_occurrences,
data_occurrences):
"""
Additional computations for the OVA annotation
"""
for k in xrange(n_iter):
(_, _, _, _, _, mat) = incoming_queue.get()
#Updating number of test occurences
if mat != None:
if data_type == 'NER': #In NER #test ~ #entities + sparse matrices
mat = np.sum([data_occurrences[x] for x in mat])
mat[mat != 0] = 1
test_occurrences += mat
############################################################################################
incoming_queue = Queue_thread()
aux_thread = threading.Thread(target=aux_computation_OVA,
args=(
incoming_queue,
n_iter,
test_occurrences,
data_occurrences,
))
aux_thread.start()
#--------- Retrieve results from queue and restat threads if needed
for k in xrange(n_iter):
#---- Retrieve and restart
(id, n_steps, counts, synth_lab, mat, b) = sim_queue.get()
threads[id].join()
#Launch Compute thread
incoming_queue.put((k, id, n_steps, counts, synth_lab, mat))
#Start next process
if len(iterations) > 0:
t = Process(name=id,
target=thread_step,
args=(iterations.pop(0), shared_coocc, cell_length,
n_samples, sim_queue, data, temp_folder,
data_type, iter_annotation_params,
classification_params, verbose, debug,
with_unique_occurrences, preclustering))
threads[id] = t
t.start()
else:
del threads[id]
#---- Additional Computations
n_samples_occurrences += counts
synthetic_labels.append(synth_lab)
score_penalty += b
#----- Verbose outputs
del mat
if verbose >= 1:
n_iterations_done += 1
print >> sys.stderr, '\n>>>> %d/%d iterations done\n' % (
n_iterations_done, n_iter)
#------------ Join remaining processes if any
for i, t in enumerate(threads.values()):
t.join()
if verbose >= 1:
print 'joined thread %d' % i
aux_thread.join()
gc.collect()
with shared_coocc[0].get_lock():
arr = np.frombuffer(shared_coocc[0].get_obj())
# Normalizing
test_occurrences[test_occurrences == 0] = 1.
co_occ = arr / test_occurrences
full_matrix[train_sample, :] = arr
# Output sorted distribution for the current sample if verbosity level high enough
if verbose >= 2:
gt = ground_truth[name]
sorted_ind = np.argsort(co_occ)
with open(os.path.join(output_folder, '%s_sorted_sim.txt' % name),
'w') as f:
f.write('\n'.join([
'%s %s\t%d-%s\t%s\t%s\t%s' %
('>' if index_to_label[i] in gt else '', name, i,
index_to_label[i], co_occ[i], arr[i], test_occurrences[i])
for i in sorted_ind
]))
# Output mAP for the current sample
if verbose >= 2:
print "######## Result for iteration %d - %s" % (train_sample,
name)
eval_retrieval(data_type,
co_occ,
label_to_index,
index_to_label,
ground_truth_file,
output_folder,
ova=train_sample,
writing=True)
return n_samples_occurrences, synthetic_labels, arr, None
import ctypes
from multiprocessing import sharedctypes, Process
import numpy
from numpy import ctypeslib
def f(cta):
from numpy import ctypeslib
npa = ctypeslib.as_array(cta._obj)
npa[0] = npa.sum()
cta = sharedctypes.Array(ctypes.c_double, numpy.arange(1e6))
npa = ctypeslib.as_array(cta._obj)
p1 = Process(target=f, args=(cta, ))
def func0(lock, a):
lock.acquire()
print a.value
func.add1(a)
#a.value += 1
time.sleep(1)
proc = current_process()
print proc.name, proc.pid
lock.release()
lock0 = Lock()
lock = Semaphore(2) # Semaphore(1) == Lock()
v = Value('f', 0.0)
a = Array('i', range(10))
sv = sharedctypes.Value('f', 0.0)
sa = sharedctypes.Array('i', range(10))
sub_proc0 = Process(target=func0, args=(lock, sa))
sub_proc1 = Process(target=func0, args=(lock, sa))
sub_proc2 = Process(target=func0, args=(lock, sa))
sub_proc0.start()
sub_proc1.start()
sub_proc2.start()
sub_proc0.join()
sub_proc1.join()
sub_proc2.join()
func0(lock, sa)
conn.poll()
a, n = conn.recv()
conn.send(f(x, a, n))
if __name__ == '__main__':
N = 10000000
numprocesses = 6
complexity = 10
print 'Array size:', N
print 'Num processors:', numprocesses
print 'Task complexity:', complexity
print
x = ones(N)
x_sct = sharedctypes.Array(ctypes.c_double, x, lock=False)
A = 1 + arange(numprocesses)
pipes = [multiprocessing.Pipe() for _ in xrange(numprocesses)]
server_conns, client_conns = zip(*pipes)
processes = [multiprocessing.Process(
target=process_listener,
args=(x_sct, conn)
) for conn in client_conns]
for p in processes:
p.start()
start = time.time()
for i, a in enumerate(A):
server_conns[i].send((a, complexity))
for i in xrange(numprocesses):
from numpy import ctypeslib
n_params = 256 * 256 * 3 * 64
def f(q, S):
x = np.random.rand(n_params) # data pre-fetched
x_ = ctypeslib.as_array(S.get_obj())
st = time.time()
#x_ = x # This does not overwrite the shared memory.
st0 = time.time()
x_[:] = x
print("SetTime:{}".format(time.time() - st0))
x_[0] = 1000
data = {"x": None, "st": st}
q.put(data)
if __name__ == '__main__':
buff = np.random.rand(n_params)
S = sharedctypes.Array("f", buff)
q = Queue()
p = Process(target=f, args=(q, S, ))
p.start()
data = q.get()
print("Parent:{}[s]".format(time.time() - data["st"]))
x_ = ctypeslib.as_array(S.get_obj())
print("Shape:{}".format(x_.shape))
print("init-val:{}, changed-val:{}".format(buff[0], x_[0]))
p.terminate()
