class Frame(object):
def __init__(self, width, height, channels, array_type_code):
self.__lock = RLock()
self.__header = Value(Header,
width,
height,
channels,
0,
lock=self.__lock)
self.__image = Array(array_type_code,
self.__header.width * self.__header.height *
channels,
lock=self.__lock)
self.__latch = StateLatch(State.READY, self.__lock)
def copy(self, dst):
memmove(addressof(dst.image.get_obj()),
addressof(self.__image.get_obj()),
sizeof(self.__image.get_obj()))
memmove(addressof(dst.header.get_obj()),
addressof(self.__header.get_obj()),
sizeof(self.__header.get_obj()))
def clear(self):
self.__header.epoch = 0
memset(addressof(self.__image.get_obj()), 0,
sizeof(self.__image.get_obj()))
memset(addressof(self.__header.detections), 0,
sizeof(self.__header.detections))
@property
def lock(self):
return self.__lock
@property
def header(self):
return self.__header
@property
def image(self):
return self.__image
@property
def latch(self):
return self.__latch
def get_numpy_image(self, dtype=None):
"""# Get numpy image from buffer.
"""
image_shape = (self.header.height, self.header.width,
self.header.channels)
image_np = frombuffer(self.image.get_obj(), dtype).reshape(image_shape)
return image_shape, image_np
def shared_array(shape):
import numpy as np
from multiprocessing.sharedctypes import Array
from ctypes import c_double
# Allocate the memory in shared space.
memory = Array(c_double, int(np.prod(shape)))
# Create and return a structure to access the shared memory (numpy array).
return np.frombuffer(memory.get_obj(), dtype=float).reshape(shape)
class CosmicMulti:
def __init__(self, n, mass, RxInit, RyInit, VxInit, VyInit, AxInit, AyInit,
interactions):
self.N = n
self.M = Value('d', mass, lock=False)
self.R = Array('d', 3 * (self.N + 1))
self.V = Array('d', 3 * (self.N + 1))
self.A = Array('d', 3 * (self.N + 1))
arrR = np.frombuffer(
self.R.get_obj()) # mp_arr and arr share the same memory
self.bR = arrR.reshape(
(3, self.N + 1)) # b and arr share the same memory
self.bR[0, 0] = RxInit
self.bR[1, 0] = RyInit
arrV = np.frombuffer(
self.V.get_obj()) # mp_arr and arr share the same memory
self.bV = arrV.reshape(
(3, self.N + 1)) # b and arr share the same memory
self.bV[0, 0] = VxInit
self.bV[1, 0] = VyInit
arrA = np.frombuffer(
self.A.get_obj()) # mp_arr and arr share the same memory
self.bA = arrA.reshape(
(3, self.N + 1)) # b and arr share the same memory
self.bA[0, 0] = AxInit
self.bA[1, 0] = AyInit
#self.R = np.zeros(shape = (3, self.N+1))
#self.V = np.zeros(shape = (3, self.N+1))
#self.A = np.zeros(shape = (3, self.N+1))
#self.R[0,0] = RxInit
#self.R[1,0] = RyInit
#self.V[0,0] = VxInit
#self.V[1,0] = VyInit
#self.A[0,0] = AxInit
#self.A[1,0] = AyInit
self.Interactions = interactions
def main():
url = 'http://192.168.1.1/videostream.cgi'
user = '******'
pwd = ''
auth = requests.auth.HTTPDigestAuth(user, pwd)
r = requests.get(url, auth=auth, stream=True, timeout=9)
h = 480
w = 640
ch = 3
img = Array('B', np.zeros(h * w * ch, dtype=np.uint8))
arr = np.frombuffer(img.get_obj(), dtype=np.uint8)
img_lock = Event()
stream = Process(target=stream_video, args=(r, img, img_lock))
stream.start()
fps = 60.0
fourcc = cv2.VideoWriter_fourcc(*'X264')
out = cv2.VideoWriter('output.mp4', fourcc, fps, (640, 480))
prev = datetime.now()
time_interval = 1.0 / fps
while True:
now = td_to_ms(datetime.now() - prev)
if now >= time_interval:
if not img_lock.is_set():
img_lock.set()
frame = arr.reshape((h, w, ch))
img_lock.clear()
cv2.imshow('camera', frame)
out.write(frame)
prev = datetime.now()
time_interval = (2.0 / fps) - now
if cv2.waitKey(1) == 27:
break
stream.terminate()
out.release()
cv2.destroyAllWindows()
return
#print z.max(), z.min(), z.std() #Comment out to verify that the 'random' seed is identical over tests
p = np.ones((w.n, 1), float)
floor_variable = p
floor = 3
'''START TIMING HERE - AFTER TEST DATA GENERATED'''
time0 = time.time()
#Multiprocessing setup
cores = mp.cpu_count()
cores = cores * 2
numP = len(p)+1
#Shared memory solution space
lockSoln = mp.Lock()
cSoln = Array(ctypes.c_double, numP*cores, lock=lockSoln)
numSoln = np.frombuffer(cSoln.get_obj())
numSoln.shape = (numP,cores)
numSoln[:] = -1
#Shared memory update flag space
lockflag = mp.Lock()
c_updateflag = Array(ctypes.c_int, 3*(cores*2), lock=lockflag) #Do I need different locks? #Why double cores again?
updateflag = np.frombuffer(c_updateflag.get_obj())
updateflag.shape=(3,cores)
updateflag[0] = 1 #True for first iteration. - whether the answer was updated
updateflag[1] = 0 #Iteration counter per core.
for index in range(len(updateflag[2])): #Define the tabu list length for each chord.
updateflag[2][index] = tabulength(numP)
_init_shared(updateflag)
neighbordict = dict(w.neighbors)#Class instances are not pickable.
def fit_model(star, nextcomp=2, guess=None, fitpar=[], order=3, niter=3*10**5,
parallel=0, factr=10**7, mvec=10):
nstar = star['mag'].shape[0]
nband = star['mag'].shape[1]
nintmagpar = len(compute_multinomials(order=order)[0])
print('Should cut out YSOs somewhere')
if guess is None:
# intmagparguess = numpy.zeros((nband, nintmagpar), dtype='f8')
intmagparguess = numpy.random.randn(nband, nintmagpar)
extcurveparguess = numpy.random.randn(nextcomp, nband)
if nband == 12:
intmagparguess[:, 0] = numpy.array([
0.2, -0.5, -0.8, -0.9, -1.0, -2.0, -2.6, -2.7, -2.7, -2.6,
-0.1, -1.1], dtype='f8')
# just get the constant term right.
if nextcomp > 0:
# probably: grizYJHK12BR
extcurveparguess[0, :] = numpy.array(
[3.5, 2.7, 2.0, 1.6, 1.3, 0.8, 0.5, 0.3, 0.2, 0.2,
3.5, 2.0])
starguess = {}
starguess['extinction'] = star['extprior'].copy()
starguess['mu'] = parallax_to_mu(
numpy.clip(star['parallax'], 0.01, numpy.inf))
for parname in fitpar:
starguess[parname] = star[parname]
else:
intmagparguess, extcurveparguess, starguess = guess
guess = wrap_param(intmagparguess, extcurveparguess,
starguess, nband, nintmagpar, nstar,
fitpar=fitpar)
def chi2_wrapper(param):
upar = unwrap_param(param, nband, nintmagpar, nstar, nextcomp,
fitpar=fitpar)
intmagpar, extcurvepar, starpar = upar
chi2, grad = fullmodel_chi2(star,
intmagpar, extcurvepar, starpar,
nband=nband,
gradient=True, fitpar=fitpar,
order=order)
return chi2, grad
if parallel > 0:
from multiprocessing import Queue, Process
from multiprocessing.sharedctypes import Array
import ctypes
qins = [Queue() for i in range(parallel)]
qout = Queue()
ind = numpy.floor(numpy.linspace(0, nstar, parallel+1, endpoint=True))
ind = ind.astype('i4')
npar = (nintmagpar+nextcomp)*nband+nstar*(1+nextcomp+len(fitpar))
grad = Array(ctypes.c_double, npar)
gradnp = numpy.frombuffer(grad.get_obj(), dtype='f8')
proclist = [
Process(target=worker,
args=(qins[i], qout, grad, star, nextcomp, fitpar, order,
ind[i:i+2], i))
for i in range(parallel)]
for p in proclist:
p.start()
def chi2_wrapper_parallel(param):
gradnp[:] = 0.
for i in range(parallel):
qins[i].put(param)
chi2 = 0.
for i in range(parallel):
tchi2 = qout.get()
chi2 += tchi2
# tgrad gets filled in by the workers; it's shared.
return chi2, gradnp
wrapper = chi2_wrapper_parallel
else:
wrapper = chi2_wrapper
from scipy.optimize import fmin_l_bfgs_b, fmin_cg
res = fmin_l_bfgs_b(wrapper, guess, m=mvec, iprint=10,
maxiter=niter, maxfun=niter, factr=factr)
# cg_chi2_wrapper = lambda x: wrapper(x)[0]
# cg_grad_wrapper = lambda x: wrapper(x)[1]
# res = fmin_cg(cg_chi2_wrapper, guess, fprime=cg_grad_wrapper,
# maxiter=niter)
return res
def start_distributed_mp():
# Setup
# To continue training, give path to state dict
state_dict_path = None
# Learner specific
learner_training_steps = 1000000
learner_learning_rate = 0.00025
learner_policy_update = 50
learner_optimizer = 'Adam'
learner_device = 'cuda'
learner_job_max_time = 60*60*24 -60*10 #2 hours 58min
learner_save_date = datetime.now().strftime("%d_%b_%Y_%H_%M_%S")
learner_eval_p_errors = [0.1, 0.2, 0.3]
learner_eval_no_episodes = 10
learner_eval_freq = -1 # -1 for no logging
# Actor specific
actor_max_actions_per_episode = 75
actor_size_local_memory_buffer = 100
actor_no_envs = 16 #number of envs/actor
no_cuda_actors = 1
no_cpu_actors = 0
actor_no_actors = no_cuda_actors + no_cpu_actors
#epsilon = calculateEpsilon(0.8, 7, actor_no_actors * actor_no_envs)
epsilon = calculateEpsilon(0.8, 7, actor_no_envs)
epsilon_delta = 0.005
env_p_error_interval_start = 0.1
env_p_error_interval_final = 0.3
env_p_error_interval_increase = 0.00005
env_p_error_strategy = 'random' # either {'random', 'linear'}
# Replay Memory specific
replay_memory_size = 1000000
replay_memory_alpha = 0.6
replay_memory_beta = 0.4
replay_memory_size_before_sampeling = 5000#replay_memory_size*0.05
replay_memory_batch_in_queue_limit = 10 #number of batches in queue to learner
log_priority_dist = True
log_write_frequency = 500
log_priority_sample_max = 10
log_priority_sample_interval_size = 0.01
# Shared
batch_size = 16
discount_factor = 0.95
env = "toric-code-v0"
env_config = { "size":9,
"min_qubit_errors": 0,
"p_error": 0.1
}
model = ResNet18
#model = NN_11
#model = NN_17
model_config = {"system_size": env_config["size"],
"number_of_actions": env_config["size"]
}
if not state_dict_path == None:
checkpoint = torch.load(state_dict_path, map_location=learner_device)
else:
checkpoint = None
# Pre-load initial network weights
if model == NN_11 or model == NN_17:
m = model(model_config["system_size"], model_config["number_of_actions"], learner_device)
else:
m = model()
# load checkpoint params
if not state_dict_path == None:
m.load_state_dict(checkpoint['model_state_dict'])
params = parameters_to_vector(m.parameters()).detach().cpu().numpy()
no_params = len(params)
#Comm setup
actor_io_queue = mp.Queue()
learner_io_queue = mp.Queue()
io_learner_queue = mp.Queue()
shared_mem_weight_id = Value('i')
shared_mem_weight_id.value = 0
# Write initial weights to shared memory
shared_mem_weights = Array('d', no_params) # Shared memory for weights
mem_reader = np.frombuffer(shared_mem_weights.get_obj()) # create memory reader for shared mem
np.copyto(mem_reader, params) # Write params to shared mem
del m # delete tmp model to load network params to free up mem
"""
Learner Process
"""
learner_args = {
"train_steps" :learner_training_steps,
"batch_size" :batch_size,
"learning_rate" :learner_learning_rate,
"policy_update" :learner_policy_update,
"discount_factor" :discount_factor,
"optimizer" :learner_optimizer,
"model" :model,
"model_config" :model_config,
"model_no_params" :no_params,
"device" :learner_device,
"env" :env,
"env_config" :env_config,
"job_max_time" :learner_job_max_time,
"save_date" :learner_save_date,
"learner_io_queue" :learner_io_queue,
"io_learner_queue" :io_learner_queue,
"shared_mem_weights" :shared_mem_weights,
"shared_mem_weight_id" :shared_mem_weight_id,
"learner_eval_p_errors" :learner_eval_p_errors,
"learner_eval_no_episodes" :learner_eval_no_episodes,
"learner_eval_freq" :learner_eval_freq,
"learner_checkpoint" :checkpoint
}
"""
Memory Process
"""
mem_args = {
"capacity" :replay_memory_size,
"alpha" :replay_memory_alpha,
"beta" :replay_memory_beta,
"batch_size" :batch_size,
"io_learner_queue" :io_learner_queue,
"learner_io_queue" :learner_io_queue,
"actor_io_queue" :actor_io_queue,
"batch_in_queue_limit" :replay_memory_batch_in_queue_limit,
"no_actors" :actor_no_actors,
"replay_size_before_sampling" :replay_memory_size_before_sampeling if not (replay_memory_size_before_sampeling is None) else min(batch_size, int(replay_memory_size*0.1)),
"save_date" :learner_save_date,
"log_priority_dist" :log_priority_dist,
"log_write_frequency" :log_write_frequency,
"log_priority_sample_max" :log_priority_sample_max,
"log_priority_sample_interval_size" :log_priority_sample_interval_size,
"start_time" :learner_save_date
}
"""
Actor Processes
"""
actor_args = {
"max_actions_per_episode" :actor_max_actions_per_episode,
"size_local_memory_buffer" :actor_size_local_memory_buffer,
"env_config" :env_config,
"model" :model,
"model_config" :model_config,
"model_no_params" :no_params,
"env" :env,
"discount_factor" :discount_factor,
"no_envs" :actor_no_envs,
"actor_io_queue" :actor_io_queue,
"shared_mem_weights" :shared_mem_weights,
"shared_mem_weight_id" :shared_mem_weight_id,
"epsilon_delta" :epsilon_delta,
"env_p_error_start" :env_p_error_interval_start,
"env_p_error_final" :env_p_error_interval_final,
"env_p_error_delta" :env_p_error_interval_increase,
"env_p_error_strategy" :env_p_error_strategy,
"no_cuda_actors" :no_cuda_actors,
"no_cpu_actors" :no_cpu_actors,
"log_actor" :log_priority_dist
}
# log header to tensorboard
if could_import_tb:
log("runs/{}/RunInfo/".format(learner_save_date), actor_args, learner_args, mem_args, state_dict_path)
io_process = mp.Process(target=io, args=(mem_args,))
actor_process = []
for i in range(actor_no_actors):
if i < no_cuda_actors :
actor_args["device"] = 'cuda'
else:
actor_args["device"] = 'cpu'
actor_args["id"] = i
#actor_args["epsilon_final"] = epsilon[i * actor_no_envs : i * actor_no_envs + actor_no_envs]
actor_args["epsilon_final"] = epsilon
actor_process.append(mp.Process(target=actor, args=(actor_args,)))
actor_process[i].start()
io_process.start()
try:
learner(learner_args)
except:
tb = SummaryWriter("runs/{}/RunInfo/".format(learner_save_date))
tb.add_text("RunInfo/Error_Message", sys.exc_info()[0])
tb.close()
time.sleep(2)
print("Training done.")
for i in range(actor_no_actors):
actor_process[i].terminate()
io_process.terminate()
print("Script complete.")
def _mutual_proximity_empiric_sparse(S: csr_matrix,
test_set_ind: np.ndarray = None,
min_nnz=0,
verbose: int = 0,
log=None,
n_jobs=None):
"""MP empiric for sparse similarity matrices.
Please do not directly use this function, but invoke via
mutual_proximity_empiric()
"""
if verbose and log:
log.message("Starting MP empiric for sparse matrices.")
self_value = 1. # similarity matrix
n = S.shape[0]
if not n_jobs:
n_jobs = 1
elif n_jobs == -1:
n_jobs = cpu_count()
else:
pass
# This will become S_mp.data
shared_data = Array(ctypes.c_double, S.data.size)
shared_data_np = np.ctypeslib.as_array(shared_data.get_obj())
if verbose and log:
log.message("Spawning processes and starting MP computation.")
with Pool(processes=n_jobs,
initializer=_mpes_init,
initargs=(S, shared_data)) as pool:
S_nonzero = filterfalse(lambda ij: ij[0] > ij[1], zip(*S.nonzero()))
for _ in pool.imap(func=partial(_mpes_sec_dist,
args=(verbose, log, n, min_nnz)),
iterable=S_nonzero,
chunksize=int(1e5)):
pass # output stored by function in shared array
pool.join()
if verbose and log:
log.message("Assemble upper-triangular MP matrix.")
S_mp = csr_matrix((shared_data_np, S.indices, S.indptr),
shape=S.shape,
copy=False).tolil()
del shared_data, shared_data_np
if verbose and log:
log.message("Symmetrizing matrix.")
S_mp += S_mp.T
# Retain original distances for objects with too few neighbors.
# That is, keep distances FROM these objects to others (rows), but
# set distances of other objects TO them to NaN (columns).
# Returned matrix is thus NOT SYMMETRIC.
if verbose and log:
log.message(("Retain original similarities for objects with too few "
"neighbors. If there are any, the output matrix will "
"not be symmetric anymore! (Rows corresponding to these "
"objects will be in original space; corresponding "
"columns will contain NaN)."))
for row in np.argwhere(S.getnnz(axis=1) <= min_nnz):
row = row[0] # use scalar for indexing instead of array
S_mp[row, :] = S.getrow(row)
if verbose and log:
log.message("Setting self similarities.")
for i in range(n):
S_mp[i, i] = self_value #need to set self values
if verbose and log:
log.message("Converting to CSR matrix and returning.")
return S_mp.tocsr()
#Setup the test data:
w = pysal.lat2W(10, 10) #A contiguity weights object
z = np.random.random_sample((w.n, 2)) #Each local is assigned two attributes
p = np.ones((w.n, 1), float) #The region that each location belongs to.
floor = 3 #The minimum bound or value for each region
#Grab the number of available core
cores = mp.cpu_count()
cores = cores *2 #Hyperthreading and testing on a dual core MBP
#Grab the length of p
numP = len(p)+1
#Setup a shared mem array for solutions with dim = numP * cores
lock = mp.Lock()
cSoln = Array(ctypes.c_double, numP*cores, lock=lock)
numSoln = np.frombuffer(cSoln.get_obj())
numSoln.shape = (numP,cores)
numSoln[:] = 1
#initSolnSpace(numSoln) #initialize the solution space as a shared memory array
'''The soln space is an array that holds node id as the index and membership as the attribute.'''
neighbordict = dict(w.neighbors) #This is interesting - we can not pass a class instance through apply_async and need to conver to a dict.
pool = mp.Pool(processes=cores) #Create a pool of workers, one for each core
for job in range(cores): #Prep to assign each core a job
pool.apply_async(initialize, args=(job,z,w,neighbordict,floor,p,numP, cores)) #Async apply each job
pool.close()
pool.join()
sharedSoln = np.frombuffer(cSoln.get_obj())
sharedSoln.shape = (numP, cores)
