def getNeighbors(self, peer):
semaphore = Semaphore()
semaphore.acquire()
neighbors = self.__layout[peer.getId()].getNeighbors()
semaphore.release()
return neighbors
def _spawnmain(
testid: TestId,
exts: List[str],
# pyre-fixme[11]: Annotation `Semaphore` is not defined as a type.
sem: multiprocessing.Semaphore,
resultqueue: multiprocessing.Queue,
):
"""run a test and report progress back
intended to be spawned via multiprocessing.Process.
"""
hasmismatch = False
def mismatchcb(mismatch: Mismatch):
nonlocal hasmismatch
hasmismatch = True
mismatch.testname = testid.name
resultqueue.put(mismatch)
result = TestResult(testid=testid)
try:
runtest(testid, exts, mismatchcb)
except TestNotFoundError as e:
result.exc = e
except Exception as e:
result.exc = e
result.tb = traceback.format_exc(limit=-1)
finally:
if result.exc is None and hasmismatch:
result.exc = MismatchError("output mismatch")
resultqueue.put(result)
resultqueue.close()
sem.release()
class Budget(object):
"""Device budget (for lack of a better term)
Details
-------
Keeps a semaphore for the total number of resources available,
and a device-specific atomic counter of available slots.
"""
def __init__(self, devices, n_per_device):
self.devices, self.n_per_device = devices, n_per_device
self.total = Semaphore(len(devices) * n_per_device)
self.alloc = Array("i", len(devices) * [n_per_device])
def acquire(self):
self.total.acquire()
with self.alloc.get_lock():
# get the largest counter
index = max(range(len(self.alloc)), key=self.alloc.__getitem__)
# assert self.alloc[index] > 0
# acquire the index and decrease the counter
self.alloc[index] -= 1
return index
def release(self, index):
with self.alloc.get_lock():
self.alloc[index] += 1
self.total.release()
def __getitem__(self, index):
return self.devices[index]
def setCurrentSimulationTime(self, currentSimulationTime):
semaphore = Semaphore()
semaphore.acquire()
self.__currentSimulationTime = currentSimulationTime
semaphore.release()
return self.__currentSimulationTime
def _readtimestepsbond(self):
# added on 2018-12-15
stepatomfiles = {}
self._mkdir(self.trajatom_dir)
with Pool(self.nproc, maxtasksperchild=10000) as pool:
semaphore = Semaphore(360)
results = pool.imap_unordered(
self.bonddetector.readatombondtype,
self._produce(
semaphore,
enumerate(
zip(self.lineiter(self.bonddetector), self.erroriter()
) if self.errorfilename is not None else self.
lineiter(self.bonddetector)),
(self.errorfilename is not None)), 100)
nstep = 0
for d, step in tqdm(results,
desc="Read trajectory",
unit="timestep"):
for bondtypebytes, atomids in d.items():
bondtype = self._bondtype(bondtypebytes)
if bondtype not in self.atombondtype:
self.atombondtype.append(bondtype)
stepatomfiles[bondtype] = open(
os.path.join(self.trajatom_dir,
f'stepatom.{bondtype}'), 'wb')
stepatomfiles[bondtype].write(
self.listtobytes([step, atomids]))
semaphore.release()
nstep += 1
pool.close()
self._nstep = nstep
for stepatomfile in stepatomfiles.values():
stepatomfile.close()
pool.join()
def _img_processor(self, sh_img_arr: Array, sem1: Semaphore,
sem2: Semaphore, ovl_arr: Array) -> None:
"""
Process images as needed.
:param sh_img_arr: The array containing the frame to work with.
:param sem1: The entrance lock.
:param sem2: The exit lock.
:param ovl_arr: The array containing the overlay work with.
:return None:
"""
img_dim = (EDIT_HEIGHT, self._cur_arr_shape[1], self._cur_arr_shape[2])
img_size = int(EDIT_HEIGHT * img_dim[1] * img_dim[2])
img_arr = frombuffer(sh_img_arr.get_obj(), count=img_size,
dtype=DTYPE).reshape(img_dim)
while self._process_imgs:
sem1.acquire()
if self._use_overlay:
img_pil = Image.fromarray(img_arr)
draw = ImageDraw.Draw(img_pil)
draw.text(OVL_POS,
text=ovl_arr.value.decode(),
font=OVL_FONT,
fill=OVL_CLR)
processed_img = asarray(img_pil)
copyto(img_arr, processed_img)
sem2.release()
def start(self, test_q: JoinableQueue, result_q: Queue) -> None:
"""
Start all worker processes
:return: this object
"""
local_test_q = JoinableQueue()
self._node_manager = Node.Manager(as_main=True,
port=self.__class__._node_port)
start_sem = Semaphore(self._max_simultaneous_connections)
# This will be used to throttle the number of connections made when makeing distributed call to get
# node-level and global-level fixtures; otherwise multiproceissing can hang on these calls if
# overwhelmed
fixture_sem = Semaphore(self._max_simultaneous_connections)
for index in range(self._num_processes):
proc = WorkerSession.start(
index,
self._host,
self._port,
start_sem,
fixture_sem,
local_test_q,
result_q,
self._node_port,
)
self._worker_procs.append(proc)
start_sem.release()
self._test_q_process = Process(target=self._process_test_q,
args=(test_q, local_test_q))
self._test_q_process.start()
def getPeerID(self, peerId):
semaphore = Semaphore()
semaphore.acquire()
peer = self.__layout[peerId]
semaphore.release()
return peer
class Msg(object):
"""
TODO: Not documenting this class because it may go away.
"""
def __init__(self, size):
self.s_e = Semaphore(1)
self.s_f = Semaphore(0)
self.s_buf = Array(ct.c_ubyte, size)
def send(self, func):
self.s_e.acquire()
self.s_buf.acquire()
send_result = func(self.s_buf._obj)
self.s_buf.release()
self.s_f.release()
return send_result
def recv(self, func):
self.s_f.acquire()
self.s_buf.acquire()
recv_result = func(self.s_buf._obj)
self.s_buf.release()
self.s_e.release()
return recv_result
def _build_single_scenario_proc(clean: bool, allow_offset_map: bool,
scenario: str, semaphore: Semaphore):
semaphore.acquire()
try:
_build_single_scenario(clean, allow_offset_map, scenario)
finally:
semaphore.release()
def generator(self, *args, **kwargs):
""" This function warp generator to ParaWrapper's generator
which is capable of multi-processing
Once the generator function was settled, we can send worker with the task then
work with full-load until meet the buff_size limit
The worker's job is to feed the list and keep it contains more than <buff_size> batches
"""
# Initialization semaphores and numbering
buff_count = Semaphore(value=0)
target_remain = Semaphore(value=self.buff_size)
number = str(self.gen_num)
self.gen_num += 1
# Initializing list
self.batch_list[number] = self.manager.list()
# Assign work and send worker
gen = self.datagen.generator(*args, **kwargs)
worker = Process(target=self.task,
args=(gen, number, target_remain, buff_count))
worker.start()
while True:
buff_count.acquire(block=True)
ret = self.batch_list[number].pop()
target_remain.release()
yield ret
class Thread_Pool_Manager(object):
def __init__(self, thread_num=cpu_count()):
self.thread_num = thread_num
print(thread_num)
self.work_queue = JoinableQueue()
self.work_num = Semaphore(0)
self.mutex = Lock()
def start_threads(self):
for i in range(self.thread_num):
thread = Process(target=self.do_job)
thread.daemon = True # set thread as daemon
thread.start()
def do_job(self):
global Numbers
while True:
# print(1)
self.work_num.acquire()
with self.mutex:
print(1, self.work_queue.qsize())
thread_job = self.work_queue.get()
print(0, self.work_queue.qsize())
thread_job.do_job(self.work_queue, self.work_num)
print(self.work_queue.qsize())
self.work_queue.task_done()
def join(self):
self.work_queue.join()
def add_job(self, job):
self.work_queue.put(job)
self.work_num.release()
def relay(semaphore: mp.Semaphore, queue: mp.Queue, output_lock: mp.Lock,
bmsg: bytes, addr: tuple, relay_dict, recv_time: datetime):
semaphore.acquire()
bmsg = bytearray(bmsg)
header = DNSHeader(bmsg[:12])
header.aa = 1
bmsg = header.bmsg + bmsg[12:]
assert header.qdcount == 1
question = DNSQuestion(bmsg, offset=12)
with output_lock:
cprint(f'[{recv_time}][recv query {bytes_to_int(bmsg[:2])}]: {bmsg} from {addr}', fore='green', style='reverse')
cprint_header(header, fore='green')
cprint_question(question, fore='green')
if question.qname in relay_dict:
if relay_dict[question.qname] == '0.0.0.0':
header.rcode = 3
answer = header.bmsg + bmsg[12:]
mode = 'intercept '
elif question.qtype == 1:
answer = fake_bmsg(bmsg, relay_dict[question.qname])
mode = 'local resolve '
else:
answer = forward(bmsg)
if answer is None:
return
mode = 'relay msg '
else:
answer = forward(bmsg)
mode = 'relay msg '
queue.put((answer, addr, recv_time, mode))
semaphore.release()
class EmbeddingWorker(Process):
def __init__(self, queue, X, y, transformation_method, embedding_args):
super().__init__()
self.pause_lock = Semaphore(value=True) # lock is free
self.embedding_args = embedding_args
self.X = X
self.y = y
self.transformation_method = transformation_method
self.queue = queue
def callback(self, command, iteration, payload):
# pausing acquires pause_lock and the following code only runs if
# pause_lock is free
with self.pause_lock:
self.queue.put((command, iteration, payload))
def run(self):
self.transformation_method(self.X, self.y, self.embedding_args,
self.callback)
def pause(self):
self.pause_lock.acquire()
def resume(self):
self.pause_lock.release()
def is_paused(self):
return not self.pause_lock.get_value()
class Msg(object):
"""
Data structure encapsulating a message.
"""
def __init__(self, size):
self.s_e = Semaphore(1)
self.s_f = Semaphore(0)
self.s_buf = Array(ct.c_ubyte, size)
def send(self, func):
self.s_e.acquire()
self.s_buf.acquire()
send_result = func(self.s_buf._obj)
self.s_buf.release()
self.s_f.release()
return send_result
def recv(self, func):
self.s_f.acquire()
self.s_buf.acquire()
recv_result = func(self.s_buf._obj)
self.s_buf.release()
self.s_e.release()
return recv_result
def _get_all(embeddings,
data_sequence,
start_iteration,
ent_type,
w_size,
batch_size,
processes,
evalutation_semaphore=None):
# The embed semaphore makes sure that the EmbedWithContext will not over produce results in relation
# to the LookUpBySurfaceAndContext creation
embed_semaphore = Semaphore(100)
for it, link_result in \
enumerate(
EmbedWithContext.run(embeddings, data_sequence, ent_type, w_size, batch_size,
processes, embed_semaphore, start_iteration=start_iteration)):
try:
if evalutation_semaphore is not None:
evalutation_semaphore.acquire(timeout=10)
yield LookUpBySurfaceAndContext(link_result)
except Exception as ex:
print(type(ex))
print("Error: ", link_result)
raise
if it % batch_size == 0:
embed_semaphore.release()
class DbWriter(Process):
def __init__(self, queue, stop_flag):
super(DbWriter, self).__init__()
self.worker_control = Semaphore(MAX_WRITER_WORKERS)
self.result_queue = queue
self.stop_flag = stop_flag
def run(self):
print(" *** DB Writer online")
while True and self.stop_flag.value != 1:
if self.worker_control.acquire(False):
task = self.result_queue.get()
if task:
try:
worker = WriterWorker(task, self.worker_control)
worker.start()
except Exception as err:
print(err)
print("Invalid task %s" % task)
self.worker_control.release()
else:
self.worker_control.release()
time.sleep(0.3)
print("stop flag: %s" % self.stop_flag.value)
def _writecoulumbmatrix(self, trajatomfilename, fc):
self.dstep = {}
with open(
os.path.join(self.trajatom_dir,
f"stepatom.{trajatomfilename}"), 'rb') as f:
for line in f:
s = self.bytestolist(line)
self.dstep[s[0]] = s[1]
n_atoms = sum(map(len, self.dstep.values()))
if n_atoms > self.n_clusters:
# undersampling
max_counter = Counter()
stepatom = np.zeros((n_atoms, 2), dtype=int)
feedvector = np.zeros((n_atoms, 0))
vector_elements = defaultdict(list)
with Pool(self.nproc, maxtasksperchild=10000) as pool:
semaphore = Semaphore(360)
results = pool.imap_unordered(
self._writestepmatrix,
self._produce(semaphore,
enumerate(self.lineiter(self.crddetector)),
None), 100)
j = 0
for result in tqdm(results,
desc=trajatomfilename,
total=self._nstep,
unit="timestep"):
for stepatoma, vector, symbols_counter in result:
stepatom[j] = stepatoma
for element in (symbols_counter -
max_counter).elements():
vector_elements[element].append(
feedvector.shape[1])
feedvector = np.pad(
feedvector, ((0, 0), (0, 1)),
'constant',
constant_values=(0,
self._coulumbdiag[element]))
feedvector[
j,
sum(
map(lambda x: vector_elements[x[0]][:x[1]],
symbols_counter.items()), [])] = vector
max_counter |= symbols_counter
j += 1
semaphore.release()
logging.info(
f"Max counter of {trajatomfilename} is {max_counter}")
pool.close()
choosedindexs = self._clusterdatas(np.sort(feedvector),
n_clusters=self.n_clusters,
n_each=self.n_each)
pool.join()
else:
stepatom = np.array([[u, vv] for u, v in self.dstep.items()
for vv in v])
choosedindexs = range(n_atoms)
fc.write(self.listtobytes(stepatom[choosedindexs]))
self._nstructure += len(choosedindexs)
def countNeighbors(self, peer):
semaphore = Semaphore()
semaphore.acquire()
count = peer.countNeighbors()
semaphore.release()
return count
def getNeighborIt(self, peer):
semaphore = Semaphore()
semaphore.acquire()
neighbors = []
for neighbor in self.__layout[peer.getId()].getNeighbors():
neighbors.append(neighbor.getTargetPeer())
neighborIt = neighbors.__iter__()
semaphore.release()
return neighborIt
class Iterator(object):
"""Provides an generic multithreaded iterator over the dataset."""
def __init__(self, dataset, batch_size, batchifier, pool,
shuffle= False, use_padding = False, no_semaphore= 20):
print("----------------- Iterator",batch_size)
# Filtered games
games = dataset.get_data()
# print("games = {}".format(games))
# print("dataset = {} ".format(dataset))
# exit()
games = batchifier.filter(games)
if shuffle:
random.shuffle(games)
self.n_examples = len(games)
self.batch_size = batch_size
self.n_batches = int(math.ceil(1. * self.n_examples / self.batch_size))
batch = split_batch(games, batch_size, use_padding)
print("++++ Iterator | n_examples = {},batch_size={},n_batches={}".format(self.n_examples,self.batch_size,self.n_batches))
print("shape({},{})".format(len(batch),len(batch[0])))
# no proc
# self.it = (batchifier.apply(b )for b in batch)
# Multi_proc
self.semaphores = Semaphore(no_semaphore)
it_batch = sem_iterator(l=batch, sem=self.semaphores)
self.process_iterator = pool.imap(batchifier.apply, it_batch)
def __len__(self):
return self.n_batches
def __iter__(self):
return self
def __next__(self):
self.semaphores.release()
return self.process_iterator.next()
#return self.it.__next__()
# trick for python 2.X
def next(self):
return self.__next__()
def iter_batches(self):
if self.parallel > 0:
with Pool(self.parallel) as pool:
semaphore = Semaphore(self.parallel * 10)
for batch in pool.imap(
self.full_construct,
self.batch_reader.read_batches(semaphore)):
yield batch
semaphore.release()
else:
for batch in self.batch_reader.read_batches():
yield self.full_construct(batch)
def addPeer(self, peer):
if self.__layout.has_key(peer.getPID()):
return False
semaphore = Semaphore()
semaphore.acquire()
self.__layout[peer.getPID()] = peer
semaphore.release()
NetworkLogger().resgiterLoggingInfo("Add peer %s in Layout Network "%(peer.getPID()))
return self.__layout.has_key(peer.getPID())
class Runner:
def __init__(self):
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
# Create the pool without special signal handling
self.pool = Pool(processes=NUM_WORKERS)
self.worker_semaphore = Semaphore(NUM_WORKERS)
self.has_received_termination_signal = False
for sig in termination_signals:
signal.signal(sig, self.handle_termination_signal)
self.units_of_work = itertools.cycle(range(100))
logging.info("Main PID {0}".format(os.getpid()))
def do_work(self):
while True:
if self.has_received_termination_signal:
logging.info(
"Stopping loop because termination signal received")
return
self.acquire_worker_semaphore()
work = (next(self.units_of_work), )
logging.info("Sending work {0}".format(work))
self.pool.apply_async(
worker_function,
work,
callback=self.release_worker_semaphore,
error_callback=self.release_worker_semaphore,
)
time.sleep(2)
def start(self):
self.do_work()
self.pool.close()
self.pool.join()
def handle_termination_signal(self, signum, frame):
my_pid = os.getpid()
logging.info("PID {0} received termination signal {1!r}".format(
my_pid, signum))
self.has_received_termination_signal = True
def acquire_worker_semaphore(self):
self.worker_semaphore.acquire()
def release_worker_semaphore(self, exception=None):
self.worker_semaphore.release()
class Contractor(Process):
def __init__(self, images, stop_flag, input_queue, result_queue,
api_key, secret_key, time_worker_run):
super(Contractor, self).__init__()
self.worker_control = Semaphore(MAX_WORKERS)
self.images = images
self.stop_flag = stop_flag
self.input_queue = input_queue
self.result_queue = result_queue
self.api_key = api_key
self.secret_key = secret_key
self.time_worker_run = time_worker_run
def run(self):
print(" ++ Contractor online ++ \n")
while True:
temp_value = None
temp_value = self.stop_flag.value
#print(">> %s temp value"% temp_value)
if temp_value != 1:
if self.worker_control.acquire(False):
try:
task = self.input_queue.get(False)
if task:
worker = Worker(self.worker_control, task,
self.input_queue, self.api_key,
self.secret_key,
self.stop_flag,
self.result_queue, self.images,
self.time_worker_run)
worker.start()
else:
self.worker_control.release()
except:
self.worker_control.release()
else:
break
time.sleep(0.2)
print("contractor done")
def _printmoleculeSMILESname(self):
mname = []
with open(self.moleculefilename, 'w') as fm, open(self.moleculetemp2filename, 'rb') as ft, Pool(self.nproc, maxtasksperchild=1000) as pool:
semaphore = Semaphore(360)
results = pool.imap(self._calmoleculeSMILESname,
self._produce(semaphore, ft, ()), 10)
for index, (name, atoms, bonds) in enumerate(results):
self._loggingprocessing(index)
mname.append(name)
print(name, ",".join([str(x) for x in atoms]), ";".join(
[",".join([str(y) for y in x]) for x in bonds]), file=fm)
semaphore.release()
self._mname = mname
def run_shard(q: Queue, sem: Semaphore, num, total):
global _MP_QUEUE
_MP_QUEUE = q
sys.argv.append("--internal-num-shards=" + str(total))
sys.argv.append("--internal-shard=" + str(num))
# sys.argv.append("--pretend")
print("shard", num, sys.argv)
try:
libcxx_main()
print("Job", num, "completed")
except Exception as e:
print("Job", num, "failed!!", e)
sem.release()
class Barrier:
def __init__(self, n):
self.n = n
self.counter = SharedCounter(0)
self.barrier = Semaphore(0)
def wait(self):
with self.counter.lock:
self.counter.val.value += 1
if self.counter.val.value == self.n:
self.barrier.release()
self.barrier.acquire()
self.barrier.release()
class Barrier:
def __init__(self, n):
self.n = n
self.counter = SharedCounter(0)
self.barrier = Semaphore(0)
def wait(self):
with self.counter.lock:
self.counter.val.value += 1
if self.counter.val.value == self.n:
self.barrier.release()
self.barrier.acquire()
self.barrier.release()
def _start_token_generation(sema: mp.Semaphore, tokens_per_second: int, stop_event: threading.Event):
"""
Releases number of tokens_per_second tokens per second on the Semaphore
"""
logging.debug('token generation thread started')
while not stop_event.is_set():
time.sleep(2 / tokens_per_second)
try:
sema.release()
sema.release()
except ValueError:
continue
logging.debug('token generation thread stoopped')
class WorkQueue(object):
def __init__(self):
self.request_rfd, self.request_wfd = os.pipe()
self.response_rfd, self.response_wfd = os.pipe()
self.response_reader = ResponseReader(self.response_rfd)
self.request_sem = Semaphore()
self.response_sem = Semaphore()
def submit_request(self, id, address, head, body):
try:
ip_str, port = address
ipa, ipb, ipc, ipd = map(int, ip_str.split("."))
except:
port = ipa = ipb = ipc = ipd = 0
os.write(
self.request_wfd, REQUEST_HEADER.pack(id, ipa, ipb, ipc, ipd, port, len(head), len(body)) + head + body
)
def get_request(self):
self.request_sem.acquire()
header = ""
bytes_to_read = REQUEST_HEADER.size
while bytes_to_read:
header += os.read(self.request_rfd, bytes_to_read)
bytes_to_read = REQUEST_HEADER.size - len(header)
id, ipa, ipb, ipc, ipd, port, head_len, body_len = REQUEST_HEADER.unpack(header)
head = StringIO()
bytes_to_read = head_len
while bytes_to_read:
head.write(os.read(self.request_rfd, bytes_to_read))
bytes_to_read = head_len - head.tell()
body = StringIO()
bytes_to_read = body_len
while bytes_to_read:
body.write(os.read(self.request_rfd, bytes_to_read))
bytes_to_read = body_len - body.tell()
self.request_sem.release()
return id, (".".join(map(str, [ipa, ipb, ipc, ipd])), port), head.getvalue(), body.getvalue()
def submit_response(self, id, response):
self.response_sem.acquire()
response_output = response.output()
keep_alive = "\x01" if response.headers.get("Connection") == "Keep-Alive" else "\x00"
os.write(self.response_wfd, RESPONSE_HEADER.pack(id, len(response_output)) + response_output + keep_alive)
self.response_sem.release()
def get_response(self):
return self.response_reader.read()
def generate_ripe_request_tokens(sema: mp.Semaphore, limit: int,
finish_event: threading.Event):
"""
Generates RIPE_REQUESTS_PER_SECOND tokens on the Semaphore
"""
logger.debug('generate thread started')
while not finish_event.is_set():
time.sleep(2 / limit)
try:
sema.release()
sema.release()
except ValueError:
continue
logger.debug('generate thread stoopped')
class ForkingWorker(BaseWorker):
def __init__(self, num_processes=1):
# Set up sync primitives, to communicate with the spawned children
self._semaphore = Semaphore(num_processes)
self._slots = Array('i', [0] * num_processes)
def spawn_child(self):
"""Forks and executes the job."""
self._semaphore.acquire() # responsible for the blocking
# Select an empty slot from self._slots (the first 0 value is picked)
# The implementation guarantees there will always be at least one empty slot
for slot, value in enumerate(self._slots):
if value == 0:
break
# The usual hardcore forking action
child_pid = os.fork()
if child_pid == 0:
# Within child
# Disable signal handlers
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
random.seed()
try:
self.fake_work()
finally:
# This is the new stuff. Remember, we're in the child process
# currently. When all work is done here, free up the current
# slot (by writing a 0 in the slot position). This
# communicates to the parent that the current child has died
# (so can safely be forgotten about).
self._slots[slot] = 0
self._semaphore.release()
os._exit(0)
else:
# Within parent, keep track of the new child by writing its PID
# into the first free slot index.
self._slots[slot] = child_pid
def wait_for_children(self):
for child_pid in self._slots:
if child_pid != 0:
os.waitpid(child_pid, 0)
def get_id(self):
return os.getpid()
class Iterator(object):
"""Provides an generic multithreaded iterator over the dataset."""
def __init__(self,
dataset,
batch_size,
batchifier,
pool,
shuffle=False,
use_padding=False,
no_semaphore=20):
# Filtered games
games = dataset.get_data()
games = batchifier.filter(games)
games = batchifier.split(games)
if shuffle:
random.shuffle(games)
self.batch_size = batch_size
self.n_batches = int(math.ceil(1. * len(games) / self.batch_size))
if use_padding:
self.n_examples = self.n_batches * self.batch_size
else:
self.n_examples = len(games)
batch = split_batch(games, batch_size, use_padding)
# no proc
# self.it = (batchifier.apply(b) for b in batch)
# Multi_proc
self.semaphores = Semaphore(no_semaphore)
it_batch = sem_iterator(l=batch, sem=self.semaphores)
self.process_iterator = pool.imap(batchifier.apply, it_batch)
def __len__(self):
return self.n_batches
def __iter__(self):
return self
def __next__(self):
self.semaphores.release()
return self.process_iterator.next()
# return self.it.__next__()
# trick for python 2.X
def next(self):
return self.__next__()
class Pool(object):
"""Create a pool of object with the given class_type, args, and kwargs.
The get method will return any objects currently unused from the pool.
The put method will put back the object, marking it as unused for future use.
Putting blocking=True will wait until an object is available, while blocking=False will return immediately if
there is no object available
"""
def __init__(self, class_type, n, blocking=True, callback=lambda x: x, args=(), kwargs={}):
self.class_type = class_type
self.items = []
self.used = Array('i', [0] * n)
self.lock = Lock()
self.count = Semaphore(n)
self.n = n
self.blocking = blocking
try:
for i in range(n):
my_item = self.class_type(*args, **kwargs)
callback(my_item)
self.items.append(my_item)
except Exception as exc:
raise type(exc)('%s: %s' % (self.class_type, exc.message))
def get(self):
self.lock.acquire(self.blocking)
try:
if not self.count.acquire(self.blocking):
return None
for i, my_item in enumerate(self.items):
if self.used[i] == 0:
self.used[i] = 1
return my_item
finally:
self.lock.release()
def put(self, item):
self.lock.acquire()
try:
for i, my_item in enumerate(self.items):
if my_item == item:
self.used[i] = 0
self.count.release()
return True
return False
finally:
self.lock.release()
class CheckableSem:
def __init__(self):
self._sem = Semaphore()
self._num_permits = 1
def acquire(self):
self._sem.acquire()
self._num_permits -= 1
def release(self):
self._sem.release()
self._num_permits += 1
def almost_perfect_get_num_permits(self):
return self._num_permits
def _printatomroute(self, atomeach):
with open(self.atomroutefilename, 'w') as f, Pool(self.nproc, maxtasksperchild=1000) as pool:
allmoleculeroute = []
semaphore = Semaphore(360)
results = pool.imap(self._getatomroute, self._produce(
semaphore, enumerate(zip(atomeach, self._atomtype), start=1), ()), 10)
for index, route in enumerate(results):
self._loggingprocessing(index)
moleculeroute, routestr = route
print(routestr, file=f)
for mroute in moleculeroute:
if not mroute in allmoleculeroute:
allmoleculeroute.append(mroute)
semaphore.release()
return allmoleculeroute
def definyPeerTrading(self):
value =0;
peerAux =""
semaphore = Semaphore()
semaphore.acquire()
for peer,trust in self.__peersTrading.iteritems():
if trust >= value:
value = trust
peerAux = peer
semaphore.release()
return (peerAux,value)
async def server_work(reader, writer):
''' Handle main communication with C#. '''
while True:
# print("wait for server work")
# Wait for an instruction from C# and act upon.
# TODO: handle disconnects (they might throw some exceptions around read).
recv = await reader.readline()
# print(recv)
try:
data = recv.decode('utf-8').split()
except (UnicodeDecodeError, AttributeError):
print(f"Invalid data received from C#: {recv}")
continue
if data[0] == "start":
num_threads = int(data[1])
model_choice = int(data[2])
image_size = int(data[3])
model_path = data[4]
visualization_type = int(data[5])
video_path = data[6]
frame_rate = int(data[7])
layers_to_see = []
layers_to_see.append("pred")
for layers in range(8, len(data)):
layers_to_see.append(data[layers])
# Start the cam worker.
image_queue = Queue(num_threads)
semaphore = Semaphore(0)
Process(target=camera_worker,
args=[
image_queue, semaphore, image_size,
visualization_type, video_path, frame_rate
]).start()
# Make the initial prediction.
image_model, layers_indices = init(model_choice, layers_to_see,
model_path)
semaphore.release()
image = image_queue.get()
# Start the prediction workers.
for index in range(num_threads):
Process(target=worker_thread,
args=(index + 1, image_queue, semaphore,
model_choice, layers_to_see,
model_path)).start()
await predict_and_write(0, image, image_model, layers_indices,
writer)
def run_mp(nproc, **arg):
pool = Pool(nproc, maxtasksperchild=1000)
semaphore = Semaphore(nproc * 150)
try:
results = multiopen(pool=pool, semaphore=semaphore, **arg)
for item in results:
yield item
semaphore.release()
except:
logging.exception("run_mp failed")
pool.terminate()
raise
else:
pool.close()
finally:
pool.join()
def removeNeighbor(self, source, target):
if (not self.__layout.has_key(source.getId())) and (not self.__layout.has_key(target.getId())) :
return False
if source.hasNeighbor(target):
return False
if target.hasNeighbor(source):
return False
semaphore = Semaphore()
semaphore.acquire()
flag = source.removeNeighbor(target)
target.removeNeighbor(source)
semaphore.release()
return flag
def start_multiprocess(self, csv_file_name):
self.get_download_task_from_csv(csv_file_name)
semaphores = Semaphore(self._thread_number)
for video_url in self._video_url:
semaphores.acquire()
process = multiprocessing.Process(
target=self.run_downloader,
args=(semaphores, video_url, self._MAX_VIDEO_DURATION_DIFF))
process.daemon = True
process.start()
# semaphores.release()
for _ in range(0, self._thread_number):
semaphores.acquire()
semaphores.release()
def _calhmm(self):
with open(self.originfilename, 'wb') if self.getoriginfile or not self.runHMM else Placeholder() as fo, open(self.hmmfilename, 'wb') if self.runHMM else Placeholder() as fh, open(self.moleculetempfilename, 'rb') as ft, open(self.moleculetemp2filename, 'wb') as ft2, Pool(self.nproc, maxtasksperchild=1000) as pool:
semaphore = Semaphore(360)
results = pool.imap_unordered(
self._getoriginandhmm, self._produce(semaphore, ft, ()), 10)
for index, (originsignal, hmmsignal, mlist) in enumerate(results):
self._loggingprocessing(index)
if 1 in hmmsignal or self.printfiltersignal or not self.runHMM:
if self.getoriginfile:
fo.write(self._compress(
"".join([str(i) for i in originsignal.tolist()])))
if self.runHMM:
fh.write(self._compress(
"".join([str(i) for i in hmmsignal.tolist()])))
ft2.write(self._compress(mlist.strip()))
semaphore.release()
def main(): gps_n = Semaphore(0) gps_s = Semaphore(1) gps_coords_stack = Manager().list() gps = GPS(gps_coords_stack, gps_n, gps_s) gps.start() # Get the first position z = gps.getPosition() dt = 0.05 range_std = 5. # Means meters # Instantiate the filter filterk = ExtendedKalmanFilter(2, 1, 0) # 1 type of value of position, but in 2 dimensions. sensor provides position in (x,y) so use 2 # Insert first position filterk.x = array(z) # Pretty sure this sets up the taylor series filterk.F = eye(2) + array([[0,1], [0,0]])*dt # Sets the uncertainty filterk.R = np.diag([range_std**2]) # Trains it using white noise? filterk.Q[0:2, 0:2] = Q_discrete_white_noise(2, dt=dt, var=0.1) filterk.Q[2, 2] = 0.1 # Covariance matrix filterk.P *= 50 for i in range(10): # Pull a value from the GPS stack gps_n.acquire() gps_s.acquire() result = gps_coords_stack.pop() gps_s.release() # Put new z value in filterk.predict_update(array(result), HJacobian_at, hx) #this maaaaay need to be formatted differently, otherwise just put the longitude and lattitude as an array [x,y] # Get the predicted value np.append(xs, filterk.x) print(filterk.x)
class EventMasterProcess(SatoriProcess):
def __init__(self):
super(EventMasterProcess, self).__init__('event master')
self.sem = Semaphore(0)
def do_run(self):
listener = Listener(address=(settings.EVENT_HOST, settings.EVENT_PORT))
master = Master(mapper=TrivialMapper())
master.listen(listener)
self.sem.release()
master.run()
def start(self, *args, **kwargs):
super(EventMasterProcess, self).start(*args, **kwargs)
while True:
if self.sem.acquire(False):
return
if not self.is_alive():
raise RuntimeError('Event master failed to start')
sleep(0)
def removePeer(self, peer):
flag = True
if not self.__layout.has_key(peer.getPID()):
return False
semaphore = Semaphore()
semaphore.acquire()
'''
pode travar pois estou chamando um sema dentro do outro?
'''
del self.__layout[peer.getPID()]
flag = not self.__layout.has_key(peer.getPID())
semaphore.release()
return flag
def connect(self, priority):
sem = Semaphore()
sem.acquire()
if self.getPeer().isConnected():
return
network = self.getPeer().getNetwork()
node = None
if netowork.countNodes() > 0:
idx = randint(0, network.countNodes() - 1)
graph = network.getGraph()
node = graph.keys()[idx]
network.addNode(self.getPeer())
if node:
network.createConnection(self.getPeer(), node)
self.getPeer().connected()
# randon time for disconnect
disconnectionTime = randint(3600, 28800)
self.getPeer().setDisconnectionTime(disconnectionTime)
sem.release()
class Barrier(object):
def __init__(self, total = 2):
self.waiting = 0
self.total = total
self.waitSem = Semaphore()
self.waitSem.acquire()
self.mutex = Semaphore()
def sync(self):
self.mutex.acquire()
if self.waiting == self.total - 1:
self.waitSem.release()
else:
self.waiting += 1
self.mutex.release()
self.waitSem.acquire()
self.waiting -= 1
if self.waiting ==0:
self.mutex.release()
else:
self.waitSem.release()
class TestLongFS3(TestLPBase):
def MOCKED_DB_mod_file_dicom_exec(self, f):
orig_isfile = os.path.isfile
def mock_isfile(s, *args, **kwargs):
return os.path.isfile(os.path.join(self.db_dir, "test_db")) if "stats.db" in s else orig_isfile(s)
with patch("os.path.isfile", side_effect=mock_isfile), patch("pysqlcipher.dbapi2.connect", lambda s, *args, **kwargs: self.db_conn if "stats.db" in s else self.jobdb_conn):
f()
def call_START(self, mock_pid_ptid_mapping=MagicMock(has_key=lambda *args, **kwargs: False)):
self.Locks = [Lock() for _ in range(0, 3)]
self.NL_L = Semaphore(2)
self.free_threads = Semaphore(5)
mock_proc_inter_instance = MagicMock(free_threads=self.free_threads,
Locks=self.Locks,
NL_L=self.NL_L,
pid_ptid_mapping=MagicMock(
has_key=lambda *args, **kwargs: False)
)
with patch("processing_interface.ProcessingInterface.Instance", side_effect=lambda *args, **kwargs: mock_proc_inter_instance):
LP = LongProcess.LongProcess(self.test_dicoms_dirs,
[os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs])
self.__test_Locks()
LP._LongProcess__START()
self.__test_Locks()
print LP._Process__success
assert(LP._Process__success == "SUCCESS")
assert(LP._Process__state == STATES.LongFS1)
return LP
def __test_Locks(self):
assert(any([L.acquire(False) for L in self.Locks]))
assert(self.NL_L.acquire(False)
and self.NL_L.acquire(False)
and not self.NL_L.acquire(False))
for L in self.Locks:
L.release()
self.NL_L.release()
self.NL_L.release()
for _ in range(0, 5):
assert(self.free_threads.acquire(False))
assert(not self.free_threads.acquire(False))
for _ in range(0, 5):
self.free_threads.release()
def vanilla_f(self, mock_make_call, mock__print_log, message1, message2, state1, state2):
with patch("FS.FreeSurfer.make_call", side_effect=mock_make_call), patch("FS.FreeSurfer._FreeSurfer__print_log", side_effect=mock__print_log), patch("path_constants.jobs", self.jobs), patch("path_constants_webinterface.jobs", self.jobs):
LP = self.call_START()
LP._LongProcess__LongFS3()
assert(LP._Process__state == state2)
assert(LP._Process__success == message2)
self.__test_Locks()
def test_LongFS3_vanilla(self):
def mock__print_log(*args):
try:
args[-1].write(args[-2])
except:
print args
def mock_make_call(*args):
return 0
f = lambda: self.vanilla_f(
mock_make_call, mock__print_log, "SUCCESS", "SUCCESS", STATES.LongFS1, STATES.GET_FS_ERROR3)
self.MOCKED_DB_mod_file_dicom_exec(f)
with open(os.path.join(self.jobs, "running"), "r") as f:
running = pickle.load(f)
assert(len(running) == 3)
assert(running[0][0] == "0810300310")
def test_LongFS3_join(self):
def mock__print_log(*args):
args[-1].write(args[-2])
def mock_make_call(*args):
# time.sleep(5)
return 0
class MockProcess(multiprocessing.Process):
count = []
def join(self, *args, **kwargs):
self.count += [1]
print "COUNT ", self.count
if(len(self.count) == 2):
raise RuntimeError("Mock join error")
super(MockProcess, self).join(*args, **kwargs)
def f():
with patch("multiprocessing.Process", MockProcess):
self.vanilla_f(mock_make_call, mock__print_log, "SUCCESS",
"Error during FreeSurfer processing: Mock join error", STATES.LongFS1, STATES.QUITTING)
self.MOCKED_DB_mod_file_dicom_exec(f)
with open(os.path.join(self.jobs, "running"), "r") as f:
running = pickle.load(f)
print running
assert(len(running) == 3)
def test_LongFS3_start(self):
def mock__print_log(*args):
args[-1].write(args[-2])
def mock_make_call(*args):
time.sleep(5)
return 0
class MockProcess(multiprocessing.Process):
count = []
def start(self, *args, **kwargs):
self.count += [1]
print self.count
if(len(self.count) == 2):
raise RuntimeError("Mock start error")
super(MockProcess, self).start(*args, **kwargs)
def f():
with patch("multiprocessing.Process", MockProcess):
self.vanilla_f(mock_make_call, mock__print_log, "SUCCESS",
"Error during FreeSurfer processing: Mock start error", STATES.LongFS1, STATES.QUITTING)
self.MOCKED_DB_mod_file_dicom_exec(f)
with open(os.path.join(self.jobs, "running"), "r") as f:
running = pickle.load(f)
print running
assert(len(running) == 3)
def test_LongFS3_make_call_error(self):
def mock__print_log(*args):
args[-1].write(args[-2])
def mock_make_call(*args):
raise RuntimeError("Mock make_call error")
def f():
self.vanilla_f(mock_make_call, mock__print_log, "SUCCESS",
"Error during FreeSurfer processing: Mock make_call error.", STATES.LongFS1, STATES.QUITTING)
self.MOCKED_DB_mod_file_dicom_exec(f)
with open(os.path.join(self.jobs, "running"), "r") as f:
running = pickle.load(f)
print running
assert(len(running) == 3)
def test_LongFS3_mock_file_write(self):
def mock__print_log(*args):
args[-1].write(args[-2])
def mock_make_call(*args):
return 0
orig_mod_file = aux_func.read_write_files.mod_file
def mock_mod_file(*args, **kwargs):
add = True
try:
tmp = args[2]([], 1)
add = len(tmp) == 1
except:
add = False
if(add):
print "%s %s %s" % (add, args[1][0], [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs][0])
if(args[1][0] == [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs][1] and add):
raise RuntimeError("Mock mod_file error")
orig_mod_file(*args, **kwargs)
def f():
with patch("aux_func.read_write_files.mod_file", mock_mod_file):
self.vanilla_f(mock_make_call, mock__print_log,
"SUCCESS", "SUCCESS", STATES.LongFS1, STATES.GET_FS_ERROR3)
self.MOCKED_DB_mod_file_dicom_exec(f)
with open(os.path.join(self.jobs, "running"), "r") as f:
running = pickle.load(f)
print running
assert(len(running) == 3)
def test_LongFS3_mock_file_remove(self):
def mock__print_log(*args):
args[-1].write(args[-2])
def mock_make_call(*args):
return 0
orig_mod_file = aux_func.read_write_files.mod_file
def mock_mod_file(*args, **kwargs):
add = True
try:
tmp = args[2]([], 1)
add = len(tmp) != 1
except:
add = False
if(add):
print "%s %s %s" % (add, args[1][0], [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs][0])
if(args[1][0] == [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs][1] and add):
raise RuntimeError("Mock mod_file error")
orig_mod_file(*args, **kwargs)
def f():
with patch("aux_func.read_write_files.mod_file", mock_mod_file):
self.vanilla_f(mock_make_call, mock__print_log,
"SUCCESS", "SUCCESS", STATES.LongFS1, STATES.GET_FS_ERROR3)
self.MOCKED_DB_mod_file_dicom_exec(f)
with open(os.path.join(self.jobs, "running"), "r") as f:
running = pickle.load(f)
print self.jobs
print running
assert(len(running) != 1)
def process(self, input_filepath, msi_loci, config):
self.__reset()
# Generate dictionary read counts for loci
counts = {}
loci = []
for locus in msi_loci:
counts[locus.locus()] = {}
loci.append(locus)
# Generate input and output queues and (mutex) semaphores
# for each.
queue_out = Queue()
queue_in = Queue()
queue_full = BoundedSemaphore(100)
full = Semaphore(0)
empty = BoundedSemaphore(40)
mutex_out = Semaphore(1)
mutex_in = Semaphore(1)
# Set amount of consumer threads; minimum one.
consumer_threads = config['threads'] - 1
if consumer_threads < 1:
consumer_threads = 1
# Create producer thread; currently only using single thread
# since I/O is more of the limiter than CPU bound processes.
self.__producer = Process(target=self.extract_reads, args=(
input_filepath,
msi_loci,
full,
empty,
mutex_out,
queue_in,
consumer_threads))
self.__producer.start()
# Spawn the set amount of threads/processes
if self.debug_output:
tprint('Main> Generating {0} analyzer process(es).'.format(consumer_threads))
for i in range(0, consumer_threads):
p = Process(target=self.read_analyzer, args=(
queue_in,
queue_out,
full,
empty,
mutex_in,
mutex_out,
queue_full))
self.__consumers.append(p)
self.__consumers[-1].start()
# Iterate through the loci, fetching any reads and pushing them to
# the pool of threads, collecting the output as they process it.
query_delay = 0.050 # In seconds
loop_counter = 0
proc_check_interval = 100
while (not queue_out.empty() or self.has_live_threads()):
# Sleep for the set amount of time so the queue isn't constantly
# getting hammered with queries
time.sleep(query_delay)
loop_counter += 1
if loop_counter % proc_check_interval is 0:
# Time to check that the consumers
# didn't die while the producer is still producing
mutex_out.acquire()
self.status_check(queue_out.qsize())
mutex_out.release()
while not queue_out.empty():
# There is data on the queue to be processed;
# the return from the queue should be a tuple
# with (locus, repeat_count)
mutex_out.acquire()
result = queue_out.get()
locus = result[0]
repeat_count = result[1]
if repeat_count >= 0:
if locus not in counts:
counts[locus] = {}
if repeat_count not in counts[locus]:
counts[locus][repeat_count] = 0
counts[locus][repeat_count] += 1
mutex_out.release()
queue_full.release()
if not self.has_live_threads():
# All processes should have terminated.
if self.debug_output:
tprint('Main> All processes complete.')
break
# end while loop
return counts
class SatoriMasterProcess(SatoriProcess):
def __init__(self, is_daemon):
super(SatoriMasterProcess, self).__init__('master')
self.name = 'master'
self.is_daemon = is_daemon
self.sem = Semaphore(0)
def do_handle_signal(self, signum, frame):
for process in reversed(self.started):
logging.info('Terminating %s', process.name)
process.terminate()
# wait for each child so it can deinitialize while other processes (like event master) still exist
process.join(5)
process.kill()
def do_run(self):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
for handler in logger.handlers:
logger.removeHandler(handler)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(process)d - %(levelname)s - %(message)s")
ensuredirs(os.path.dirname(settings.LOG_FILE))
file_handler = logging.FileHandler(settings.LOG_FILE)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
ensuredirs(os.path.dirname(settings.PID_FILE))
fp = open(settings.PID_FILE, 'a+')
try:
fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
fp.close()
raise RuntimeError('Satori already running')
fp.seek(0)
fp.truncate()
fp.write(str(os.getpid()))
fp.flush()
os.chdir('/')
os.umask(0)
os.setsid()
logging.info('Loading ARS interface...')
import satori.core.api
logging.info('ARS interface loaded.')
if self.is_daemon:
logger.removeHandler(console_handler)
os.close(sys.stdin.fileno())
os.open('/dev/null', os.O_RDWR)
os.close(sys.stdout.fileno())
os.open('/dev/null', os.O_RDWR)
os.close(sys.stderr.fileno())
os.open('/dev/null', os.O_RDWR)
self.sem.release()
from satori.core.management.processes import EventMasterProcess, DbevNotifierProcess, ThriftServerProcess, TwistedHttpServerProcess, UwsgiHttpServerProcess, DebugQueueProcess, CheckingMasterProcess, PrintingMasterProcess
to_start = [
EventMasterProcess(),
DebugQueueProcess(),
DbevNotifierProcess(),
CheckingMasterProcess(),
PrintingMasterProcess(),
ThriftServerProcess(),
# choose one of the two following:
TwistedHttpServerProcess(),
#UwsgiHttpServerProcess(),
]
self.started = []
for process in to_start:
process.start()
self.started.append(process)
for process in reversed(self.started):
process.join()
fp.close()
os.remove(settings.PID_FILE)
def start(self, *args, **kwargs):
super(SatoriMasterProcess, self).start(*args, **kwargs)
while True:
if self.sem.acquire(False):
return
if not self.is_alive():
raise RuntimeError('Satori master failed to start')
sleep(0)
def stop(self):
fp = open(settings.PID_FILE, 'a+')
try:
fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
fp.seek(0)
pid = int(fp.read())
fp.close()
try:
os.kill(pid, SIGINT)
except OSError:
pass
else:
fp.close()
print 'Satori not running'
class TestQUITTING(TestLPBase):
def MOCKED_DB_mod_file_dicom_exec(self, f):
orig_isfile = os.path.isfile
def mock_isfile(s, *args, **kwargs):
return orig_isfile(os.path.join(self.test_installation, "stats.db")) if "stats.db" in s else orig_isfile(s)
with patch("os.path.isfile", side_effect=mock_isfile), patch("path_constants.installation", self.test_installation):
f()
def call_START(self, mock_pid_ptid_mapping=MagicMock(has_key=lambda *args, **kwargs: False)):
self.Locks = [Lock() for _ in range(0, 3)]
self.NL_L = Semaphore(2)
self.free_threads = Semaphore(5)
mock_proc_inter_instance = MagicMock(free_threads=self.free_threads,
Locks=self.Locks,
NL_L=self.NL_L,
pid_ptid_mapping=MagicMock(
has_key=lambda *args, **kwargs: False)
)
with patch("processing_interface.ProcessingInterface.Instance", side_effect=lambda *args, **kwargs: mock_proc_inter_instance):
LP = LongProcess.LongProcess(self.test_dicoms_dirs,
[os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs])
self.__test_Locks()
LP._LongProcess__START()
self.__test_Locks()
assert(LP._Process__success == "SUCCESS")
assert(LP._Process__state == STATES.LongFS1)
return LP
def __test_Locks(self):
assert(any([L.acquire(False) for L in self.Locks]))
assert(self.NL_L.acquire(False)
and self.NL_L.acquire(False)
and not self.NL_L.acquire(False))
for L in self.Locks:
L.release()
self.NL_L.release()
self.NL_L.release()
for _ in range(0, 5):
assert(self.free_threads.acquire(False))
assert(not self.free_threads.acquire(False))
for _ in range(0, 5):
self.free_threads.release()
def vanilla_f(self, mock__print_log, message1, message2, state1, state2):
with patch("path_constants.jobs", self.jobs), patch("path_constants_webinterface.jobs", self.jobs):
LP = self.call_START()
LP._LongProcess__QUITTING()
print LP._Process__success
assert(LP._Process__state == state2)
assert(LP._Process__success == message2)
self.__test_Locks()
def test_QUITTING_vanilla(self):
def mock__print_log(*args):
try:
args[-1].write(args[-2])
except:
print args
f = lambda: self.vanilla_f(
mock__print_log, "SUCCESS", "SUCCESS", STATES.LongFS1, STATES.DONE)
with patch("path_constants.fs_subjectsdir",
self.test_fs_sub),\
patch("path_constants_webinterface.fs_subjectsdir",
self.test_fs_sub), \
patch("path_constants.fs_dir",
FS_DIR):
self.MOCKED_DB_mod_file_dicom_exec(f)
with open(os.path.join(self.jobs, "running"), "r") as f:
running = pickle.load(f)
assert(len(running) == 0)
with open(os.path.join(self.jobs, "done"), "r") as f:
done = pickle.load(f)
assert(len(done) == 3)
for i in range(0, len(TEST_FS_RES)):
assert(done[i][0] == str(2911752899 + i))
for i in range(1, len(TEST_FS_RES) + 1):
assert(not os.path.isdir(os.path.join(
self.test_fs_sub, (TEMPLATE_NAME +
str(i) +
".long." +
TEMPLATE_NAME))))
def test_QUITTING_db(self):
def mock__print_log(*args):
try:
args[-1].write(args[-2])
except:
print args
f = lambda: self.vanilla_f(
mock__print_log, "SUCCESS", "SUCCESS", STATES.LongFS1, STATES.DONE)
with\
patch("path_constants.fs_subjectsdir",
self.test_fs_sub),\
patch("path_constants_webinterface.fs_subjectsdir",
self.test_fs_sub),\
patch("path_constants.fs_dir",
FS_DIR):
self.MOCKED_DB_mod_file_dicom_exec(f)
with open(os.path.join(self.jobs, "running"), "r") as f:
running = pickle.load(f)
assert(len(running) == 0)
with open(os.path.join(self.jobs, "done"), "r") as f:
done = pickle.load(f)
assert(len(done) == 3)
for i in range(0, len(TEST_FS_RES)):
assert(done[i][0] == str(2911752899 + i))
for i in range(1, len(TEST_FS_RES) + 1):
assert(not os.path.isdir(os.path.join(
self.test_fs_sub, (TEMPLATE_NAME +
str(i) +
".long." +
TEMPLATE_NAME))))
with sqlite3.connect(os.path.join(self.test_installation, "stats.db")) as conn:
cur = conn.cursor()
cur.execute("PRAGMA key='f$$pm->>>'")
ret = cur.execute(
"select ptid from errors group by ptid").fetchall()
assert(len(ret) == 3)
def countPeers(self):
semaphore = Semaphore()
semaphore.acquire()
tamPeers = len(self.__layout)
semaphore.release()
return tamPeers
class DownloadManager:
"""Class to start and track status of download processes."""
def __init__(self, api):
self.api = api
self.config = self.api.session.config
self.download_progress = dict()
self.download_progress['active'] = dict()
self.download_progress['completed'] = list()
self.download_progress['failed'] = list()
self.pool = Pool(CORES_TO_USE, initializer=initialize_download, maxtasksperchild=1)
self.sema = Semaphore()
def start_dl(self, broadcast):
"""Adds a download task to the multiprocessing pool"""
print("[{0}] Adding Download: {1}".format(current_datetimestring(), broadcast.title))
self.pool.apply_async(Download(broadcast).start, (), callback=self._callback_dispatcher)
self.sema.acquire()
self.active_downloads[broadcast.id] = broadcast
self.sema.release()
def review_broadcast_status(self, broadcast, download_ok):
"""Starts download of broadcast replay if not already gotten; or, resumes interrupted
live download. Print status to console.
"""
old_title = broadcast.title
broadcast.lock_name = False
broadcast.update_info()
if broadcast.isreplay and broadcast.replay_downloaded:
return None
elif download_ok and broadcast.islive:
broadcast.dl_failures += 1
failure_message = None
if broadcast.dl_failures > MAX_DOWNLOAD_ATTEMPTS:
if broadcast.islive and broadcast.available:
broadcast.wait_for_replay = True
broadcast.dl_failures = 0
print("[{0}] Too many live resume attempts, waiting for replay: {1} {2}".format(
current_datetimestring(), old_title, failure_message))
else:
failure_message = "\n\tExceeded maximum download attempts "
if broadcast.failure_reason is not None:
failure_message += "with the following error:\n\t" + \
str(broadcast.failure_reason)
elif not (broadcast.islive or broadcast.isreplay or broadcast.dl_failures == 0):
failure_message = "\n\tBroadcast no longer available."
elif broadcast.dl_failures > 0:
print("[{0}] Resuming download (Attempt {1} of {2}): {3}".format(
current_datetimestring(), broadcast.dl_failures, MAX_DOWNLOAD_ATTEMPTS,
broadcast.title))
elif broadcast.isreplay and not broadcast.replay_downloaded:
print("[{0}] Downloading replay of: "
"{1}".format(current_datetimestring(), broadcast.title))
else:
return None
if failure_message is not None:
print("[{0}] Failed: {1} {2}".format(current_datetimestring(),
old_title, failure_message))
self.sema.acquire()
self.failed_downloads.append((current_datetimestring(), broadcast))
self.sema.release()
else:
self.start_dl(broadcast)
def _callback_dispatcher(self, results):
"""Unpacks callback argument and passes to appropriate cleanup method"""
download_ok, broadcast = results
self.sema.acquire()
del self.active_downloads[broadcast.id]
self.sema.release()
if download_ok:
print("[{0}] Completed: {1}".format(current_datetimestring(), broadcast.title))
self.sema.acquire()
self.completed_downloads.append((current_datetimestring(), broadcast))
self.sema.release()
else:
broadcast.dl_failures += 1
self.review_broadcast_status(broadcast, download_ok)
@property
def status(self):
"""Retrieve status string for printing to console"""
self.sema.acquire()
active = len(self.active_downloads)
complete = len(self.completed_downloads)
failed = len(self.failed_downloads)
self.sema.release()
cur_status = "{0} active downloads, {1} completed downloads, " \
"{2} failed downloads".format(active, complete, failed)
return "[{0}] {1}".format(current_datetimestring(), cur_status)
@property
def currently_downloading(self):
"""Returns list of the broadcast.title property of all active broadcast downloads"""
self.sema.acquire()
_ = [broadcast.title for _, broadcast in self.active_downloads.items()]
self.sema.release()
return _
@property
def active_downloads(self):
"""Return dictionary of active downloads"""
return self.download_progress['active']
@property
def completed_downloads(self):
"""Return list of completed downloads"""
return self.download_progress['completed']
@property
def failed_downloads(self):
"""Return list of failed downloads"""
return self.download_progress['failed']
def getCurrentSimulationTime(self):
semaphore = Semaphore()
semaphore.acquire()
time = self.__currentSimulationTime
semaphore.release()
return time
class ForkingWorker(BaseWorker):
##
# Overridden from BaseWorker
def __init__(self, num_processes=1):
# Set up sync primitives, to communicate with the spawned children
self.num_processes = num_processes
# This semaphore is used as a "worker pool guard" to keep the number
# of spawned workers in the pool to the specified maximum (and block
# the .spawn_child() call after that)
self._semaphore = Semaphore(num_processes)
# This array of integers represents a slot per worker and holds the
# actual pids (process ids) of the worker's children. Initially, the
# array-of-pids is all zeroes. When a new child is spawned, the pid
# is written into the slot. WHen a child finishes, it resets its own
# slot to 0 again, effectively freeing up the slot (and allowing new
# children to be spawned).
self._pids = Array('i', [0] * num_processes)
# This array of integers also represents a slot per worker and also
# holds the actual pids of the worker's children. The difference with
# _pids, however, is that this array's slots don't get reset
# immediately when the children end. In order for Unix subprocesses
# to actually disappear from the process list (and freeing up the
# memory), they need to be waitpid()'ed for by the parent process.
# When each new child is spawned, it waitpid()'s for the (finished)
# child that was previously in that slot before it claims the new
# slot. This mainly avoids ever-growing process lists and slowly
# growing the memory footprint.
self._waitfor = Array('i', [0] * num_processes)
# This array of booleans represent workers that are in their idle
# state (i.e. they are waiting for work). During this time, it is
# safe to terminate them when the user requests so. Once they start
# processing work, they flip their idle state and won't be terminated
# while they're still doing work.
self._idle = Array('b', [False] * num_processes)
def get_ident(self):
return os.getpid()
def spawn_child(self):
"""Forks and executes the job."""
# Responsible for the blocking, may be interrupted by SIGINT or
# SIGTERM, the worker's main loop will catch it
with Interruptable():
self._semaphore.acquire()
self._fork()
def wait_for_children(self):
"""
Wait for children to finish their execution. This function should
block until all children are finished. Must be interruptable by
another press of Ctrl+C, which kicks off forceful termination.
"""
# As soon as we can acquire all slots, we're done executing
with Interruptable():
for pid in self._pids:
if pid != 0:
print 'waiting for pid %d to finish gracefully...' % (pid,)
waitpid(pid)
def terminate_idle_children(self):
for slot, idle in enumerate(self._idle):
pid = self._pids[slot]
if idle:
print '==> Killing idle pid {}'.format(pid)
kill(pid, signal.SIGKILL)
#os.waitpid(pid, 0) # necessary?
else:
print '==> Waiting for pid {} (still busy)'.format(pid)
def kill_children(self):
"""
Force-kill all children. This function should block until all
children are terminated.
"""
# As soon as we can acquire all slots, we're done executing
for pid in self._pids:
if pid != 0:
print 'killing pid %d...' % (pid,)
kill(pid, signal.SIGKILL)
self.wait_for_children()
##
# Helper methods (specific to forking workers)
def _fork(self): # noqa
slot = self._claim_slot()
# The usual hardcore forking action
child_pid = os.fork()
if child_pid == 0:
random.seed()
# Within child
try:
def _mark_busy(slot):
def _inner():
self._idle[slot] = False
return _inner
self._idle[slot] = True
self.main_child(_mark_busy(slot))
finally:
# Remember, we're in the child process currently. When all
# work is done here, free up the current slot (by writing
# a 0 in the slot position). This communicates to the parent
# that the current child has died (so can safely be forgotten
# about).
self._pids[slot] = 0
self._semaphore.release()
os._exit(0)
else:
# Within parent, keep track of the new child by writing its PID
# into the first free slot index.
self._pids[slot] = child_pid
self._waitfor[slot] = child_pid
def _claim_slot(self):
slot = self._find_empty_slot()
self._wait_for_previous_worker(slot)
return slot
def _find_empty_slot(self):
# Select an empty slot from self._pids (the first 0 value is picked)
# The implementation guarantees there will always be at least one empty slot
for slot, value in enumerate(self._pids):
if value == 0:
return slot
raise RuntimeError('This should never happen.')
def _wait_for_previous_worker(self, slot):
if self._waitfor[slot] > 0:
os.waitpid(self._waitfor[slot], 0)
self._waitfor[slot] = 0
class Scheduler():
def __init__(self):
self.ready_list = []
self.last_run = None;\
self.semafore=Semaphore(1);
# Add a process to the run list
def add_process(self, process):
#set the index as the last element of the list at the begining
#RACE condition here!!
'''
Say a process with priority 1 comes into the ready_list with 5 elements
The ready_list priorities are as shown
ready_list_priorities={10,8,8,6,4}
the index=len(self.ready_list) line executes
and gets the value 5. This means index =5
then say before the next line is executed, another process is added to the ready_list, its priority is 3
then ready_list_priorities={10,8,8,6,4,3}
but index is still 5 so new process will be inserted here rather than at the end of the list
so ready_list_priorities={10,8,8,6,4,1,3}
this is incorrect
we must lock this method and only allow one process to access it at a time
maybe with a semaphore
'''
self.semafore.acquire()
index=len(self.ready_list)
#find the first priority that is less than the priority of the process
for i in range (len(self.ready_list)-1,-1,-1):
if self.ready_list[i].priority<process.priority:
index = i
#add the item there
self.ready_list.insert(index, process)
self.semafore.release()
return
def remove_process(self, process):
#do more shit to ensure that all things remain the way they are
self.ready_list.remove(process)
# Selects the process with the best priority.
# If more than one have the same priority these are selected in round-robin fashion.
def select_process(self):
#return none if the process list is empty
if len(self.ready_list)==0:
return None
#otherwise check to see if the lastrun process process is the same as the current running process
if self.last_run == self.ready_list[0]:
#make sure the list doesnt contain only one process
if len(self.ready_list)>1:
#check to see that here are no more processes with the same or higher priority level
#because the ready_list is already sorted, we only need to check the next element and check that it is not
#of equal or higher priority
if(self.last_run.priority<=self.ready_list[1].priority):
#if it is, remove this process from the readylist and put it back in
#This has the effect of placing it behind all the processes with the same priority level as this process
self.remove_process(self.last_run)
self.add_process(self.last_run)
#return the process that is at the head of the queue and move it to the tail of the queue
self.last_run = self.ready_list[0]
return self.ready_list[0]
# Suspends the currently running process by sending it a STOP signal.
@staticmethod
def suspend(process):
os.kill(process.pid, signal.SIGSTOP)
# Resumes a process by sending it a CONT signal.
@staticmethod
def resume(process):
if process.pid: # if the process has a pid it has started
os.kill(process.pid, signal.SIGCONT)
else:
process.run()
def run(self):
current_process = None
while True:
#print('length of ready_list:', len(self.ready_list))
next_process = self.select_process()
if next_process == None: # no more processes
controller_write.write('terminate\n')
sys.exit()
if next_process != current_process:
if current_process:
self.suspend(current_process)
current_process = next_process
self.resume(current_process)
time.sleep(1)
# need to remove dead processes from the list
try:
current_process_finished = (
os.waitpid(current_process.pid, os.WNOHANG) != (0, 0)
)
except ChildProcessError:
current_process_finished = True
if current_process_finished:
print('remove process', current_process.pid, 'from ready list')
self.remove_process(current_process)
current_process = None
def getPeers(self):
semaphore = Semaphore()
semaphore.acquire()
peers = self.__layout.itervalues()
semaphore.release()
return peers
class URLGather(object):
def __init__(self, *args, **kwargs):
self.url = kwargs.get("url")
if not self.url:
raise Exception("No URL to gather")
self.max_depth = kwargs.get("depth", 1)
self.workers = kwargs.get("workers", 1)
self.max_errors = kwargs.get("acceptable_errors", None)
self.out = kwargs.get("out", "/tmp/")
if not self.out.endswith("/"):
self.out += "/"
self.out += "url_gather/"
if not os.path.exists(self.out):
os.makedirs(self.out)
self.collector_file = kwargs.get("collector_file")
self.collector_class = kwargs.get("collector_class")
self._load_collector()
self._gathered_urls = set()
# initiate multiprocessing resources
self._pool = Pool(self.workers)
self._semaphore = Semaphore(self.workers)
self._manager = Manager()
self._url_children = self._manager.dict()
self._url_errors = self._manager.dict()
self._url_events = {}
def _load_collector(self):
if self.collector_file:
if os.path.isfile(self.collector_file):
if self.collector_class:
# TODO load custom collector
with open(self.collector_file, "r") as custom_code:
import_code(custom_code.read(), "custom_collector", 1)
m = __import__("custom_collector")
self.collector = m.__getattribute__(self.collector_class)
else:
raise Exception("Undefined custom collector class name.")
else:
raise Exception("Custom collector file %s not found." % self.collector_file)
else:
self.collector = DefaultCollector
def run(self):
self._gather_url(self.url)
self._pool.close()
self._pool.join()
if self._exceed_max_errors():
error_log = "%serror.log" % self.out
with open(error_log, "w+") as f:
f.write(self._url_errors)
print "REACHED MAX ERRORS. SEE %s FILE TO MORE DETAILS" % error_log
def task_done(self, result):
self._semaphore.release()
def _exceed_max_errors(self):
return self.max_errors is not None and self.max_errors >= 0 and len(self._url_errors) > self.max_errors
def _wait_children_in_thread(self, url):
while url not in self._url_children:
time.sleep(0.1)
event = self._url_events.pop(url)
event.set()
def _wait_for_children(self, url):
event = Event()
self._url_events[url] = event
t = Thread(target=self._wait_children_in_thread, args=(url,))
t.start()
event.wait()
def _gather_url(self, url, current_depth=0):
if self._exceed_max_errors() or url in self._gathered_urls:
return
self._gathered_urls.add(url)
gather_children = current_depth < self.max_depth
self._semaphore.acquire()
if self._exceed_max_errors():
return
self._pool.apply_async(
run_in_child,
(url, self.collector, self._url_children, gather_children, self.out, self._url_errors),
callback=self.task_done,
)
# gather children links
if gather_children:
self._wait_for_children(url)
if self._url_children[url]:
for child in self._url_children[url]:
self._gather_url(child, current_depth + 1)
class Barrier:
"""Simple reusable semaphore barrier.
Python 2.6 doesn't have multiprocessing barriers so we implement this.
See http://greenteapress.com/semaphores/downey08semaphores.pdf, p. 41.
"""
def __init__(self, n, timeout=None):
self.n = n
self.to = timeout
self.count = Value('i', 0)
self.mutex = Semaphore(1)
self.turnstile1 = Semaphore(0)
self.turnstile2 = Semaphore(1)
def wait(self):
if not self.mutex.acquire(timeout=self.to):
raise BarrierTimeoutError()
self.count.value += 1
if self.count.value == self.n:
if not self.turnstile2.acquire(timeout=self.to):
raise BarrierTimeoutError()
self.turnstile1.release()
self.mutex.release()
if not self.turnstile1.acquire(timeout=self.to):
raise BarrierTimeoutError()
self.turnstile1.release()
if not self.mutex.acquire(timeout=self.to):
raise BarrierTimeoutError()
self.count.value -= 1
if self.count.value == 0:
if not self.turnstile1.acquire(timeout=self.to):
raise BarrierTimeoutError()
self.turnstile2.release()
self.mutex.release()
if not self.turnstile2.acquire(timeout=self.to):
raise BarrierTimeoutError()
self.turnstile2.release()
