def __init__(self, config, event_queue, debug, rib, policy_handler, test,
no_notifications, rib_timing, notification_timing):
super(LoopDetector, self).__init__(config, event_queue, debug)
self.config = config
self.cib = CIB(self.config.id)
self.rib = rib
self.policy_handler = policy_handler
# mapping of participant and prefix to list of forbidden forward participants
self.forbidden_paths = defaultdict(lambda: defaultdict(list))
self.run = False
self.listener = Listener(
(self.config.sdx.address, self.config.loop_detector.port),
authkey=None)
self.msg_in_queue = Queue(1000)
self.msg_out_queue = Queue(1000)
self.no_notifications = no_notifications
self.rib_timing = rib_timing
if self.rib_timing:
self.rib_timing_file = 'rib_timing_' + str(int(time())) + '.log'
self.notification_timing = notification_timing
if self.notification_timing:
self.notification_timing_file = 'notification_timing_' + str(
int(time())) + '.log'
def compute_sp(self):
from queue import Queue
queue = Queue()
datalen = len(self.D['coords'])
self(queue,0,datalen, True, False)
self(queue,0,datalen, False, False)
return queue.get() + queue.get()
class TestBlockingMethods(unittest.TestCase):
def setUp(self):
self.quiet=True
self.random = Random()
self._timers = []
self.namespaces = []
self.iface = PyRQIface(quiet=self.quiet, ref="test")
self.dummyQueue = Queue()
self.marshaller = MarshallerFactory.get(MarshallerFactory.DEFAULT, quiet=self.quiet)
desiredPort = "19001"
self.r = SubprocessQueueServer(
desiredPort=desiredPort,
handlerClazz=Linkage.create(MockHandler),
quiet=self.quiet
# includePydevd="/home/francis/.eclipse/org.eclipse.platform_3.7.0_155965261/plugins/org.python.pydev.debug_2.5.0.2012040618/pysrc"
)
PyRQIface.setGlobalPYRQ(self.r.details())
self.r.start().waitUntilRunning()
pass
def tearDown(self):
try:
self.dummyQueue.close()
del self.dummyQueue
except Exception, _e:
pass
for namespace in self.namespaces:
self.iface.setNamespace(namespace)
try: self.iface.close()
except ClosedError, _e:
pass
def main():
result_queue = Queue()
crawler = CrawlerWorker(CanberraWealtherSpider(), result_queue)
crawler.start()
for item in result_queue.get():
#print datetime.datetime.now(),item
print item
class Multiprocess(object):
# THE COMPLICATION HERE IS CONNECTING THE DISPARATE LOGGING TO
# A CENTRAL POINT
def __init__(self, functions):
self.outbound = Queue()
self.inbound = Queue()
self.inbound = Queue()
#MAKE
#MAKE THREADS
self.threads = []
for t, f in enumerate(functions):
thread = worker(
"worker " + unicode(t),
f,
self.inbound,
self.outbound,
)
self.threads.append(thread)
def __enter__(self):
return self
#WAIT FOR ALL QUEUED WORK TO BE DONE BEFORE RETURNING
def __exit__(self, a, b, c):
try:
self.inbound.close() # SEND STOPS TO WAKE UP THE WORKERS WAITING ON inbound.pop()
except Exception, e:
Log.warning("Problem adding to inbound", e)
self.join()
def parallel_process_by_queue(num_p,
data_iter,
target,
args,
ctx: BaseContext = None,
task_unit_size=5000,
print_out=__debug__):
if isinstance(target, MPTarget):
target.use_queue = True
if ctx is None:
ctx = get_context('spawn')
iq = Queue(ctx=ctx)
oq: Manager = ctx.Manager().Queue()
tic(f"Creating input queue with task unit size {task_unit_size}",
verbose=print_out)
cnt_task_unit = 0
for item in tqdm(slices__(data_iter, task_unit_size)):
iq.put(item)
cnt_task_unit += 1
jobs = [None] * num_p
for i in range(num_p):
jobs[i] = ctx.Process(target=target, args=(i, iq, oq) + args)
toc()
tic(f"Working on {cnt_task_unit} task units with {num_p} processes",
verbose=print_out)
start_and_wait_jobs(jobs)
out = []
while not oq.empty():
out.append(oq.get_nowait())
toc()
return out
def put(self, contents: QT, block=True, timeout=None):
self.lock.acquire()
while not self.empty():
Queue.get(self, block=False)
# NOTE/TODO: this, because multiprocessing Queues are stupid, is
# necessary. Explained in short, if you try to q.put_nowait() too
# quickly, it breaks. For example, say you were in ipython,
# and you typed to following
# - q = MonoQueue()
# - q.put_nowait(2)
# - q.put_nowait(3)
# - q.put_nowait(4)
# - q.put_nowait(5)
# EVEN THOUGH there is a Lock() to atomize the access to the Queue,
# one of the non-first 'put_nowait()' calls will acquire the lock,
# the 'self.empty()' call is apparently True, even though something is
# actually in the queue, and then it will not '.get()' it and try to
# put something in the queue, raise a 'Full' exception.
# So basically, apparently if something tries to put in the queue too
# quickly, everything breaks. And yes, I made a pytest to test this,
# guess what, if you try to run a trace (debugger), aka you jus step
# through, it works fine, but as soon as you just run it, it breaks.
# UGH, maybe I'm dumb and am doing something wrong
with suppress(Full):
Queue.put(self, contents, block=block, timeout=timeout)
self.lock.release()
def compute_sp(self):
from Queue import Queue
queue = Queue()
datalen = len(self.D['coords'])
self(queue, 0, datalen, True, False)
self(queue, 0, datalen, False, False)
return queue.get() + queue.get()
def _calculate_rmse_mp(self, population, process_count):
i = 0
process_pop = dict()
while i < len(population):
for j in range(process_count):
if str(j) not in process_pop.keys():
process_pop[str(j)] = []
if i < len(population):
process_pop[str(j)].append(population[i])
i += 1
final_population = []
queue = Queue()
processes = []
for i in range(process_count):
pop = process_pop[str(i)]
process = Process(target=self._calculate_rmse,
name="%d" % i,
args=(pop, queue))
process.start()
processes.append(process)
for i in range(process_count):
final_population += queue.get()
for process in processes:
process.join()
return final_population
def __init__(self, name=None):
self.name = name
self._state = None
ctx = SpawnContext()
self.alive = Value('b', True)
self.in_queue = Queue(ctx=ctx, maxsize=120)
self.out_queue = Queue(ctx=ctx, maxsize=110)
def main():
result_queue = Queue()
crawler = CrawlerWorker(CanberraWealtherSpider(), result_queue)
crawler.start()
for item in result_queue.get():
#print datetime.datetime.now(),item
print item
class Multiprocess(object):
# THE COMPLICATION HERE IS CONNECTING THE DISPARATE LOGGING TO
# A CENTRAL POINT
def __init__(self, functions):
self.outbound = Queue()
self.inbound = Queue()
self.inbound = Queue()
#MAKE
#MAKE THREADS
self.threads = []
for t, f in enumerate(functions):
thread = worker(
"worker " + unicode(t),
f,
self.inbound,
self.outbound,
)
self.threads.append(thread)
def __enter__(self):
return self
#WAIT FOR ALL QUEUED WORK TO BE DONE BEFORE RETURNING
def __exit__(self, a, b, c):
try:
self.inbound.close(
) # SEND STOPS TO WAKE UP THE WORKERS WAITING ON inbound.pop()
except Exception, e:
Log.warning("Problem adding to inbound", e)
self.join()
def process_init(self):
self.event_queue = Queue(1000)
self.event_queue_name = str(id(self))
PmakeManager.queues[self.event_queue_name] = self.event_queue
# info('Starting %d processes' % self.num_processes)
self.subs = {} # name -> sub
# available + processing + aborted = subs.keys
self.sub_available = set()
self.sub_processing = set()
self.sub_aborted = set()
db = self.context.get_compmake_db()
storage = db.basepath # XXX:
logs = os.path.join(storage, 'logs')
#self.signal_queue = Queue()
for i in range(self.num_processes):
name = 'parmake_sub_%02d' % i
write_log = os.path.join(logs, '%s.log' % name)
make_sure_dir_exists(write_log)
signal_token = name
self.subs[name] = PmakeSub(name=name,
signal_queue=None,
signal_token=signal_token,
write_log=write_log)
self.job2subname = {}
# all are available
self.sub_available.update(self.subs)
self.max_num_processing = self.num_processes
def process_init(self):
self.event_queue = Queue()
self.event_queue_name = str(id(self))
from compmake.plugins.backend_pmake.pmake_manager import PmakeManager
PmakeManager.queues[self.event_queue_name] = self.event_queue
# info('Starting %d processes' % self.num_processes)
self.subs = {} # name -> sub
# available + processing + aborted = subs.keys
self.sub_available = set()
self.sub_processing = set()
self.sub_aborted = set()
self.signal_queue = Queue()
db = self.context.get_compmake_db()
storage = db.basepath # XXX:
logs = os.path.join(storage, 'logs')
for i in range(self.num_processes):
name = 'w%02d' % i
write_log = os.path.join(logs, '%s.log' % name)
make_sure_dir_exists(write_log)
signal_token = name
self.subs[name] = PmakeSub(name,
signal_queue=self.signal_queue,
signal_token=signal_token,
write_log=write_log)
self.job2subname = {}
self.subname2job = {}
# all are available at the beginning
self.sub_available.update(self.subs)
self.max_num_processing = self.num_processes
def _setup(self):
if isinstance(self._instruction, I.ATM):
self._code = self._instruction.code
self._instruction_pipelines.append(Pipeline(self._code.instructions, self._tables))
self._atomic_process = Process(target=self.run_atomic)
self._is_atomic_enabled = True
elif isinstance(self._instruction, I.SEQ):
self._code = self._instruction.code
self._instruction_pipelines.append(Pipeline(self._code.instructions, self._tables))
self._sequential_ingress_process = Process(target=self.run_sequential_ingress)
self._sequential_egress_process = Process(target=self.run_sequential_egress)
self._metadata_queue = Queue()
self._is_sequential_enabled = True
elif isinstance(self._instruction, I.CNC):
# Note: CNC can't have PUSH/POP instructions in its code blocks. They violate the concurrency invariant.
self._codes = self._instruction.codes
self._modified_locations = []
self._modified_reserved_fields = []
self._modified_fields = []
for code in self._codes:
self._instruction_pipelines.append(Pipeline(code.instructions, self._tables))
self._modified_locations.append(get_modified_locations(code.instructions))
self._modified_reserved_fields.append(get_modified_reserved_fields(code.instructions))
self._modified_fields.append(get_modified_fields(code.instructions, code.argument_fields))
self._concurrent_ingress_process = Process(target=self.run_concurrent_ingress)
self._concurrent_egress_process = Process(target=self.run_concurrent_egress)
self._metadata_queue = Queue()
self._is_concurrent_enabled = True
else:
raise RuntimeError()
def __init__(self, process: Subprocess, chunk_size=CHUNK_SIZE_DEFAULT):
if not isinstance(process, Subprocess):
raise TypeError("process must be Subprocess")
if not process.is_alive():
raise ValueError("Process wasn't working.")
if chunk_size <= 0:
raise ValueError("Chunk size must be > 0.")
if self.process.stdout is None and self.process.stdin is None:
raise RuntimeError("Process IO are unavailable.")
self.process = process
self.chunk_size = chunk_size
self.read_buffer_cache = b""
if self.process.stdin is not None:
self.queue_write = Queue()
self.thread_write = Thread(target=self._write)
self.thread_write.start()
else:
self.queue_write = None
self.thread_write = None
if self.process.stdout is not None:
self.queue_read = Queue()
self.thread_read = Thread(target=self._read)
self.thread_read.start()
else:
self.queue_read = None
self.thread_read = None
def force_stop(self):
self._stop_requested = True
# Just in case the user calls this directly. Will signal all threads to
# stop (save _com_thread).
self._stop.set()
self._log_queue.put(None)
self._monitor_queue.put((None, None))
# Set _force_stop to stop _com_thread.
self._force_stop.set()
# Terminate the process and make sure all threads stopped properly.
self._process.terminate()
self._log_thread.join()
self._monitor_thread.join()
self._com_thread.join()
self.active = False
if self._processing.is_set():
self.done = ('INTERRUPTED', 'The user forced the system to stop')
self._processing.clear()
# Discard the queues as they may have been corrupted when the process
# was terminated.
self._log_queue = Queue()
self._monitor_queue = Queue()
class WorkerThreads(object):
def __init__(self, threads=1):
"""
Initialize the thread pool and queues.
"""
self.pools = ThreadPool(processes=threads)
self.updater_queue = Queue()
def get_updater_queue(self):
return self.updater_queue
def updater(self, ident, state, meta):
"""
Updater function: This just post a message to a queue.
"""
self.updater_queue.put({'id': ident, 'state': state, 'meta': meta})
def pull(self, request, updater, testmode=0):
try:
pull(request, updater, testmode=testmode)
except Exception as err:
resp = {'error_type': str(type(err)),
'message': str(err)}
updater.update_status('FAILURE', 'FAILURE', response=resp)
def expire(self, request, updater):
try:
remove_image(request, updater)
except Exception as err:
resp = {'error_type': str(type(err)),
'message': str(err)}
updater.update_status('FAILURE', 'FAILURE', response=resp)
def wrkimport(self, request, updater, testmode=0):
try:
img_import(request, updater, testmode=testmode)
except Exception as err:
resp = {'error_type': str(type(err)),
'message': str(err)}
updater.update_status('FAILURE', 'FAILURE', response=resp)
def dopull(self, ident, request, testmode=0):
"""
Kick off a pull operation.
"""
updater = Updater(ident, self.updater)
self.pools.apply_async(self.pull, [request, updater],
{'testmode': testmode})
def doexpire(self, ident, request, testmode=0):
updater = Updater(ident, self.updater)
self.pools.apply_async(self.expire, [request, updater])
def dowrkimport(self, ident, request, testmode=0):
logging.debug("wrkimport starting")
updater = Updater(ident, self.updater)
self.pools.apply_async(self.wrkimport, [request, updater],
{'testmode': testmode})
class WorkerThreads(object):
def __init__(self, threads=1):
"""
Initialize the thread pool and queues.
"""
self.pools = ThreadPool(processes=threads)
self.updater_queue = Queue()
def get_updater_queue(self):
return self.updater_queue
def updater(self, ident, state, meta):
"""
Updater function: This just post a message to a queue.
"""
self.updater_queue.put({'id': ident, 'state': state, 'meta': meta})
def pull(self, request, updater, testmode=0):
try:
pull(request, updater, testmode=testmode)
except Exception as err:
resp = {'error_type': str(type(err)),
'message': str(err)}
updater.update_status('FAILURE', 'FAILURE', response=resp)
def expire(self, request, updater):
try:
remove_image(request, updater)
except Exception as err:
resp = {'error_type': str(type(err)),
'message': str(err)}
updater.update_status('FAILURE', 'FAILURE', response=resp)
def wrkimport(self, request, updater, testmode=0):
try:
img_import(request, updater, testmode=testmode)
except Exception as err:
resp = {'error_type': str(type(err)),
'message': str(err)}
updater.update_status('FAILURE', 'FAILURE', response=resp)
def dopull(self, ident, request, testmode=0):
"""
Kick off a pull operation.
"""
updater = Updater(ident, self.updater)
self.pools.apply_async(self.pull, [request, updater],
{'testmode': testmode})
def doexpire(self, ident, request, testmode=0):
updater = Updater(ident, self.updater)
self.pools.apply_async(self.expire, [request, updater])
def dowrkimport(self, ident, request, testmode=0):
logging.debug("wrkimport starting")
updater = Updater(ident, self.updater)
self.pools.apply_async(self.wrkimport, [request, updater],
{'testmode': testmode})
def setUp(self):
pg_connector = PostgresConnector(_POSTGRES_DSN)
self.notif_queue = Queue(1)
self.listener = PostgresNotificationListener(
pg_connector, _NOTIF_CHANNEL, self.notif_queue,
CommonErrorStrategy(), Queue(), fire_on_start=False
)
self.listener.log = MagicMock()
def empty_queue(queue_: Queue) -> None:
while True:
try:
queue_.get(block=False)
except queue.Empty:
break
queue_.close()
def setUp(self):
self.notif_queue = Queue(1)
self.error_queue = Queue()
self.component = Component(self.notif_queue._reader,
CommonErrorStrategy(),
self.error_queue,
PostgresConnector(_POSTGRES_DSN))
self.component.log = MagicMock()
def __init__(self, name, machine, ready_func, workers=1):
self.id = name
self.machine = machine
self.ready_func = ready_func
self.name = name
self.readq = Queue(maxsize=-1, ctx=multiprocessing.get_context())
self.num_workers = max(1, workers)
self.workers = []
def __enter__(self):
self.smtp_process_queue = Queue()
self.smtp_process = Process(
target=get_otp_mail, args=(self.smtp_process_queue, self.timeout))
self.smtp_process.start()
self.port = self.smtp_process_queue.get(True, 5)
self._do_lintop_config()
return self
def __init__(self, maxsize=0):
Queue.__init__(self, maxsize, ctx=multiprocessing)
self.__timer_ctrl_func = None
self._flow_ctrl = icom_flow_ctrl.flow_ctrl(current_thread().ident)
self._win_msg = None
self.__buffer_msg = []
self._flow_ctrl.set_timer_req_function(self.req_timer_ctrl_Q_cmd)
self._flow_ctrl.set_flow_ctrl_req_function(self.req_flow_ctrl_Q_cmd)
self._flow_ctrl.set_enable(True)
def __init__(self, maxsize=0):
"""Constructor
Parameters:
maxsize -- Maximium size of this Pile
"""
Queue.__init__(self, maxsize)
self._tasks = RawValue('i',0)
self._tasks_lock = Lock()
def put(self, obj, block=True, timeout=None):
Queue.put(self, obj, block, timeout)
self._put_counter.value += 1
if self.qsize() != 0:
self.cond_notempty.acquire()
try:
self.cond_notempty.notify_all()
finally:
self.cond_notempty.release()
def setUp(self):
self.filter_queue = Queue()
self.message_queue = Queue()
self.client_cfg = {"ip_address": b"127.0.0.1",
"filename": b"/dev/null",
"verbose": 0,
"port": "1234",
}
self.stop_event = Event()
self.handler = DLTMessageHandler(self.filter_queue, self.message_queue, self.stop_event, self.client_cfg)
def start(self,url):
# raise BadFormatError
items = []
# The part below can be called as often as you want
results = Queue()
crawler = CrawlerWorker(LinkedinSpider(url), results)
crawler.start()
for item in results.get():
items.append(dict(item))
return items
def put_while(queue: Queue,
task: Any,
predicate: Callable[[], bool],
timeout: int = 3):
while predicate():
try:
queue.put(task, block=True, timeout=timeout)
break
except Full:
pass
class ServerSink(iMockDebuggerSink):
def __init__(self, peerName, theTime, details, quiet):
self._peerName = peerName
self._methods = []
methods = iMockDebuggerSink()._getMethods()
self._methods = methods
self._terminate = False
self._details = details
self._qw = None
self._startMutex = Semaphore(0)
self._q = Queue()
self.quiet= quiet
self._marshaller = MarshallerFactory.get(MarshallerFactory.DEFAULT, quiet=quiet)
self._qw = QueueWriter(target=details, autoConnect=True, marshaller=self._marshaller, quiet=quiet)
self._qw.start()
self.thread = None
def start(self):
t = threading.Thread(target=self.run, args=[self._startMutex])
t.setName("ServerSink.%(P)s"%{"P":self._peerName})
t.setDaemon(True)
self.thread = t
self.thread.start()
return "server.sink.started"
def close(self):
self._terminate = True
try: self.thread.join()
except: pass
try: self._qw.close()
except: pass
try: self._q.close()
except: pass
return "server.sink.closed"
def waitUntilRunning(self, block=True, timeout=None):
self._startMutex.acquire(block=block, timeout=timeout)
return self
def __getattribute__(self, name):
if name in object.__getattribute__(self, "_methods"):
q = self._q
def wrapper(self, *args, **kwargs):
ServerSink._testPickleability((name, args, kwargs))
q.put((name, args, kwargs))
return wrapper
return object.__getattribute__(self, name)
def run(self, startMutex):
startMutex.release()
while self._terminate==False:
try:
data = self._q.get(block=True, timeout=1)
except Empty: pass
else:
ServerSink._testPickleability(data)
try:
self._qw.put(data, block=True, timeout=10)
except Exception, _e:
break
def run(worker_fn: Callable[[], DataWorker], max_tasks: int, in_queue: Queue,
out_queue: Queue, cancel_process: Value):
# This is the worker function of a spawned process.
# While the task does not receive a Finished flag, if fetches an input sample which is yielded in the worker.
# The outputs are then written to the output queue.
# Each thread is stopping if max_tasks is reached.
logger.debug("Worker starting")
worker = worker_fn()
worker.initialize_thread()
def generator():
# Transform the input queue to a generator
# Note that other processes read from the sample in_queue
while True:
try:
s = in_queue.get(timeout=0.01)
except Empty:
logger.debug("In queue empty.")
if cancel_process.value:
logger.debug("Canceling working generator.")
break
else:
if isinstance(s, Finished):
logger.debug(
"Received Finished. Stopping working generator")
break
yield s
for out in worker.process(generator()):
# Process the data and write it to the queue
while True:
if cancel_process.value:
logger.debug("Canceling working processor (inner).")
break
try:
out_queue.put(out, timeout=0.01)
except Full:
logger.debug("Out queue Full.")
continue
else:
break
if cancel_process.value:
logger.debug("Canceling working processor (outer).")
break
max_tasks -= 1
if max_tasks == 0:
logger.debug("Max tasks reached for this worker. Stopping.")
break
logger.debug("Worker finished")
if cancel_process.value:
out_queue.cancel_join_thread(
) # this prevents a deadlock if the generator is stopped prematurely
class BasicActor(ABC):
name: Optional[str]
in_queue: Queue
out_queue: Queue
alive: Value
_loop_task: Optional[Task]
def __init__(self, name=None):
self.name = name
self._state = None
ctx = SpawnContext()
self.alive = Value('b', True)
self.in_queue = Queue(ctx=ctx, maxsize=120)
self.out_queue = Queue(ctx=ctx, maxsize=110)
async def runner(self):
loop = get_running_loop()
self._state = await self.handle_started()
self._loop_task = loop.create_task(self._do_loop(loop))
async def _do_loop(self, loop: AbstractEventLoop):
while loop.is_running():
try:
sent_from, message = self.in_queue.get(timeout=0.1)
loop.create_task(
self._handle_message(message, sent_from, self._state))
except Empty:
pass
await asyncio.sleep(0)
def send_message(self, to, message):
if self.out_queue.qsize() > 100:
logger.warning("Shedding excess outgoing message")
return
self.out_queue.put((to, message))
async def _handle_message(self, message, sent_from, state):
try:
self._state = await self.handle_message(message, sent_from, state)
except:
self.stop()
raise
def stop(self):
self.alive.value = False
if self._loop_task:
self._loop_task.cancel()
@abstractmethod
async def handle_message(self, message, sent_from, state) -> Any:
pass
@abstractmethod
async def handle_started(self) -> Any:
pass
def __put(self, obj, block=True, timeout=None):
if win_gui_Queue.__sync_flow_ctrl_count <= win_gui_Queue.__SAMPLE_SIZE:
Queue.put(self, obj, block, timeout)
if win_gui_Queue.__real_proc_data_msg_count > win_gui_Queue.__MAX_MSG_RATE_THREDHOLD:
self.flow_ctrl(win_gui_Queue.__real_proc_data_msg_count *
win_gui_Queue.__SAMPLE_SIZE //
(win_gui_Queue.__MAX_MSG_RATE_THREDHOLD))
return self.do_send_ctrl_msg(ICOM_CTRL_MSG.ID_PROC_DATA_MSG,
win_gui_Queue.__SAMPLE_SIZE)
self.send_data_sync_msg(obj)
def __init__(
self,
name: str = "UdderQueue",
num_feeders: int = 1,
context: str = "spawn",
):
self.name = name
self.parse_arguments(num_feeders, name, context)
Queue.__init__(maxsize=0, ctx=multiprocessing.get_context(context))
self.num_feeders = num_feeders
self.lock = Lock
def launchSequentialProcess(worldRunner, observableQueue, updateQueue,
worldNumber, runNumber, useGUI):
""" Run world in the same process since processes run sequentially. """
forkQueue = Queue()
worldRunner(manager, observableQueue, updateQueue, forkQueue,
worldNumber, runNumber, useGUI)
# Empty fork queue
try:
while True:
forkQueue.get(False)
except:
pass
def getResult(self,key):
"This method return the result in imdb for given key"
spider = ImdbSpider(key)
result_queue = Queue()
crawler = CrawlerWorker(spider, result_queue)
crawler.start()
results = result_queue.get()
if len(results)>self.maxResult :
del results[self.maxResult:]
logging.debug('%s results', len(results))
return results
def __init__(self, settings=None):
"""
Parms:
settings (scrapy.settings.Settings) - settings to apply. Defaults
to Scrapy default settings.
"""
kwargs = {"ctx": __import__("billiard.synchronize")}
self.results = Queue(**kwargs)
self.items = []
self.settings = settings or Settings()
dispatcher.connect(self._item_scraped, signals.item_scraped)
def __init__(self, pull_interval=5):
self.input = None
self.filter = None
self.output = None
# for input write and filter read
self.iqueue = Queue()
# for filter write and output read
self.oqueue = Queue()
self.pull_interval = pull_interval
self.__init_all()
def findCalibrationChessboard(image):
findTimeout = 10
patternSize = (7, 7) # Internal corners of 8x8 chessboard
grayImg = cv.CreateMat(image.rows, image.cols, cv.CV_8UC1)
cv.CvtColor(image, grayImg, cv.CV_RGB2GRAY)
cv.AddWeighted(grayImg, -1, grayImg, 0, 255, grayImg)
cornerListQueue = Queue()
def getCorners(idx, inImg, cornersQueue):
"""Search for corners in image and put them in the queue"""
print "{} Searching".format(idx)
_, corners = cv.FindChessboardCorners(inImg,
patternSize)
print "{} found {} corners".format(idx, len(corners))
saveimg(inImg, name="Chessboard_Search_{}".format(idx))
cornersQueue.put(corners)
for i in range(0, 12, 3):
img = cv.CloneMat(grayImg)
cv.Erode(img, img, iterations=i)
cv.Dilate(img, img, iterations=i)
p = multiprocessing.Process(target=lambda: getCorners(i, img, cornerListQueue))
p.daemon = True
p.start()
corners = []
while len(corners) != 49 and i > 0:
corners = cornerListQueue.get(True)
print "Got Result {}".format(i)
i -= 1
if len(corners) == 49:
# Debug Image
debugImg = cv.CreateMat(grayImg.rows, grayImg.cols, cv.CV_8UC3)
cv.CvtColor(grayImg, debugImg, cv.CV_GRAY2RGB)
for pt in corners:
pt = (int(pt[0]), int(pt[1]))
cv.Circle(debugImg, pt, 4, (255, 0, 0))
saveimg(debugImg, name="Corners_Found")
# //Debug Image
# Figure out the correct corner mapping
points = sorted([corners[42], corners[0], corners[6], corners[48]], key=lambda pt: pt[0] + pt[1])
if points[1][0] < points[2][0]:
points[1], points[2] = points[2], points[1] # swap tr/bl as needed
(tl, tr, bl, br) = points
warpCorners = [tl, tr, br, bl]
else:
print "Could not find corners"
warpCorners = []
return warpCorners
def testDodgyActor(self):
queue = Queue()
yield self.spawn(actor_class=DodgyActor, max_requests=1,
ioqueue=queue, on_event=on_event)
proxy = pulsar.get_actor().get_actor(self.a.aid)
self.assertEqual(proxy.name, 'dodgyactor')
queue.put(('request', 'Hello'))
c = 0
while c < 20:
if not proxy.is_alive():
break
else:
c += 1
yield pulsar.NOT_DONE
self.assertFalse(proxy.is_alive())
def __init__(self):
self._storage = AuthorsStorage("author_generator_data_former")
self._r = praw.Reddit(user_agent=choice(USER_AGENTS))
self._queue = Queue()
adder = ActionGeneratorDataFormer.AuthorAdder(self._queue, self)
adder.start()
def _start(self,name,cpu, module_name, class_name, params):
fn = None
self._processes = []
self._in_queue = Queue()
self._out_queue = Queue()
self._log_queue = Queue()
if name == "mapper":
fn = q_run_mapper
elif name == "reducer":
fn = q_run_reducer
for i in range(cpu):
process = Process(target=fn,args=(module_name, class_name ,params, self._in_queue, self._out_queue, self._log_queue))
self._processes.append(process)
process.start()
def test_spy():
"""Test the measure spy working.
"""
q = Queue()
data = TaskDatabase()
spy = MeasureSpy(queue=q, observed_database=data,
observed_entries=('test',))
data.notifier(('test', 1))
assert q.get()
data.notifier(('test2', 1))
assert q.empty()
spy.close()
assert q.get() == ('', '')
def __enter__(self):
self.smtp_process_queue = Queue()
self.smtp_process = Process(
target=get_otp_mail, args=(self.smtp_process_queue, self.timeout))
self.smtp_process.start()
self.port = self.smtp_process_queue.get(True, 5)
self._do_lintop_config()
return self
def __init__(self, solvers, lock):
self._solvers = solvers
self._lock = lock
self._q = Queue()
self._qDistributor = Queue()
self._queues = {}
self._type = SolverImplType.THREADED
super(AsynchronousSolver, self).__init__()
self._go()
class UDPServer(Process):
def __init__(self, queue):
Process.__init__(self, name = "UDPServer")
#self.daemon = True
self.queue = queue
self.shutdownQueue = Queue()
self.start()
def __checkShutdown(self):
try:
self.shutdownQueue.get(block = False)
except Empty, _e:
return self.reactor.callLater(1, self.__checkShutdown)
self.shutdownQueue.close()
if self.reactor.running:
self.reactor.stop()
self.queue.close()
def get(self, block=True, timeout=None):
ret = Queue.get(self, block, timeout)
if self.qsize() == 0:
self.cond_empty.acquire()
try:
self.cond_empty.notify_all()
finally:
self.cond_empty.release()
return ret
def start_work(self, worker, work, num_jobs, *args, **kwargs):
'''work should be and indexable sequence'''
wlen = len(work)
if not wlen: return
if self._counter is not None: self._counter.set_work(wlen)
#determine number of jobs to start
if not num_jobs: num_jobs = cpu_count
#prepare jobs
in_queue = Queue(wlen+num_jobs)
self._jobs = [None]*num_jobs
for j in xrange(num_jobs):
queue = Queue()
job = UProcess(target=worker, args=(queue, self._abort_event,
in_queue, work)+args, kwargs=kwargs)
job.daemon = self._daemonic
self._jobs[j] = Job(job,queue)
self.start_jobs()
for i in xrange(wlen): in_queue.put(i, False)
for j in xrange(num_jobs): in_queue.put(None, False)
def test_multiprocess_tasks():
wait_until_convenient()
TAG = "message_q"
def fetch_task(queue):
pid = os.getpid()
count = 0
for dq in q.listen(TAG, timeout=1):
s = { 'pid': pid, 'data': dq }
if dq:
count += 1
queue.put(s)
sleep(uniform(0.1, 0.5)) # sleep 0.1~0.5 seconds randomly
elif q.count(TAG) == 0:
return count # the number of tasks done by this process
test_items = range(0, 10000) # enqueue 10000 tasks
for i in test_items:
q.enqueue(TAG, i + 1)
while q.count(TAG) != len(test_items): # wait until test data is ready
wait_until_convenient()
jobs = []
wait_until_convenient()
queue = Queue()
start = timer()
num_p = 30 # the number of processes to use
for i in range(0, num_p):
job = Process(target=fetch_task, args=(queue,))
jobs.append(job)
job.start() # start task process
remaining = q.count(TAG)
while remaining > 0: # wait until the queue is consumed completely
remaining = q.count(TAG)
sys.stdout.write('\rRunning test_multiprocess_tasks - remaining %5d/%5d' % (remaining, len(test_items),))
sys.stdout.flush()
wait_until_convenient()
processed_data = set()
qsize = 0
while not queue.empty():
item = queue.get()
data = item.get('data')
qsize += 1
assert data not in processed_data, "failed test_multiprocess_tasks - data %s has been processed already" % (data, )
processed_data.add(item.get('data'))
queue.close()
queue.join_thread()
for j in jobs:
j.join()
assert qsize == len(test_items), "failed test_multiprocess_tasks - tasks are not complete %d/%d" % (qsize, len(test_items), )
end = timer()
print("\rOK test_multiprocess_tasks - %d done in %5d seconds" % (qsize, end - start))
def put(self,element):
'''
Put the element in the queue
Raises an exception if too many errors are
encountered
'''
dt = 1e-3
while dt < 1:
try:
Queue.put(self,element)
return
except IOError:
logger.warning('IOError encountered in SafeQueue put()')
try:
time.sleep(dt)
except:pass
dt *= 2
e = IOError('Unrecoverable error')
raise e
def __init__(self, name, ns="", solicited=True, ignoreUnhandled=False, maxAsync=None):
super(ApiBase, self).__init__(ns=ns, solicited=solicited, name=name)
self._setup(ns=self._getNamespace(), solicited=self.solicited, ipc=self.ipc)
self._dataRxCount = itertools.count(0)
self._ignoreUnhandled = ignoreUnhandled
if (maxAsync == None) or (maxAsync < 1):
maxAsync = ApiBase.DEFAULT_MAX_ASYNC_HANDLERS
self._maxAsync = maxAsync
self._q = Queue()
self._workers = []
self._createAsyncWorkers()
self.isAlive = True
def __init__(self, peerName, theTime, filename, quiet):
self._peerName = peerName
self._fp = open(filename, "w")
self.fp.write("File debugger started at: %(T)s for client: %(C)s"%{"T":theTime, "C":peerName})
self.fp.flush()
self._methods = []
methods = iMockDebuggerSink()._getMethods()
self._methods = methods
self._terminate = False
self.quiet= quiet
self._startMutex = Semaphore(0)
self._q = Queue()
self.thread = None
def parallel_sort(bam, out, n_workers):
lb = BGZFReader(bam)
mem = lb.uncompressed_size
buf = RawArray(ctypes.c_char, mem)
q = Queue()
procs = []
block_allocs = chunk(lb.blocks, n_workers)
offsets = [0] + list(accumulate(sum(b.offset for b in blocks)
for blocks in block_allocs))[:-1]
ary_szs = [sum([b.size_u for b in blocks]) for blocks in block_allocs]
bufs = [RawArray(ctypes.c_char,mem) for mem in ary_szs]
z = zip(chunk(lb.blocks, n_workers), offsets, bufs)
for i,(blocks,off,buf) in enumerate(z):
args = (i, bam, blocks, off, buf, q)
p = Process(target=sort_read_ary, args=args)
procs.append(p)
p.start()
combined = []
for _ in procs:
combined += q.get(True)
logging.debug("Starting combined sort on %i reads" % len(combined))
combined.sort()
logging.debug("Finished combined sort")
for p in procs:
p.join()
logging.debug("Returned from " + str(p))
hdr = RawBAM(gzip.GzipFile(bam), header=True).rawheader
with open(out, 'wb') as f:
write_bgzf_block(f, hdr)
for creads in grouper(READS_PER_BLOCK, combined):
data = ""
for i,cr in enumerate(creads):
data += bufs[cr.worker_num][cr.ptr:(cr.ptr+cr.bs+4)]
write_bgzf_block(f, data)
write_bam_eof(f)
def yk_monitor(self, mon_l):
# forming command to run parallel monitoring processes
mon_cmd = ' & '.join(["xinput test {}".format(y_id) for y_id in mon_l])
monitor = subprocess.Popen(mon_cmd, shell=True, stdout=subprocess.PIPE)
stdout_queue = Queue()
stdout_reader = AsynchronousFileReader(monitor.stdout, stdout_queue)
stdout_reader.start()
triggered = False
timestamp = time.time()
while not stdout_reader.eof and time.time() - timestamp < TIMEOUT:
while stdout_queue.qsize() > 0:
stdout_queue.get() # emptying queue
triggered = True
time.sleep(.04)
if triggered:
print('YubiKey triggered. Now disabling.')
break
time.sleep(.001)
if not triggered:
print('No YubiKey triggered. Timeout.')
def __init__(self, scrlck_mode=False):
self.scrlck_mode = scrlck_mode
self.id_q = Queue()
self.on_q = Queue()
self.pi_q = Queue()
# init processes
gi_proc = Process(target=self.get_ids)
gi_proc.daemon = True
cs_proc = Process(target=self.change_state)
# no daemon, or main program will terminate before Keys can be unlocked
cs_proc.daemon = False
zmq_lis = ZmqListener(
self.on_q) # somehow works ony with threads not processes
zmq_lis_thr = Thread(target=zmq_lis.start_listener)
zmq_lis_thr.setDaemon(True)
pi = PanelIndicator(self.pi_q, self.on_q)
# starting processes and catching exceptions:
try:
gi_proc.start()
cs_proc.start()
zmq_lis_thr.start()
pi.run_pi() # main loop of root process
except (KeyboardInterrupt, SystemExit):
print('Caught exit event.')
finally:
# send exit signal, will reactivate YubiKey slots
print('Sending EXIT_SIGNAL')
self.on_q.put(EXIT_SIGNAL)
