class Multiprocess(object): # THE COMPLICATION HERE IS CONNECTING THE DISPARATE LOGGING TO # A CENTRAL POINT def __init__(self, functions): self.outbound = Queue() self.inbound = Queue() self.inbound = Queue() #MAKE #MAKE THREADS self.threads = [] for t, f in enumerate(functions): thread = worker( "worker " + unicode(t), f, self.inbound, self.outbound, ) self.threads.append(thread) def __enter__(self): return self #WAIT FOR ALL QUEUED WORK TO BE DONE BEFORE RETURNING def __exit__(self, a, b, c): try: self.inbound.close() # SEND STOPS TO WAKE UP THE WORKERS WAITING ON inbound.pop() except Exception, e: Log.warning("Problem adding to inbound", e) self.join()
class TestBlockingMethods(unittest.TestCase): def setUp(self): self.quiet=True self.random = Random() self._timers = [] self.namespaces = [] self.iface = PyRQIface(quiet=self.quiet, ref="test") self.dummyQueue = Queue() self.marshaller = MarshallerFactory.get(MarshallerFactory.DEFAULT, quiet=self.quiet) desiredPort = "19001" self.r = SubprocessQueueServer( desiredPort=desiredPort, handlerClazz=Linkage.create(MockHandler), quiet=self.quiet # includePydevd="/home/francis/.eclipse/org.eclipse.platform_3.7.0_155965261/plugins/org.python.pydev.debug_2.5.0.2012040618/pysrc" ) PyRQIface.setGlobalPYRQ(self.r.details()) self.r.start().waitUntilRunning() pass def tearDown(self): try: self.dummyQueue.close() del self.dummyQueue except Exception, _e: pass for namespace in self.namespaces: self.iface.setNamespace(namespace) try: self.iface.close() except ClosedError, _e: pass
class Multiprocess(object): # THE COMPLICATION HERE IS CONNECTING THE DISPARATE LOGGING TO # A CENTRAL POINT def __init__(self, functions): self.outbound = Queue() self.inbound = Queue() self.inbound = Queue() #MAKE #MAKE THREADS self.threads = [] for t, f in enumerate(functions): thread = worker( "worker " + unicode(t), f, self.inbound, self.outbound, ) self.threads.append(thread) def __enter__(self): return self #WAIT FOR ALL QUEUED WORK TO BE DONE BEFORE RETURNING def __exit__(self, a, b, c): try: self.inbound.close( ) # SEND STOPS TO WAKE UP THE WORKERS WAITING ON inbound.pop() except Exception, e: Log.warning("Problem adding to inbound", e) self.join()
def empty_queue(queue_: Queue) -> None: while True: try: queue_.get(block=False) except queue.Empty: break queue_.close()
def test_multiprocess_tasks(): wait_until_convenient() TAG = "message_q" def fetch_task(queue): pid = os.getpid() count = 0 for dq in q.listen(TAG, timeout=1): s = { 'pid': pid, 'data': dq } if dq: count += 1 queue.put(s) sleep(uniform(0.1, 0.5)) # sleep 0.1~0.5 seconds randomly elif q.count(TAG) == 0: return count # the number of tasks done by this process test_items = range(0, 10000) # enqueue 10000 tasks for i in test_items: q.enqueue(TAG, i + 1) while q.count(TAG) != len(test_items): # wait until test data is ready wait_until_convenient() jobs = [] wait_until_convenient() queue = Queue() start = timer() num_p = 30 # the number of processes to use for i in range(0, num_p): job = Process(target=fetch_task, args=(queue,)) jobs.append(job) job.start() # start task process remaining = q.count(TAG) while remaining > 0: # wait until the queue is consumed completely remaining = q.count(TAG) sys.stdout.write('\rRunning test_multiprocess_tasks - remaining %5d/%5d' % (remaining, len(test_items),)) sys.stdout.flush() wait_until_convenient() processed_data = set() qsize = 0 while not queue.empty(): item = queue.get() data = item.get('data') qsize += 1 assert data not in processed_data, "failed test_multiprocess_tasks - data %s has been processed already" % (data, ) processed_data.add(item.get('data')) queue.close() queue.join_thread() for j in jobs: j.join() assert qsize == len(test_items), "failed test_multiprocess_tasks - tasks are not complete %d/%d" % (qsize, len(test_items), ) end = timer() print("\rOK test_multiprocess_tasks - %d done in %5d seconds" % (qsize, end - start))
class ServerSink(iMockDebuggerSink): def __init__(self, peerName, theTime, details, quiet): self._peerName = peerName self._methods = [] methods = iMockDebuggerSink()._getMethods() self._methods = methods self._terminate = False self._details = details self._qw = None self._startMutex = Semaphore(0) self._q = Queue() self.quiet= quiet self._marshaller = MarshallerFactory.get(MarshallerFactory.DEFAULT, quiet=quiet) self._qw = QueueWriter(target=details, autoConnect=True, marshaller=self._marshaller, quiet=quiet) self._qw.start() self.thread = None def start(self): t = threading.Thread(target=self.run, args=[self._startMutex]) t.setName("ServerSink.%(P)s"%{"P":self._peerName}) t.setDaemon(True) self.thread = t self.thread.start() return "server.sink.started" def close(self): self._terminate = True try: self.thread.join() except: pass try: self._qw.close() except: pass try: self._q.close() except: pass return "server.sink.closed" def waitUntilRunning(self, block=True, timeout=None): self._startMutex.acquire(block=block, timeout=timeout) return self def __getattribute__(self, name): if name in object.__getattribute__(self, "_methods"): q = self._q def wrapper(self, *args, **kwargs): ServerSink._testPickleability((name, args, kwargs)) q.put((name, args, kwargs)) return wrapper return object.__getattribute__(self, name) def run(self, startMutex): startMutex.release() while self._terminate==False: try: data = self._q.get(block=True, timeout=1) except Empty: pass else: ServerSink._testPickleability(data) try: self._qw.put(data, block=True, timeout=10) except Exception, _e: break
class TestWithTimeouts(_BaseReader): def setUp(self): self.logger = self._getLogger() self.quiet=True self.random = Random() self.timerTerminate = 0 self._timers = [] self.namespaces = [] self.dummyQueue = Queue() self.iface = PyRQIface(ref="test", quiet=self.quiet, loggingModule=testLoggingModule) self.marshaller = MarshallerFactory.get(MarshallerFactory.DEFAULT, quiet=self.quiet) desiredPort = "19001" self.r = SubprocessQueueServer( desiredPort=desiredPort, handlerClazz=Linkage.create(TimeoutMockHandler), quiet=self.quiet, ) PyRQIface.setGlobalPYRQ(self.r.details()) def _createInterface(self): namespace = self.iface.create() self.iface = PyRQIface(namespace=namespace, ref="test", quiet=self.quiet, loggingModule=testLoggingModule) self.namespaces.append(namespace) return namespace def tearDown(self): self.timerTerminate = 1 time.sleep(2) try: self.dummyQueue.close() del self.dummyQueue except Exception, _e: pass for namespace in self.namespaces: self.iface.setNamespace(namespace) try: self.iface.close() except ClosedError, _e: pass
class TestRRQReader(_BaseReader): def setUp(self): self.logger = self._getLogger() self.quiet = True self._queues = [] self.random = Random() self.timerTerminate = 0 self._timers = [] self.namespaces = [] self.iface = PyRQIface(quiet=self.quiet, ref="default", loggingModule=testLoggingModule) self.dummyQueue = Queue() self.marshaller = MarshallerFactory.get(MarshallerFactory.DEFAULT, quiet=self.quiet) desiredPort = "19001" self.r = SubprocessQueueServer( desiredPort=desiredPort, quiet=self.quiet, handlerClazz=Linkage.create(MockHandler), # includePydevd="/home/francis/.eclipse/org.eclipse.platform_3.7.0_155965261/plugins/org.python.pydev.debug_2.5.0.2012040618/pysrc" ) PyRQIface.setGlobalPYRQ(self.r.details()) self.r.start().waitUntilRunning() pass def tearDown(self): self.logger.info("------------ TEST END ------------") self.timerTerminate = 1 time.sleep(2) try: self.dummyQueue.close() del self.dummyQueue except Exception, _e: pass for namespace in self.namespaces: self.iface.setNamespace(namespace) try: self.iface.close() except ClosedError, _e: pass
class MultiCoreEngine(): _mapred = None _out_queue = None _in_queue = None _log_queue = None _processes = None def __init__(self,mapred): self._mapred = mapred def _start(self,name,cpu, module_name, class_name, params): fn = None self._processes = [] self._in_queue = Queue() self._out_queue = Queue() self._log_queue = Queue() if name == "mapper": fn = q_run_mapper elif name == "reducer": fn = q_run_reducer for i in range(cpu): process = Process(target=fn,args=(module_name, class_name ,params, self._in_queue, self._out_queue, self._log_queue)) self._processes.append(process) process.start() def _stop(self): for process in self._processes: self._in_queue.put("STOP") while not self._log_queue.empty(): print self._log_queue.get() def _get_data_chunks(self): chunks = [] for process in self._processes: chunks.append(self._out_queue.get()) return chunks def _set_data_chunks(self, chunks): map(self._in_queue.put,chunks) def _send_lines(self,lines, cpu, lines_len ): line_splits = [lines[i* lines_len / cpu : (i+1)* lines_len / cpu] for i in range(cpu) ] for i in range(cpu): self._in_queue.put(line_splits[i]) def _terminate(self): for process in self._processes: process.join() process.terminate() self._in_queue.close() self._out_queue.close() self._processes = None def _force_terminate(self): for process in self._processes: process.terminate() def _merge_data(self, data): self._mapred.data = merge_kv_dict(self._mapred.data,data) def _merge_reduced_data(self, data): self._mapred.data_reduced = merge_kv_dict(self._mapred.data_reduced,data) def _split_data(self, num_splits): splits = [] index = 0 len_data = len(self._mapred.data) chunk_len = int(math.ceil(len_data / float(num_splits))) if chunk_len == 0: splits.append(self._mapred.data) else: for i in range(int(math.ceil(len_data/float(chunk_len)))): splits.append({}) for (key, value) in self._mapred.data.items(): i = int(math.floor(index / float(chunk_len))) splits[i][key]=value index = index + 1 return splits def _run_map(self,cpu,cache_line,input_reader ): self._start("mapper",cpu, self._mapred.__class__.__module__,self._mapred.__class__.__name__ ,self._mapred.params) try: map_len = 0 lines = [] lines_len = 0 f = input_reader.read() for line in f: if lines_len > 0 and lines_len % cache_line == 0: self._send_lines(lines, cpu, lines_len) lines = [] lines_len = 0 lines.append(line) lines_len += 1 map_len += 1 input_reader.close() self._send_lines(lines, cpu, lines_len) self._stop() map(self._merge_data, self._get_data_chunks()) self._terminate() except Exception,e: print "ERROR: Exception while mapping : %s\n%s" % (e,traceback.print_exc()) self._force_terminate() return map_len
class SSHClient(Process): TIMEOUT = 10 PING_RECEIVED = re.compile("1 received") def __init__(self, username, password, host, cmdsToSend, port = 22, exitCmd = "exit", timeout = None): Process.__init__(self, name = "SSHClient") self.logger = LogManager().getLogger('SSHClient-%s' % host) self.username = username self.password = password self.host = host self.port = int(port) self.cmdsToSend = cmdsToSend if isinstance(cmdsToSend, list) else [cmdsToSend] self.exitCmd = exitCmd self.queue = Queue() self.msg = "" self.status = Status.FAILURE self.startTime = Value('d', 0.0) self.endTime = Value('d', 0.0) self.timeout = timeout or SSHClient.TIMEOUT self.cmdsSend = False self.start() def isFinished(self): """ True if the process has finished """ return not self.is_alive() def updateOutput(self): """ Update the msg to include the latest output from the given commands """ try: while True: msg = self.queue.get(timeout = 0.5) self.msg += msg except Empty: pass except IOError: pass if self.isFinished(): self.queue.close() return self.msg def run(self): factory = SSHFactory(self) factory.protocol = ClientTransport reactor.connectTCP(self.host, self.port, factory) self.startTime.value = time.time() check = task.LoopingCall(self.__ping) check.start(2.0) reactor.callLater(self.timeout, self.__timeout) log.defaultObserver.stop() reactor.run() self.endTime.value = time.time() self.queue.close() sys.exit(self.status) def __timeout(self): """ Timeout checker """ if self.status != Status.FAILURE: return self.logger.error('Connection timeout to peer %s:%s' %(self.host, self.port)) reactor.stop() def __ping(self): with open('/dev/null') as null: ping = subprocess.Popen(["ping", "-c1", "-W1", self.host], stdout = null, stderr = null) ping.wait() if ping.returncode != 0 and reactor.running: if self.cmdsSend == False: self.status = Status.FAILURE reactor.stop() def cleanup(self): self.queue.close() def shutdown(self): """ Terminate the SSH process """ self.terminate() self.join() self.endTime.value = time.time()
class MVacManager(Manager): """ Multyvac backend. """ @contract(num_processes='int') def __init__(self, context, cq, num_processes, recurse=False, show_output=False, new_process=False, volumes=[], rdb=False, rdb_vol=None, rdb_db=None): Manager.__init__(self, context=context, cq=cq, recurse=recurse) self.num_processes = num_processes self.last_accepted = 0 self.cleaned = False self.show_output = show_output self.new_process = new_process self.volumes = volumes self.rdb = rdb self.rdb_db = rdb_db self.rdb_vol = rdb_vol def process_init(self): self.event_queue = Queue() self.event_queue_name = str(id(self)) from compmake.plugins.backend_pmake.pmake_manager import PmakeManager PmakeManager.queues[self.event_queue_name] = self.event_queue # info('Starting %d processes' % self.num_processes) self.subs = {} # name -> sub # available + processing + aborted = subs.keys self.sub_available = set() self.sub_processing = set() self.sub_aborted = set() self.signal_queue = Queue() db = self.context.get_compmake_db() storage = db.basepath # XXX: logs = os.path.join(storage, 'logs') for i in range(self.num_processes): name = 'w%02d' % i write_log = os.path.join(logs, '%s.log' % name) make_sure_dir_exists(write_log) signal_token = name self.subs[name] = PmakeSub(name, signal_queue=self.signal_queue, signal_token=signal_token, write_log=write_log) self.job2subname = {} self.subname2job = {} # all are available at the beginning self.sub_available.update(self.subs) self.max_num_processing = self.num_processes def check_any_finished(self): # We make a copy because processing is updated during the loop try: token = self.signal_queue.get(block=False) except Empty: return False #print('received %r' % token) job_id = self.subname2job[token] self.subs[token].last self.check_job_finished(job_id, assume_ready=True) return True # XXX: boiler plate def get_resources_status(self): resource_available = {} assert len(self.sub_processing) == len(self.processing) if not self.sub_available: msg = 'already %d nproc' % len(self.sub_processing) if self.sub_aborted: msg += ' (%d workers aborted)' % len(self.sub_aborted) resource_available['nproc'] = (False, msg) # this is enough to continue return resource_available else: resource_available['nproc'] = (True, '') return resource_available @contract(reasons_why_not=dict) def can_accept_job(self, reasons_why_not): if len(self.sub_available) == 0 and len(self.sub_processing) == 0: # all have failed msg = 'All workers have aborted.' raise MakeHostFailed(msg) resources = self.get_resources_status() some_missing = False for k, v in resources.items(): if not v[0]: some_missing = True reasons_why_not[k] = v[1] if some_missing: return False return True def instance_job(self, job_id): publish(self.context, 'worker-status', job_id=job_id, status='apply_async') assert len(self.sub_available) > 0 name = sorted(self.sub_available)[0] self.sub_available.remove(name) assert not name in self.sub_processing self.sub_processing.add(name) sub = self.subs[name] self.job2subname[job_id] = name self.subname2job[name] = job_id job = get_job(job_id, self.db) if self.rdb: f = mvac_job_rdb args = (job_id, self.context, self.event_queue_name, self.show_output, self.volumes, self.rdb_vol.name, self.rdb_db, os.getcwd()) else: if job.needs_context: # if self.new_process: # f = parmake_job2_new_process # args = (job_id, self.context) # # else: f = parmake_job2 args = (job_id, self.context, self.event_queue_name, self.show_output) else: f = mvac_job args = (job_id, self.context, self.event_queue_name, self.show_output, self.volumes, os.getcwd()) if True: async_result = sub.apply_async(f, args) else: warnings.warn('Debugging synchronously') async_result = f(args) return async_result def event_check(self): if not self.show_output: return while True: try: event = self.event_queue.get(block=False) # @UndefinedVariable event.kwargs['remote'] = True broadcast_event(self.context, event) except Empty: break def process_finished(self): if self.cleaned: return self.cleaned = True #print('Clean up...') for name in self.sub_processing: self.subs[name].proc.terminate() for name in self.sub_available: self.subs[name].terminate() elegant = False # XXX: in practice this never works well if elegant: timeout = 1 for name in self.sub_available: self.subs[name].proc.join(timeout) # XXX: ... so we just kill them mercilessly else: # print('killing') for name in self.sub_processing: pid = self.subs[name].proc.pid os.kill(pid, signal.SIGKILL) self.event_queue.close() self.signal_queue.close() from compmake.plugins.backend_pmake.pmake_manager import PmakeManager del PmakeManager.queues[self.event_queue_name] # Normal outcomes def job_failed(self, job_id, deleted_jobs): Manager.job_failed(self, job_id, deleted_jobs) self._clear(job_id) def job_succeeded(self, job_id): Manager.job_succeeded(self, job_id) self._clear(job_id) def _clear(self, job_id): assert job_id in self.job2subname name = self.job2subname[job_id] del self.job2subname[job_id] del self.subname2job[name] assert name in self.sub_processing assert name not in self.sub_available self.sub_processing.remove(name) self.sub_available.add(name) def host_failed(self, job_id): Manager.host_failed(self, job_id) assert job_id in self.job2subname name = self.job2subname[job_id] del self.job2subname[job_id] del self.subname2job[name] assert name in self.sub_processing assert name not in self.sub_available self.sub_processing.remove(name) # put in sub_aborted self.sub_aborted.add(name) def cleanup(self): self.process_finished()
class MVacManager(Manager): """ Multyvac backend. """ @contract(num_processes='int') def __init__(self, context, cq, num_processes, recurse=False, show_output=False, new_process=False, volumes=[], rdb=False, rdb_vol=None, rdb_db=None): Manager.__init__(self, context=context, cq=cq, recurse=recurse) self.num_processes = num_processes self.last_accepted = 0 self.cleaned = False self.show_output = show_output self.new_process = new_process self.volumes = volumes self.rdb = rdb self.rdb_db = rdb_db self.rdb_vol = rdb_vol def process_init(self): self.event_queue = Queue() self.event_queue_name = str(id(self)) from compmake.plugins.backend_pmake.pmake_manager import PmakeManager PmakeManager.queues[self.event_queue_name] = self.event_queue # info('Starting %d processes' % self.num_processes) self.subs = {} # name -> sub # available + processing + aborted = subs.keys self.sub_available = set() self.sub_processing = set() self.sub_aborted = set() self.signal_queue = Queue() db = self.context.get_compmake_db() storage = db.basepath # XXX: logs = os.path.join(storage, 'logs') for i in range(self.num_processes): name = 'w%02d' % i write_log = os.path.join(logs, '%s.log' % name) make_sure_dir_exists(write_log) signal_token = name self.subs[name] = PmakeSub(name, signal_queue=self.signal_queue, signal_token=signal_token, write_log=write_log) self.job2subname = {} self.subname2job = {} # all are available at the beginning self.sub_available.update(self.subs) self.max_num_processing = self.num_processes def check_any_finished(self): # We make a copy because processing is updated during the loop try: token = self.signal_queue.get(block=False) except Empty: return False #print('received %r' % token) job_id = self.subname2job[token] self.subs[token].last self.check_job_finished(job_id, assume_ready=True) return True # XXX: boiler plate def get_resources_status(self): resource_available = {} assert len(self.sub_processing) == len(self.processing) if not self.sub_available: msg = 'already %d nproc' % len(self.sub_processing) if self.sub_aborted: msg += ' (%d workers aborted)' % len(self.sub_aborted) resource_available['nproc'] = (False, msg) # this is enough to continue return resource_available else: resource_available['nproc'] = (True, '') return resource_available @contract(reasons_why_not=dict) def can_accept_job(self, reasons_why_not): if len(self.sub_available) == 0 and len(self.sub_processing) == 0: # all have failed msg = 'All workers have aborted.' raise MakeHostFailed(msg) resources = self.get_resources_status() some_missing = False for k, v in resources.items(): if not v[0]: some_missing = True reasons_why_not[k] = v[1] if some_missing: return False return True def instance_job(self, job_id): publish(self.context, 'worker-status', job_id=job_id, status='apply_async') assert len(self.sub_available) > 0 name = sorted(self.sub_available)[0] self.sub_available.remove(name) assert not name in self.sub_processing self.sub_processing.add(name) sub = self.subs[name] self.job2subname[job_id] = name self.subname2job[name] = job_id job = get_job(job_id, self.db) if self.rdb: f = mvac_job_rdb args = (job_id, self.context, self.event_queue_name, self.show_output, self.volumes, self.rdb_vol.name, self.rdb_db, os.getcwd()) else: if job.needs_context: # if self.new_process: # f = parmake_job2_new_process # args = (job_id, self.context) # # else: f = parmake_job2 args = (job_id, self.context, self.event_queue_name, self.show_output) else: f = mvac_job args = (job_id, self.context, self.event_queue_name, self.show_output, self.volumes, os.getcwd()) if True: async_result = sub.apply_async(f, args) else: warnings.warn('Debugging synchronously') async_result = f(args) return async_result def event_check(self): if not self.show_output: return while True: try: event = self.event_queue.get(block=False) # @UndefinedVariable event.kwargs['remote'] = True broadcast_event(self.context, event) except Empty: break def process_finished(self): if self.cleaned: return self.cleaned = True #print('Clean up...') for name in self.sub_processing: self.subs[name].proc.terminate() for name in self.sub_available: self.subs[name].terminate() elegant = False # XXX: in practice this never works well if elegant: timeout = 1 for name in self.sub_available: self.subs[name].proc.join(timeout) # XXX: ... so we just kill them mercilessly else: # print('killing') for name in self.sub_processing: pid = self.subs[name].proc.pid os.kill(pid, signal.SIGKILL) self.event_queue.close() self.signal_queue.close() from compmake.plugins.backend_pmake.pmake_manager import PmakeManager del PmakeManager.queues[self.event_queue_name] # Normal outcomes def job_failed(self, job_id, deleted_jobs): Manager.job_failed(self, job_id, deleted_jobs) self._clear(job_id) def job_succeeded(self, job_id): Manager.job_succeeded(self, job_id) self._clear(job_id) def _clear(self, job_id): assert job_id in self.job2subname name = self.job2subname[job_id] del self.job2subname[job_id] del self.subname2job[name] assert name in self.sub_processing assert name not in self.sub_available self.sub_processing.remove(name) self.sub_available.add(name) def host_failed(self, job_id): Manager.host_failed(self, job_id) assert job_id in self.job2subname name = self.job2subname[job_id] del self.job2subname[job_id] del self.subname2job[name] assert name in self.sub_processing assert name not in self.sub_available self.sub_processing.remove(name) # put in sub_aborted self.sub_aborted.add(name) def cleanup(self): self.process_finished()
class SmtpMessageServer(object): """ This class can start an SMTP debugging server, configure LinOTP to talk to it and read the results back to the parent tester. On open, an SMTP server is set up to listen locally. Derived classes can define a hook to set the LinOTP configuration to point to this server. Example usage: with SmtpMessageServer(testcase) as smtp: get_otp() """ def __init__(self, testcase, message_timeout): self.testcase = testcase # We need a minimum version of 2.9.2 to set the SMTP port number, so # skip if testing an earlier version self.testcase.need_linotp_version('2.9.2') self.timeout = message_timeout self.set_config = SetConfig(testcase.http_protocol, testcase.http_host, testcase.http_port, testcase.http_username, testcase.http_password) # We advertise the local SMTP server hostname # using the IP address that connects to LinOTP self.addr = self._get_local_ip() self.msg_payload = None def __enter__(self): self.smtp_process_queue = Queue() self.smtp_process = Process(target=get_otp_mail, args=(self.smtp_process_queue, self.timeout)) self.smtp_process.start() self.port = self.smtp_process_queue.get(True, 5) self._do_lintop_config() return self def _do_lintop_config(self): parameters = self.get_config_parameters() logger.debug("Configuration parameters: %s", parameters) result = self.set_config.setConfig(parameters) assert result, "It was not possible to set the config. Result:%s" % result def get_config_parameters(self): # This function can be overridden to provide configuration parameters to configure # specific parts of LinOTP assert False, "This function should be overridden" def get_otp(self): messagestr = self.smtp_process_queue.get(True, 10) msg = email.message_from_string(messagestr) otp = msg.get_payload() logger.debug("Received email message payload:%s", otp) return otp def __exit__(self, *args): self.smtp_process_queue.close() self.smtp_process.terminate() self.smtp_process.join(5) def _get_local_ip(self): """ Get the IP address of the interface that connects to LinOTP """ with closing( socket.create_connection( (self.testcase.http_host, int(self.testcase.http_port)), 10)) as s: addr = s.getsockname()[0] return addr
class ApiBase(iApi, iIpcTransportDataReceiveListener): r""" @summary: The base-class to the top-level api object ONLY, not sub-apis. """ DEFAULT_MAX_ASYNC_HANDLERS = 1 def __init__(self, name, ns="", solicited=True, ignoreUnhandled=False, maxAsync=None): super(ApiBase, self).__init__(ns=ns, solicited=solicited, name=name) self._setup(ns=self._getNamespace(), solicited=self.solicited, ipc=self.ipc) self._dataRxCount = itertools.count(0) self._ignoreUnhandled = ignoreUnhandled if (maxAsync == None) or (maxAsync < 1): maxAsync = ApiBase.DEFAULT_MAX_ASYNC_HANDLERS self._maxAsync = maxAsync self._q = Queue() self._workers = [] self._createAsyncWorkers() self.isAlive = True def _createAsyncWorkers(self, start=True): # Create the thread pool to handle the api calls. for _ in range(0, self._maxAsync): thread = ApiAsyncWorker.create(self._q, self, start=start) self._workers.append(thread) self._logger.debug("Created workers.") def __del__(self): self.teardown() def teardown(self): if self is threading.current_thread(): return if not self.isAlive: return self.isAlive = False # Unfortunately we require time to stop the workers. self._logger.debug("Stopping async workers...") for _ in range(0, self._maxAsync): self._q.put(STOP()) time.sleep(1) for worker in self._workers: worker.stop() for worker in self._workers: if worker.isAlive(): worker.join() self._workers = [] self._q.close() time.sleep(1) del self._q self._q = None self._logger.debug("Stopped async workers (all daemon anyway).") # Now un-bind our data-receive listener from the IPC: if self._ipc != None: self._ipc.setTransportDataReceiveListener(self) self._ipc = None def _newIpc(self): super(ApiBase, self)._newIpc() # Now bind our data-receive listener to the IPC: self._ipc.setTransportDataReceiveListener(self) def transportDataReceive(self, tId, data): r""" @summary: Data is received that is NOT part of an existing transaction. We need to decide what to do with it... Recursively ask each of our sub-api's to decode the data and handle it. If no one can, then return UnsupportedApiError() (unless we consume it with: self._ignoreUnhandled==True). The handlers will have previously been set by the controlling entity, ie: ExecutionOrganiser, Head. This method always returns NoResponseRequired, making the call asynchronous. """ myNsPrefix = self._getNamespacePrefix() try: count = self._dataRxCount.next() if isinstance(data, iApiTransportItem): ns = data.ns() if self._isInMyNamespace(ns): self._findHandler(ns) args = data.args() kwargs = data.kwargs() synchronous = True self._q.put(KNOWN(ns, tId, synchronous, count, args, kwargs)) raise NoResponseRequired(ns) else: # Inform our listener about the data that we can't handle: handler = self.transportDataReceiveListener if handler != None: self._q.put(UNKNOWN(tId, data)) raise NoResponseRequired(myNsPrefix) except UnsupportedApiError, e: if self._ignoreUnhandled == False: # Propagate exception directly as before. raise # Consume silently: self._logger.debug("UnsupportedApiError: %(NS)s" % {"NS":e.ns()}) raise NoResponseRequired(myNsPrefix, e)
class PmakeManager(Manager): """ Specialization of Manager for local multiprocessing, using an adhoc implementation of "pool" because of bugs of the Python 2.7 implementation of pool multiprocessing. """ queues = {} @contract(num_processes='int') def __init__(self, context, cq, num_processes, recurse=False, new_process=False, show_output=False): Manager.__init__(self, context=context, cq=cq, recurse=recurse) self.num_processes = num_processes self.last_accepted = 0 self.new_process = new_process self.show_output = show_output if new_process and show_output: msg = ('Compmake does not yet support echoing stdout/stderr ' 'when jobs are run in a new process.') warning(msg) self.cleaned = False def process_init(self): self.event_queue = Queue(1000) self.event_queue_name = str(id(self)) PmakeManager.queues[self.event_queue_name] = self.event_queue # info('Starting %d processes' % self.num_processes) self.subs = {} # name -> sub # available + processing + aborted = subs.keys self.sub_available = set() self.sub_processing = set() self.sub_aborted = set() db = self.context.get_compmake_db() storage = db.basepath # XXX: logs = os.path.join(storage, 'logs') #self.signal_queue = Queue() for i in range(self.num_processes): name = 'parmake_sub_%02d' % i write_log = os.path.join(logs, '%s.log' % name) make_sure_dir_exists(write_log) signal_token = name self.subs[name] = PmakeSub(name=name, signal_queue=None, signal_token=signal_token, write_log=write_log) self.job2subname = {} # all are available self.sub_available.update(self.subs) self.max_num_processing = self.num_processes # XXX: boiler plate def get_resources_status(self): resource_available = {} assert len(self.sub_processing) == len(self.processing) if not self.sub_available: msg = 'already %d processing' % len(self.sub_processing) if self.sub_aborted: msg += ' (%d workers aborted)' % len(self.sub_aborted) resource_available['nproc'] = (False, msg) # this is enough to continue return resource_available else: resource_available['nproc'] = (True, '') return resource_available @contract(reasons_why_not=dict) def can_accept_job(self, reasons_why_not): if len(self.sub_available) == 0 and len(self.sub_processing) == 0: # all have failed msg = 'All workers have aborted.' raise MakeHostFailed(msg) resources = self.get_resources_status() some_missing = False for k, v in resources.items(): if not v[0]: some_missing = True reasons_why_not[k] = v[1] if some_missing: return False return True def instance_job(self, job_id): publish(self.context, 'worker-status', job_id=job_id, status='apply_async') assert len(self.sub_available) > 0 name = sorted(self.sub_available)[0] self.sub_available.remove(name) assert not name in self.sub_processing self.sub_processing.add(name) sub = self.subs[name] self.job2subname[job_id] = name if self.new_process: f = parmake_job2_new_process args = (job_id, self.context) else: f = parmake_job2 args = (job_id, self.context, self.event_queue_name, self.show_output) async_result = sub.apply_async(f, args) return async_result def event_check(self): if not self.show_output: return while True: try: event = self.event_queue.get(block=False) # @UndefinedVariable event.kwargs['remote'] = True broadcast_event(self.context, event) except Empty: break def process_finished(self): if self.cleaned: return self.cleaned = True # print('process_finished()') for name in self.sub_processing: self.subs[name].proc.terminate() for name in self.sub_available: self.subs[name].terminate() # XXX: in practice this never works well if False: # print('joining') timeout = 1 for name in self.sub_available: self.subs[name].proc.join(timeout) # XXX: ... so we just kill them mercilessly if True: # print('killing') for name in self.sub_processing: pid = self.subs[name].proc.pid os.kill(pid, signal.SIGKILL) #print('process_finished() finished') self.event_queue.close() del PmakeManager.queues[self.event_queue_name] # Normal outcomes def job_failed(self, job_id, deleted_jobs): Manager.job_failed(self, job_id, deleted_jobs) self._clear(job_id) def job_succeeded(self, job_id): Manager.job_succeeded(self, job_id) self._clear(job_id) def _clear(self, job_id): assert job_id in self.job2subname name = self.job2subname[job_id] del self.job2subname[job_id] assert name in self.sub_processing assert name not in self.sub_available self.sub_processing.remove(name) self.sub_available.add(name) def host_failed(self, job_id): Manager.host_failed(self, job_id) assert job_id in self.job2subname name = self.job2subname[job_id] del self.job2subname[job_id] assert name in self.sub_processing assert name not in self.sub_available self.sub_processing.remove(name) # put in sub_aborted self.sub_aborted.add(name) def cleanup(self): self.process_finished()
class Channel(object): # Maximum size of shared memory array, just a safety check it's not set to something ridiculous MAX_BUFFER_SIZE = 256 * 1024 * 1024 def __init__(self, buffer_size): # type: (int) -> None self.buffer_size = buffer_size if self.buffer_size > Channel.MAX_BUFFER_SIZE: raise Exception( "{} exceeds allowable size ({}) of shared memory to allocate". format(self.buffer_size, Channel.MAX_BUFFER_SIZE)) else: self.__queue = Queue() # Create the shared memory array self.__shared_array = RawArray(c_char, self.buffer_size) # Tracks who has been given permission to write to the shared array self.__owner = None def put(self, sending_worker_name, packet): # type: (str, PacketBase) -> None # if sending_worker_name is None: # warnings.warn("Put packet with no sender {}".format(packet)) packet.set_sender(sending_worker_name) if not isinstance(packet, PacketBase): raise Exception("Packet {} is not of type {}".format( packet, PacketBase)) self.__pickle_and_put(packet) def get(self): packet = self.__get_and_unpickle() # if packet.sender_name is None: # warnings.warn("Get packet with no sender {}".format(packet)) return packet def __pickle_and_put(self, packet): # type: (PacketBase) -> None pickled_packet = self.__pickle(packet) self.__queue.put(pickled_packet) @staticmethod def __pickle(packet): return cPickle.dumps(packet, cPickle.HIGHEST_PROTOCOL) def available(self): return self.__owner is None def release(self): self.__owner = None def acquire(self, worker_name): if self.__owner is None or self.__owner == worker_name: self.__owner = worker_name else: raise Exception( "Shared array is owned by '{}'. '{}' cannot acquire.".format( self.__owner, worker_name)) def copy_array_to_buffer(self, ndarray): # type: (np.ndarray) -> None total_bytes = ndarray.nbytes if total_bytes > self.buffer_size: raise Exception( "Numpy array ({} bytes) exceeds capacity of shared memory ({} bytes)." .format(total_bytes, self.buffer_size)) else: # Reshape the source array into 1 row, wrap the shared array and copy the ndarray into the wrapper ndarray = ndarray.reshape(1, ndarray.shape[0] * ndarray.shape[1]) shared_array_wrapper = np.frombuffer(self.__shared_array, dtype=ndarray.dtype, count=ndarray.size) shared_array_wrapper[0:ndarray.shape[0] * ndarray.shape[1]] = ndarray def copy_array_from_buffer(self, df_shape, dtype): # type: (tuple, str) -> pd.DataFrame num_elements = df_shape[0] * df_shape[1] num_bytes = num_elements * np.dtype(dtype).itemsize if num_bytes > self.buffer_size: raise Exception( "Array size {} exceeds capacity of shared memory {}.".format( num_bytes, self.buffer_size)) else: # Wrap the shared array, reshape the shared array and reconstruct the dataframe shared_array_wrapper = np.frombuffer(self.__shared_array, dtype=dtype, count=num_elements) shaped_array = shared_array_wrapper.reshape( df_shape[0], df_shape[1]) df = pd.DataFrame(shaped_array, copy=False) return df def __get_and_unpickle(self): pickled_msg = self.__queue.get() msg = self.__unpickle(pickled_msg) return msg @staticmethod def __unpickle(pickled_msg): return cPickle.loads(pickled_msg) def close(self): self.__queue.close()
class PmakeManager(Manager): """ Specialization of Manager for local multiprocessing, using an adhoc implementation of "pool" because of bugs of the Python 2.7 implementation of pool multiprocessing. """ queues = {} @contract(num_processes='int') def __init__(self, context, cq, num_processes, recurse=False, new_process=False, show_output=False): Manager.__init__(self, context=context, cq=cq, recurse=recurse) self.num_processes = num_processes self.last_accepted = 0 self.new_process = new_process self.show_output = show_output if new_process and show_output: msg = ('Compmake does not yet support echoing stdout/stderr ' 'when jobs are run in a new process.') warning(msg) self.cleaned = False def process_init(self): self.event_queue = Queue(1000) self.event_queue_name = str(id(self)) PmakeManager.queues[self.event_queue_name] = self.event_queue # info('Starting %d processes' % self.num_processes) self.subs = {} # name -> sub # available + processing + aborted = subs.keys self.sub_available = set() self.sub_processing = set() self.sub_aborted = set() db = self.context.get_compmake_db() storage = db.basepath # XXX: logs = os.path.join(storage, 'logs') #self.signal_queue = Queue() for i in range(self.num_processes): name = 'parmake_sub_%02d' % i write_log = os.path.join(logs, '%s.log' % name) make_sure_dir_exists(write_log) signal_token = name self.subs[name] = PmakeSub(name=name, signal_queue=None, signal_token=signal_token, write_log=write_log) self.job2subname = {} # all are available self.sub_available.update(self.subs) self.max_num_processing = self.num_processes # XXX: boiler plate def get_resources_status(self): resource_available = {} assert len(self.sub_processing) == len(self.processing) if not self.sub_available: msg = 'already %d nproc' % len(self.sub_processing) if self.sub_aborted: msg += ' (%d workers aborted)' % len(self.sub_aborted) resource_available['nproc'] = (False, msg) # this is enough to continue return resource_available else: resource_available['nproc'] = (True, '') return resource_available @contract(reasons_why_not=dict) def can_accept_job(self, reasons_why_not): if len(self.sub_available) == 0 and len(self.sub_processing) == 0: # all have failed msg = 'All workers have aborted.' raise MakeHostFailed(msg) resources = self.get_resources_status() some_missing = False for k, v in resources.items(): if not v[0]: some_missing = True reasons_why_not[k] = v[1] if some_missing: return False return True def instance_job(self, job_id): publish(self.context, 'worker-status', job_id=job_id, status='apply_async') assert len(self.sub_available) > 0 name = sorted(self.sub_available)[0] self.sub_available.remove(name) assert not name in self.sub_processing self.sub_processing.add(name) sub = self.subs[name] self.job2subname[job_id] = name if self.new_process: f = parmake_job2_new_process args = (job_id, self.context) else: f = parmake_job2 args = (job_id, self.context, self.event_queue_name, self.show_output) async_result = sub.apply_async(f, args) return async_result def event_check(self): if not self.show_output: return while True: try: event = self.event_queue.get(block=False) # @UndefinedVariable event.kwargs['remote'] = True broadcast_event(self.context, event) except Empty: break def process_finished(self): if self.cleaned: return self.cleaned = True # print('process_finished()') for name in self.sub_processing: self.subs[name].proc.terminate() for name in self.sub_available: self.subs[name].terminate() # XXX: in practice this never works well if False: # print('joining') timeout = 1 for name in self.sub_available: self.subs[name].proc.join(timeout) # XXX: ... so we just kill them mercilessly if True: # print('killing') for name in self.sub_processing: pid = self.subs[name].proc.pid os.kill(pid, signal.SIGKILL) #print('process_finished() finished') self.event_queue.close() del PmakeManager.queues[self.event_queue_name] # Normal outcomes def job_failed(self, job_id, deleted_jobs): Manager.job_failed(self, job_id, deleted_jobs) self._clear(job_id) def job_succeeded(self, job_id): Manager.job_succeeded(self, job_id) self._clear(job_id) def _clear(self, job_id): assert job_id in self.job2subname name = self.job2subname[job_id] del self.job2subname[job_id] assert name in self.sub_processing assert name not in self.sub_available self.sub_processing.remove(name) self.sub_available.add(name) def host_failed(self, job_id): Manager.host_failed(self, job_id) assert job_id in self.job2subname name = self.job2subname[job_id] del self.job2subname[job_id] assert name in self.sub_processing assert name not in self.sub_available self.sub_processing.remove(name) # put in sub_aborted self.sub_aborted.add(name) def cleanup(self): self.process_finished()
class FileSink(iMockDebuggerSink): def __init__(self, peerName, theTime, filename, quiet): self._peerName = peerName self._fp = open(filename, "w") self.fp.write("File debugger started at: %(T)s for client: %(C)s"%{"T":theTime, "C":peerName}) self.fp.flush() self._methods = [] methods = iMockDebuggerSink()._getMethods() self._methods = methods self._terminate = False self.quiet= quiet self._startMutex = Semaphore(0) self._q = Queue() self.thread = None def start(self): t = threading.Thread(target=self.run, args=[self._startMutex]) t.setName("FileSink.%(P)s"%{"P":self._peerName}) t.setDaemon(True) self.thread = t self.thread.start() return "file.sink.started" def close(self): self._terminate = True try: self.thread.join() except: pass try: self._fp.close() except: pass try: self._fp.close() except: pass try: self._q.close() except: pass self._fp = None return "file.sink.closed" def waitUntilRunning(self, block=True, timeout=None): self._startMutex.acquire(block=block, timeout=timeout) return self def __getattribute__(self, name): if name in object.__getattribute__(self, "_methods"): q = self._q def wrapper(self, *args, **kwargs): q.put((name, args, kwargs)) return wrapper return object.__getattribute__(self, name) def run(self, startMutex): startMutex.release() while self._terminate==False: try: data = self._q.get(block=True, timeout=1) except Empty: pass else: try: (methodName, args, kwargs) = data peerName = args[0] relativeTime = args[1] args = args[2:] ss = ["PEER:", peerName, "REL-TIME:", relativeTime, "METHOD", methodName, "ARGS:", str(args), "KWARGS", str(kwargs)] s = "\n".join(ss) except: pass else: try: self._fp.write(s) except: break
class SmtpMessageServer(object): """ This class can start an SMTP debugging server, configure LinOTP to talk to it and read the results back to the parent tester. On open, an SMTP server is set up to listen locally. Derived classes can define a hook to set the LinOTP configuration to point to this server. Example usage: with SmtpMessageServer(testcase) as smtp: get_otp() """ def __init__(self, testcase, message_timeout): self.testcase = testcase # We need a minimum version of 2.9.2 to set the SMTP port number, so # skip if testing an earlier version self.testcase.need_linotp_version('2.9.2') self.timeout = message_timeout self.set_config = SetConfig(testcase.http_protocol, testcase.http_host, testcase.http_port, testcase.http_username, testcase.http_password) # We advertise the local SMTP server hostname # using the IP address that connects to LinOTP self.addr = self._get_local_ip() self.msg_payload = None def __enter__(self): self.smtp_process_queue = Queue() self.smtp_process = Process( target=get_otp_mail, args=(self.smtp_process_queue, self.timeout)) self.smtp_process.start() self.port = self.smtp_process_queue.get(True, 5) self._do_lintop_config() return self def _do_lintop_config(self): parameters = self.get_config_parameters() logger.debug("Configuration parameters: %s", parameters) result = self.set_config.setConfig(parameters) assert result, "It was not possible to set the config. Result:%s" % result def get_config_parameters(self): # This function can be overridden to provide configuration parameters to configure # specific parts of LinOTP assert False, "This function should be overridden" def get_otp(self): messagestr = self.smtp_process_queue.get(True, 10) msg = email.message_from_string(messagestr) otp = msg.get_payload() logger.debug("Received email message payload:%s", otp) return otp def __exit__(self, *args): self.smtp_process_queue.close() self.smtp_process.terminate() self.smtp_process.join(5) def _get_local_ip(self): """ Get the IP address of the interface that connects to LinOTP """ with closing(socket.create_connection((self.testcase.http_host, int(self.testcase.http_port)), 10)) as s: addr = s.getsockname()[0] return addr