def lookup(self, buildingID): from building.dummy import dummy ''' @buildingID: int ''' building = Building.objects.get(bid=buildingID) print building if building is None: return None if self.binstances.has_key(buildingID) is False: ins = None if building.bri == BUILDING_REPRESENTATION_DEBUG: ins = dummy.DummyBif(buildingID) elif building.bri == BUILDING_REPRESENTATION_IFC: raise NotImplementedError, 'Missing implementation of IFC plugin' elif building.bri == BUILDING_REPRESENTATION_REVIT: raise NotImplementedError, 'Missing implementation of Revit plugin' # Fire up the simulator for the building if necessary. self.comm_queues[buildingID] = Queue() p = Process(target=_Manager__init_simulator, args=(ins, self.comm_queues[buildingID])) p.start() self.binstances[buildingID] = ins return self.binstances[buildingID]
def test_litmus_with_authentication(self): """Run litmus test suite on HTTP with authentification. This test passes """ try: proc = Process(target=run_wsgidav_server, args=(True, False)) proc.daemon = True proc.start() time.sleep(1) try: self.assertEqual( subprocess.call([ "litmus", "http://127.0.0.1:8080/", "tester", "secret" ]), 0, "litmus suite failed: check the log") except OSError: print "*" * 70 print "This test requires the litmus test suite." print "See http://www.webdav.org/neon/litmus/" print "*" * 70 raise finally: proc.terminate() proc.join()
def main(gcm="", rcm="", out_folder=""): pc = None pf = None kwargs = { "start_year": 1970, "end_year": 1999, "rcm": rcm, "gcm": gcm, "out_folder": out_folder } in_folder = "data/narccap/{0}-{1}/current".format(gcm, rcm) if os.path.isdir(in_folder): pc = Process(target=interpolate_to_amno, args=(in_folder, ), kwargs=kwargs) pc.start() else: print "{0} does not exist, ignoring the period ...".format(in_folder) kwargs = { "start_year": 2041, "end_year": 2070, "rcm": rcm, "gcm": gcm, "out_folder": out_folder } in_folder = "data/narccap/{0}-{1}/future".format(gcm, rcm) if os.path.isdir(in_folder): pf = Process(target=interpolate_to_amno, args=(in_folder, ), kwargs=kwargs) pf.start() else: print "{0} does not exist, ignoring the period ...".format(in_folder) #do current and future climates in parallel if pc is not None: pc.join() if pf is not None: pf.join()
def service_background_jobs(self): logger.info('service_background_jobs') # NOTE: paths must begin with a "/", indicating that the first part of # the URI is a script name (which each app, i.e "reports" serves as). # see django.core.handlers.wsgi.__init__ uri = '/' + '/'.join([SCHEMA.REPORTS_API_URI, JOB.resource_name ]) data = { JOB.STATE: SCHEMA.VOCAB.job.state.PENDING } kwargs = {} logger.info('get jobs: %r', uri) resp = self.api_client.get( uri, data=data, **kwargs) job_listing = self.api_client.deserialize(resp) if API_RESULT_DATA in job_listing: job_listing = job_listing[API_RESULT_DATA] for job in job_listing: logger.info('found job: %r', job) job_id =job[JOB.ID] logger.info('Process the job: %r', job_id) p = Process(target=self.client_processor.service,args=(job_id,) ) # make the parent process wait: # p.daemon = True # if set to true, then the parent process won't wait. logger.info('start') p.start(); logger.info('started...') logger.debug('servicing completed')
def test_litmus_with_authentication(self): """Run litmus test suite on HTTP with authentification. This test passes """ try: proc = Process(target=run_wsgidav_server, args=(True, False)) proc.daemon = True proc.start() time.sleep(1) try: self.assertEqual(subprocess.call(["litmus", "http://127.0.0.1:8080/", "tester", "secret"]), 0, "litmus suite failed: check the log") except OSError: print "*" * 70 print "This test requires the litmus test suite." print "See http://www.webdav.org/neon/litmus/" print "*" * 70 raise finally: proc.terminate() proc.join()
def process_input_dir(args, input_path, output_path): patt = input_path + os.sep + "*" + args.extension files = glob.glob(patt) docs_num = len(files) if docs_num > args.threads: slice_size = docs_num / args.threads else: slice_size = 1 print "Threads:", args.threads print "Documents number:", docs_num print "Documents per thread:", slice_size start = 0 jobs = [] for job_num in range(args.threads): print "Initializing process", job_num end = start + slice_size p = Process(target=lemmatize_files, args=(files[start:end], output_path, args)) print files[start:end] jobs.append(p) p.start() start += slice_size for p in jobs: p.join() if (docs_num % 2) == 1: lemmatize_files(files, output_path, args)
def process(host,port): from multiprocessing.process import Process p = Process(target=run, args=(host,port)) p.daemon = True p.start() return p
def service_background_jobs(self): logger.info('service_background_jobs') # NOTE: paths must begin with a "/", indicating that the first part of # the URI is a script name (which each app, i.e "reports" serves as). # see django.core.handlers.wsgi.__init__ uri = '/' + '/'.join([SCHEMA.REPORTS_API_URI, JOB.resource_name]) data = {JOB.STATE: SCHEMA.VOCAB.job.state.PENDING} kwargs = {} logger.info('get jobs: %r', uri) resp = self.api_client.get(uri, data=data, **kwargs) job_listing = self.api_client.deserialize(resp) if API_RESULT_DATA in job_listing: job_listing = job_listing[API_RESULT_DATA] for job in job_listing: logger.info('found job: %r', job) job_id = job[JOB.ID] logger.info('Process the job: %r', job_id) p = Process(target=self.client_processor.service, args=(job_id, )) # make the parent process wait: # p.daemon = True # if set to true, then the parent process won't wait. logger.info('start') p.start() logger.info('started...') logger.debug('servicing completed')
def __init__(self, accessLevel, eightApi): self.accessLevel = accessLevel self._eightApi = eightApi pygame.init() self.check() thread = Process(target=self.inputHandler) thread.start()
def async_file_reading(fd, callback): """Helper which instantiate and run an AsynchronousFileReader.""" queue = SimpleQueue() reader = AsynchronousFileReader(fd, queue) reader.start() consummer = Process(target=consume_queue, args=(queue, callback)) consummer.start() return (reader, consummer)
def add_server(self, app_name, service_name, host, port, processor, use_simple_server=True, wait=1, use_ssl=False, ca_certs=None, cert=None, key=None): self.sd_client.register_endpoint(app_name, service_name, host, port) server_process = Process(target=self.__thrift_server, args=(processor, host, port, use_simple_server, use_ssl, ca_certs, cert, key)) server_process.start() time.sleep(wait) self.server_processes.append(server_process)
def main(): auth_server = Process(target=run_auth_server) auth_server.start() def sigint_handler(signal, frame): print("Terminating servers...") auth_server.terminate() auth_server.join() signal.signal(signal.SIGINT, sigint_handler)
def test_mcdpweb_server(dirname): port = random.randint(11000, 15000) base = 'http://127.0.0.1:%s' % port p = Process(target=start_server, args=(dirname, port,)) p.start() print('sleeping') time.sleep(5) try: url_wrong = base + '/not-existing' urllib2.urlopen(url_wrong).read() except HTTPError: pass else: raise Exception('Expected 404') # now run the spider tmpdir = tempfile.mkdtemp(prefix='wget-output') cwd = '.' cmd = ['wget', '-nv', '-P', tmpdir, '-m', base] # res = system_cmd_result( # cwd, cmd, # display_stdout=True, # display_stderr=True, # raise_on_error=True) sub = subprocess.Popen( cmd, bufsize=0, cwd=cwd) sub.wait() exc = get_exceptions(port) if len(exc) == 0: msg = 'Expected at least a not-found error' raise Exception(msg) if not 'not-existing' in exc[0]: raise Exception('Could not find 404 error') exc = exc[1:] if exc: msg = 'Execution raised errors:\n\n' msg += str("\n---\n".join(exc)) raise_desc(Exception, msg) url_exit = base + '/exit' urllib2.urlopen(url_exit).read() print('waiting for start_server() process to exit...') p.join() print('...clean exit')
def test_mcdpweb_server(dirname): port = random.randint(11000, 15000) base = 'http://127.0.0.1:%s' % port p = Process(target=start_server, args=( dirname, port, )) p.start() print('sleeping') time.sleep(5) try: url_wrong = base + '/not-existing' urllib2.urlopen(url_wrong).read() except HTTPError: pass else: raise Exception('Expected 404') # now run the spider tmpdir = tempfile.mkdtemp(prefix='wget-output') cwd = '.' cmd = ['wget', '-nv', '-P', tmpdir, '-m', base] # res = system_cmd_result( # cwd, cmd, # display_stdout=True, # display_stderr=True, # raise_on_error=True) sub = subprocess.Popen(cmd, bufsize=0, cwd=cwd) sub.wait() exc = get_exceptions(port) if len(exc) == 0: msg = 'Expected at least a not-found error' raise Exception(msg) if not 'not-existing' in exc[0]: raise Exception('Could not find 404 error') exc = exc[1:] if exc: msg = 'Execution raised errors:\n\n' msg += str("\n---\n".join(exc)) raise_desc(Exception, msg) url_exit = base + '/exit' urllib2.urlopen(url_exit).read() print('waiting for start_server() process to exit...') p.join() print('...clean exit')
def main(): auth_server = Process(target=run_auth_server) auth_server.start() print "Access http://10.10.112.59:8081/app in your browser" def sigint_handler(signal, frame): print "Terminating servers..." auth_server.terminate() auth_server.join() signal.signal(signal.SIGINT, sigint_handler)
def main(): app_server = Process(target=run_app_server) app_server.start() print("Access http://localhost:8081/app in your browser") def sigint_handler(signal, frame): print("Terminating servers...") app_server.terminate() app_server.join() signal.signal(signal.SIGINT, sigint_handler)
def _launch(type_name, config, is_important): plugin_clazz = _get_type(type_name) if not plugin_clazz: logging.warn('could not find %s plugin' % type_name) d = plugin_clazz(config) p = Process(target=d.start) p.daemon = not is_important p.name = 'plugin: %s' % d.name p.start()
def __init__(self): self.__actualSpeed = Value('f', 0.0) self._targetSpeed = Value('f', 0.0) self.__actualDirection = Value('f', 0.0) self._targetDirection = Value('f', 0.0) self.__setupMotors() steering = Process(target=self.steer) #steering.daemon = True steering.start()
def import_library_in_another_process(path, args): q = Queue(maxsize=1) p = Process(target=library_initializer, args=(q, path, args)) p.start() while True: try: result = q.get(timeout=0.1) if isinstance(result, Exception): raise ImportError(result) return result except Empty: if not p.is_alive(): raise ImportError()
def main(): ''' parse the command line - new up the appl and listen on port ''' if os.path.isfile("../kew_pe.conf"): print ("Loading config file ../kew_pe.conf") options.parse_config_file("../kew_pe.conf") options.parse_command_line() logging.basicConfig(level=logging.DEBUG) #report_control.process('localhost',8081) process = Process(target=report_control.run, name="report_control", kwargs={'host':'localhost', 'port':8081}) process.daemon = True process.start()
def start_sched(): j_logger.info(u'starting job scheduler ...') jobs = get_all_jobs() for job in jobs: j_logger.info(u'starting job %s ' % job.name) job.run_status = Value('i', 0) #job的状态值 try: p = Process(target=_inner_job, name=job.name,args=(job,)) p.start() job.process_id = p.pid j_logger.info(u'job %s started !' % job.name) except Exception as e: j_logger.error(u'job %s fail to start,due to [%s]!' % (job.name,e)) register_signal_notify() j_logger.info(u'job scheduler started !')
def main(): auth_server = Process(target=run_auth_server) auth_server.start() app_server = Process(target=run_app_server) app_server.start() print("Access http://localhost:8081/app in your browser") def sigint_handler(signal, frame): print("Terminating servers...") auth_server.terminate() auth_server.join() app_server.terminate() app_server.join() signal.signal(signal.SIGINT, sigint_handler)
class Downloader(object): def __init__(self, timeout=30, retries=100, wait=1): self.timeout = timeout self.retries = retries self.wait = wait self.manager = SyncManager() self.manager.start() def retry_fetch_data(self, url): market_data = self.fetch_data(url) retries = 1 while not market_data and retries < self.retries: print "Retry #%s..." % str(retries) market_data = self.fetch_data(url) if market_data: print "Fetched: " + str(len(market_data)) else: print "Fetched nothing!" retries += 1 return market_data def fetch_data(self, url): limit = 60 msg = "Downloading " + url[0: min(limit, len(url))] if len(url) > limit: msg += "(+" + str(len(url) - limit) + ")" print msg return_dict = self.manager.dict() self.job = Process(target=get_page_data, args=(url, return_dict)) self.job.start() self.job.join(self.timeout) if self.job.is_alive(): self.job.terminate() self.job = None market_data = None if 'page' in return_dict: market_data = return_dict['page'] if self.wait > 0: time.sleep(self.wait) return market_data
def start_sched(): j_logger.info(u'starting job scheduler ...') jobs = get_all_jobs() for job in jobs: j_logger.info(u'starting job %s ' % job.name) job.run_status = Value('i', 0) #job的状态值 try: p = Process(target=_inner_job, name=job.name, args=(job, )) p.start() job.process_id = p.pid j_logger.info(u'job %s started !' % job.name) except Exception as e: j_logger.error(u'job %s fail to start,due to [%s]!' % (job.name, e)) register_signal_notify() j_logger.info(u'job scheduler started !')
def main(): auth_server = Process(target=run_auth_server) auth_server.start() print("To test getting an auth token, execute the following curl command:") print( "curl --ipv4 -v -X POST" " -d 'grant_type=client_credentials&client_id=abc&client_secret=xyz' " "http://localhost:8080/token" ) def sigint_handler(signal, frame): print("Terminating server...") auth_server.terminate() auth_server.join() signal.signal(signal.SIGINT, sigint_handler)
def set_from_file(self, varfile_path, args): q = Queue() p = Process(target=set_from_file, args=(q, varfile_path, args)) p.start() p.join() there_are_results = False while True: try: results = q.get_nowait() there_are_results = True if len(results) == 1: raise DataError(results[0]) self.set(*results) except Empty: if not there_are_results: raise DataError('No variables') return
def _start(self,name,cpu, module_name, class_name, params): fn = None self._processes = [] self._in_queue = Queue() self._out_queue = Queue() self._log_queue = Queue() if name == "mapper": fn = q_run_mapper elif name == "reducer": fn = q_run_reducer for i in range(cpu): process = Process(target=fn,args=(module_name, class_name ,params, self._in_queue, self._out_queue, self._log_queue)) self._processes.append(process) process.start()
class FileResultsLogger(ResultsLogger): process = None queue = None logFileName = None def __init__(self, args): self.queue = Queue() if args.no_p2p: fileName = '_'.join([args.mem_algo, str(args.mem_size), 'no_p2p']) elif args.no_mem: fileName = '_'.join( [args.p2p_net, 'no_mem', args.p2p_algo, str(args.p2p_size)]) else: fileName = '_'.join([ args.p2p_net, args.mem_algo, str(args.mem_size), args.p2p_algo, str(args.p2p_size) ]) self.logFileName = 'logs/' + fileName + '.logs' def logRequest(self, nodeId, address, latency, cacheLevel=-1): ResultsLogger.logRequest(self, nodeId, address, latency, cacheLevel=cacheLevel) self.queue.put("{}\t{}\t{}\t{}\n".format(nodeId, address, latency, cacheLevel)) def finish(self): self.queue.put("STOP") self.process.join() def writerProcess(self): fileObject = open(self.logFileName, 'w+') for logEntry in iter(self.queue.get, "STOP"): fileObject.write(logEntry) fileObject.close() def start(self): self.process = Process(target=self.writerProcess) self.process.start()
def test_cleanup(self): """ Test the cleanup thread """ # We want the cleanup thread to run in this context # but it is an infitite loop. So let's delay a shtudown. sd2 = "/tmp/shut2" rec = { 'address': '1.2.3.4', 'ip': '10.128.0.1', 'router': 'router', 'last_associated': time(), 'end_time': 0, 'uid': 501, 'user': '******', 'jobid': '1233', 'status': 'used' } self.db.routes2.insert(rec) # We want to shutdown the thread that is started on init with open(sd2, 'w') as f: f.write('1') settings = self.settings.copy() settings['COLLECTION'] = 'routes2' settings['SHUTFILE'] = sd2 rt = router.Router(settings) # Wait for the init thread to shutdown sleep(0.2) # Now let's start our own if os.path.exists(sd2): os.remove(sd2) shut = Process(target=self._shutdown) shut.start() rt.cleanup() shut.terminate() r = self.db.routes2.find_one({'address': '1.2.3.4'}) self.assertEquals(r['status'], 'available') self.db.routes2.remove({}) rv = rt.cleanup() self.assertEquals(-1, rv)
class ChordNode: process = None node = None def createNode(self, nodeId, port, cacheStorage, knownHosts): BlockingChordNode(nodeId, knownHosts, port, cacheStorage) def __init__(self, knownHosts, port=4000, cacheStorage=None): # TODO id? nodeId = random.randrange(NETWORK_SIZE) self.process = Process(target=self.createNode, args=(nodeId, port, cacheStorage, knownHosts)) self.process.start() self.node = ServerProxy(('', port), nodeId, ('localhost', port)) def terminate(self): """ Kill node process """ self.process.terminate()
def prepare_proxies(configdata): if configdata[const.PROXY_CONFIG].get(const.PROXY_CONFIG_SOURCE_TYPE, u'1') != u'2': return p = Process(group=None, target=fetch_proxy,) p.start() p.join() print u'%s get %d free proxy' % (datetime.datetime.now(), len(open(u'proxy.txt', u'r').readlines())) c = Process(group=None, target=valid_proxy,) c.start() valid_time = int(configdata[const.PROXY_CONFIG].get(const.PROXY_VALID_TIME)) print u'%s following %d seconds will valid the proxy' % (datetime.datetime.now(), valid_time) time.sleep(valid_time) c.terminate() print u'%s get %d effective proxy' % (datetime.datetime.now(), len(open(u'enable_proxies.txt', u'r').readlines()))
def start_actor_server( actor, host_port_tuple=None, start_in_background_thread=False, log_requests=False, server_starter=start_bjoern_server ): if server_starter != start_bjoern_server: print("Using builtin server (slow)") if not host_port_tuple: # if no specific port is given, # run on free port host_port_tuple = ('', 0) port = host_port_tuple[1] if host_port_tuple[1] == 0: port = get_free_port(host_port_tuple) host_name = get_host_name(host_port_tuple[0]) host_uri = 'http://{0}:{1}'.format(host_name, port) print("Running server on {0}".format(host_uri)) wsgi_application = get_wsgi_application( actor, host_uri, log_requests=log_requests ) if start_in_background_thread: process = Process( target=server_starter, args=(wsgi_application, host_name, port) ) process.daemon = True process.start() return host_uri, process server_starter(wsgi_application, host_name, port)
def run(self): items = get_file_list(self.path_list) #取得所有文件列表,读取文件交给了map_workers self.file_list = items self.total_count = len(items) item_step_count = self.total_count / self.map_worker_count #每个worker分得的任务数量 print u'文件总数:%d' % self.total_count for i in range(self.map_worker_count): name = 'pre_worker@%d' % (i + 1) p = Process(target=add_worker,args=(name,self.task_queue,self.result_queue)) self.worker_process_list.append(p) self.start_time = time.clock() self.last_refresh_time = time.clock() print u'开始时间:',datetime.now() for p in self.worker_process_list: p.start() t_reader = threading.Thread(target=self.put_task_thread) t_writer = threading.Thread(target=self.get_result_thread) t_reader.start() t_writer.start() print '------------------------'
def process_mongo(args, output_path): # connects to the MongoDB server if args.port: connection = Connection(args.address, args.port) else: connection = Connection(args.address) # gets the DB db = connection[args.db_name] # gets the collection collec = db[args.collection] # sets the number of documents to be processed by each thread docs_num = collec.count() slice_size = docs_num / args.threads print "Threads:", args.threads print "Documents number:", docs_num print "Documents per thread:", slice_size # initiates a thread for each slice of documents # the slices are controlled using the base and offset variables base = 0 offset = slice_size jobs = [] for thread_num in range(args.threads): print "Initializing process", thread_num p = Process(target=lemmatize_slice, args=(collec, base, offset, args, output_path)) jobs.append(p) p.start() base += offset for p in jobs: p.join() if (docs_num % 2) == 1: lemmatize_slice(collec, base, offset, args, output_path)
def get_stream(hdf5_file, which_set, batch_size=None): dataset = TrajectoryDataset(which_sets=(which_set, )) if batch_size == None: batch_size = dataset.num_examples data_stream = DataStream(dataset=dataset, iteration_scheme=ShuffledScheme( examples=dataset.num_examples, batch_size=batch_size)) load_in_memory = os.path.getsize( hdf5_file) < 14 * 10**9 or which_set == 'test' if not load_in_memory: port = 5557 if which_set == 'train' else 5558 print port server_process = Process(target=start_server, args=(data_stream, port, 10)) server_process.start() data_stream = ServerDataStream(dataset.sources, False, host='localhost', port=port, hwm=10) return data_stream
class RemoteTest(unittest.TestCase): """ Test the Component, DataFlow, and VAs when shared remotely. The test cases are run as "clients" and at start a server is started. """ container_name = "test" def setUp(self): # Use Thread for debug: if USE_THREADS: self.server = threading.Thread(target=ServerLoop, args=(self.container_name, )) else: self.server = Process(target=ServerLoop, args=(self.container_name, )) self.server.start() self.count = 0 self.data_arrays_sent = 0 time.sleep(0.1) # give it some time to start self.rdaemon = Pyro4.Proxy("PYRO:Pyro.Daemon@./u:" + self.container_name) self.comp = self.rdaemon.getObject("mycomp") def tearDown(self): self.comp.stopServer() time.sleep(0.1) # give it some time to terminate if self.server.is_alive(): if not USE_THREADS: print "Warning: killing server still alive" self.server.terminate() # @unittest.skip("simple") def test_simple(self): """ start a component, ping, and stop it """ ret = self.comp.ping() self.assertEqual(ret, "pong", "Ping failed") # @unittest.skip("simple") def test_exception(self): # test it raises self.assertRaises(MyError, self.comp.bad_call) # test it raises when wrong argument self.assertRaises(TypeError, self.comp.ping, ("non needed arg", )) # non existing method self.assertRaises(AttributeError, self.comp.non_existing_method) # @unittest.skip("simple") def test_roattributes(self): """ check roattributes """ val = self.comp.my_value self.assertEqual(val, "ro", "Reading attribute failed") # @unittest.skip("simple") def test_async(self): """ test futures MyComponent queues the future in order of request """ self.comp.set_number_futures(0) ft1 = self.comp.do_long(2) # long enough we can cancel ft2 ft2 = self.comp.do_long(1) # shorter than ft1 self.assertFalse(ft1.done(), "Future finished too early") self.assertFalse(ft2.done(), "Future finished too early") self.assertFalse(ft2.cancelled(), "future doesn't claim being cancelled") self.assertFalse(ft2.cancelled(), "future doesn't claim being cancelled") self.assertGreater(ft2.result(), 1) # wait for ft2 self.assertFalse(ft2.cancel(), "could cancel the finished future") self.assertTrue(ft1.done(), "Future not finished") self.assertGreater(ft1.result(), 2) self.assertEqual(self.comp.get_number_futures(), 2) # @unittest.skip("simple") def test_unref_futures(self): """ test many futures which don't even get referenced It should behave as if the function does not return anything """ self.comp.set_number_futures(0) expected = 100 # there was a bug with expected > threadpool size (=24) start = time.time() for i in range(expected): self.comp.do_long(0.1) ft_last = self.comp.do_long(0.1) ft_last.result() duration = time.time() - start self.assertGreaterEqual(duration, expected * 0.1) self.assertEqual(self.comp.get_number_futures(), expected + 1) # @unittest.skip("simple") def test_ref_futures(self): """ test many futures which get referenced and accumulated It should behave as if the function does not return anything """ self.comp.set_number_futures(0) small_futures = [] expected = 100 # there was a bug with expected > threadpool size (=24) start = time.time() for i in range(expected): small_futures.append(self.comp.do_long(0.1)) ft_last = self.comp.do_long(0.1) ft_last.result() duration = time.time() - start self.assertGreaterEqual(duration, expected * 0.1) for f in small_futures: self.assertTrue(f.done()) self.assertEqual(self.comp.get_number_futures(), expected + 1) # @unittest.skip("simple") def test_async_cancel(self): """ test futures """ self.comp.set_number_futures(0) ft1 = self.comp.do_long(2) # long enough we can cancel ft2 ft2 = self.comp.do_long(1) # shorter than ft1 self.assertTrue(ft2.cancel(), "couldn't cancel the future") self.assertTrue(ft2.cancelled(), "future doesn't claim being cancelled") self.assertRaises(futures.CancelledError, ft2.result) # wait for ft1 res1a = ft1.result() self.assertGreater(res1a, 2) self.assertTrue(ft1.done(), "Future not finished") res1b = ft1.result() self.assertEqual(res1a, res1b) self.assertGreater(res1b, 2) self.assertEqual(self.comp.get_number_futures(), 2) def test_prog_future(self): """ Test ProgressiveFuture (remotely) """ self._done_calls = 0 self._progess_calls = 0 self._start = None self._end = None ft = self.comp.do_progressive_long(5) ft.add_update_callback(self._on_future_proress) ft.add_done_callback(self._on_future_done) # we should have received at least one progress update already self.assertGreaterEqual(self._progess_calls, 1) ft.result() self.assertGreaterEqual(self._progess_calls, 5) self.assertEqual(self._done_calls, 1) def _on_future_done(self, f): self._done_calls += 1 def _on_future_proress(self, f, start, end): self._progess_calls += 1 logging.info("Received future update for %f -> %f", start, end) self._start = start self._end = end # @unittest.skip("simple") def test_subcomponents(self): # via method and via roattributes # need to test cycles p = self.rdaemon.getObject("parent") self.assertEqual(len(p.children.value), 1, "parent doesn't have one child") c = list(p.children.value)[0] # self.assertEqual(c.parent, p, "Component and parent of child is different") self.assertEqual(p.value, 42) self.assertEqual(c.value, 43) self.assertEqual(len(c.children.value), 0, "child shouldn't have children") # @unittest.skip("simple") def test_dataflow_subscribe(self): self.count = 0 self.expected_shape = (2048, 2048) self.data_arrays_sent = 0 self.comp.data.reset() self.comp.data.subscribe(self.receive_data) time.sleep(0.5) self.comp.data.unsubscribe(self.receive_data) count_end = self.count print "received %d arrays over %d" % (self.count, self.data_arrays_sent) time.sleep(0.1) self.assertEqual(count_end, self.count) self.assertGreaterEqual(count_end, 1) def test_synchronized_df(self): """ Tests 2 dataflows, one synchronized on the event of acquisition started on the other dataflow. """ number = 20 self.count = 0 self.left = number self.expected_shape = (2, 2) self.expected_shape_au = (2048, 2048) self.data_arrays_sent = 0 dfe = self.comp.data dfs = self.comp.datas dfe.reset() dfs.synchronizedOn(self.comp.startAcquire) dfs.subscribe(self.receive_data) time.sleep( 0.2) # long enough to be after the first data if it generates data # ensure that datas hasn't generated anything yet self.assertEqual(self.count, 0) dfe.subscribe(self.receive_data_auto_unsub) for i in range(number): # end early if it's already finished if self.left == 0: break time.sleep(0.2) # 0.2s should be more than enough in any case self.assertEqual(0, self.left) self.assertEqual(number, self.count) print "received %d arrays over %d" % (self.count, self.data_arrays_sent) max_lat = dfs.get_max_lat() if max_lat: print "latency: %r, max= %f, avg= %f" % ( max_lat, max(max_lat), sum(max_lat) / len(max_lat)) time.sleep(0.1) self.assertEqual(number, self.count) dfs.unsubscribe(self.receive_data) dfs.synchronizedOn(None) time.sleep(0.1) self.assertEqual(number, self.count) # @unittest.skip("simple") def test_dataflow_stridden(self): # test that stridden array can be passed (even if less efficient) self.count = 0 self.data_arrays_sent = 0 self.expected_shape = (2048, 2045) self.comp.cut.value = 3 self.comp.data.reset() self.comp.data.subscribe(self.receive_data) time.sleep(0.5) self.comp.data.unsubscribe(self.receive_data) self.comp.cut.value = 0 # put it back count_end = self.count print "received %d stridden arrays over %d" % (self.count, self.data_arrays_sent) time.sleep(0.1) self.assertEqual(count_end, self.count) self.assertGreaterEqual(count_end, 1) def test_dataflow_empty(self): """ test passing empty DataArray """ self.count = 0 self.data_arrays_sent = 0 self.comp.data.setShape((0, ), 16) self.expected_shape = (0, ) self.comp.data.subscribe(self.receive_data) time.sleep(0.5) self.comp.data.unsubscribe(self.receive_data) count_end = self.count print "received %d stridden arrays over %d" % (self.count, self.data_arrays_sent) time.sleep(0.1) self.assertEqual(count_end, self.count) self.assertGreaterEqual(count_end, 1) def receive_data(self, dataflow, data): self.count += 1 self.assertEqual(data.shape, self.expected_shape) if data.ndim >= 2: self.data_arrays_sent = data[0][0] self.assertGreaterEqual(self.data_arrays_sent, self.count) def receive_data_auto_unsub(self, dataflow, data): """ callback for df """ self.assertEqual(data.shape, self.expected_shape_au) self.left -= 1 if self.left <= 0: dataflow.unsubscribe(self.receive_data_auto_unsub) def receive_data_and_unsubscribe(self, dataflow, data): self.count += 1 self.assertEqual(data.shape, (2048, 2048)) self.data_arrays_sent = data[0][0] self.assertGreaterEqual(self.data_arrays_sent, self.count) dataflow.unsubscribe(self.receive_data_and_unsubscribe) # @unittest.skip("simple") def test_dataflow_unsubscribe_from_callback(self): self.count = 0 self.data_arrays_sent = 0 self.comp.data.reset() self.comp.data.subscribe(self.receive_data_and_unsubscribe) time.sleep(0.3) self.assertEqual(self.count, 1) # It should be 1, or if the generation went very fast, it might be bigger self.assertGreaterEqual(self.data_arrays_sent, 1) # print "received %d arrays over %d" % (self.count, self.data_arrays_sent) # data = comp.data # del comp # print gc.get_referrers(data) # gc.collect() # print gc.get_referrers(data) # @unittest.skip("simple") def test_dataflow_get(self): self.comp.data.reset() array = self.comp.data.get() self.assertEqual(array.shape, (2048, 2048)) self.assertEqual(array[0][0], 0) array = self.comp.data.get() self.assertEqual(array.shape, (2048, 2048)) self.assertEqual(array[0][0], 0) # @unittest.skip("simple") def test_va_update(self): prop = self.comp.prop self.assertIsInstance(prop, VigilantAttributeBase) self.assertEqual(prop.value, 42) prop.value += 1 self.assertEqual(prop.value, 43) self.called = 0 self.last_value = None prop.subscribe(self.receive_va_update) # now count prop.value = 3 # +1 prop.value = 0 # +1 prop.value = 0 # nothing because same value time.sleep(0.1) # give time to receive notifications prop.unsubscribe(self.receive_va_update) self.assertEqual(prop.value, 0) self.assertEqual(self.last_value, 0) # called once or twice depending if the brief 3 was seen self.assertTrue(1 <= self.called and self.called <= 2) called_before = self.called # check we are not called anymore prop.value = 3 # +1 self.assertEqual(self.called, called_before) # re-subscribe prop.subscribe(self.receive_va_update) # change remotely self.comp.change_prop(45) time.sleep(0.1) # give time to receive notifications self.assertEqual(prop.value, 45) prop.unsubscribe(self.receive_va_update) self.assertEqual(self.called, called_before + 1) try: prop.value = 7.5 self.fail("Assigning float to a int should not be allowed.") except TypeError: pass # as it should be def receive_va_update(self, value): self.called += 1 self.last_value = value self.assertIsInstance(value, (int, float)) # @unittest.skip("simple") def test_enumerated_va(self): # enumerated self.assertEqual(self.comp.enum.value, "a") self.assertEqual(self.comp.enum.choices, set(["a", "c", "bfds"])) self.comp.enum.value = "c" self.assertEqual(self.comp.enum.value, "c") try: self.comp.enum.value = "wfds" self.fail("Assigning out of bound should not be allowed.") except IndexError: pass # as it should be def test_continuous_va(self): # continuous self.assertEqual(self.comp.cont.value, 2) self.assertEqual(self.comp.cont.range, (-1, 3.4)) self.comp.cont.value = 3.0 self.assertEqual(self.comp.cont.value, 3) try: self.comp.cont.value = 4.0 self.fail("Assigning out of bound should not be allowed.") except IndexError: pass # as it should be def test_list_va(self): # List l = self.comp.listval self.assertEqual(len(l.value), 2) self.called = 0 l.subscribe(self.receive_listva_update) l.value += [3] self.assertEqual(len(l.value), 3) time.sleep(0.01) l.value[-1] = 4 self.assertEqual(l.value[-1], 4) time.sleep(0.01) l.value.reverse() self.assertEqual(l.value[0], 4) time.sleep(0.1) self.assertEqual(self.called, 3) l.unsubscribe(self.receive_listva_update) def receive_listva_update(self, value): self.called += 1 self.last_value = value self.assertIsInstance(value, list)
class AuthorizationCodeTestCase(unittest.TestCase): def setUp(self): self.client = None self.server = None def test_tornado(self): def run_provider(queue): try: provider = create_provider() app = TornadoApplication([ url(r"/authorize", OAuth2Handler, dict(provider=provider)), url(r"/token", OAuth2Handler, dict(provider=provider)) ], debug=True) app.listen(15486) queue.put({"result": 0}) IOLoop.current().start() except Exception as e: queue.put({"result": 1, "error_message": str(e)}) ready_queue = Queue() self.server = Process(target=run_provider, args=(ready_queue,)) self.server.start() provider_started = ready_queue.get() if provider_started["result"] != 0: raise Exception("Error starting Provider process with message" "'{0}'".format(provider_started["error_message"])) self.client = Process(target=run_client, args=(ready_queue,)) self.client.start() client_started = ready_queue.get() if client_started["result"] != 0: raise Exception("Error starting Client Application process with " "message '{0}'" .format(client_started["error_message"])) self.access_token() def test_wsgi(self): def run_provider(queue): try: provider = create_provider() app = Application(provider=provider) httpd = make_server('', 15486, app, handler_class=NoLoggingHandler) queue.put({"result": 0}) httpd.serve_forever() except Exception as e: queue.put({"result": 1, "error_message": str(e)}) ready_queue = Queue() self.server = Process(target=run_provider, args=(ready_queue,)) self.server.start() provider_started = ready_queue.get() if provider_started["result"] != 0: raise Exception("Error starting Provider process with message" "'{0}'".format(provider_started["error_message"])) self.client = Process(target=run_client, args=(ready_queue,)) self.client.start() client_started = ready_queue.get() if client_started["result"] != 0: raise Exception("Error starting Client Application process with " "message '{0}'" .format(client_started["error_message"])) self.access_token() def test_wsgi_404(self): def run_provider(queue): try: provider = create_provider() app = Application(provider=provider) httpd = make_server('', 15486, app, handler_class=NoLoggingHandler) queue.put({"result": 0}) httpd.serve_forever() except Exception as e: queue.put({"result": 1, "error_message": str(e)}) ready_queue = Queue() self.server = Process(target=run_provider, args=(ready_queue,)) self.server.start() provider_started = ready_queue.get() if provider_started["result"] != 0: raise Exception("Error starting Provider process with message" "'{0}'".format(provider_started["error_message"])) try: urlopen("http://127.0.0.1:15486/invalid-path").read() except HTTPError as e: self.assertEqual(404, e.code) def access_token(self): uuid_regex = "^[a-z0-9]{8}\-[a-z0-9]{4}\-[a-z0-9]{4}\-[a-z0-9]{4}-[a-z0-9]{12}$" try: access_token_result = urlopen("http://127.0.0.1:15487/app").read() except HTTPError as e: print(e.read()) exit(1) access_token_data = json.loads(access_token_result.decode('utf-8')) self.assertEqual(access_token_data["token_type"], "Bearer") self.assertEqual(access_token_data["expires_in"], 120) self.assertRegexpMatches(access_token_data["access_token"], uuid_regex) self.assertRegexpMatches(access_token_data["refresh_token"], uuid_regex) request_data = {"grant_type": "refresh_token", "refresh_token": access_token_data["refresh_token"], "client_id": "abc", "client_secret": "xyz"} refresh_token_result = urlopen( "http://127.0.0.1:15486/token", urlencode(request_data).encode('utf-8') ) refresh_token_data = json.loads(refresh_token_result.read().decode('utf-8')) self.assertEqual(refresh_token_data["token_type"], "Bearer") self.assertEqual(refresh_token_data["expires_in"], 120) self.assertRegexpMatches(refresh_token_data["access_token"], uuid_regex) def tearDown(self): if self.client is not None: self.client.terminate() self.client.join() if self.server is not None: self.server.terminate() self.server.join()
toBeAdded.append(server) if len(toBeAdded) == how_many: break for server in toBeAdded: rest.add_node('Administrator', 'password', server.ip) #check if its added ? nodes = rest.node_statuses() otpNodes = [node.id for node in nodes] started = rest.rebalance(otpNodes, []) msg = "rebalance operation started ? {0}" self.log.info(msg.format(started)) if started: result = rest.monitorRebalance() msg = "successfully rebalanced out selected nodes from the cluster ? {0}" self.log.info(msg.format(result)) return result return False if __name__ == "__main__": process1 = Process(target=start_load, args=(sys.argv,)) process1.start() process2 = Process(target=start_combo, args=(sys.argv,)) process2.start() process3 = Process(target=start_backup, args=(sys.argv,)) process3.start() process1.join() process2.join() process3.join()
def evaluate(self, session, starting_set): step = len(self.names) / self.nr_cores procs = [] conns = [] results = [] invert = False invert_count = False if self.wildcard_rule.startswith('!'): corrected_rule = self.wildcard_rule[1:] if self.rule_enforcement == 'all': target_rule = 'any' invert = True elif self.rule_enforcement == 'any': target_rule = 'all' invert = True elif self.rule_enforcement == 'none': target_rule = 'all' elif self.rule_enforcement.startswith('count'): target_rule = 'count' invert_count = True else: target_rule = self.rule_enforcement corrected_rule = self.wildcard_rule if starting_set == "*" and (invert or target_rule == 'none' or target_rule == 'count'): correct_starting_set = array.array('i', rows_as_set(session.execute("SELECT variant_id FROM variants"))) elif starting_set != "*": correct_starting_set = array.array('i', starting_set) else: correct_starting_set = starting_set for i in range(self.nr_cores): parent_conn, child_conn = Pipe() conns.append(parent_conn) p = Process(target=eval(target_rule +'_query'),\ args=(child_conn, self.column, corrected_rule,\ correct_starting_set, self.db_contact_points, self.keyspace)) procs.append(p) p.start() #Split names in chunks and communicate to procs for i in range(self.nr_cores): n = len(self.names) begin = i*step + min(i, n % self.nr_cores) end = begin + step if i < n % self.nr_cores: end += 1 conns[i].send(self.names[begin:end]) #Collect results for i in range(self.nr_cores): results.append(conns[i].recv()) conns[i].close() for i in range(self.nr_cores): procs[i].join() res = set() if target_rule == 'any': for r in results: res = res | r elif target_rule in ['all', 'none']: res = results[0] for r in results[1:]: res = res & r if invert: res = set(correct_starting_set) - res if target_rule == 'count': res_dict = {x: 0 for x in correct_starting_set} for sub_result_dict in results: for var, count in sub_result_dict.iteritems(): res_dict[var] += count if invert_count: total = len(self.names) for variant, count in res_dict.iteritems(): res_dict[variant] = total - count res = set([variant for variant, count in res_dict.iteritems() \ if eval(str(count) + self.count_comp)]) return res
def start(self): print("starting test") testing = Process(target=self.test) #testing.daemon = True testing.start()
class Worker(_th.Thread): def __new__(cls, name=None): if name is None: name = 'default' if name in _workers.keys(): return _workers[name] return super(Worker, cls).__new__(cls) def __init__(self, name=None): if name is None: name = 'default' if name in _workers.keys(): return _workers[name] = self super(Worker, self).__init__() self.daemon = True self.name = name self._queue = _ver.queue.Queue(1) self.last_exception = None self._pon = _mp.Value('b', True) tsk, self.task = _mp.Pipe(False) self.out, res = _mp.Pipe(False) self.process = Process(target=process, args=(self._pon, tsk, res), name=name) self.process.start() self._on = True self.start() _time.sleep(1) def run(self): _sup.debug('%s started' % (str(self.name), )) while self._on or not self._queue.empty(): try: result, target, args, kwargs = self._queue.get(True, .1) _sup.debug('%s: %s-task received' % (str(self.name), target.__name__)) self.task.send((target, args, kwargs)) res = self.out.recv() del (result[self.name]) _sup.debug(res) result[target.__name__] = res _sup.debug('%s: %s-task done' % (str(self.name), target.__name__)) self._queue.task_done() except _ver.queue.Empty: continue except KeyboardInterrupt as ki: raise ki except Exception as exc: _sup.debug('%s: %s' % (str(self.name), str(exc)), 0) if result is not None: result[self.name] = exc self.last_exception = exc _sup.debug('%s: done!' % (str(self.name), )) self._pon.value = False del (_workers[self.name]) def join(self): self._on = False self._queue.join() super(Worker, self).join() self._pon.value = False self.process.join() def put(self, target, *args, **kwargs): result = {self.name: target.__name__} self._queue.put((result, target, args, kwargs)) _time.sleep(.1) return result
start_time = time.time() proc = Process(target=progress) if (os.path.exists('validationLog.log')): print "Log File exists. Making new one" os.remove('validationLog.log') logFile = open( 'validationLog.log', 'a' ) #open a log file for output -> in case it already exists delete old one? try: TO_DO = [ "Data", "Data_HLT", "FullSim", "FullSim_HLT", "FullSim_PU", "FullSim_PU_HLT", "FastSim", "FastSim_HLT", "FastSim_PU", "FastSim_PU_HLT", "FullFastSim", "FullFastSim_HLT", "Generator" ] proc.start() for i in TO_DO: do_some_work(i, logFile) #Do some work here proc.terminate() logFile.close() #os.system("echo 'Done.\nElapsed time:\n"+secondsToStr(time.time() - start_time)+"' | mail -s 'RelMon\n' [email protected]") os.system( "echo 'Done.\nElapsed time:\n%s' | mail -s 'RelMon' [email protected]" % (secondsToStr(time.time() - start_time))) except SystemExit: os.system( "echo 'Error' | mail -s 'RelMon' [email protected]") sys.exit() except:
def start_example_app_process3(): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED p = Process(target=start_example_server3) p.start() return p
def build_process(target): f51fp = Process(target=target, name=target.__name__) f51fp.daemon = 1 f51fp.start() return f51fp
class WhenFunctionalTestingGameClient(unittest.TestCase): def setUp(self): #setup game server to run on a seperate process self.game_server = Process(target=start_server) self.game_server.start() #create the game client self.client = GameClient(host="127.0.0.1", port="5000") self.player_one = "Arthur" self.player_one_suspect = game_state.PEACOCK self.player_two = "Steven" self.player_two_suspect = game_state.PLUM def test_game_client(self): try: #give the game server process a chance to start time.sleep(3) #test registering players and choosing suspects self.client.register_player(self.player_one) self.client.choose_suspect(self.player_one, self.player_one_suspect) self.client.register_player(self.player_two) self.client.choose_suspect(self.player_two, self.player_two_suspect) #retreive the registered players with the client and validate the #return values players = self.client.get_players() for player in players: self.assertIsInstance(player, game_state.Player) self.assertTrue( self.player_one in [player.username for player in players]) self.assertTrue( self.player_two in [player.username for player in players]) self.assertTrue(self.player_one_suspect in [player.suspect for player in players]) self.assertTrue(self.player_two_suspect in [player.suspect for player in players]) #start a new game with the client and validate a GameState object #is returned game = self.client.start_new_game() self.assertTrue(game, game_state.GameState) game = self.client.get_game_state(game.game_id) self.assertTrue(game, game_state.GameState) #move player 1 from start space to hallway player = game.current_player player_1_current_space = game.game_board[player.suspect] move_space = player_1_current_space.connected_spaces[0] game = self.client.move_player(player.username, player.suspect, move_space) self.assertEqual(game.turn_status, game_state.AWAITING_ACCUSATION_OR_END_TURN) game = self.client.end_turn(player.username) player_1_current_space = game.game_board[move_space] self.assertEqual(game.turn_status, game_state.AWAITING_MOVE) #move player 2 from start space to hallway player = game.current_player player_2_current_space = game.game_board[player.suspect] move_space = player_2_current_space.connected_spaces[0] game = self.client.move_player(player.username, player.suspect, move_space) self.assertEqual(game.turn_status, game_state.AWAITING_ACCUSATION_OR_END_TURN) game = self.client.end_turn(player.username) player_2_current_space = game.game_board[move_space] self.assertEqual(game.turn_status, game_state.AWAITING_MOVE) #move player 1 from hallway to room player = game.current_player move_space = player_1_current_space.connected_spaces[0] game = self.client.move_player(player.username, player.suspect, move_space) self.assertEqual(game.turn_status, game_state.AWAITING_SUGGESTION) #make suggestion based on room player is currently in game = self.client.make_suggestion(player.username, game_state.MUSTARD, game_state.REVOLVER, move_space) #if there is a player that can prove the suggestion false #then test the suggestion response if game.suggestion_response_player: with self.assertRaises(errors.GameClientException): game = self.client.move_player(player.username, player.suspect, move_space) self.assertEqual(game.turn_status, game_state.AWAITING_SUGGESTION_RESPONSE) response_player = game.suggestion_response_player suggestion = game.current_suggestion gamecard_item = list( {suggestion.weapon, suggestion.room, suggestion.suspect} & set(card.item for card in response_player.game_cards))[0] game = self.client.make_suggestion_response( response_player.username, gamecard_item) self.assertEqual(game.turn_status, game_state.AWAITING_ACCUSATION_OR_END_TURN) game = self.client.end_turn(player.username) self.assertEqual(game.turn_status, game_state.AWAITING_MOVE) last_player = player player = game.current_player self.assertNotEqual(player.username, last_player.username) #test accusation suspect = [ card.item for card in game.case_file if card.type == game_state.SUSPECT ][0] weapon = [ card.item for card in game.case_file if card.type == game_state.WEAPON ][0] room = [ card.item for card in game.case_file if card.type == game_state.ROOM ][0] game = self.client.make_accusation(player.username, suspect, weapon, room) for message in game.player_messages: print message self.client.destroy_game(game.game_id) finally: self.game_server.terminate()
args = parser().parse_args() initLogger(args.log) #log.startLogging(open('logs/twisted.logs', 'w+')) factory = ArgsClientFactory(args) if args.spawn: nodes = [] logging.info('Spawning {} processes'.format(args.spawn)) startNode('Main node', factory, args.p2p_port, getKnownNodes(args)) for i in range(1, args.spawn): process = Process(target=startNode, args=('Node {}'.format(i), factory, args.p2p_port + i, [('localhost', args.p2p_port)])) process.start() nodes.append(process) try: while 1: time.sleep(1) finally: for n in nodes: logging.info('Killing node {}'.format(n)) n.terminate() n.join() else: node = startNode('P2P node', factory, args.p2p_port, getKnownNodes(args)) # pylint: disable=E1101
class MongoRedis(object): def __init__(self, mongo_db, collection_name='cache'): # Ensure index if not isinstance(mongo_db, Database): raise ValueError( 'mongo_db must be instance of pymongo.database.Database') self.col = mongo_db[collection_name] self.col.ensure_index('k', unique=True) self.col.ensure_index('exp') self.prune_expired() def start(self): """ Starts the background process that prunes expired items """ def task(): while True: self.prune_expired() pytime.sleep(.5) self.processs = Process(target=task) self.processs.start() def end(self): """ End the background process that prunes expired items """ if not hasattr(self, 'process'): self.processs.terminate() def prune_expired(self): """ Deletes expired keys from the db, returns count deleted """ now = pytime.time() result = self.col.remove({'exp': {'$exists': True, '$lte': now}}) return result['n'] ### REDIS COMMANDS ### def delete(self, *names): """ Delete one or more keys specified by ``names`` """ return self.col.remove({'k': {'$in': names}})['n'] __delitem__ = delete def expire(self, name, time): """ Set an expire flag on key ``name`` for ``time`` seconds. ``time`` can be represented by an integer or a Python timedelta object. """ expire_at = pytime.time() if isinstance(time, datetime.timedelta): time = time.seconds + time.days * 24 * 3600 expire_at += time return bool( self.col.update({'k': name}, {'$set': { 'exp': expire_at }})['n']) def flushdb(self): """ Delete all keys in the current database """ self.col.remove() return True def get(self, name): """ Return the value at key ``name``, or None if the key doesn't exist """ now = pytime.time() result = self.col.find_one({'k': name}) or {} if result.get('exp', now) < now: return None return result.get('v') def set(self, name, value, ex=None, px=None, nx=False, xx=False): """ Set the value at key ``name`` to ``value`` ``ex`` sets an expire flag on key ``name`` for ``ex`` seconds. ``px`` sets an expire flag on key ``name`` for ``px`` milliseconds. ``nx`` if set to True, set the value at key ``name`` to ``value`` if it does not already exist. ``xx`` if set to True, set the value at key ``name`` to ``value`` if it already exists. """ upsert = True expire_at = pytime.time() if px: # if isinstance(px, datetime.timedelta): # ms = int(px.microseconds / 1000) # px = (px.seconds + px.days * 24 * 3600) * 1000 + ms # expire_at += px * 0.001 raise NotImplementedError # Millis to fine grained elif ex: if isinstance(ex, datetime.timedelta): ex = ex.seconds + ex.days * 24 * 3600 expire_at += ex if nx: try: data = {'k': name, 'v': value, 'exp': expire_at} if ex: data['exp'] = expire_at self.col.save(data) return True except DuplicateKeyError: return None elif xx: upsert = False data = {'v': value} if ex: data['exp'] = expire_at result = self.col.update({'k': name}, {'$set': data}, upsert=upsert) return True if result['n'] == 1 else None __setitem__ = set def ttl(self, name): """ Returns the number of seconds until the key ``name`` will expire """ now = pytime.time() exp = (self.col.find_one({'k': name}) or {}).get('exp', now) diff = exp - now return long(-1) if diff <= 0 else diff
def start_echo_server_process(): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED p = Process(target=start_echo_server) p.start() return p
class SmtpMessageServer(object): """ This class can start an SMTP debugging server, configure LinOTP to talk to it and read the results back to the parent tester. On open, an SMTP server is set up to listen locally. Derived classes can define a hook to set the LinOTP configuration to point to this server. Example usage: with SmtpMessageServer(testcase) as smtp: get_otp() """ def __init__(self, testcase, message_timeout): self.testcase = testcase # We need a minimum version of 2.9.2 to set the SMTP port number, so # skip if testing an earlier version self.testcase.need_linotp_version('2.9.2') self.timeout = message_timeout self.set_config = SetConfig(testcase.http_protocol, testcase.http_host, testcase.http_port, testcase.http_username, testcase.http_password) # We advertise the local SMTP server hostname # using the IP address that connects to LinOTP self.addr = self._get_local_ip() self.msg_payload = None def __enter__(self): self.smtp_process_queue = Queue() self.smtp_process = Process(target=get_otp_mail, args=(self.smtp_process_queue, self.timeout)) self.smtp_process.start() self.port = self.smtp_process_queue.get(True, 5) self._do_lintop_config() return self def _do_lintop_config(self): parameters = self.get_config_parameters() logger.debug("Configuration parameters: %s", parameters) result = self.set_config.setConfig(parameters) assert result, "It was not possible to set the config. Result:%s" % result def get_config_parameters(self): # This function can be overridden to provide configuration parameters to configure # specific parts of LinOTP assert False, "This function should be overridden" def get_otp(self): messagestr = self.smtp_process_queue.get(True, 10) msg = email.message_from_string(messagestr) otp = msg.get_payload() logger.debug("Received email message payload:%s", otp) return otp def __exit__(self, *args): self.smtp_process_queue.close() self.smtp_process.terminate() self.smtp_process.join(5) def _get_local_ip(self): """ Get the IP address of the interface that connects to LinOTP """ with closing( socket.create_connection( (self.testcase.http_host, int(self.testcase.http_port)), 10)) as s: addr = s.getsockname()[0] return addr
class Fetcher(object): __metaclass__ = ABCMeta def __init__(self, worker_list): from .worker import Worker self.worker_list = worker_list assert isinstance(self.worker_list, list) for worker in self.worker_list: assert isinstance(worker, Worker) self.process = Process(target=self.run_forever, args=()) self.running = False def start(self): self.process.start() @abstractmethod def shutdown(self): # 处理进程关闭 self.running = False def setup_shutdown(self): """ 设置优雅退出 :return: """ import signal def on_sigterm(*ignore): self.shutdown() signal.signal(signal.SIGTERM, on_sigterm) signal.signal(signal.SIGINT, on_sigterm) @abstractmethod def choose(self, msg): """ 选择进程的处理进程下标 如:hash(msg) % len(self.worker_list) :param msg: :return: """ pass def do_feed(self, msg): index = self.choose(msg) assert isinstance(index, int) and index < len(self.worker_list) while True: try: self.worker_list[index].feed(msg) except Full: p = self.worker_list[index].process assert isinstance(p, Process) # 队列满了重试发送 logging.error( "pid:%s worker is full. Please check the Thread blocking situation." % str(p.pid)) continue break @abstractmethod def run_forever(self): """ 启动处理 :return: """ self.running = True def join(self): self.process.join()
class ImageMngr(object): """ This class handles most of the backend work for the image gateway. It uses a Mongo Database to track state, uses threads to dispatch work, and has public functions to lookup, pull and expire images. """ def __init__(self, config, logger=None, logname='imagemngr'): """ Create an instance of the image manager. """ if logger is None and logname is None: self.logger = logging.getLogger(logname) log_handler = logging.StreamHandler() logfmt = '%(asctime)s [%(name)s] %(levelname)s : %(message)s' log_handler.setFormatter(logging.Formatter(logfmt)) log_handler.setLevel(logging.INFO) self.logger.addHandler(log_handler) elif logname is not None: self.logger = logging.getLogger(logname) self.logger.info('ImageMngr using logname %s' % (logname)) else: print "Using upstream logger" self.logger = logger print logger self.logger.info('ImageMngr using upstream logger') self.logger.debug('Initializing image manager') self.config = config if 'Platforms' not in self.config: raise NameError('Platforms not defined') self.systems = [] # Time before another pull can be attempted self.pullupdatetimeout = 300 if 'PullUpdateTime' in self.config: self.pullupdatetimeout = self.config['PullUpdateTimeout'] # Max amount of time to allow for a pull self.pulltimeout = self.pullupdatetimeout # This is not intended to provide security, but just # provide a basic check that a session object is correct self.magic = 'imagemngrmagic' if 'Authentication' not in self.config: self.config['Authentication'] = "munge" self.auth = Authentication(self.config) self.platforms = self.config['Platforms'] for system in self.config['Platforms']: self.systems.append(system) # Connect to database if 'MongoDBURI' not in self.config: raise NameError('MongoDBURI not defined') self.workers = WorkerThreads() self.status_queue = self.workers.get_updater_queue() self.status_proc = Process(target=self.status_thread, name='StatusThread') self.status_proc.start() atexit.register(self.shutdown) self.mongo_init() def shutdown(self): self.status_queue.put('stop') def mongo_init(self): client = MongoClient(self.config['MongoDBURI']) db_ = self.config['MongoDB'] self.images = client[db_].images self.metrics = None if 'Metrics' in self.config and self.config['Metrics'] is True: self.metrics = client[db_].metrics def status_thread(self): """ This listens for update messages from a queue. """ self.mongo_init() while True: message = self.status_queue.get() if message == 'stop': self.logger.info("Shutting down Status Thread") break ident = message['id'] state = message['state'] meta = message['meta'] # TODO: Handle a failed expire if state == "FAILURE": self.logger.warn("Operation failed for %s", ident) # print "Status: %s" % (state) # A response message if state != 'READY': self.update_mongo_state(ident, state, meta) continue if 'response' in meta and meta['response']: response = meta['response'] self.logger.debug(response) if 'meta_only' in response: self.logger.debug('Updating ACLs') self.update_acls(ident, response) else: self.complete_pull(ident, response) self.logger.debug('meta=%s', str(response)) def check_session(self, session, system=None): """Check if this is a valid session session is a session handle """ if 'magic' not in session: self.logger.warn("request recieved with no magic") return False elif session['magic'] is not self.magic: self.logger.warn("request received with bad magic %s", session['magic']) return False if system is not None and session['system'] != system: self.logger.warn("request received with a bad system %s!=%s", session['system'], system) return False return True def _isadmin(self, session, system=None): """ Check if this is an admin user. Returns true if is an admin or false if not. """ if 'admins' not in self.platforms[system]: return False admins = self.platforms[system]['admins'] user = session['user'] if user in admins: self.logger.info('user %s is an admin', user) return True return False def _isasystem(self, system): """Check if system is a valid platform.""" return bool(system in self.systems) def _get_groups(self, uid, gid): """Look up auxilary groups. """ proc = Popen(['id', '-G', '%d' % (uid)], stdout=PIPE, stderr=PIPE) if proc is None: self.logger.warn("Group lookup failed") return [] stdout, stderr = proc.communicate() groups = [] for group in stdout.split(): groups.append(int(group)) return groups def _checkread(self, session, rec): """ Checks if the user has read permissions to the image. """ # Start by checking if the image is public (no ACLs) if 'private' in rec and rec['private'] is False: return True iUACL = None iGACL = None if 'userACL' in rec: iUACL = rec['userACL'] if 'groupACL' in rec: iGACL = rec['groupACL'] if iUACL is None and iGACL is None: return True if iUACL == [] and iGACL == []: return True uid = session['uid'] gid = session['gid'] self.logger.debug('uid=%s iUACL=%s' % (uid, str(iUACL))) self.logger.debug('sessions = ' + str(session)) groups = self._get_groups(uid, gid) if iUACL is not None and uid in iUACL: return True if iGACL is not None and gid in iGACL: return True for group in groups: if iGACL is not None and group in iGACL: return True return False def _resetexpire(self, ident): """Reset the expire time. (Not fully implemented).""" # Change expire time for image # TODO shore up expire-time parsing expire_timeout = self.config['ImageExpirationTimeout'] (days, hours, minutes, secs) = expire_timeout.split(':') expire = time() + int(secs) + 60 * (int(minutes) + 60 * (int(hours) + 24 * int(days))) self._images_update({'_id': ident}, {'$set': {'expiration': expire}}) return expire def _make_acl(self, acllist, id): if id not in acllist: acllist.append(id) return acllist def _compare_list(self, a, b, key): """" look at the key element of two objects and compare the list of ids. return True if everything matches return False if anything is different """ # If the key isn't in the objects or # something else fails, then it must # have changed. try: if key not in a: return False if key not in b: return False except: return True aitems = a[key] bitems = b[key] if len(aitems) != len(bitems): return False for item in aitems: if item not in bitems: return False return True def _add_metrics(self, session, request, record): """ Add a row to mongo for this lookup request. """ try: r = { 'user': session['user'], 'uid': session['uid'], 'system': request['system'], 'type': request['itype'], 'tag': request['tag'], 'id': record['id'], 'time': time() } self._metrics_insert(r) except: self.logger.warn('Failed to log lookup.') def get_metrics(self, session, system, limit): """ Return the last <limit> lookup records. """ recs = [] if not self._isadmin(session, system): return recs if self.metrics is None: return recs count = self.metrics.count() skip = count - limit if skip < 0: skip = 0 for r in self.metrics.find().skip(skip): r.pop('_id', None) recs.append(r) return recs def new_session(self, auth_string, system): """ Creates a session context that can be used for multiple transactions. auth is an auth string that will be passed to the authenication layer. Returns a context that can be used for subsequent operations. """ if auth_string is None: return {'magic': self.magic, 'system': system} arec = self.auth.authenticate(auth_string, system) if arec is None and isinstance(arec, dict): raise OSError("Authenication returned None") else: if 'user' not in arec: raise OSError("Authentication returned invalid response") session = arec session['magic'] = self.magic session['system'] = system return session def lookup(self, session, image): """ Lookup an image. Image is dictionary with system,itype and tag defined. """ if not self.check_session(session, image['system']): raise OSError("Invalid Session") query = { 'status': 'READY', 'system': image['system'], 'itype': image['itype'], 'tag': { '$in': [image['tag']] } } self.update_states() rec = self._images_find_one(query) if rec is not None: if self._checkread(session, rec) is False: return None self._resetexpire(rec['_id']) if self.metrics is not None: self._add_metrics(session, image, rec) return rec def imglist(self, session, system): """ list images for a system. Image is dictionary with system defined. """ if not self.check_session(session, system): raise OSError("Invalid Session") if self._isasystem(system) is False: raise OSError("Invalid System") query = {'status': 'READY', 'system': system} self.update_states() records = self._images_find(query) resp = [] for record in records: if self._checkread(session, record): resp.append(record) # verify access return resp def show_queue(self, session, system): """ list queue for a system. Image is dictionary with system defined. """ if not self.check_session(session, system): raise OSError("Invalid Session") query = {'status': {'$ne': 'READY'}, 'system': system} self.update_states() records = self._images_find(query) resp = [] for record in records: resp.append({ 'status': record['status'], 'image': record['pulltag'] }) return resp def _isready(self, image): """Helper function to determine if an image is READY.""" query = { 'status': 'READY', 'system': image['system'], 'itype': image['itype'], 'tag': { '$in': [image['tag']] } } rec = self._images_find_one(query) if rec is not None: return True return False def _pullable(self, rec): """ An image is pullable when: -There is no existing record -The status is a FAILURE -The status is READY and it is past the update time -The state is something else and the pull has expired """ # if rec is None then do a pull if rec is None: return True # Okay there has been a pull before # If the status flag is missing just repull (shouldn't happen) if 'status' not in rec: return True status = rec['status'] # EXPIRED images can be pulled if status == 'EXPIRED': return True # Need to deal with last_pull for a READY record if 'last_pull' not in rec: return True nextpull = self.pullupdatetimeout + rec['last_pull'] # It has been a while, so re-pull to see if it is fresh if status == 'READY' and (time() > nextpull): return True # Repull failed pulls if status == 'FAILURE' and (time() > nextpull): return True # Last thing... What if the pull somehow got hung or died in the middle # See if heartbeat is old # TODO: add pull timeout. For now use 1 hour if status != 'READY' and 'last_heartbeat' in rec: if (time() - rec['last_heartbeat']) > 3600: return True return False def new_pull_record(self, image): """ Creates a new image in mongo. If the pull already exist it removes it first. """ # Clean out any existing records for rec in self._images_find(image): if rec['status'] == 'READY': continue else: self._images_remove({'_id': rec['_id']}) newimage = { 'format': 'invalid', # <ext4|squashfs|vfs> 'arch': 'amd64', # <amd64|...> 'os': 'linux', # <linux|...> 'location': '', # urlencoded location 'remotetype': 'dockerv2', # <file|dockerv2|amazonec2> 'ostcount': '0', # integer, number of OSTs (future) 'replication': '1', # integer, number of copies to deploy 'userACL': [], 'groupACL': [], 'private': None, 'tag': [], 'status': 'INIT' } if 'DefaultImageFormat' in self.config: newimage['format'] = self.config['DefaultImageFormat'] for param in image: if param is 'tag': continue newimage[param] = image[param] self._images_insert(newimage) return newimage def pull(self, session, image, testmode=0): """ pull the image Takes an auth token, a request object Optional: testmode={0,1,2} See below... """ request = { 'system': image['system'], 'itype': image['itype'], 'pulltag': image['tag'] } self.logger.debug('Pull called Test Mode=%d', testmode) if not self.check_session(session, request['system']): self.logger.warn('Invalid session on system %s', request['system']) raise OSError("Invalid Session") # If a pull request exist for this tag # check to see if it is expired or a failure, if so remove it # otherwise # return the record rec = None # find any pull record self.update_states() # let's lookup the active image query = { 'status': 'READY', 'system': image['system'], 'itype': image['itype'], 'tag': { '$in': [image['tag']] } } rec = self._images_find_one(query) for record in self._images_find(request): status = record['status'] if status == 'READY' or status == 'SUCCESS': continue rec = record break inflight = False recent = False if rec is not None and rec['status'] != 'READY': inflight = True elif rec is not None: # if an image has been pulled in the last 60 seconds # let's consider that "recent" if (time() - rec['last_pull']) < 10: recent = True request['userACL'] = [] request['groupACL'] = [] if 'userACL' in image and image['userACL'] != []: request['userACL'] = self._make_acl(image['userACL'], session['uid']) if 'groupACL' in image and image['groupACL'] != []: request['groupACL'] = self._make_acl(image['groupACL'], session['gid']) if self._compare_list(request, rec, 'userACL') and \ self._compare_list(request, rec, 'groupACL'): acl_changed = False else: self.logger.debug("No ACL change detected.") acl_changed = True # We could hit a key error or some other edge case # so just do our best and update if there are problems update = False if not recent and not inflight and acl_changed: self.logger.debug("ACL change detected.") update = True if self._pullable(rec): self.logger.debug("Pullable image") update = True if update: self.logger.debug("Creating New Pull Record") rec = self.new_pull_record(request) ident = rec['_id'] self.logger.debug("ENQUEUEING Request") self.update_mongo_state(ident, 'ENQUEUED') request['tag'] = request['pulltag'] request['session'] = session self.logger.debug("Calling do pull with queue=%s", request['system']) self.workers.dopull(ident, request, testmode=testmode) memo = "pull request queued s=%s t=%s" \ % (request['system'], request['tag']) self.logger.info(memo) self.update_mongo(ident, {'last_pull': time()}) return rec def mngrimport(self, session, image, testmode=0): """ import the image directly from a file Only for allowed users Takes an auth token, a request object """ meta = {} fp = image['filepath'] request = { 'system': image['system'], 'itype': image['itype'], 'pulltag': image['tag'], 'filepath': image['filepath'], 'format': image['format'], 'meta': meta } self.logger.debug('mngrmport called for file %s' % (fp)) # self.logger.debug(image) if not self.check_session(session, request['system']): self.logger.warn('Invalid session on system %s', request['system']) raise OSError("Invalid Session") # Skip checks about previous requests for now # Future work could check the fasthash and # not import if they're the same q = { 'system': image['system'], 'itype': image['itype'], 'pulltag': image['tag'] } rec = self._images_find_one(q) if not self._pullable(rec): return rec # rec = None # request['userACL'] = [] # request['groupACL'] = [] # if 'userACL' in image and image['userACL'] != []: # request['userACL'] = self._make_acl(image['userACL'], # session['uid']) # if 'groupACL' in image and image['groupACL'] != []: # request['groupACL'] = self._make_acl(image['groupACL'], # session['gid']) # if self._compare_list(request, rec, 'userACL') and \ # self._compare_list(request, rec, 'groupACL'): # acl_changed = False # else: # self.logger.debug("No ACL change detected.") # acl_changed = True # We could hit a key error or some other edge case # so just do our best and update if there are problems self.logger.debug("Creating New Import Record") # new_pull_record works for import too rec = self.new_pull_record(request) ident = rec['_id'] self.logger.debug("ENQUEUEING Request, ident %s" % (ident)) self.update_mongo_state(ident, 'ENQUEUED') request['tag'] = request['pulltag'] request['session'] = session self.logger.debug("Calling wrkimport with queue=%s", request['system']) self.workers.dowrkimport(ident, request, testmode=testmode) memo = "import request queued s=%s t=%s" \ % (request['system'], request['tag']) self.logger.info(memo) self.update_mongo(ident, {'last_pull': time()}) return rec def update_mongo_state(self, ident, state, info=None): """ Helper function to set the mongo state for an image with _id==ident to state=state. """ if state == 'SUCCESS': state = 'READY' set_list = {'status': state, 'status_message': ''} if info is not None and isinstance(info, dict): if 'heartbeat' in info: set_list['last_heartbeat'] = info['heartbeat'] if 'message' in info: set_list['status_message'] = info['message'] self._images_update({'_id': ident}, {'$set': set_list}) def add_tag(self, ident, system, tag): """ Helper function to add a tag to an image. ident is the mongo id (not image id) """ # Remove the tag first self.remove_tag(system, tag) # see if tag isn't a list rec = self._images_find_one({'_id': ident}) if rec is not None and 'tag' in rec and \ not isinstance(rec['tag'], (list)): memo = 'Fixing tag for non-list %s %s' % (ident, str(rec['tag'])) self.logger.info(memo) curtag = rec['tag'] self._images_update({'_id': ident}, {'$set': {'tag': [curtag]}}) self._images_update({'_id': ident}, {'$addToSet': {'tag': tag}}) return True def remove_tag(self, system, tag): """ Helper function to remove a tag to an image. """ self._images_update({ 'system': system, 'tag': { '$in': [tag] } }, {'$pull': { 'tag': tag }}, multi=True) return True def update_acls(self, ident, response): self.logger.debug("Update ACLs called for %s %s", ident, str(response)) pullrec = self._images_find_one({'_id': ident}) if pullrec is None: self.logger.error('ERROR: Missing pull request (r=%s)', str(response)) return # Check that this image ident doesn't already exist for this system rec = self._images_find_one({ 'id': response['id'], 'status': 'READY', 'system': pullrec['system'] }) if rec is None: # This means the image already existed, but we didn't have a # record of it. That seems odd (it happens in tests). Let's # note it and power on through. msg = "WARNING: No image record found for an ACL update" self.logger.warn(msg) response['last_pull'] = time() response['status'] = 'READY' self.update_mongo(ident, response) self.add_tag(ident, pullrec['system'], pullrec['pulltag']) else: updates = { 'userACL': response['userACL'], 'groupACL': response['groupACL'], 'private': response['private'], 'last_pull': time() } self.logger.debug("Doing ACLs update") response['last_pull'] = time() response['status'] = 'READY' self.update_mongo(rec['_id'], updates) self._images_remove({'_id': ident}) def complete_pull(self, ident, response): """ Transition a completed pull request to an available image. """ self.logger.debug("Complete called for %s %s", ident, str(response)) pullrec = self._images_find_one({'_id': ident}) if pullrec is None: self.logger.warn('Missing pull request (r=%s)', str(response)) return # Check that this image ident doesn't already exist for this system rec = self._images_find_one({ 'id': response['id'], 'system': pullrec['system'], 'status': 'READY' }) tag = pullrec['pulltag'] if rec is not None: # So we already had this image. # Let's delete the pull record. # TODO: update the pull time of the matching id self.logger.warn('Duplicate image') update_rec = {'last_pull': time()} self.update_mongo(rec['_id'], update_rec) self._images_remove({'_id': ident}) # However it could be a new tag. So let's update the tag try: rec['tag'].index(response['tag']) except: self.add_tag(rec['_id'], pullrec['system'], tag) return True else: response['last_pull'] = time() response['status'] = 'READY' self.update_mongo(ident, response) self.add_tag(ident, pullrec['system'], tag) def update_mongo(self, ident, resp): """ Helper function to set the mongo values for an image with _id==ident. """ setline = dict() # This maps from the key name in the response to the # key name used in mongo mappings = { 'id': 'id', 'entrypoint': 'ENTRY', 'env': 'ENV', 'workdir': 'WORKDIR', 'last_pull': 'last_pull', 'userACL': 'userACL', 'groupACL': 'groupACL', 'private': 'private', 'status': 'status' } if 'private' in resp and resp['private'] is False: resp['userACL'] = [] resp['groupACL'] = [] for key in mappings.keys(): if key in resp: setline[mappings[key]] = resp[key] self._images_update({'_id': ident}, {'$set': setline}) def get_state(self, ident): """ Lookup the state of the image with _id==ident in Mongo. Returns the state. """ self.update_states() rec = self._images_find_one({'_id': ident}, {'status': 1}) if rec is None: return None elif 'status' not in rec: return None return rec['status'] def update_states(self): """ Cleanup failed transcations after a period """ for rec in self._images_find({'status': 'FAILURE'}): nextpull = self.pullupdatetimeout + rec['last_pull'] # It it has been a while then let's clean up if time() > nextpull: self._images_remove({'_id': rec['_id']}) def autoexpire(self, session, system, testmode=0): """Auto expire images and do cleanup""" # While this should be safe, let's restrict this to admins if not self._isadmin(session, system): return False # Cleanup - Lookup for things stuck in non-READY state self.update_states() removed = [] for rec in self._images_find({ 'status': { '$ne': 'READY' }, 'system': system }): if 'last_pull' not in rec: self.logger.warning('Image missing last_pull for pulltag:' + rec['pulltag']) continue if time() > rec['last_pull'] + self.pulltimeout: removed.append(rec['_id']) self._images_remove({'_id': rec['_id']}) expired = [] # Look for READY images that haven't been pulled recently for rec in self._images_find({'status': 'READY', 'system': system}): if 'expiration' not in rec: continue elif rec['expiration'] < time(): self.logger.debug("expiring %s", rec['id']) ident = rec.pop('_id') self.expire_id(rec, ident) if 'id' in rec: expired.append(rec['id']) else: expired.append('unknown') self.logger.debug(rec['expiration'] > time()) return expired def expire_id(self, rec, ident, testmode=0): """ Helper function to expire by id """ memo = "Calling do expire id=%s TM=%d" \ % (ident, testmode) self.logger.debug(memo) self.workers.doexpire(ident, rec) self.logger.info("expire request queued s=%s t=%s", rec['system'], ident) def expire(self, session, image, testmode=0): """Expire an image. (Not Implemented)""" if not self._isadmin(session, image['system']): return False query = { 'system': image['system'], 'itype': image['itype'], 'tag': { '$in': [image['tag']] } } rec = self._images_find_one(query) if rec is None: return None ident = rec.pop('_id') memo = "Calling do expire with queue=%s id=%s TM=%d" \ % (image['system'], ident, testmode) self.logger.debug(memo) self.workers.doexpire(ident, rec) memo = "expire request queued s=%s t=%s" \ % (image['system'], image['tag']) self.logger.info(memo) return True @mongo_reconnect_reattempt def _images_remove(self, *args, **kwargs): """ Decorated function to remove images from mongo """ return self.images.remove(*args, **kwargs) @mongo_reconnect_reattempt def _images_update(self, *args, **kwargs): """ Decorated function to updates images in mongo """ return self.images.update(*args, **kwargs) @mongo_reconnect_reattempt def _images_find(self, *args, **kwargs): """ Decorated function to find images in mongo """ return self.images.find(*args, **kwargs) @mongo_reconnect_reattempt def _images_find_one(self, *args, **kwargs): """ Decorated function to find one image in mongo """ return self.images.find_one(*args, **kwargs) @mongo_reconnect_reattempt def _images_insert(self, *args, **kwargs): """ Decorated function to insert an image in mongo """ return self.images.insert(*args, **kwargs) @mongo_reconnect_reattempt def _metrics_insert(self, *args, **kwargs): """ Decorated function to insert an image in mongo """ if self.metrics is not None: return self.metrics.insert(*args, **kwargs)
class WhenFunctionalTestingGameClient(unittest.TestCase): def setUp(self): #setup game server to run on a seperate process self.game_server = Process(target=start_server) self.game_server.start() #create the game client self.client = GameClient(host="127.0.0.1", port="5000") self.player_one = "Arthur" self.player_one_suspect = game_state.PEACOCK self.player_two = "Steven" self.player_two_suspect = game_state.PLUM def test_game_client(self): try: #give the game server process a chance to start time.sleep(3) #test registering players and choosing suspects self.client.register_player(self.player_one) self.client.choose_suspect( self.player_one, self.player_one_suspect) self.client.register_player( self.player_two) self.client.choose_suspect( self.player_two, self.player_two_suspect) #retreive the registered players with the client and validate the #return values players = self.client.get_players() for player in players: self.assertIsInstance(player, game_state.Player) self.assertTrue( self.player_one in [player.username for player in players]) self.assertTrue( self.player_two in [player.username for player in players]) self.assertTrue( self.player_one_suspect in [player.suspect for player in players]) self.assertTrue( self.player_two_suspect in [player.suspect for player in players]) #start a new game with the client and validate a GameState object #is returned game = self.client.start_new_game() self.assertTrue(game, game_state.GameState) game = self.client.get_game_state(game.game_id) self.assertTrue(game, game_state.GameState) #move player 1 from start space to hallway player = game.current_player player_1_current_space = game.game_board[player.suspect] move_space = player_1_current_space.connected_spaces[0] game = self.client.move_player( player.username, player.suspect, move_space) self.assertEqual( game.turn_status, game_state.AWAITING_ACCUSATION_OR_END_TURN) game = self.client.end_turn(player.username) player_1_current_space = game.game_board[move_space] self.assertEqual(game.turn_status, game_state.AWAITING_MOVE) #move player 2 from start space to hallway player = game.current_player player_2_current_space = game.game_board[player.suspect] move_space = player_2_current_space.connected_spaces[0] game = self.client.move_player( player.username, player.suspect, move_space) self.assertEqual( game.turn_status, game_state.AWAITING_ACCUSATION_OR_END_TURN) game = self.client.end_turn(player.username) player_2_current_space = game.game_board[move_space] self.assertEqual(game.turn_status, game_state.AWAITING_MOVE) #move player 1 from hallway to room player = game.current_player move_space = player_1_current_space.connected_spaces[0] game = self.client.move_player( player.username, player.suspect, move_space) self.assertEqual( game.turn_status, game_state.AWAITING_SUGGESTION) #make suggestion based on room player is currently in game = self.client.make_suggestion( player.username, game_state.MUSTARD, game_state.REVOLVER, move_space ) #if there is a player that can prove the suggestion false #then test the suggestion response if game.suggestion_response_player: with self.assertRaises(errors.GameClientException): game = self.client.move_player( player.username, player.suspect, move_space) self.assertEqual( game.turn_status, game_state.AWAITING_SUGGESTION_RESPONSE) response_player = game.suggestion_response_player suggestion = game.current_suggestion gamecard_item = list( {suggestion.weapon, suggestion.room, suggestion.suspect} & set(card.item for card in response_player.game_cards))[0] game = self.client.make_suggestion_response( response_player.username, gamecard_item) self.assertEqual( game.turn_status, game_state.AWAITING_ACCUSATION_OR_END_TURN) game = self.client.end_turn(player.username) self.assertEqual(game.turn_status, game_state.AWAITING_MOVE) last_player = player player = game.current_player self.assertNotEqual(player.username, last_player.username) #test accusation suspect = [ card.item for card in game.case_file if card.type == game_state.SUSPECT ][0] weapon = [ card.item for card in game.case_file if card.type == game_state.WEAPON ][0] room = [ card.item for card in game.case_file if card.type == game_state.ROOM ][0] game = self.client.make_accusation( player.username, suspect, weapon, room) for message in game.player_messages: print message self.client.destroy_game(game.game_id) finally: self.game_server.terminate()
def action_import_apks(storage, apk_paths, copy_apk = False, copy_to_mongodb = False, update = False, tag = None, # shared memory cnt_imported_apks = None, total_apk_count = None, import_finished = None, # concurrent settings concurrency = None ): ''' Import the apks from the `apk_paths` and create the file system structure where the results will be kept, specified by `storage`. Parameters ---------- storage : RedundantStorage The store to use. apk_paths : iterable<str> The apk files and/or directories. copy_apk : bool Import the apk file to the `import_dir` (copy it). copy_to_mongodb : bool, optional (default is False) Also import into MongoDB. Useful for the distributed analysis. update : bool Update apks that have already been imported. tag : str, optional (default is None) Some tag cnt_imported_apks : multiprocessing.Value<int>, optional (default is None) If given, use for progress updating. total_apk_count : multiprocessing.Value<int>, optional (default is None) If given, use for total count of apks. import_finished : multiprocessing.Value<byte>, optional (default is None) If given, use to signal that import has been completed. concurrency : int, optional (default is number of cpus) Number of processes to use for the import. ''' from androlyze.loader.ApkImporter import ApkImporter # get single paths to apks so we get the correct total count of apks clilog.info("looking for apks in given paths ... ") apk_paths = ApkImporter.get_apks_from_list_or_dir(apk_paths) if total_apk_count is not None: # may be time consuming for recursive lookup apk_paths, total_apk_count.value = Util.count_iterable_n_clone(apk_paths) # create count if not given if cnt_imported_apks is None: cnt_imported_apks = Value('i', 0, lock = RLock()) # set concurrency if concurrency is None: concurrency = cpu_count() log.warn("Using %d processes", concurrency) clilog.info("Storage dir is %s" % storage.fs_storage.store_root_dir) if copy_apk: clilog.info("Copying APKs to %s ..." % storage.fs_storage.store_root_dir) def import_apks(apk_paths): apk_importer = ApkImporter(apk_paths, storage) for apk in apk_importer.import_apks(copy_apk = copy_apk, copy_to_mongodb = copy_to_mongodb, update = update, tag = tag): clilog.info("imported %s", apk.short_description()) # use shared memory counter if given if cnt_imported_apks is not None: with cnt_imported_apks.get_lock(): cnt_imported_apks.value += 1 pool = [] # don't convert generator to list if only 1 process wanted apk_paths = [apk_paths] if concurrency == 1 else Util.split_n_uniform_distri(list(apk_paths), concurrency) # start parallel import # multiprocessing's pool causes pickle errors for i in range(concurrency): p = Process(target = import_apks, args = (apk_paths[i], )) log.debug("starting process %s", p) pool.append(p) p.start() for it in pool: log.debug("joined on process %s", p) it.join() apks_imported = cnt_imported_apks.value != 0 # show some message that no APK has been imported if not apks_imported: log.warn("No .apk file has been imported! This means no .apk file has been found or they already have been imported.") else: clilog.info("done") # because not all apks may be importable, we cannot use we count for signal that the import is done if import_finished is not None: import_finished.value = 1 clilog.info("Imported %d apks", cnt_imported_apks.value)
class AuthorizationCodeTestCase(unittest.TestCase): def setUp(self): self.client = None self.provider = None def test_request_access_token(self): def run_provider(queue): try: redirect_uri = "http://127.0.0.1:15487/callback" stores = store_factory(client_identifier="abc", client_secret="xyz", redirect_uris=[redirect_uri]) provider = Provider( access_token_store=stores["access_token_store"], auth_code_store=stores["auth_code_store"], client_store=stores["client_store"], site_adapter=TestSiteAdapter(), token_generator=Uuid4()) provider.add_grant(AuthorizationCodeGrant(expires_in=120)) provider.add_grant(RefreshToken(expires_in=60)) app = Wsgi(server=provider) httpd = make_server('', 15486, app, handler_class=NoLoggingHandler) queue.put({"result": 0}) httpd.serve_forever() except Exception as e: queue.put({"result": 1, "error_message": str(e)}) def run_client(queue): try: app = ClientApplication( callback_url="http://127.0.0.1:15487/callback", client_id="abc", client_secret="xyz", provider_url="http://127.0.0.1:15486") httpd = make_server('', 15487, app, handler_class=NoLoggingHandler) queue.put({"result": 0}) httpd.serve_forever() except Exception as e: queue.put({"result": 1, "error_message": str(e)}) uuid_regex = "^[a-z0-9]{8}\-[a-z0-9]{4}\-[a-z0-9]{4}\-[a-z0-9]{4}-[a-z0-9]{12}$" ready_queue = Queue() self.provider = Process(target=run_provider, args=(ready_queue, )) self.provider.start() provider_started = ready_queue.get() if provider_started["result"] != 0: raise Exception("Error starting Provider process with message" "'{0}'".format(provider_started["error_message"])) self.client = Process(target=run_client, args=(ready_queue, )) self.client.start() client_started = ready_queue.get() if client_started["result"] != 0: raise Exception("Error starting Client Application process with " "message '{0}'".format( client_started["error_message"])) access_token_result = urlopen("http://127.0.0.1:15487/app").read() access_token_data = json.loads(access_token_result.decode('utf-8')) self.assertEqual(access_token_data["token_type"], "Bearer") self.assertEqual(access_token_data["expires_in"], 120) self.assertRegexpMatches(access_token_data["access_token"], uuid_regex) self.assertRegexpMatches(access_token_data["refresh_token"], uuid_regex) request_data = { "grant_type": "refresh_token", "refresh_token": access_token_data["refresh_token"], "client_id": "abc", "client_secret": "xyz" } refresh_token_result = urlopen("http://127.0.0.1:15486/token", urlencode(request_data).encode('utf-8')) refresh_token_data = json.loads( refresh_token_result.read().decode('utf-8')) self.assertEqual(refresh_token_data["token_type"], "Bearer") self.assertEqual(refresh_token_data["expires_in"], 120) self.assertRegexpMatches(refresh_token_data["access_token"], uuid_regex) def tearDown(self): if self.client is not None: self.client.terminate() self.client.join() if self.provider is not None: self.provider.terminate() self.provider.join()
def start_example_app_process(): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED p = Process(target=start_example_server) p.start() return p
class AuthorizationCodeTestCase(unittest.TestCase): def setUp(self): self.client = None self.provider = None def test_request_access_token(self): def run_provider(queue): try: redirect_uri = "http://127.0.0.1:15487/callback" stores = store_factory(client_identifier="abc", client_secret="xyz", redirect_uris=[redirect_uri]) provider = Provider(access_token_store=stores["access_token_store"], auth_code_store=stores["auth_code_store"], client_store=stores["client_store"], site_adapter=TestSiteAdapter(), token_generator=Uuid4()) provider.add_grant(AuthorizationCodeGrant(expires_in=120)) provider.add_grant(RefreshToken(expires_in=60)) app = Wsgi(server=provider) httpd = make_server('', 15486, app, handler_class=NoLoggingHandler) queue.put({"result": 0}) httpd.serve_forever() except Exception as e: queue.put({"result": 1, "error_message": str(e)}) def run_client(queue): try: app = ClientApplication( callback_url="http://127.0.0.1:15487/callback", client_id="abc", client_secret="xyz", provider_url="http://127.0.0.1:15486") httpd = make_server('', 15487, app, handler_class=NoLoggingHandler) queue.put({"result": 0}) httpd.serve_forever() except Exception as e: queue.put({"result": 1, "error_message": str(e)}) uuid_regex = "^[a-z0-9]{8}\-[a-z0-9]{4}\-[a-z0-9]{4}\-[a-z0-9]{4}-[a-z0-9]{12}$" ready_queue = Queue() self.provider = Process(target=run_provider, args=(ready_queue,)) self.provider.start() provider_started = ready_queue.get() if provider_started["result"] != 0: raise Exception("Error starting Provider process with message" "'{0}'".format(provider_started["error_message"])) self.client = Process(target=run_client, args=(ready_queue,)) self.client.start() client_started = ready_queue.get() if client_started["result"] != 0: raise Exception("Error starting Client Application process with " "message '{0}'" .format(client_started["error_message"])) access_token_result = urlopen("http://127.0.0.1:15487/app").read() access_token_data = json.loads(access_token_result.decode('utf-8')) self.assertEqual(access_token_data["token_type"], "Bearer") self.assertEqual(access_token_data["expires_in"], 120) self.assertRegexpMatches(access_token_data["access_token"], uuid_regex) self.assertRegexpMatches(access_token_data["refresh_token"], uuid_regex) request_data = {"grant_type": "refresh_token", "refresh_token": access_token_data["refresh_token"], "client_id": "abc", "client_secret": "xyz"} refresh_token_result = urlopen( "http://127.0.0.1:15486/token", urlencode(request_data).encode('utf-8') ) refresh_token_data = json.loads(refresh_token_result.read().decode('utf-8')) self.assertEqual(refresh_token_data["token_type"], "Bearer") self.assertEqual(refresh_token_data["expires_in"], 120) self.assertRegexpMatches(refresh_token_data["access_token"], uuid_regex) def tearDown(self): if self.client is not None: self.client.terminate() self.client.join() if self.provider is not None: self.provider.terminate() self.provider.join()