def test_pause_stop(self): """ Verify that the pause method actually works. In this case, working means that the process doesn't send any more HTTP requests after we, pause and that stop works when paused. """ core_start = Process(target=self.w3afcore.start, name="TestRunner") core_start.daemon = True core_start.start() # Let the core start, and the count plugin send some requests. time.sleep(5) count_before_pause = self.count_plugin.count self.assertGreater(self.count_plugin.count, 0) # Pause and measure self.w3afcore.pause(True) count_after_pause = self.count_plugin.count time.sleep(2) count_after_sleep = self.count_plugin.count all_equal = count_before_pause == count_after_pause == count_after_sleep self.assertTrue(all_equal) # Unpause and verify that all requests were sent self.w3afcore.stop() core_start.join() # No more requests sent after pause self.assertEqual(self.count_plugin.count, count_after_sleep)
def test_pause_unpause(self): """ Verify that the pause method actually works. In this case, working means that the process doesn't send any more HTTP requests, fact that is verified with the "fake" count plugin. """ core_start = Process(target=self.w3afcore.start, name='TestRunner') core_start.daemon = True core_start.start() # Let the core start, and the count plugin send some requests. time.sleep(5) count_before_pause = self.count_plugin.count self.assertGreater(self.count_plugin.count, 0) # Pause and measure self.w3afcore.pause(True) count_after_pause = self.count_plugin.count time.sleep(2) count_after_sleep = self.count_plugin.count all_equal = count_before_pause == count_after_pause == count_after_sleep self.assertTrue(all_equal) # Unpause and verify that all requests were sent self.w3afcore.pause(False) core_start.join() self.assertEqual(self.count_plugin.count, self.count_plugin.loops)
def main(): snesorDict = {'imu': 'LSM6DS3TR-C'} readObj = ReadData(snesorDict) # outputDataSigma = multiprocessing.Array('f', [0] * len(snesorDict) * 24) outputDataSigma = None magBg = multiprocessing.Array('f', [0] * 6) outputData = multiprocessing.Array('f', [0] * len(snesorDict) * 24) state = multiprocessing.Array('f', [0, 0, 0, 1, 0, 0, 0]) # Wait a second to let the port initialize # readObj.send() # receive data in a new process pRec = Process(target=readObj.receive, args=(outputData, magBg, outputDataSigma)) pRec.daemon = True pRec.start() pTrack3D = multiprocessing.Process(target=track3D, args=(state, )) pTrack3D.daemon = True pTrack3D.start() mp = MahonyPredictor(q=state[3:], Kp=100, Ki=0.01, dt=0.002) while True: # print("a={}, w={}".format(np.round(outputData[:3], 2), np.round(outputData[3:6], 2))) mp.getGyroOffset(outputData[3:6]) mp.IMUupdate(outputData[:3], outputData[3:6]) state[3:] = mp.q time.sleep(0.08)
def test_pause_unpause(self): output = Queue.Queue() self.uri_opener.pause(True) def send(uri_opener, output): url = URL(get_moth_http()) try: http_response = uri_opener.GET(url) output.put(http_response) except: output.put(None) th = Process(target=send, args=(self.uri_opener, output)) th.daemon = True th.start() self.assertRaises(Queue.Empty, output.get, True, 2) self.uri_opener.pause(False) http_response = output.get() self.assertNotIsInstance(http_response, types.NoneType, 'Error in send thread.') th.join() self.assertEqual(http_response.get_code(), 200) self.assertIn(self.MOTH_MESSAGE, http_response.body)
def test_pause_stop(self): ''' Verify that the pause method actually works. In this case, working means that the process doesn't send any more HTTP requests after we, pause and that stop works when paused. ''' core_start = Process(target=self.w3afcore.start, name='TestRunner') core_start.daemon = True core_start.start() # Let the core start, and the count plugin send some requests. time.sleep(5) count_before_pause = self.count_plugin.count self.assertGreater(self.count_plugin.count, 0) # Pause and measure self.w3afcore.pause(True) count_after_pause = self.count_plugin.count time.sleep(2) count_after_sleep = self.count_plugin.count all_equal = count_before_pause == count_after_pause == count_after_sleep self.assertTrue(all_equal) # Unpause and verify that all requests were sent self.w3afcore.stop() core_start.join() # No more requests sent after pause self.assertEqual(self.count_plugin.count, count_after_sleep)
def test_stop(self): """ Verify that the stop method actually works. In this case, working means that the process doesn't send any more HTTP requests after we stop(). This test seems to be failing @ CircleCI because of a test dependency issue. If run alone in your workstation it will PASS, but if run at CircleCI the count plugin doesn't seem to start. """ core_start = Process(target=self.w3afcore.start, name='TestRunner') core_start.daemon = True core_start.start() # Let the core start, and the count plugin send some requests. time.sleep(5) count_before_stop = self.count_plugin.count self.assertGreater(count_before_stop, 0) # Stop now, self.w3afcore.stop() core_start.join() count_after_stop = self.count_plugin.count self.assertEqual(count_after_stop, count_before_stop)
def test_pause_unpause(self): """ Verify that the pause method actually works. In this case, working means that the process doesn't send any more HTTP requests, fact that is verified with the "fake" count plugin. """ core_start = Process(target=self.w3afcore.start, name='TestRunner') core_start.daemon = True core_start.start() # Let the core start, and the count plugin send some requests. time.sleep(5) count_before_pause = self.count_plugin.count self.assertGreater(self.count_plugin.count, 0) # Pause and measure self.w3afcore.pause(True) count_after_pause = self.count_plugin.count time.sleep(2) count_after_sleep = self.count_plugin.count all_equal = count_before_pause == count_after_pause == count_after_sleep self.assertTrue(all_equal) # Unpause and verify that all requests were sent self.w3afcore.pause(False) core_start.join() self.assertEqual(self.count_plugin.count, self.count_plugin.loops)
def new_process(cls, *args, **kwargs): process = Process(target=BaseCoroutine.start_coroutine, args=(cls, ) + args, kwargs=kwargs) process.daemon = True process.start() return process
def test_pause_unpause(self): output = Queue.Queue() self.uri_opener.pause(True) def send(uri_opener, output): url = URL(get_moth_http()) try: http_response = uri_opener.GET(url) output.put(http_response) except: output.put(None) th = Process(target=send, args=(self.uri_opener, output)) th.daemon = True th.start() self.assertRaises(Queue.Empty, output.get, True, 2) self.uri_opener.pause(False) http_response = output.get() self.assertNotIsInstance(http_response, types.NoneType, 'Error in send thread.') th.join() self.assertEqual(http_response.get_code(), 200) self.assertIn(self.MOTH_MESSAGE, http_response.body)
def test_stop(self): """ Verify that the stop method actually works. In this case, working means that the process doesn't send any more HTTP requests after we stop(). This test seems to be failing @ CircleCI because of a test dependency issue. If run alone in your workstation it will PASS, but if run at CircleCI the count plugin doesn't seem to start. """ core_start = Process(target=self.w3afcore.start, name='TestRunner') core_start.daemon = True core_start.start() # Let the core start, and the count plugin send some requests. time.sleep(5) count_before_stop = self.count_plugin.count self.assertGreater(count_before_stop, 0) # Stop now, self.w3afcore.stop() core_start.join() count_after_stop = self.count_plugin.count self.assertEqual(count_after_stop, count_before_stop)
def process_request(self, request, client_address): """ Start a new thread to process the request. Override here """ t = Process(target=self.process_request_thread, args=(request, client_address)) t.daemon = self.daemon_threads t.start()
def new_send_email(email): if not IS_ACTIVE: new_send_email.queue = Manager().Queue() process = Process(target=process_sent_queue, args=(new_send_email.queue, )) process.daemon = True process.start() models.IS_ACTIVE = True new_send_email.queue.put(email)
def process_request(self, request, client_address): """ Start a new thread to process the request. Override here """ t = Process(target=self.process_request_thread, args=(request, client_address)) t.daemon = self.daemon_threads t.start()
def main(): snesorDict = {'imu': 'LSM6DS3TR-C'} readObj = ReadData(snesorDict) outputDataSigma = None magBg = multiprocessing.Array('f', [0] * 6) outputData = multiprocessing.Array('f', [0] * len(snesorDict) * 24) state = multiprocessing.Array('f', [0, 0, 0, 1, 0, 0, 0]) # Wait a second to let the port initialize # readObj.send() # receive data in a new process pRec = Process(target=readObj.receive, args=(outputData, magBg, outputDataSigma)) pRec.daemon = True pRec.start() time.sleep(0.5) pTrack3D = multiprocessing.Process(target=track3D, args=(state, )) pTrack3D.daemon = True pTrack3D.start() i = 0 bw = np.zeros(3) qEKF = QEKF() while True: for j in range(4): # print("w={}".format(np.round(outputData[3+6*j:6*(j+1)], 2))) if i < 100: bw += outputData[3 + 6 * j:6 * (j + 1)] i += 1 if i == 100: bw /= i qEKF.bw = bw print("get gyroscope bias:{}deg/s".format(bw)) else: w = outputData[3 + 6 * j:6 * (j + 1)] wb = w - bw qEKF.F = qEKF.Fx(qEKF.dt, wb) print('time={:.4f}: wb={}, q={}'.format( time.time(), np.round(qEKF.wb, 2), np.round(qEKF.x, 3))) qEKF.predict() qNorm = np.linalg.norm(qEKF.x) qEKF.x = qEKF.x / qNorm state[3:7] = qEKF.x[:] aNorm = np.linalg.norm(outputData[6 * j:6 * j + 3]) qEKF.z = np.array(outputData[6 * j:6 * j + 3]) / aNorm qEKF.update(qEKF.z, HJacobian, Hx, qEKF.R) qNorm = np.linalg.norm(qEKF.x) qEKF.x = qEKF.x / qNorm state[3:7] = qEKF.x[:] time.sleep(0.037)
def test_pause(self): output = Queue.Queue() self.uri_opener.pause(True) def send(uri_opener, output): url = URL('http://moth/') http_response = uri_opener.GET(url) output.put(http_response) th = Process(target=send, args=(self.uri_opener, output)) th.daemon = True th.start() self.assertRaises(Queue.Empty, output.get, True, 2)
def test_pause(self): output = Queue.Queue() self.uri_opener.pause(True) def send(uri_opener, output): url = URL('http://moth/') http_response = uri_opener.GET(url) output.put(http_response) th = Process(target=send, args=(self.uri_opener, output)) th.daemon = True th.start() self.assertRaises(Queue.Empty, output.get, True, 2)
def __init__(self): with socket.socket() as tcp_socket: # 防止端口占用 tcp_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # 绑定端口 tcp_socket.bind(('', 8080)) # 监听 tcp_socket.listen() # 等待客户端连接 while True: self.client_socket, self.client_addr = tcp_socket.accept() t = Process(target=self.handle) t.daemon = True t.run()
def scan_stop(scan_id): """ Stop a scan :param scan_id: The scan ID to stop :return: Empty result if success, 403 if the current state indicates that the scan can't be stopped. """ scan_info = get_scan_info_from_id(scan_id) if scan_info is None: abort(404, 'Scan not found') if not scan_info.w3af_core.can_stop(): abort(403, 'Scan can not be stop') t = Process(target=scan_info.w3af_core.stop, name='ScanStopThread', args=()) t.daemon = True t.start() return jsonify({'message': 'Stopping scan'})
def scan_stop(scan_id): """ Stop a scan :param scan_id: The scan ID to stop :return: Empty result if success, 403 if the current state indicates that the scan can't be stopped. """ scan_info = get_scan_info_from_id(scan_id) if scan_info is None: abort(404, 'Scan not found') if not scan_info.w3af_core.can_stop(): abort(403, 'Scan can not be stop') t = Process(target=scan_info.w3af_core.stop, name='ScanStopThread', args=()) t.daemon = True t.start() return jsonify({'message': 'Stopping scan'})
def test_stop(self): """ Verify that the stop method actually works. In this case, working means that the process doesn't send any more HTTP requests after we stop(). """ core_start = Process(target=self.w3afcore.start, name="TestRunner") core_start.daemon = True core_start.start() # Let the core start, and the count plugin send some requests. time.sleep(5) count_before_stop = self.count_plugin.count self.assertGreater(count_before_stop, 0) # Stop now, self.w3afcore.stop() core_start.join() count_after_stop = self.count_plugin.count self.assertEqual(count_after_stop, count_before_stop)
def test_pause_unpause(self): output = Queue.Queue() self.uri_opener.pause(True) def send(uri_opener, output): url = URL('http://moth/') http_response = uri_opener.GET(url) output.put(http_response) th = Process(target=send, args=(self.uri_opener, output)) th.daemon = True th.start() self.assertRaises(Queue.Empty, output.get, True, 2) self.uri_opener.pause(False) http_response = output.get() th.join() self.assertEqual(http_response.get_code(), 200) self.assertIn(self.MOTH_MESSAGE, http_response.body)
def test_stop(self): ''' Verify that the stop method actually works. In this case, working means that the process doesn't send any more HTTP requests after we stop(). ''' core_start = Process(target=self.w3afcore.start, name='TestRunner') core_start.daemon = True core_start.start() # Let the core start, and the count plugin send some requests. time.sleep(5) count_before_stop = self.count_plugin.count self.assertGreater(count_before_stop, 0) # Stop now, self.w3afcore.stop() core_start.join() count_after_stop = self.count_plugin.count self.assertEqual(count_after_stop, count_before_stop)
def MouseCallback(event, x, y, flags, param): # 四个点的顺序一定要按照左上,右上,右下,左下的顺时针顺序点 if event == cv.EVENT_LBUTTONDBLCLK: clickTime = param.clickTime cv.circle(param.drawnImage, (x, y), 8, (0, 0, 255), -1) param.dots.append((x, y)) cv.imshow(mainWinName, param.drawnImage) if clickTime + 1 == 4: global windowCount p = Process(target=WarpImage, args=(param.sourceImage, param.dots.copy(), windowCount)) p.daemon = True p.start() param.drawnImage = param.sourceImage.copy() cv.imshow(mainWinName, param.sourceImage) param.dots.clear() windowCount += 1 param.clickTime = (clickTime + 1) % 4
def test_pause_unpause(self): output = Queue.Queue() self.uri_opener.pause(True) def send(uri_opener, output): url = URL('http://moth/') http_response = uri_opener.GET(url) output.put(http_response) th = Process(target=send, args=(self.uri_opener, output)) th.daemon = True th.start() self.assertRaises(Queue.Empty, output.get, True, 2) self.uri_opener.pause(False) http_response = output.get() th.join() self.assertEqual(http_response.get_code(), 200) self.assertIn(self.MOTH_MESSAGE, http_response.body)
def test_pause_stop(self): """ Verify that the pause method actually works. In this case, working means that the process doesn't send any more HTTP requests after we, pause and that stop works when paused. This test seems to be failing @ CircleCI because of a test dependency issue. If run alone in your workstation it will PASS, but if run at CircleCI the count plugin doesn't seem to start. """ core_start = Process(target=self.w3afcore.start, name='TestRunner') core_start.daemon = True core_start.start() # Let the core start, and the count plugin send some requests. time.sleep(5) count_before_pause = self.count_plugin.count self.assertGreater(self.count_plugin.count, 0) # Pause and measure self.w3afcore.pause(True) count_after_pause = self.count_plugin.count time.sleep(2) count_after_sleep = self.count_plugin.count all_equal = count_before_pause == count_after_pause == count_after_sleep self.assertTrue(all_equal) # Unpause and verify that all requests were sent self.w3afcore.stop() core_start.join() # No more requests sent after pause self.assertEqual(self.count_plugin.count, count_after_sleep)
def test_pause_stop(self): """ Verify that the pause method actually works. In this case, working means that the process doesn't send any more HTTP requests after we, pause and that stop works when paused. This test seems to be failing @ CircleCI because of a test dependency issue. If run alone in your workstation it will PASS, but if run at CircleCI the count plugin doesn't seem to start. """ core_start = Process(target=self.w3afcore.start, name='TestRunner') core_start.daemon = True core_start.start() # Let the core start, and the count plugin send some requests. time.sleep(5) count_before_pause = self.count_plugin.count self.assertGreater(self.count_plugin.count, 0) # Pause and measure self.w3afcore.pause(True) count_after_pause = self.count_plugin.count time.sleep(2) count_after_sleep = self.count_plugin.count all_equal = count_before_pause == count_after_pause == count_after_sleep self.assertTrue(all_equal) # Unpause and verify that all requests were sent self.w3afcore.stop() core_start.join() # No more requests sent after pause self.assertEqual(self.count_plugin.count, count_after_sleep)
Windows only. Checking if process kodi is running. Initial exit if isn't. """ print('Exit daemon started') while True: kodi_process = False time.sleep(5) for process in list(psutil.process_iter()): if re.search('kodi', process.name().lower()): kodi_process = True break if kodi_process: continue else: break print('Kodi process not found') print('EXITING...') os._exit(1) if __name__ == '__main__': check_exit = Process(target=exit) check_exit.daemon = True check_exit.start() run()
def run(self, func=None, workhorse=True): if self.kwargs.get('socket'): try: from emross.utility.socket import establish_connection self.socket = establish_connection() socket_handler = JsonSocketHandler(self.socket, self.bots) socket_handler.daemon = True socket_handler.start() except Exception as e: logger.exception(e) self.initialise_bots( socket_writer=getattr(self.socket, 'queue_out', None), settings=self.settings ) workers = [] for bot in self.bots: bot.session.start_time = time.time() if func: logger.info('Starting new bot thread for {0}'.format(bot.api.player)) worker = threading.Thread(target=func, args=(bot,)) worker.bot = bot worker.daemon = True worker.start() else: logger.debug('No need to use a main thread for this worker!') worker = EmrossBaseObject(bot) workers.append(worker) """ Now we can start the main event loop. If we are running a console then the `code.interact` will block so we need to spawn a thread to process each bot's error queue. """ error_thread = Process(target=_error_checker, args=(workers, self.bots)) error_thread.daemon = True error_thread.start() processes = self.kwargs.get('processes') or DEFAULT_POOL_SIZE queue = Queue.Queue(processes) for i in range(processes): t = Process(target=_bot_consumer, args=(queue,)) t.daemon = True t.start() if self.console: worker = Process(target=_bot_runner, args=(queue, self.bots), kwargs=self.kwargs) worker.daemon = True worker.start() import emross.utility.settings sandbox = dict( manager=self, settings=emross.utility.settings, bot=self.bot ) code.interact(banner='EmrossWar Bot Management console', local=sandbox) raise KeyboardInterrupt elif workhorse: # We can run this directly in this thread _bot_runner(queue, self.bots, **self.kwargs)
def start_scan(): """ Starts a new w3af scan Receive a JSON containing: - A list with the target URLs - The profile (eg. the content of fast_scan.pw3af) :return: A JSON containing: - The URL to the newly created scan (eg. /scans/1) - The newly created scan ID (eg. 1) """ if not request.json or 'scan_profile' not in request.json: abort(400, 'Expected scan_profile in JSON object') if 'target_urls' not in request.json: abort(400, 'Expected target_urls in JSON object') scan_profile = request.json['scan_profile'] target_urls = request.json['target_urls'] # # First make sure that there are no other scans running, remember that this # REST API is an MVP and we can only run one scan at the time (for now) # scan_infos = SCANS.values() if not all([si is None for si in scan_infos]): abort( 400, 'This version of the REST API does not support' ' concurrent scans. Remember to DELETE finished scans' ' before starting a new one.') # # Before trying to start a new scan we verify that the scan profile is # valid and return an informative error if it's not # # scan_profile_file_name, profile_path = create_temp_profile(scan_profile) # w3af_core = w3afCore() # # try: # w3af_core.profiles.use_profile(scan_profile_file_name, # workdir=profile_path) # except BaseFrameworkException, bfe: # abort(400, str(bfe)) # # Now that we know that the profile is valid I verify the scan target info # if not len(target_urls): abort(400, 'No target URLs specified') for target_url in target_urls: try: URL(target_url) except ValueError: abort(400, 'Invalid URL: "%s"' % target_url) # target_options = w3af_core.target.get_options() # target_option = target_options['target'] # try: # target_option.set_value([URL(u) for u in target_urls]) # w3af_core.target.set_options(target_options) # except BaseFrameworkException, bfe: # abort(400, str(bfe)) # # Finally, start the scan in a different thread # scan_id = get_new_scan_id() scan_info_setup = Event() args = (target_urls, scan_profile, scan_info_setup) t = Process(target=start_scan_helper, name='ScanThread', args=args) t.daemon = True t.start() # Wait until the thread starts scan_info_setup.wait() return jsonify({ 'message': 'Success', 'id': scan_id, 'href': '/scans/%s' % scan_id }), 201
if __name__ == '__main__': # 不能同时显示IMU和magSensor的数值 # snesorDict = {'imu': 'LSM6DS3TR-C', 'magSensor': 'AK09970d'} snesorDict = {'magSensor': 'AK09970d'} # snesorDict = {'imu': 'LSM6DS3TR-C'} readObj = ReadData(snesorDict) # 创建读取数据的对象 # def data struct outputData = multiprocessing.Array('f', [0] * len(snesorDict) * 24) magBg = multiprocessing.Array('f', [0] * 6) # 平滑曲线只能在singleCurve模式下绘图,且只有magSensor有 outputDataSmooth = multiprocessing.Array('f', [0] * len(snesorDict) * 24) # outputDataSmooth = None # 标准差曲线只能在multiCurve模式下绘图 # outputDataSigma = multiprocessing.Array('f', [0] * len(snesorDict) * 24) outputDataSigma = None # Wait a second to let the port initialize # readObj.send() # receive data in a new process pRec = Process(target=readObj.receive, args=(outputData, outputDataSmooth, magBg, outputDataSigma)) pRec.daemon = True pRec.start() time.sleep(1) # plot sensor data plotSensor(snesorDict, outputData, outputDataSmooth, outputDataSigma)
def process_dump( # noqa: C901 input_file, template_file, out_file, file_size, file_compress, process_count): # pylint: disable=R0912,R0914,R0915 """ :param input_file: name of the wikipedia dump file; '-' to read from stdin :param template_file: optional file with template definitions. :param out_file: directory where to store extracted data, or '-' for stdout :param file_size: max size of each extracted file, or None for no max (one file) :param file_compress: whether to compress files with bzip. :param process_count: number of extraction processes to spawn. """ global urlbase # global knownNamespaces global templateNamespace, templatePrefix global moduleNamespace, modulePrefix if input_file == "-": input_source = sys.stdin else: input_source = fileinput.FileInput(input_file, openhook=fileinput.hook_compressed) # collect siteinfo for line in input_source: line = line.decode("utf-8") m = TAG_RE.search(line) if not m: continue tag = m.group(2) if tag == "base": # discover urlbase from the xml dump file # /mediawiki/siteinfo/base base = m.group(3) urlbase = base[:base.rfind("/")] elif tag == "namespace": known_namespaces.add(m.group(3)) if re.search('key="10"', line): templateNamespace = m.group(3) templatePrefix = templateNamespace + ":" elif re.search('key="828"', line): moduleNamespace = m.group(3) modulePrefix = moduleNamespace + ":" elif tag == "/siteinfo": break if expand_templates: # preprocess template_load_start = default_timer() if template_file and os.path.exists(template_file): logger.info( "Preprocessing '%s' to collect template definitions: this may take some time.", template_file) file_input = fileinput.FileInput( template_file, openhook=fileinput.hook_compressed) load_templates(file_input) file_input.close() else: if input_file == "-": # can't scan then reset stdin; must error w/ suggestion to specify template_file raise ValueError( "to use templates with stdin dump, must supply explicit template-file" ) logger.info( "Preprocessing '%s' to collect template definitions: this may take some time.", input_file) load_templates(input_source, template_file) input_source.close() input_source = fileinput.FileInput( input_file, openhook=fileinput.hook_compressed) template_load_elapsed = default_timer() - template_load_start # FIXME: get rid of this horrible global variable logger.info( "Loaded %d templates in %.1fs", len(templates), template_load_elapsed # noqa: F821 # pylint: disable=E0602 ) if out_file == "-": output = sys.stdout if file_compress: logger.warning( "writing to stdout, so no output compression (use an external tool)" ) else: next_file = NextFile(out_file) output = OutputSplitter(next_file, file_size, file_compress) # process pages logger.info("Starting page extraction from %s.", input_file) extract_start = default_timer() # Parallel Map/Reduce: # - pages to be processed are dispatched to workers # - a reduce process collects the results, sort them and print them. maxsize = 10 * process_count # output queue output_queue = Queue(maxsize=maxsize) # Reduce job that sorts and prints output reduce_job = Process(target=reduce_process, args=(output_queue, output)) reduce_job.start() # initialize jobs queue jobs_queue = Queue(maxsize=maxsize) # start worker processes logger.info("Using %d extract processes.", process_count) workers = [] for _ in range(max(1, process_count)): extractor = Process(target=extract_process, args=(jobs_queue, output_queue)) extractor.daemon = True # only live while parent process lives extractor.start() workers.append(extractor) # Mapper process # we collect individual lines, since str.join() is significantly faster # than concatenation page = [] article_id = None last_id = None ordinal = 0 # page count in_text = False redirect = False for line in input_source: line = line.decode("utf-8") if "<" not in line: # faster than doing re.search() if in_text: page.append(line) continue m = TAG_RE.search(line) if not m: continue tag = m.group(2) if tag == "page": page = [] redirect = False elif tag == "id" and not article_id: article_id = m.group(3) elif tag == "title": title = m.group(3) elif tag == "redirect": redirect = True elif tag == "text": in_text = True line = line[m.start(3):m.end(3)] page.append(line) if m.lastindex == 4: # open-close in_text = False elif tag == "/text": if m.group(1): page.append(m.group(1)) in_text = False elif in_text: page.append(line) elif tag == "/page": colon = title.find(":") if ((colon < 0 or title[:colon] in acceptedNamespaces) and article_id != last_id and not redirect and not title.startswith(templateNamespace)): job = (article_id, title, page, ordinal) jobs_queue.put(job) # goes to any available extract_process last_id = article_id ordinal += 1 article_id = None page = [] input_source.close() # signal termination for _ in workers: jobs_queue.put(None) # wait for workers to terminate for w in workers: w.join() # signal end of work to reduce_job process output_queue.put(None) # wait for it to finish reduce_job.join() if output != sys.stdout: output.close() extract_duration = default_timer() - extract_start extract_rate = ordinal / extract_duration logger.info( "Finished %d-process extraction of %d articles in %.1fs (%.1f art/s)", process_count, ordinal, extract_duration, extract_rate, )
(face_inp_q, face_res_q, facemdl_continue_evt, svm_reload_1, "./profiles/svm_model.bin"))) dummycontinue = True worker_handres_mailman = DummyProcess(target=resMailMan, args=(hand_res_q, 'hand_res')) worker_faceres_mailman = DummyProcess(target=resMailMan, args=(face_res_q, 'face_res')) worker_pref_writeman = DummyProcess(target=savePref, args=(pref_wrt_q, 'pref_wrt')) worker_svm_trainer = DummyProcess(target=updateFaceClassifier, args=(10, )) # worker_preprocess_p.daemon = True worker_hand_p1.daemon = True worker_face_p1.daemon = True worker_handres_mailman.daemon = True worker_faceres_mailman.daemon = True worker_pref_writeman.daemon = True worker_svm_trainer.daemon = True # worker_preprocess_p.start() worker_hand_p1.start() worker_face_p1.start() worker_handres_mailman.start() worker_faceres_mailman.start() worker_pref_writeman.start() worker_svm_trainer.start() with open('./profiles/profiles.pkl', 'rb') as pref_fd: try: pref_db = pkl.load(pref_fd)
abort(400, str(bfe)) scan_id = get_new_scan_id() scan_info = ScanInfo() scan_info.w3af_core = w3af_core scan_info.target_urls = target_urls scan_info.profile_path = scan_profile_file_name scan_info.output = RESTAPIOutput() SCANS[scan_id] = scan_info # # Finally, start the scan in a different thread # args = (scan_info,) t = Process(target=start_scan_helper, name='ScanThread', args=args) t.daemon = True t.start() return jsonify({'message': 'Success', 'id': scan_id, 'href': '/scans/%s' % scan_id}), 201 @app.route('/scans/', methods=['GET']) @requires_auth def list_scans(): """ :return: A JSON containing a list of: - Scan resource URL (eg. /scans/1) - Scan target
def start_scan(): """ Starts a new w3af scan Receive a JSON containing: - A list with the target URLs - The profile (eg. the content of fast_scan.pw3af) :return: A JSON containing: - The URL to the newly created scan (eg. /scans/1) - The newly created scan ID (eg. 1) """ if not request.json or 'scan_profile' not in request.json: abort(400, 'Expected scan_profile in JSON object') if 'target_urls' not in request.json: abort(400, 'Expected target_urls in JSON object') scan_profile = request.json['scan_profile'] target_urls = request.json['target_urls'] # # First make sure that there are no other scans running, remember that this # REST API is an MVP and we can only run one scan at the time (for now) # scan_infos = SCANS.values() if not all([si is None for si in scan_infos]): abort(400, 'This version of the REST API does not support' ' concurrent scans. Remember to DELETE finished scans' ' before starting a new one.') # # Before trying to start a new scan we verify that the scan profile is # valid and return an informative error if it's not # # scan_profile_file_name, profile_path = create_temp_profile(scan_profile) # w3af_core = w3afCore() # # try: # w3af_core.profiles.use_profile(scan_profile_file_name, # workdir=profile_path) # except BaseFrameworkException, bfe: # abort(400, str(bfe)) # # Now that we know that the profile is valid I verify the scan target info # if not len(target_urls): abort(400, 'No target URLs specified') for target_url in target_urls: try: URL(target_url) except ValueError: abort(400, 'Invalid URL: "%s"' % target_url) # target_options = w3af_core.target.get_options() # target_option = target_options['target'] # try: # target_option.set_value([URL(u) for u in target_urls]) # w3af_core.target.set_options(target_options) # except BaseFrameworkException, bfe: # abort(400, str(bfe)) # # Finally, start the scan in a different thread # scan_id = get_new_scan_id() scan_info_setup = Event() args = (target_urls, scan_profile, scan_info_setup) t = Process(target=start_scan_helper, name='ScanThread', args=args) t.daemon = True t.start() # Wait until the thread starts scan_info_setup.wait() return jsonify({'message': 'Success', 'id': scan_id, 'href': '/scans/%s' % scan_id}), 201
def main(): p = Process(target=angel) p.daemon = True p.start() input('press any key to exit ...')
format(process_number, repr(e))) sleep(3) return my_func @error_catching def f(process_number): print("starting worker:", process_number) while True: sleep(2) print("Worker {} checks in.".format(process_number)) if __name__ == '__main__': processes = [] manager = Manager() for i in range(3): p = Process(target=f) p.daemon = True p.start() processes.append(p) try: while True: sleep(1) except KeyboardInterrupt: print("Keyboard interrupt in main") sys.exit()
("../modelzoo/hand/test.prototxt", "../modelzoo/hand/vgg16_faster_rcnn_handGesdet_aug_fulldata_iter_50000.caffemodel", 0.4, 0.8, ('natural', 'yes', 'no')), (hand_inp_q, hand_res_q))) svm_reload_1 = Value('i', 0) worker_face_p1 = Process(target = modelPrepare, args = ('faceRecModel', ("../modelzoo/face/haarcascade_frontalface_alt.xml", "./align/shape_predictor_68_face_landmarks.dat", "../modelzoo/face/LightenedCNN_B_deploy.prototxt", "../modelzoo/face/LightenedCNN_B.caffemodel", "eltwise_fc1", "./profiles/svm_model.bin"), (face_inp_q, face_res_q, facemdl_continue_evt, svm_reload_1, "./profiles/svm_model.bin"))) dummycontinue = True worker_handres_mailman = DummyProcess(target = resMailMan, args = (hand_res_q, 'hand_res')) worker_faceres_mailman = DummyProcess(target = resMailMan, args = (face_res_q, 'face_res')) worker_pref_writeman = DummyProcess(target = savePref, args = (pref_wrt_q, 'pref_wrt')) worker_svm_trainer = DummyProcess(target = updateFaceClassifier, args = (10,)) # worker_preprocess_p.daemon = True worker_hand_p1.daemon = True worker_face_p1.daemon = True worker_handres_mailman.daemon = True worker_faceres_mailman.daemon = True worker_pref_writeman.daemon = True worker_svm_trainer.daemon = True # worker_preprocess_p.start() worker_hand_p1.start() worker_face_p1.start() worker_handres_mailman.start() worker_faceres_mailman.start() worker_pref_writeman.start() worker_svm_trainer.start() with open('./profiles/profiles.pkl', 'rb') as pref_fd: try: pref_db = pkl.load(pref_fd)