def run(self): thread_list = [] # 创建获取url地址的子线程 t_url = DummyProcess(target=self.get_url) thread_list.append(t_url) # 创建获取html页面的子线程 for i in range(86): t_html = DummyProcess(target=self.get_html) thread_list.append(t_html) # 创建获取数据的子线程 for i in range(86): t_parse = DummyProcess(target=self.get_data) thread_list.append(t_parse) # 创建保存数据的子线程 t_save = DummyProcess(target=self.save_data) thread_list.append(t_save) # 将所有子进程启动 for t in thread_list: # 让主线程结束之后子线程页结束 t.setDaemon(True) t.start() # 让主进程等待子进程结束在结束 for q in [self.url_queue, self.html_queue, self.get_data_queue]: q.join() print('\nCrawl Over...') print("Crawl Time:", time.clock())
def outer(): try: t = DummyProcess(target=start_w3af_core, args=(self._exception_handler,)) t.start() except AttributeError: pass
def spawn(function, fargs, fkwargs): """ Execute the function now in an asynchronous process/thread """ #logger.debug('Spawning execution of %s' % function) p = DummyProcess(target=function, args=fargs) p.start() return p
def test_multiple_instances_in_different_dummy_processes(self): """ Create different w3afCore instances, in different threads. https://github.com/andresriancho/w3af-module/issues/5 """ t = DummyProcess(target=start_w3af_core, args=(self._exception_handler,)) t.start() t.join() self.assertEqual(self._exceptions, [])
def main(): out_folder = 'results' temp_folder = os.path.join(out_folder, 'tmp') if not os.path.exists(out_folder): os.makedirs(out_folder) if os.path.exists(temp_folder): shutil.rmtree(temp_folder, onerror=remove_readonly) os.makedirs(temp_folder) if not os.path.exists(temp_folder): os.makedirs(temp_folder) try: while os.path.isfile('pipefile2'): time.sleep(10) with open('pipefile', 'w') as f: f.write('0') directory = os.path.join('uploads', 'input.xlsx') p = DummyProcess(target=run_process, args=(directory, 'по_расписанию')) p.start() settings.is_running = True while settings.is_running: time.sleep(5) with open('pipefile', 'w') as f: f.write(str(calculate_progress())) except: print('Произошла ошибка: ', traceback.print_exc()) print('error') finally: if os.path.isfile('pipefile'): os.remove('pipefile')
def test_multiple_instances_in_different_dummy_processes(self): """ Create different w3afCore instances, in different threads. https://github.com/andresriancho/w3af-module/issues/5 """ t = DummyProcess(target=start_w3af_core, args=(self._exception_handler, )) t.start() t.join() self.assertEqual(self._exceptions, [])
def start(self): if not hasattr(self._parent, "_children"): setattr(self._parent, "_children", weakref.WeakKeyDictionary()) DummyProcess.start(self)
def outer(): try: t = DummyProcess(target=start_w3af_core) t.start() except AttributeError: pass
def outer(): t = DummyProcess(target=start_w3af_core, args=(self._exception_handler, )) t.start() t.join()
def outer(): t = DummyProcess(target=start_w3af_core, args=(self._exception_handler,)) t.start() t.join()
0.4, 0.8, ('natural', 'yes', 'no')), (hand_inp_q, hand_res_q))) svm_reload_1 = Value('i', 0) worker_face_p1 = Process( target=modelPrepare, args=('faceRecModel', ("../modelzoo/face/haarcascade_frontalface_alt.xml", "./align/shape_predictor_68_face_landmarks.dat", "../modelzoo/face/LightenedCNN_B_deploy.prototxt", "../modelzoo/face/LightenedCNN_B.caffemodel", "eltwise_fc1", "./profiles/svm_model.bin"), (face_inp_q, face_res_q, facemdl_continue_evt, svm_reload_1, "./profiles/svm_model.bin"))) dummycontinue = True worker_handres_mailman = DummyProcess(target=resMailMan, args=(hand_res_q, 'hand_res')) worker_faceres_mailman = DummyProcess(target=resMailMan, args=(face_res_q, 'face_res')) worker_pref_writeman = DummyProcess(target=savePref, args=(pref_wrt_q, 'pref_wrt')) worker_svm_trainer = DummyProcess(target=updateFaceClassifier, args=(10, )) # worker_preprocess_p.daemon = True worker_hand_p1.daemon = True worker_face_p1.daemon = True worker_handres_mailman.daemon = True worker_faceres_mailman.daemon = True worker_pref_writeman.daemon = True worker_svm_trainer.daemon = True # worker_preprocess_p.start()
def make(): global p if request.form.get('stop_button'): settings.is_terminating = True return redirect('/') folder = os.path.join(current_app.root_path, app.config['UPLOAD_FOLDER']) if request.form.get('schedule_button'): f = request.files['schedule_file'] if not f: return redirect('/') filename = 'input.xlsx' filepath = os.path.join(folder, filename) if os.path.isfile(filepath): os.remove(filepath) f.save(filepath) return redirect('/') if request.form.get('report_button'): start_date = request.form.get('start_date') end_date = request.form.get('end_date') if start_date and end_date: run_process(None, None, 'report', start_date, end_date) return redirect('/') if not os.path.exists(folder): os.makedirs(folder) out_folder = os.path.join(current_app.root_path, 'results') temp_folder = os.path.join(out_folder, 'tmp') if not os.path.exists(out_folder): os.makedirs(out_folder) if os.path.exists(temp_folder): shutil.rmtree(temp_folder, onerror=remove_readonly) os.makedirs(temp_folder) if not os.path.exists(temp_folder): os.makedirs(temp_folder) f = request.files['file'] if not f: return redirect('/') filename = f.filename f.save(os.path.join(folder, filename)) if p and p.is_alive(): return redirect('/') xlsx_name = os.path.join( os.path.join(current_app.root_path, app.config['UPLOAD_FOLDER']), filename) if request.form.get('add_checkbox'): DummyProcess(target=run_process, args=(xlsx_name, filename, 'add')).start() if request.form.get('parse_checkbox'): p = DummyProcess(target=run_process, args=(xlsx_name, filename)) p.start() return redirect('/')