def run(self):
thread_list = []
# 创建获取url地址的子线程
t_url = DummyProcess(target=self.get_url)
thread_list.append(t_url)
# 创建获取html页面的子线程
for i in range(86):
t_html = DummyProcess(target=self.get_html)
thread_list.append(t_html)
# 创建获取数据的子线程
for i in range(86):
t_parse = DummyProcess(target=self.get_data)
thread_list.append(t_parse)
# 创建保存数据的子线程
t_save = DummyProcess(target=self.save_data)
thread_list.append(t_save)
# 将所有子进程启动
for t in thread_list:
# 让主线程结束之后子线程页结束
t.setDaemon(True)
t.start()
# 让主进程等待子进程结束在结束
for q in [self.url_queue, self.html_queue, self.get_data_queue]:
q.join()
print('\nCrawl Over...')
print("Crawl Time:", time.clock())
def outer():
try:
t = DummyProcess(target=start_w3af_core,
args=(self._exception_handler,))
t.start()
except AttributeError:
pass
def spawn(function, fargs, fkwargs):
"""
Execute the function now in an asynchronous process/thread
"""
#logger.debug('Spawning execution of %s' % function)
p = DummyProcess(target=function, args=fargs)
p.start()
return p
def test_multiple_instances_in_different_dummy_processes(self):
"""
Create different w3afCore instances, in different threads.
https://github.com/andresriancho/w3af-module/issues/5
"""
t = DummyProcess(target=start_w3af_core,
args=(self._exception_handler,))
t.start()
t.join()
self.assertEqual(self._exceptions, [])
def main():
out_folder = 'results'
temp_folder = os.path.join(out_folder, 'tmp')
if not os.path.exists(out_folder):
os.makedirs(out_folder)
if os.path.exists(temp_folder):
shutil.rmtree(temp_folder, onerror=remove_readonly)
os.makedirs(temp_folder)
if not os.path.exists(temp_folder):
os.makedirs(temp_folder)
try:
while os.path.isfile('pipefile2'):
time.sleep(10)
with open('pipefile', 'w') as f:
f.write('0')
directory = os.path.join('uploads', 'input.xlsx')
p = DummyProcess(target=run_process, args=(directory, 'по_расписанию'))
p.start()
settings.is_running = True
while settings.is_running:
time.sleep(5)
with open('pipefile', 'w') as f:
f.write(str(calculate_progress()))
except:
print('Произошла ошибка: ', traceback.print_exc())
print('error')
finally:
if os.path.isfile('pipefile'):
os.remove('pipefile')
def test_multiple_instances_in_different_dummy_processes(self):
"""
Create different w3afCore instances, in different threads.
https://github.com/andresriancho/w3af-module/issues/5
"""
t = DummyProcess(target=start_w3af_core,
args=(self._exception_handler, ))
t.start()
t.join()
self.assertEqual(self._exceptions, [])
def start(self):
if not hasattr(self._parent, "_children"):
setattr(self._parent, "_children",
weakref.WeakKeyDictionary())
DummyProcess.start(self)
def outer():
try:
t = DummyProcess(target=start_w3af_core)
t.start()
except AttributeError:
pass
def outer():
t = DummyProcess(target=start_w3af_core,
args=(self._exception_handler, ))
t.start()
t.join()
def outer():
try:
t = DummyProcess(target=start_w3af_core)
t.start()
except AttributeError:
pass
def outer():
t = DummyProcess(target=start_w3af_core,
args=(self._exception_handler,))
t.start()
t.join()
0.4, 0.8, ('natural', 'yes', 'no')), (hand_inp_q, hand_res_q)))
svm_reload_1 = Value('i', 0)
worker_face_p1 = Process(
target=modelPrepare,
args=('faceRecModel',
("../modelzoo/face/haarcascade_frontalface_alt.xml",
"./align/shape_predictor_68_face_landmarks.dat",
"../modelzoo/face/LightenedCNN_B_deploy.prototxt",
"../modelzoo/face/LightenedCNN_B.caffemodel", "eltwise_fc1",
"./profiles/svm_model.bin"),
(face_inp_q, face_res_q, facemdl_continue_evt, svm_reload_1,
"./profiles/svm_model.bin")))
dummycontinue = True
worker_handres_mailman = DummyProcess(target=resMailMan,
args=(hand_res_q, 'hand_res'))
worker_faceres_mailman = DummyProcess(target=resMailMan,
args=(face_res_q, 'face_res'))
worker_pref_writeman = DummyProcess(target=savePref,
args=(pref_wrt_q, 'pref_wrt'))
worker_svm_trainer = DummyProcess(target=updateFaceClassifier, args=(10, ))
# worker_preprocess_p.daemon = True
worker_hand_p1.daemon = True
worker_face_p1.daemon = True
worker_handres_mailman.daemon = True
worker_faceres_mailman.daemon = True
worker_pref_writeman.daemon = True
worker_svm_trainer.daemon = True
# worker_preprocess_p.start()
def make():
global p
if request.form.get('stop_button'):
settings.is_terminating = True
return redirect('/')
folder = os.path.join(current_app.root_path, app.config['UPLOAD_FOLDER'])
if request.form.get('schedule_button'):
f = request.files['schedule_file']
if not f:
return redirect('/')
filename = 'input.xlsx'
filepath = os.path.join(folder, filename)
if os.path.isfile(filepath):
os.remove(filepath)
f.save(filepath)
return redirect('/')
if request.form.get('report_button'):
start_date = request.form.get('start_date')
end_date = request.form.get('end_date')
if start_date and end_date:
run_process(None, None, 'report', start_date, end_date)
return redirect('/')
if not os.path.exists(folder):
os.makedirs(folder)
out_folder = os.path.join(current_app.root_path, 'results')
temp_folder = os.path.join(out_folder, 'tmp')
if not os.path.exists(out_folder):
os.makedirs(out_folder)
if os.path.exists(temp_folder):
shutil.rmtree(temp_folder, onerror=remove_readonly)
os.makedirs(temp_folder)
if not os.path.exists(temp_folder):
os.makedirs(temp_folder)
f = request.files['file']
if not f:
return redirect('/')
filename = f.filename
f.save(os.path.join(folder, filename))
if p and p.is_alive():
return redirect('/')
xlsx_name = os.path.join(
os.path.join(current_app.root_path, app.config['UPLOAD_FOLDER']),
filename)
if request.form.get('add_checkbox'):
DummyProcess(target=run_process,
args=(xlsx_name, filename, 'add')).start()
if request.form.get('parse_checkbox'):
p = DummyProcess(target=run_process, args=(xlsx_name, filename))
p.start()
return redirect('/')
