def sju_post(session, url, form_data, max_retries, query = None):
http_res = ''
retries = 0
ui_stream = sju_models.UI_Stream('search')
if query == None:
query = ('None','','Sejong Univ')
while (retries < max_retries):
try:
http_res = session.post(url, form_data, verify=False)
break
except NewConnectionError as nce:
sleep(5)
finally:
retries += 1
try:
if retries >= max_retries:
raise sju_exceptions.NewConnectionError()
except sju_exceptions.NewConnectionError:
ui_stream.push(command='err', msg=sju_CONSTANTS.STATE_MSG[5000])
ui_stream.push(
command='res', target='errQuery',
res={'query': query, 'msg': sju_CONSTANTS.STATE_MSG[5000]}
)
return
# # 요청 초과 시 익셉션 발생
# ConnectionError 정의 안되있으니 정의 따로 하시고, 에러 핸들 하셔야합니다.
# if retries >= max_retries:
# raise new ConnectionError()
return http_res
def __init__(self, cookies=None):
'''
'''
self.res_name = 'mres'
self.threading_amount = 10
self.ui_stream = sju_models.UI_Stream('multi_search', 'multi main',
self.res_name)
containers = {}
ui_stream = self.ui_stream
threading_amount = self.threading_amount
ui_stream.push(command='log', msg=sju_CONSTANTS.STATE_MSG[2101])
with concurrent.futures.ThreadPoolExecutor(
max_workers=threading_amount) as executor:
future_containers = {
executor.submit(SingleSearchContainer,
thread_id,
cookies=cookies): thread_id
for thread_id in range(threading_amount)
}
for future in concurrent.futures.as_completed(future_containers):
thread_id = future_containers[future]
try:
containers[thread_id] = future.result()
ui_stream.push(command='log',
msg='multi sub%d의 %s' %
(thread_id, sju_CONSTANTS.STATE_MSG[2201]))
except sju_exceptions.InitSessionError as ise:
ui_stream.push(command='err',
msg='multi sub%d %s' %
(thread_id, '접근 거부 User-Agent 삭제'))
# del containers[thread_id]
self.threading_amount -= 1
except Exception as e:
ui_stream.push(
command='err',
msg='multi sub%d의 %s' %
(thread_id, sju_CONSTANTS.STATE_MSG[2301][0]))
raise e
ui_stream.push(command='log',
msg='가용 스레드 수 = %d' % self.threading_amount)
if self.threading_amount < 2:
ui_stream.push(command='err',
msg='서버가 거부한 초기화가 너무 많아 검색을 수행할 수 없습니다.')
raise sju_exceptions.InitMultiSessionErr()
self.containers = containers
ui_stream.push(command='log',
msg='%s' % (sju_CONSTANTS.STATE_MSG[2201]))
def __init__(self, thread_id, cookies = None):
'''
'''
self.qid = 0
self.thread_id = thread_id
self.query_package = []
self.container_lock = threading.Lock()
self.session = False
self.res_name = 'mres'
self.base_url = "http://apps.webofknowledge.com"
self.ui_stream = sju_models.UI_Stream('multi_search', 'multi sub%d'%thread_id, self.res_name)
self.single_search = sju_single_search.SingleSearch(thread_id = thread_id, cookies=cookies)
def __init__(self, cookies=None):
'''
DuplSearch constructor
:param cookies: 쿠키 값 저장변수
:return:
'''
self.qid = 0
self.session = False
self.base_url = "http://apps.webofknowledge.com"
self.res_name = 'res'
self.threading_amount = 10
self.ui_stream = sju_models.UI_Stream('dupl_search', 'dupl main',
self.res_name)
self.set_session(cookies)
def __init__(self, thread_id=None, cookies=None):
'''
SingleSearch constructor
:param thread_id: 쓰레드 네임
:Param cookies: 쿠키 값 변수
:param session: 세션 변수
:param base_url: 메인 도메인 URL
:param res_name: push command 변수
:param ui_stream: logging을 위한 함수 호출 후 저장
:return:
'''
self.qid = 0
self.session = False
self.base_url = "http://apps.webofknowledge.com"
self.res_name = 'res' if thread_id == None else 'mres'
self.thread_id = 'single main' if thread_id == None else thread_id
ui_stream_name = 'single_search' if thread_id == None else 'multi sub'
self.ui_stream = sju_models.UI_Stream(ui_stream_name, self.thread_id,
self.res_name)
self.set_session(cookies)
def __init__(self, thread_id = None, cookies = None):
'''
FastSearch constructor
:param thread_id: 쓰레드 네임
:Param cookies: 쿠키 값 변수
:param qid: session 재호출을 위한 카운트변수
:param session: 세션 변수
:param base_url: 메인 도메인 URL
:param res_name: push command 변수
:param ui_stream: logging을 위한 함수 호출 후 저장
:return:
'''
self.qid = 0
self.session = False
self.base_url = "http://apps.webofknowledge.com"
self.res_name = 'fres'
self.thread_id = 'fast main'
ui_stream_name = 'fast_search'
self.ui_stream = sju_models.UI_Stream(ui_stream_name, self.thread_id, self.res_name)
self.set_session(cookies=cookies)
import sju_exceptions as _EXCEP
# std_libs
import re
import sys
import traceback
import threading
import concurrent.futures
from sju_utiles import os
from sju_utiles import datetime
from sju_utiles import requests
if __name__ == "__main__":
# ui 출력 객체 초기화
ui_stream = _models.UI_Stream('dispatcher', 'main', 'res')
# 디스패처 활성화, 로딩 활성화
ui_stream.push(command='res', target='loading', res=True)
ui_stream.push(command='log', msg=_CONS.STATE_MSG[101])
cookies = None
multiSearchObj = None
singleSearchObj = None
authorSearchObj = None
compactSearchObj = None
# 서비스 초기화
while True:
try:
service_list = [
{ 'name': 'singleSearch', 'init': sju_single_search.SingleSearch },