def get_document_list(self):
"""
Firebase로부터 Document Code List를 가져온다
:return: Firebase로부터 가져온 Code List
"""
try:
code_list = []
target_url = self.firestore_api_url + self.database_route + '/documents:runQuery?key=' + self.firebase_web_key
structed_query = {
"structuredQuery": {
"from": [{
"collectionId": 'hiyobi'
}],
}
}
headers = {'Authorization': 'Bearer ' + self.idToken}
response = requests.post(target_url,
headers=headers,
json=structed_query)
for item in response.json():
code_list.append(
str(item['document']['fields']['code']['stringValue']))
return code_list
except requests.exceptions.RequestException:
Logger.LOGGER.error("Firebase Requests Error..")
Dialog.ErrorDialog().open_dialog("Firebase Error",
"Firebase Request Error!")
sys.exit()
except Exception:
Logger.LOGGER.error("Unexpected Error in Firebase Request")
def get_firebase_id_token(self, email, password):
"""
firebase 인증 토큰을 발급받는다
:param email: 인증 Email
:param password: 인증 Password
:return: Firebase ID Token
"""
try:
auth_url = self.auth_url + self.firebase_web_key
auth_data = {
u'email': email,
u'password': password,
u'returnSecureToken': True
}
response = requests.post(
auth_url,
headers={'Content-Type': 'application/json'},
data=json.dumps(auth_data))
token = response.json()['idToken']
return token
except requests.exceptions.RequestException:
Logger.LOGGER.error("Firebase Authorization Error..")
Dialog.ErrorDialog().open_dialog("Authorization Error",
"Firebase Authorization Error!")
sys.exit()
except Exception:
Logger.LOGGER.error("Unexpected Error in Firebase Authorization")
def get_download_list(crawl_page, parent):
"""
Get Download List from Hiyobi
:param crawl_page: 수집할 페이지 범위
:param parent: 진행 상황을 표시할 화면
:return: 수집된 Gallery List
"""
Logger.LOGGER.info("Load exception list from Firebase")
parent.current_state.emit("Load exception list from Firebase")
exception_list = FirebaseClient.fbclient.get_document_list()
# TODO Remove next line on build
# exception_list = []
parent.notifyProgress.emit(100 * 1 / (crawl_page+2))
try:
gallery_list = []
Logger.LOGGER.info("Get cookie data for Cloudflare..")
parent.current_state.emit("Get cookie data for Cloudflare..")
cookie_value, user_agent = cfscrape.get_cookie_string(URL_HIYOBI)
headers = {'User-Agent': user_agent}
cookies = {'session_id': cookie_value}
parent.notifyProgress.emit(100 * 2 / (crawl_page+2))
except Exception:
Logger.LOGGER.error("Unexpected Exception Error..")
Logger.LOGGER.error(traceback.format_exc())
Dialog.ErrorDialog().open_dialog("Unexpected Exception Error", "Unexpected Exception Request Error!")
try:
for i in tqdm(range(1, crawl_page+1)):
# print("[SYSTEM]: Load From Page %d.." % i)
parent.current_state.emit("Load From Page %d.." % i)
soup = BeautifulSoup(requests.get(URL_HIYOBI+'list/'+str(i), cookies=cookies, headers=headers).content, 'html.parser')
galleries = soup.find_all('div', class_="gallery-content")
for data in galleries:
gallery = Gallery.Gallery()
gallery.initialize(data)
if gallery.code not in exception_list:
gallery_list.append(gallery)
parent.notifyProgress.emit(100 * (i+2) / (crawl_page+2))
return gallery_list
except requests.exceptions.RequestException:
Logger.LOGGER.error("Hiyobi Requests Error..")
Dialog.ErrorDialog().open_dialog("Hiyobi Error", "Hiyobi Request Error!")
except Exception:
Logger.LOGGER.error("Unexpected Error in Hiyobi Request")
Logger.LOGGER.error(traceback.format_exc())
Dialog.ErrorDialog().open_dialog("Unexpected Exception Error", "Unexpected Exception Request Error!")