Esempio n. 1
0
 def get_document_list(self):
     """
     Firebase로부터 Document Code List를 가져온다
     :return: Firebase로부터 가져온 Code List
     """
     try:
         code_list = []
         target_url = self.firestore_api_url + self.database_route + '/documents:runQuery?key=' + self.firebase_web_key
         structed_query = {
             "structuredQuery": {
                 "from": [{
                     "collectionId": 'hiyobi'
                 }],
             }
         }
         headers = {'Authorization': 'Bearer ' + self.idToken}
         response = requests.post(target_url,
                                  headers=headers,
                                  json=structed_query)
         for item in response.json():
             code_list.append(
                 str(item['document']['fields']['code']['stringValue']))
         return code_list
     except requests.exceptions.RequestException:
         Logger.LOGGER.error("Firebase Requests Error..")
         Dialog.ErrorDialog().open_dialog("Firebase Error",
                                          "Firebase Request Error!")
         sys.exit()
     except Exception:
         Logger.LOGGER.error("Unexpected Error in Firebase Request")
Esempio n. 2
0
 def get_firebase_id_token(self, email, password):
     """
     firebase 인증 토큰을 발급받는다
     :param email: 인증 Email
     :param password: 인증 Password
     :return: Firebase ID Token
     """
     try:
         auth_url = self.auth_url + self.firebase_web_key
         auth_data = {
             u'email': email,
             u'password': password,
             u'returnSecureToken': True
         }
         response = requests.post(
             auth_url,
             headers={'Content-Type': 'application/json'},
             data=json.dumps(auth_data))
         token = response.json()['idToken']
         return token
     except requests.exceptions.RequestException:
         Logger.LOGGER.error("Firebase Authorization Error..")
         Dialog.ErrorDialog().open_dialog("Authorization Error",
                                          "Firebase Authorization Error!")
         sys.exit()
     except Exception:
         Logger.LOGGER.error("Unexpected Error in Firebase Authorization")
Esempio n. 3
0
def get_download_list(crawl_page, parent):
    """
    Get Download List from Hiyobi
    :param crawl_page: 수집할 페이지 범위
    :param parent: 진행 상황을 표시할 화면
    :return: 수집된 Gallery List
    """
    Logger.LOGGER.info("Load exception list from Firebase")
    parent.current_state.emit("Load exception list from Firebase")
    exception_list = FirebaseClient.fbclient.get_document_list()
    # TODO Remove next line on build
    # exception_list = []
    parent.notifyProgress.emit(100 * 1 / (crawl_page+2))

    try:
        gallery_list = []
        Logger.LOGGER.info("Get cookie data for Cloudflare..")
        parent.current_state.emit("Get cookie data for Cloudflare..")
        cookie_value, user_agent = cfscrape.get_cookie_string(URL_HIYOBI)
        headers = {'User-Agent': user_agent}
        cookies = {'session_id': cookie_value}
        parent.notifyProgress.emit(100 * 2 / (crawl_page+2))
    except Exception:
        Logger.LOGGER.error("Unexpected Exception Error..")
        Logger.LOGGER.error(traceback.format_exc())
        Dialog.ErrorDialog().open_dialog("Unexpected Exception Error", "Unexpected Exception Request Error!")

    try:
        for i in tqdm(range(1, crawl_page+1)):
            # print("[SYSTEM]: Load From Page %d.." % i)
            parent.current_state.emit("Load From Page %d.." % i)
            soup = BeautifulSoup(requests.get(URL_HIYOBI+'list/'+str(i), cookies=cookies, headers=headers).content, 'html.parser')
            galleries = soup.find_all('div', class_="gallery-content")
            for data in galleries:
                gallery = Gallery.Gallery()
                gallery.initialize(data)
                if gallery.code not in exception_list:
                    gallery_list.append(gallery)
            parent.notifyProgress.emit(100 * (i+2) / (crawl_page+2))
        return gallery_list
    except requests.exceptions.RequestException:
        Logger.LOGGER.error("Hiyobi Requests Error..")
        Dialog.ErrorDialog().open_dialog("Hiyobi Error", "Hiyobi Request Error!")
    except Exception:
        Logger.LOGGER.error("Unexpected Error in Hiyobi Request")
        Logger.LOGGER.error(traceback.format_exc())
        Dialog.ErrorDialog().open_dialog("Unexpected Exception Error", "Unexpected Exception Request Error!")
Esempio n. 4
0
 def open_roll_option_dialog(self):
     dlg = Dialog.RollOptionDialog()
     dlg.exec_()
     if dlg.roll_cnt is not None:
         self.mainWidget.max_item_count = dlg.roll_cnt
         self.mainWidget.toggle_loading = dlg.is_reload
         self.mainWidget.toggle_shuffle = True
         self.mainWidget.init_table_data()
Esempio n. 5
0
 def load_table_data(self):
     if self.input_crawl_pages.text() != "":
         self.index = -1
         while self.download_table.rowCount() > 0:
             self.download_table.removeRow(0)
         global TARGET_CRAWL_PAGE
         TARGET_CRAWL_PAGE = int(self.input_crawl_pages.text())
         dlg = Dialog.ProgressBar(LoadDownloadData())
         dlg.exec_()
         for item in download_list:
             self.index = self.index + 1
             self.download_table.insertRow(self.index)
             self.download_table.setItem(self.index, 0,
                                         QTableWidgetItem(item.code))
             self.download_table.setItem(self.index, 1,
                                         QTableWidgetItem(item.title))
             self.download_table.setItem(self.index, 2,
                                         QTableWidgetItem("Ready.."))
Esempio n. 6
0
def open_manual_input():
    dlg = Dialog.ManualInputDialog()
    dlg.exec_()
Esempio n. 7
0
def open_loading_dialog(thread):
    dlg = Dialog.ProgressBar(thread)
    dlg.show()
    dlg.exec_()