def __init__(self):
self._serial_num = 0
self._idset = set()
self.task_dict = {}
self.finished_list = {}
self.logger = logger.Logger()
dt = datetime.now()
self.current_date = "{:4d}{:02d}{:02d}".format(dt.year, dt.month, dt.day)
self.active_task = None
self.task_stat = []
self.last_update = 0
self.schedule = {}
import mmap
import time
import queue
import threading
import argparse
import itertools
import traceback
import validators
import requests
import requests_ntlm
from libs import logger
# Initialize logger
log = logger.Logger()
# Disable InsecureRequestWarning for unverified HTTPS request
requests.packages.urllib3.disable_warnings(
requests.packages.urllib3.exceptions.InsecureRequestWarning)
def parse_args():
def check_arg_threads(value):
try:
value = int(value)
except:
raise Exception('Buffer size must be a integer')
if value < 1 or value > 30:
raise Exception('Buffer size must be an interger between 1 and 30')
return value
from lxml.html import etree
from urllib import parse
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Image, Table, TableStyle, Frame, ListFlowable, ListItem
from reportlab.lib.enums import TA_JUSTIFY
from reportlab.lib import colors
from reportlab.lib.colors import CMYKColor
from reportlab.lib.units import mm
import os
from libs import logger
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
log = logger.Logger('logs/DownloadBooks.out.log', level='info')
headers = {
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Referer":
"http://www.ruiwen.com/jiaocai/shuxue/bubianban/yinianjishangce/shangce11.html",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"
}
values = {'request_type': 'GET'}
main_url = 'http://www.ruiwen.com'
except Exception as e:
node.logger.app_log.warning("Exception {}".format(e))
node.logger.app_log.warning(
"Processing of {} finished".format(token))
except:
node.logger.app_log.warning("Error parsing")
db_handler_instance.index.commit()
if __name__ == "__main__":
from libs import node, logger
import dbhandler
node = node.Node()
node.debug_level = "WARNING"
node.terminal_output = True
node.logger = logger.Logger()
node.logger.app_log = log.log("local_test.log", node.debug_level,
node.terminal_output)
node.logger.app_log.warning("Configuration settings loaded")
db_handler = dbhandler.DbHandler("static/index_local_test.db",
"static/ledger.db", "static/hyper.db",
False, None, node.logger, False)
tokens_update(node, db_handler)
# tokens_update("tokens.db","reindex")