help="Using test data.")
parser.add_argument("-l",
"--headless",
action="store_true",
dest="headless",
help="Cast headless mode.")
args = parser.parse_args()
path = args.path
THREADS_NUM = args.thread_num
RETRIES = args.retries
TIMEOUT = args.timeout
RENDER_TIME = args.render_time
is_debug = args.debug
is_test = args.test
is_headless = args.headless
RUN_PATH = sys.path[0]
LOG, RQ = Logger('CrawlerLog', is_debug).getlog()
sch = Scheduler(path)
try:
sch.scheduling()
except KeyboardInterrupt:
LOG.info("Quit.")
finally:
LOG.info('Done.')
for w, _, _ in sch.d.workers_info:
w.exit() # TODO 非阻塞退出
ex_cmd('TASKKILL /IM chromedriver.exe /F', None, True)
from main import ex_cmd
if __name__ == '__main__':
ex_cmd("rm -rf */datas/*")
ex_cmd("rm -rf */Logs/*")
ex_cmd("rm -rf */logs/*")
ex_cmd("rm -rf */tmp/*")
ex_cmd("rm -rf */pics/*")
ex_cmd("rm -rf 4/noiseFigure/*")
ex_cmd("rm -rf 4/rawFigure/*")
ex_cmd("rm -rf 4/result_figure/*")
sbin_path = getcwd()
chdir(abslt_root_path)
tgz_list = glob("*.tgz")
for tgz in tgz_list:
decrypt(tgz)
db_tgz_list = glob("_*.tgz")
for _tgz in db_tgz_list:
# _分割
num = _tgz[1:].split("_")[0]
_tgz_name = splitext(_tgz)[0]
if not exists(_tgz_name):
mkdir(_tgz_name)
cmd = "tar -zxvf {} -C {}".format(_tgz, _tgz_name)
print(cmd)
# 需要等待子线程结束，要不然下面重命名会找不到文件直接主进程关掉了
ex_cmd(cmd)
# 重命名
dot_files_path = join(_tgz_name, "_queryResult_db_")
chdir(dot_files_path)
print("change dir to {}".format(dot_files_path))
ex_cmd("ls result*. | sh -c \"xargs -n1 -i mv {} {}tar\"")
# Popen("ls |sh -c \"xargs -n1 -i sed -i '1d' {}\"", shell=True).communicate()
ex_cmd("ls result*.tar | sh -c \"xargs -n1 -i tar -zxvf {}\"")
ex_cmd("ls _queryResult_db_/result*.txt | sh -c \"xargs -n1 -i cat {} >>../../" + num + ".txt\"")
print("change dir to {}".format(abslt_root_path))
chdir(abslt_root_path)
from main import ex_cmd
from os import mkdir, makedirs
from os.path import exists, join
import time
def safe_mkdir(path: str) -> None:
if not exists(path):
mkdir(path)
def safe_makedirs(path: str) -> None:
if not exists(path):
makedirs(path)
if __name__ == '__main__':
safe_mkdir("backup")
date = time.strftime('%Y%m%d%H%M', time.localtime(time.time()))
root = join("backup", date)
safe_makedirs(root)
for path in (join(root, "1",
"datas"), join(root, "2",
"datas"), join(root, "3", "datas")):
safe_makedirs(path)
ex_cmd("cp -r 2/datas/*.txt backup/{}/2/datas".format(date))
ex_cmd("cp -r 2/datas/[!_]*_ret.csv backup/{}/2/datas".format(date))
ex_cmd("cp -r 3/datas/* backup/{}/3/datas".format(date))