def my_copy_file(source_file,target): i_file_count = 0 try: str_dir = source_file if os.path.isfile(str_dir): str_dir = os.path.dirname(str_dir) arr_files = get_dir_files(str_dir) i_file_count = len(arr_files) for name in arr_files: copy_file(name,target) return str(i_file_count) + ",.," except Exception as e: LogHelper.error("CopyError1:" + e.message) return i_file_count + ",.," + "CopyError1:" + e.message
def copy_file(source_file,target_file): try: source = getRootDir(source_file) desfilename=source_file.replace('/',os.sep).replace(source,target_file,1).replace('\\\\',os.sep) LogHelper.debug(source_file + " copy to "+desfilename) if not os.path.exists(os.path.dirname(desfilename)): os.makedirs(os.path.dirname(desfilename)) if not os.path.exists(desfilename): shutil.copy(source_file,desfilename)#如果要改为移动,而不是拷贝,可以将copy改为move return "1,.," except Exception as e: LogHelper.error("CopyError0:" + e.message) return "0,.,CopyError0:" + e.message
# 读取外部配置 configInfo = ReadConfigIni() while configInfo.repeatTimes > 0 or configInfo.repeatTimes == -1: logger = LogHelper('Bili', cmdLevel='INFO', fileLevel="DEBUG").logger try: logger.info('repeatTimes = ' + str(configInfo.repeatTimes)) # -------------------------------------------------------------- # 设置需要下载的信息 # 每个 UP 主视频 downloadlistfile = 'DownloadList.txt' if os.path.exists(downloadlistfile) == True: filmList = ReadDownloadList(downloadlistfile) else: logger.error("DownloadList.txt not found") raise Exception("DownloadList.txt not found") uperList = ReadDownloadList(downloadlistfile) MainProcess(logger, uperList, configInfo.saveRootPath, configInfo.concurrency) BarkMe(configInfo.barkurl, configInfo.barkapikey, 'Job 4 Bilibli', configInfo.notifyurl) except Exception as ex: logger.error(ex) finally: logger.info('MainProcess One Time Done.') # 等待
def MainProcess(uperList, saveRootPath, concurrency = 3): logger = LogHelper('Bili', cmdLevel='INFO', fileLevel="DEBUG").logger pp = None try: # -------------------------------------------------------------- # 进行每个 UP 主视频页数的获取 pp = PreProcess(logger = logger, uperList=uperList) pp.ScanLoclInfo(saveRootPath) pp.Process() # -------------------------------------------------------------- # 爬取要下载视频的 url for uper in pp.uperList: logger.info(uper.UserName + " Spider Start···") OneSpiderRetryTimes = 0 # 打算下载的数量,要去网络动态获取的数量进行对比 while ((uper.NeedDownloadFilmCount > len(uper.VideoInfoDic_NetFileName) or len(uper.ErrorUrl_Dic) > 0) and OneSpiderRetryTimes <= 10): # dd = BiliSpider() # GithubDeveloperSpider BiliSpider.start(logger = logger, uper = uper, saveRootPath = saveRootPath, concurrency = concurrency, middleware=middleware) OneSpiderRetryTimes = OneSpiderRetryTimes + 1 logger.info("Try Spider " + uper.UserName + " " + str(OneSpiderRetryTimes) + " times.") RandomSleep() logger.info(uper.UserName + " Spider Done.") if OneSpiderRetryTimes > 10: logger.error(uper.UserName + " Spider Retry " + str(OneSpiderRetryTimes) + "times.") logger.error("Error Url:") for eUrl in uper.ErrorUrl_Dic: logger.error(eUrl) else: # 本地应该原有+准备要下载的 != 网络总数,需要提示 if len(uper.VideoInfoDic_NetFileName) != len(uper.VideoInfoDic_loaclFileName): logger.warn("VideoInfoDic_NetFileName Count: " + str(len(uper.VideoInfoDic_NetFileName)) + " != VideoInfoDic_loaclFileName Count: " + str(len(uper.VideoInfoDic_loaclFileName)) ) uper.ErrorUrl_Dic.clear() logger.info("Spider All Done.") # -------------------------------------------------------------- logger.info("Start Download"+ "----" * 20) # 开始下载 # 先对 local 与 net 的字典进行同步 logger.info("Start Sync Dic") for uper in pp.uperList: iNeedDl = 0 for fileName, oneVideo in zip(uper.VideoInfoDic_loaclFileName.keys(), uper.VideoInfoDic_loaclFileName.values()): if fileName in uper.VideoInfoDic_NetFileName: uper.VideoInfoDic_NetFileName[fileName].isDownloaded = oneVideo.isDownloaded if oneVideo.isDownloaded == False: iNeedDl = iNeedDl + 1 logger.info(uper.UserName + "NetFile / LocalFile -- NeedDl: " + str(len(uper.VideoInfoDic_NetFileName)) + " / " + str(len(uper.VideoInfoDic_loaclFileName)) + " -- " + str(iNeedDl)) logger.info("End Sync Dic") for uper in pp.uperList: directory = os.path.join(saveRootPath, uper.UserName) for fileName, oneVideo in zip(uper.VideoInfoDic_NetFileName.keys(), uper.VideoInfoDic_NetFileName.values()): if oneVideo.isDownloaded == True: continue DownloadRetryTimes = 0 oneRe = False while oneRe is False and DownloadRetryTimes <= 10: oneRe = Downloader(logger, directory, oneVideo.time, oneVideo.title, oneVideo.url).ProcessOne() DownloadRetryTimes = DownloadRetryTimes + 1 logger.info("Try Download " + str(DownloadRetryTimes) + " times.") RandomSleep() if OneSpiderRetryTimes > 10: logger.error("Retry Download " + str(DownloadRetryTimes) + " times.") logger.error("Error Url: " + oneVideo.url) # 标记下载完成 if oneRe: oneVideo.isDownloaded = True uper.ThisTimeDownloadCount = uper.ThisTimeDownloadCount + 1 except Exception as ex: errInfo = "Catch Exception: " + str(ex) logger.error(errInfo) finally: logger.info("finally"+ "----" * 20) for uper in pp.uperList: logger.info("This Time Download: " + uper.UserName + " -- " + str(uper.ThisTimeDownloadCount)) for uper in pp.uperList: for fileName, oneVideo in zip(uper.VideoInfoDic_NetFileName.keys(), uper.VideoInfoDic_NetFileName.values()): if oneVideo.isDownloaded == False: logger.error('Download Fail:' + uper.UserName) logger.error(oneVideo.url) logger.info("All Done.")