def download_url(url, filename): try: logger.debug('download_url : %s', url) save_path = ModelSetting.get('save_path') max_pf_count = ModelSetting.get('max_pf_count') tmp = Wavve.get_prefer_url(url) proxy = None if ModelSetting.get_bool('use_proxy'): proxy = ModelSetting.get('proxy_url') f = ffmpeg.Ffmpeg(tmp, filename, plugin_id=-1, listener=LogicBasic.ffmpeg_listener, max_pf_count=max_pf_count, call_plugin='wavve_basic', save_path=save_path, proxy=proxy) #f.start_and_wait() f.start() #time.sleep(60) return True except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc())
def download_thread_function(): import plugin while True: try: while True: if LogicQueue.current_ffmpeg_count < int( ModelSetting.get('max_ffmpeg_process_count')): break #logger.debug(LogicQueue.current_ffmpeg_count) time.sleep(5) entity = LogicQueue.download_queue.get() if entity.cancel: continue episode = ModelLinkkf('auto', info=entity.info) db.session.add(episode) db.session.commit() from .logic_linkkf import LogicLinkkf entity.url = LogicLinkkf.get_video_url(entity.info['code']) if entity.url is None: entity.ffmpeg_status_kor = 'URL실패' plugin.socketio_list_refresh() continue import ffmpeg max_pf_count = 0 save_path = ModelSetting.get('download_path') if ModelSetting.get('auto_make_folder') == 'True': program_path = os.path.join(save_path, entity.info['save_folder']) save_path = program_path try: if not os.path.exists(save_path): os.makedirs(save_path) except: logger.debug('program path make fail!!') # 파일 존재여부 체크 if os.path.exists( os.path.join(save_path, entity.info['filename'])): entity.ffmpeg_status_kor = '파일 있음' entity.ffmpeg_percent = 100 plugin.socketio_list_refresh() continue f = ffmpeg.Ffmpeg(entity.url, entity.info['filename'], plugin_id=entity.entity_id, listener=LogicQueue.ffmpeg_listener, max_pf_count=max_pf_count, call_plugin=package_name, save_path=save_path) f.start() LogicQueue.current_ffmpeg_count += 1 LogicQueue.download_queue.task_done() except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc())
def download_thread_function(self): while True: try: while True: try: if self.current_ffmpeg_count < self.max_ffmpeg_count: break time.sleep(5) except Exception as exception: self.P.logger.error('Exception:%s', exception) self.P.logger.error(traceback.format_exc()) self.P.logger.error('current_ffmpeg_count : %s', self.current_ffmpeg_count) self.P.logger.error('max_ffmpeg_count : %s', self.max_ffmpeg_count) break entity = self.download_queue.get() if entity.cancel: continue #from .logic_ani24 import LogicAni24 #entity.url = LogicAni24.get_video_url(entity.info['code']) video_url = entity.get_video_url() if video_url is None: entity.ffmpeg_status_kor = 'URL실패' entity.refresh_status() #plugin.socketio_list_refresh() continue import ffmpeg #max_pf_count = 0 #save_path = ModelSetting.get('download_path') #if ModelSetting.get('auto_make_folder') == 'True': # program_path = os.path.join(save_path, entity.info['filename'].split('.')[0]) # save_path = program_path #try: # if not os.path.exists(save_path): # os.makedirs(save_path) #except: # logger.debug('program path make fail!!') # 파일 존재여부 체크 filepath = entity.get_video_filepath() if os.path.exists(filepath): entity.ffmpeg_status_kor = '파일 있음' entity.ffmpeg_percent = 100 entity.refresh_status() #plugin.socketio_list_refresh() continue dirname = os.path.dirname(filepath) if not os.path.exists(dirname): os.makedirs(dirname) f = ffmpeg.Ffmpeg(video_url, os.path.basename(filepath), plugin_id=entity.entity_id, listener=self.ffmpeg_listener, call_plugin=self.P.package_name, save_path=dirname, headers=entity.headers) f.start() self.current_ffmpeg_count += 1 self.download_queue.task_done() except Exception as exception: self.P.logger.error('Exception:%s', exception) self.P.logger.error(traceback.format_exc())
def movie_download(url, filename): try: save_path = ModelSetting.get('save_path') logger.debug(save_path) max_pf_count = ModelSetting.get('max_pf_count') f = ffmpeg.Ffmpeg(url, filename, plugin_id=-1, listener=None, max_pf_count=max_pf_count, call_plugin='tving_basic', save_path=save_path) #f.start_and_wait() f.start() #time.sleep(60) return True except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc())
def download_url(url, filename): try: save_path = ModelSetting.get('save_path') max_pf_count = ModelSetting.get('max_pf_count') TvingBasic.current_episode.start_time = datetime.now() db.session.add(TvingBasic.current_episode) db.session.commit() f = ffmpeg.Ffmpeg(url, TvingBasic.current_episode.filename, plugin_id=TvingBasic.current_episode.id, listener=TvingBasic.ffmpeg_listener, max_pf_count=max_pf_count, call_plugin='tving_basic', save_path=save_path) #f.start_and_wait() f.start() #time.sleep(60) return True except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc())
def download_url(url, filename): try: save_path = ModelSetting.get('save_path') max_pf_count = ModelSetting.get('max_pf_count') f = ffmpeg.Ffmpeg(url, filename, plugin_id=-1, listener=LogicBasic.ffmpeg_listener, max_pf_count=max_pf_count, call_plugin='wavve_basic', save_path=save_path) #f.start_and_wait() f.start() #time.sleep(60) return True except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc())
def download_thread_function(): from . import plugin while True: #while self.flag_stop == False: try: while True: if TvingProgram.current_ffmpeg_count < ModelSetting.get_int( 'program_auto_count_ffmpeg'): break time.sleep(5) entity = TvingProgram.download_queue.get() if entity.cancel: continue # 초기화 if entity is None: return #Log('* 스캔 큐 AWAKE : %s', self.current_scan_entity.filename) #self.current_scan_entity.status = 'SCAN_START' #self.current_scan_entity.scan_start_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') #Log('* 스캔 큐 scan_start_time : %s', self.current_scan_entity.scan_start_time) Tving._token = ModelSetting.get('token') data, vod_url = TvingBasic.get_episode_json( entity.episode_code, entity.quality) #logger.debug(data) episode = Episode('basic') episode = TvingBasic.make_episode_by_json( episode, data, vod_url) #TvingBasic.current_episode = episode entity.json_data['filename'] = episode.filename #if TvingProgram.current_ffmpeg_count < TvingProgram.max_ffmpeg_count: import ffmpeg max_pf_count = ModelSetting.get('max_pf_count') save_path = ModelSetting.get('program_auto_path') if ModelSetting.get('program_auto_make_folder') == 'True': program_path = os.path.join( save_path, data['body']['content']['program_name']) save_path = program_path try: if not os.path.exists(save_path): os.makedirs(save_path) except: logger.debug('program path make fail!!') # 파일 존재여부 체크 if os.path.exists(os.path.join(save_path, episode.filename)): entity.ffmpeg_status_kor = '파일 있음' entity.ffmpeg_percent = 100 plugin.socketio_list_refresh() continue f = ffmpeg.Ffmpeg(vod_url, episode.filename, plugin_id=entity.entity_id, listener=TvingProgram.ffmpeg_listener, max_pf_count=max_pf_count, call_plugin='%s_program' % package_name, save_path=save_path) f.start() TvingProgram.current_ffmpeg_count += 1 TvingProgram.download_queue.task_done() """ self.current_scan_t = ScanThread() self.current_scan_t.set(self.current_scan_entity, self.wait_event) self.current_scan_t.start() Log('* 스캔 큐 thread 종료 대기') self.current_scan_t.join(60*10) if self.current_scan_t.is_alive(): Log('* 스캔 큐 still Alive') self.current_scan_t.process.terminate() self.current_scan_t.join() Log('process returncode %s', self.current_scan_t.process.returncode) self.current_scan_t = None # 초기화 한번 체크 if self.flag_stop: return self.current_scan_entity.status = 'SCAN_COMPLETED' self.current_scan_entity.scan_end_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') self.scan_queue.task_done() Log('* 남은 큐 사이즈 : %s', self.scan_queue.qsize()) self.current_scan_entity = None """ #time.sleep(100) except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc())
def scheduler_function(): try: #logger.debug('start scheduler_function') import ffmpeg page = ModelSetting.get_int('auto_page') max_pf_count = ModelSetting.get('max_pf_count') save_path = ModelSetting.get('auto_save_path') default_quality = Tving.get_quality_to_tving(ModelSetting.get('auto_quality')) retry_user_abort = ModelSetting.get_bool('retry_user_abort') except_channel = ModelSetting.get('except_channel') except_program = ModelSetting.get(key='except_program') download_qvod = ModelSetting.get_bool('download_qvod') download_program_in_qvod = ModelSetting.get('download_program_in_qvod') download_mode = ModelSetting.get(key='download_mode') whitelist_program = ModelSetting.get('whitelist_program') whitelist_first_episode_download = ModelSetting.get_bool('whitelist_first_episode_download') except_channels = [x.strip() for x in except_channel.replace('\n', ',').split(',')] except_programs = [x.strip().replace(' ', '') for x in except_program.replace('\n', ',').split(',')] download_program_in_qvods = [x.strip().replace(' ', '') for x in download_program_in_qvod.replace('\n', ',').split(',')] whitelist_programs = [x.strip().replace(' ', '') for x in whitelist_program.replace('\n', ',').split(',')] except_channels = Util.get_list_except_empty(except_channels) except_programs = Util.get_list_except_empty(except_programs) download_program_in_qvods = Util.get_list_except_empty(download_program_in_qvods) whitelist_programs = Util.get_list_except_empty(whitelist_programs) logger.debug('except_channels:%s', except_channels) logger.debug('except_programs:%s', except_programs) logger.debug('download_qvod :%s %s', download_qvod, type(download_qvod)) logger.debug('download_program_in_qvods:%s', download_program_in_qvods) for i in range(1, page+1): vod_list = Tving.get_vod_list(page=i)["body"]["result"] for vod in vod_list: try: if not scheduler.is_include('tving_recent'): logger.debug('not in scheduler') return code = vod["episode"]["code"] with db.session.no_autoflush: # 2019-01-11 720권한만 있을 때 1080p를 받으려고 하면 계속 episode를 생성 #episode = db.session.query(Episode).filter_by(episode_code=code, quality=default_quality).with_for_update().first() #episode = db.session.query(Episode).filter_by(episode_code=code).with_for_update().first() # 2020-02-14 qvod episode_code 고정 episode = db.session.query(Episode).filter_by(episode_code=code, broadcast_date=str(vod["episode"]["broadcast_date"])[2:]).with_for_update().first() if episode is not None: logger.debug('program_name:%s frequency:%s %s %s', episode.program_name, episode.frequency, episode.user_abort, episode.retry) if episode.completed: logger.debug('COMPLETED') continue elif episode.user_abort: if retry_user_abort: episode.user_abort = False else: continue elif episode.etc_abort > 10: # 1:알수없는이유 시작실패, 2 타임오버, 3, 강제스톱.킬 # 12:제외채널, 13:제외프로그램 # 14:화이트리스트 # 9 : retry # 8 : qvod logger.debug('ETC ABORT:%s', episode.etc_abort) continue elif episode.retry > 20: logger.debug('retry 20') episode.etc_abort = 15 continue # URL때문에 DB에 있어도 다시 JSON을 받아야함. json_data, url = TvingBasic.get_episode_json(code, default_quality) if episode is None: logger.debug('EPISODE is none') episode = Episode('auto') episode = TvingBasic.make_episode_by_json(episode, json_data, url) db.session.add(episode) else: episode = TvingBasic.make_episode_by_json(episode, json_data, url) # qvod 체크 is_qvod = False if url.find('quickvod') != -1: is_qvod = True # 채널, 프로그램 체크 flag_download = True if is_qvod: if not download_qvod: flag_download = False for program_name in download_program_in_qvods: if episode.program_name.replace(' ', '').find(program_name) != -1: episode.etc_abort = 0 flag_download = True logger.debug('is qvod.. %s %s', program_name, flag_download) break if not flag_download: episode.etc_abort = 8 db.session.commit() logger.debug('is qvod.. pass') continue if download_mode == '0': for program_name in except_programs: if episode.program_name.replace(' ', '').find(program_name) != -1: episode.etc_abort = 13 flag_download = False break if episode.channel_name in except_channels: episode.etc_abort = 12 flag_download = False else: if flag_download: #무조건 탐 find_in_whitelist = False for program_name in whitelist_programs: if episode.program_name.replace(' ', '').find(program_name) != -1: find_in_whitelist = True break if not find_in_whitelist: episode.etc_abort = 14 flag_download = False if not flag_download and whitelist_first_episode_download and episode.frequency == 1: flag_download = True if flag_download: episode.etc_abort = 0 episode.retry += 1 episode.pf = 0 # 재시도 episode.save_path = save_path episode.start_time = datetime.now() db.session.commit() else: db.session.commit() time.sleep(2) continue logger.debug('FFMPEG Start.. id:%s', episode.id) if episode.id is None: logger.debug('PROGRAM:%s', episode.program_name) f = ffmpeg.Ffmpeg(url, episode.filename, plugin_id=episode.id, listener=TvingBasic.ffmpeg_listener, max_pf_count=max_pf_count, call_plugin='%s_recent' % package_name, save_path=save_path) f.start_and_wait() except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc()) #break #logger.debug('end scheduler_function') except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc())
def scheduler_function(): try: logger.debug('Wavve scheduler_function start..') LogicBasic.login() page = int(ModelSetting.get('auto_page')) max_pf_count = ModelSetting.get('max_pf_count') save_path = ModelSetting.get('auto_save_path') auto_quality = ModelSetting.get('auto_quality') retry_user_abort = (ModelSetting.get('retry_user_abort') == 'True') credential = ModelSetting.get('credential') qvod_download = (ModelSetting.get('qvod_download') == 'True') except_channel = ModelSetting.get('except_channel') except_program = ModelSetting.get('except_program') download_program_in_qvod = ModelSetting.get( 'download_program_in_qvod') download_mode = ModelSetting.get('download_mode') whitelist_program = ModelSetting.get('whitelist_program') whitelist_first_episode_download = ( ModelSetting.get('whitelist_first_episode_download') == 'True') except_channels = [ x.strip() for x in except_channel.replace('\n', ',').split(',') ] except_programs = [ x.strip().replace(' ', '') for x in except_program.replace('\n', ',').split(',') ] download_program_in_qvods = [ x.strip().replace(' ', '') for x in download_program_in_qvod.replace('\n', ',').split(',') ] whitelist_programs = [ x.strip().replace(' ', '') for x in whitelist_program.replace('\n', ',').split(',') ] Util.get_list_except_empty(except_channels) except_channels = Util.get_list_except_empty(except_channels) except_programs = Util.get_list_except_empty(except_programs) download_program_in_qvods = Util.get_list_except_empty( download_program_in_qvods) whitelist_programs = Util.get_list_except_empty(whitelist_programs) logger.debug('except_channels:%s', except_channels) logger.debug('except_programs:%s', except_programs) logger.debug('qvod_download :%s %s', qvod_download, type(qvod_download)) for i in range(1, page + 1): vod_list = Wavve.vod_newcontents(page=i)['list'] #logger.debug(vod_list) logger.debug('Page:%s vod len:%s', page, len(vod_list)) for vod in vod_list: try: while True: if LogicRecent.current_auto_count_ffmpeg < int( ModelSetting.get('auto_count_ffmpeg')): break time.sleep(10) #logger.debug('wavve wait : %s', LogicRecent.current_auto_count_ffmpeg) #logger.debug(vod) contentid = vod["contentid"] contenttype = 'onairvod' if vod[ 'type'] == 'onair' else 'vod' episode = db.session.query(ModelWavveEpisode) \ .filter((ModelWavveEpisode.call == 'auto') | (ModelWavveEpisode.call == None)) \ .filter_by(contentid=contentid) \ .with_for_update().first() \ if episode is not None: if episode.completed: continue elif episode.user_abort: if retry_user_abort: episode.user_abort = False else: #사용자 중지로 중단했고, 다시받기가 false이면 패스 continue elif episode.etc_abort > 10: # 1:알수없는이유 시작실패, 2 타임오버, 3, 강제스톱.킬 # 11:제외채널, 12:제외프로그램 # 13:장르제외, 14:화이트리스트 제외, 7:권한없음, 6:화질다름 #logger.debug('EPC Abort : %s', episode.etc_abort) continue elif episode.retry > 20: logger.debug('retry 20') episode.etc_abort = 9 continue # URL때문에 DB에 있어도 다시 JSON을 받아야함. for episode_try in range(3): json_data = Wavve.streaming( contenttype, contentid, auto_quality, credential) try: tmp = json_data['playurl'] except: try: LogicBasic.login() json_data = Wavve.streaming( contenttype, contentid, auto_quality, credential) except: pass if json_data is None: logger.debug('episode fail.. %s', episode_try) time.sleep(20) else: break if episode is None: if json_data is None: episode = ModelWavveEpisode( 'auto', info=vod, streaming=json_data) db.session.add(episode) db.session.commit() continue else: episode = ModelWavveEpisode( 'auto', info=vod, streaming=json_data) db.session.add(episode) else: if json_data is None: continue else: episode.set_streaming(json_data) if json_data['playurl'].find('preview') != -1: episode.etc_abort = 7 db.session.commit() continue # 채널, 프로그램 체크 flag_download = True if contenttype == 'onairvod': if not qvod_download: episode.etc_abort = 11 flag_download = False for programtitle in download_program_in_qvods: if episode.programtitle.replace( ' ', '').find(programtitle) != -1: flag_download = True episode.etc_abort = 0 break # 시간체크 if flag_download: logger.debug(episode.episodetitle) match = re.compile( r'Quick\sVOD\s(?P<time>\d{2}\:\d{2})\s' ).search(episode.episodetitle) if match: dt_now = datetime.now() logger.debug(dt_now) dt_tmp = datetime.strptime( match.group('time'), '%H:%M') dt_start = datetime( dt_now.year, dt_now.month, dt_now.day, dt_tmp.hour, dt_tmp.minute, 0, 0) logger.debug(dt_start) if (dt_now - dt_start).seconds < 0: dt_start = dt_start + timedelta( days=-1) #detail = Wavve.vod_contents_contentid(episode.contentid) if 'detail' not in episode.contents_json: episode.contents_json[ 'detail'] = Wavve.vod_contents_contentid( episode.contentid) qvod_playtime = episode.contents_json[ 'detail']['playtime'] delta = (dt_now - dt_start).seconds if int(qvod_playtime) > delta: flag_download = False episode.etc_abort = 8 logger.debug('QVOD %s %s %s %s', flag_download, match.group('time'), qvod_playtime, delta) else: logger.debug('QVOD fail..') flag_download = False episode.etc_abort = 7 if download_mode == '0': for program_name in except_programs: if episode.programtitle.replace( ' ', '').find(program_name) != -1: episode.etc_abort = 13 flag_download = False break if episode.channelname in except_channels: episode.etc_abort = 12 flag_download = False else: if flag_download: find_in_whitelist = False for program_name in whitelist_programs: if episode.programtitle.replace( ' ', '').find(program_name) != -1: find_in_whitelist = True break if not find_in_whitelist: episode.etc_abort = 14 flag_download = False if not flag_download and whitelist_first_episode_download and episode.episodenumber == '1': flag_download = True #logger.debug(episode.quality) if flag_download and episode.quality != auto_quality: if auto_quality == '2160p' and episode.quality == '1080p' and ModelSetting.get_bool( '2160_receive_1080'): if episode.created_time + timedelta( minutes=ModelSetting.get_int( '2160_wait_minute') ) < datetime.now(): logger.debug('1080p download') pass else: episode.etc_abort = 5 db.session.commit() continue else: episode.etc_abort = 6 db.session.commit() continue if flag_download: episode.etc_abort = 0 episode.retry += 1 episode.pf = 0 # 재시도 episode.save_path = save_path episode.start_time = datetime.now() db.session.commit() else: db.session.commit() continue logger.debug('FFMPEG Start.. id:%s', episode.id) if episode.id is None: logger.debug('PROGRAM:%s', episode.programtitle) tmp = Wavve.get_prefer_url(episode.playurl) f = ffmpeg.Ffmpeg(tmp, episode.filename, plugin_id=episode.id, listener=LogicBasic.ffmpeg_listener, max_pf_count=max_pf_count, call_plugin='%s_recent' % package_name, save_path=save_path) f.start() LogicRecent.current_auto_count_ffmpeg += 1 #f.start_and_wait() time.sleep(20) #return except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc()) #db.session.rollback() logger.debug( 'ROLLBACK!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') finally: #logger.debug('wait..') pass logger.debug('=======================================') except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc())
def download_thread_function(): while True: try: while True: if LogicProgram.current_ffmpeg_count < int( ModelSetting.get('program_auto_count_ffmpeg')): break time.sleep(5) entity = LogicProgram.download_queue.get() if entity.cancel: continue # 초기화 if entity is None: return contenttype = 'onairvod' if entity.json_data[ 'type'] == 'onair' else 'vod' count = 0 while True: count += 1 streaming_data = Wavve.streaming( contenttype, entity.episode_code, entity.quality, ModelSetting.get('credential')) try: tmp = data2['playurl'] except: try: from .logic_basic import LogicBasic LogicBasic.login() streaming_data = Wavve.streaming( contenttype, entity.episode_code, entity.quality, ModelSetting.get('credential')) except: pass entity.json_data['filename'] = Wavve.get_filename( entity.json_data, streaming_data['quality']) if streaming_data is not None: break else: time.sleep(20) if count > 10: entity.ffmpeg_status_kor = u'URL실패' break if streaming_data is None: continue import ffmpeg max_pf_count = ModelSetting.get('max_pf_count') save_path = ModelSetting.get('program_auto_path') if ModelSetting.get('program_auto_make_folder') == 'True': program_path = os.path.join( save_path, entity.json_data['programtitle']) save_path = program_path try: if not os.path.exists(save_path): os.makedirs(save_path) except: logger.debug('program path make fail!!') ### edit by lapis ModelWavveProgram.update(entity) ### # 파일 존재여부 체크 if os.path.exists( os.path.join(save_path, entity.json_data['filename'])): entity.ffmpeg_status_kor = '파일 있음' entity.ffmpeg_percent = 100 ### edit by lapis ModelWavveProgram.delete(entity.episode_code, entity.quality) ### plugin.socketio_list_refresh() continue tmp = Wavve.get_prefer_url(streaming_data['playurl']) f = ffmpeg.Ffmpeg(tmp, entity.json_data['filename'], plugin_id=entity.entity_id, listener=LogicProgram.ffmpeg_listener, max_pf_count=max_pf_count, call_plugin='wavve_program', save_path=save_path) f.start() LogicProgram.current_ffmpeg_count += 1 LogicProgram.download_queue.task_done() except Exception as e: logger.error('Exception:%s', e) logger.error(traceback.format_exc())