def scheduler_failed_play_url():
# run per hour
logging.debug('scheduler_failed_play_url begin')
redis_queue = RedisMsgQueue()
if redis_queue.size('demo') <= 0 and redis_queue.size('download_queue') <= 0 \
and redis_queue.size('controller_queue') <= 0:
logging.debug('start re_crawl urls')
urls = get_paly_urls()
write_urls(urls)
urls = [
url.replace('\n', '')
for url in open('/data/my_ant/play_urls').readlines()
]
urls_downloaded_subtitle = select_original_url_downloaded_subtitle(
urls)
urls_downloaded_media = select_original_url_downloaded_video_audio(
urls)
urls_downloaded_merged_media = select_original_url_downloaded_merged_media(
urls)
urls_to_parse_subtitle = set(urls) - set(urls_downloaded_subtitle)
urls_to_parse_media = set(urls) - set(urls_downloaded_media) - set(
urls_downloaded_merged_media)
urls_to_parse = urls_to_parse_subtitle | urls_to_parse_media
# scheduler_remote_service(urls_to_parse)
for url in urls_to_parse:
scheduler_controller_queue(url)
def demo_controller(urls):
for url in urls:
scheduler_controller_queue(url)
task_queue = RedisMsgQueue()
controller_worker = Controller(
consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 1)
controller_worker.start()
controller_worker.wait()
task_queue.addQueue(consts.constant_manager.DOWNLOAD_QUEUE_NAME, 20480)
task_queue.addQueue('demo', 20480)
# downloader_worker = Downloader(consts.constant_manager.DOWNLOAD_QUEUE_NAME, task_queue, 1)
# downloader_worker.start()
# downloader_worker.wait()
downloader_worker = Downloader('demo', task_queue, 1)
downloader_worker.start()
downloader_worker.wait()
# task_queue = RedisMsgQueue()
# controller_worker = Controller(consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 1)
# controller_worker.start()
while True:
logging.info(
"=================> queue_size: %d",
task_queue.size(consts.constant_manager.CONTROLLER_QUEUE_NAME))
time.sleep(10)
pass
def run():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.CONTROLLER_QUEUE_NAME, 20480)
controller_worker = Controller(consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 10)
controller_worker.start()
while True:
logging.info("=================> queue_size: %d", task_queue.size(consts.constant_manager.CONTROLLER_QUEUE_NAME))
time.sleep(10)
def run(queue_name):
task_queue = RedisMsgQueue()
task_queue.addQueue(queue_name, 20480)
downloader_worker = Downloader(queue_name, task_queue, 10)
downloader_worker.start()
while True:
logging.info("queue_name %s==========> queue_size: %d" %
(queue_name, task_queue.size(queue_name)))
time.sleep(10)
def run():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.DB_SAVE_QUEUE_NAME, 20480)
db_save_worker = DBSaveWorker(consts.constant_manager.DB_SAVE_QUEUE_NAME,
task_queue, 3)
db_save_worker.start()
while True:
logging.info(
"=================> queue_size: %d",
task_queue.size(consts.constant_manager.DB_SAVE_QUEUE_NAME))
time.sleep(10)
def run():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.DOWNLOAD_QUEUE_NAME, 20480)
downloader_worker = Downloader(consts.constant_manager.DOWNLOAD_QUEUE_NAME,
task_queue, 3)
downloader_worker.start()
while True:
logging.info(
"=================> queue_size: %d",
task_queue.size(consts.constant_manager.DOWNLOAD_QUEUE_NAME))
time.sleep(10)