def downloading(file_name):
# todo:兼容其他是否下載的驗證方式
redis_queue = RedisMsgQueue()
if get_hash_sign(file_name) in redis_queue.hash_get_all(
consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME).keys():
return True
return False
def scheduler_failed_play_url():
# run per hour
logging.debug('scheduler_failed_play_url begin')
redis_queue = RedisMsgQueue()
if redis_queue.size('demo') <= 0 and redis_queue.size('download_queue') <= 0 \
and redis_queue.size('controller_queue') <= 0:
logging.debug('start re_crawl urls')
urls = get_paly_urls()
write_urls(urls)
urls = [
url.replace('\n', '')
for url in open('/data/my_ant/play_urls').readlines()
]
urls_downloaded_subtitle = select_original_url_downloaded_subtitle(
urls)
urls_downloaded_media = select_original_url_downloaded_video_audio(
urls)
urls_downloaded_merged_media = select_original_url_downloaded_merged_media(
urls)
urls_to_parse_subtitle = set(urls) - set(urls_downloaded_subtitle)
urls_to_parse_media = set(urls) - set(urls_downloaded_media) - set(
urls_downloaded_merged_media)
urls_to_parse = urls_to_parse_subtitle | urls_to_parse_media
# scheduler_remote_service(urls_to_parse)
for url in urls_to_parse:
scheduler_controller_queue(url)
def run():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.CONTROLLER_QUEUE_NAME, 20480)
controller_worker = Controller(consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 10)
controller_worker.start()
while True:
logging.info("=================> queue_size: %d", task_queue.size(consts.constant_manager.CONTROLLER_QUEUE_NAME))
time.sleep(10)
def run(queue_name):
task_queue = RedisMsgQueue()
task_queue.addQueue(queue_name, 20480)
downloader_worker = Downloader(queue_name, task_queue, 10)
downloader_worker.start()
while True:
logging.info("queue_name %s==========> queue_size: %d" %
(queue_name, task_queue.size(queue_name)))
time.sleep(10)
def being_download(hash_sign):
# todo:兼容其他是否下載的驗證方式
redis_queue = RedisMsgQueue()
all_files_json = redis_queue.set_get_all(
consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME)
for file_str in all_files_json:
file_json = from_string_to_json(file_str)
if hash_sign == file_json['hash_sign']:
return True
return False
def run():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.DB_SAVE_QUEUE_NAME, 20480)
db_save_worker = DBSaveWorker(consts.constant_manager.DB_SAVE_QUEUE_NAME,
task_queue, 3)
db_save_worker.start()
while True:
logging.info(
"=================> queue_size: %d",
task_queue.size(consts.constant_manager.DB_SAVE_QUEUE_NAME))
time.sleep(10)
def run():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.DOWNLOAD_QUEUE_NAME, 20480)
downloader_worker = Downloader(consts.constant_manager.DOWNLOAD_QUEUE_NAME,
task_queue, 3)
downloader_worker.start()
while True:
logging.info(
"=================> queue_size: %d",
task_queue.size(consts.constant_manager.DOWNLOAD_QUEUE_NAME))
time.sleep(10)
def downloading_status_manager(self):
if self.status == consts.constant_manager.DOWNLOADING:
download_file_status_json = copy.deepcopy(
tmp_download_file_status_json)
download_file_status_json['hash_sign'] = self.hash_sign
download_file_status_json['absolute_path'] = self.title
download_file_status_json['total_size'] = self.total
download_file_status_json['now_size'] = self.now_size
download_file_status_json['download_speed'] = self.speed
download_file_status_json['status'] = self.status
scheduler_download_status_queue(download_file_status_json)
elif self.status == consts.constant_manager.DOWNLOAD_OVER:
redis_queue = RedisMsgQueue()
redis_queue.hash_del(
consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME,
self.hash_sign)
def scheduler_download_queue(download_file_json, priority=False):
redis_queue = RedisMsgQueue()
redis_queue.addQueue(consts.constant_manager.DOWNLOAD_QUEUE_NAME, 1)
if priority:
redis_queue.l_push(consts.constant_manager.DOWNLOAD_QUEUE_NAME, from_json_to_string(download_file_json))
else:
# todo:下载任务细粒度管理
redis_queue.push('demo', from_json_to_string(download_file_json))
logging.debug('push download_file download_url %s' % download_file_json['download_url'])
pass
def demo_controller(urls):
for url in urls:
scheduler_controller_queue(url)
task_queue = RedisMsgQueue()
controller_worker = Controller(
consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 1)
controller_worker.start()
controller_worker.wait()
task_queue.addQueue(consts.constant_manager.DOWNLOAD_QUEUE_NAME, 20480)
task_queue.addQueue('demo', 20480)
# downloader_worker = Downloader(consts.constant_manager.DOWNLOAD_QUEUE_NAME, task_queue, 1)
# downloader_worker.start()
# downloader_worker.wait()
downloader_worker = Downloader('demo', task_queue, 1)
downloader_worker.start()
downloader_worker.wait()
# task_queue = RedisMsgQueue()
# controller_worker = Controller(consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 1)
# controller_worker.start()
while True:
logging.info(
"=================> queue_size: %d",
task_queue.size(consts.constant_manager.CONTROLLER_QUEUE_NAME))
time.sleep(10)
pass
def demo_controller(urls):
for url in urls:
scheduler_controller_queue(url)
task_queue = RedisMsgQueue()
controller_worker = Controller(
consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 3)
controller_worker.start()
controller_worker.wait()
task_queue.addQueue(consts.constant_manager.DOWNLOAD_QUEUE_NAME, 20480)
downloader_worker = Downloader(
consts.constant_manager.DOWNLOAD_QUEUE_NAME, task_queue, 3)
downloader_worker.start()
downloader_worker.wait()
# task_queue.addQueue(consts.constant_manager.DB_SAVE_QUEUE_NAME, 20480)
# db_save_worker = DBSaveWorker(consts.constant_manager.DB_SAVE_QUEUE_NAME, task_queue, 3)
# db_save_worker.start()
# db_save_worker.wait()
pass
def merge_download_media():
download_media_dict_tuple = select_to_merge()
redis_queue = RedisMsgQueue()
order_dict = {}
if not download_media_dict_tuple:
logging.debug('no media to merged')
return
for download_media_dict in download_media_dict_tuple:
if download_media_dict['merged_sign'] not in order_dict.keys():
order_dict[download_media_dict['merged_sign']] = []
order_dict[download_media_dict['merged_sign']].append(
download_media_dict)
for merged_sign in order_dict.keys():
redis_queue.push(consts.constant_manager.FILE_MERGE_QUEUE_NAME,
from_json_to_string(order_dict[merged_sign]))
media_process_worker = MediaProcess(
consts.constant_manager.FILE_MERGE_QUEUE_NAME, redis_queue, 10)
media_process_worker.start()
media_process_worker.wait()
pass
def monitor_download_status():
redis_queue = RedisMsgQueue()
while True:
all_files_old_json = redis_queue.hash_get_all(
consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME)
for hash_sign in all_files_old_json.keys():
all_files_old_json[hash_sign] = from_string_to_json(
all_files_old_json[hash_sign])
all_files_old_json[hash_sign]['now_size'] = get_file_size(
all_files_old_json[hash_sign]['absolute_path'])
time.sleep(10) # 5分钟下载中文件大小不变化，认为下载服务异常挂掉，删除下载队列
all_files_new_json = redis_queue.hash_get_all(
consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME)
for hash_sign in all_files_new_json:
all_files_new_json[hash_sign] = from_string_to_json(
all_files_new_json[hash_sign])
all_files_new_json[hash_sign]['now_size'] = get_file_size(
all_files_new_json[hash_sign]['absolute_path'])
if int(all_files_new_json[hash_sign]['now_size']) - int(
all_files_old_json[hash_sign]['now_size']) == 0:
redis_queue.hash_del(
consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME,
hash_sign)
logging.debug('monitor download queue')
pass
def scheduler_download_status_queue(download_file_status_json):
redis_queue = RedisMsgQueue()
redis_queue.addQueue(consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME, 1)
redis_queue.set_add(consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME,
from_json_to_string(download_file_status_json))
logging.debug('push download_file_status hash_sign %s' %
download_file_status_json['hash_sign'])
pass
def monitor_download_status():
"""
1.读取内存中的即下载文件，以及总大小
2.读取这些文件目前大小，以及间隔时间的下载速度(每秒讀取一次)
:return:
"""
# todo:性能优化，通用下载器的文件下载监控方式
redis_queue = RedisMsgQueue()
while True:
all_files_old_json = redis_queue.set_get_all(
consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME)
for single_file_json in all_files_old_json:
single_file_json['now_size'] = '读取文件大小'
time.sleep(3)
all_files_new_json = redis_queue.set_get_all(
consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME)
for single_file_json in all_files_new_json:
single_file_json['now_size'] = '读取文件大小'
single_file_json['download_speed'] = ('now_size' - 'now_size') / 3
# 修改最新值，重新一个个放入内存中
if 'download_speed' <= 0:
'塞入队列重新下载'
pass
def scheduler_controller_queue(url):
try:
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.CONTROLLER_QUEUE_NAME, 1)
task_queue.push(consts.constant_manager.CONTROLLER_QUEUE_NAME, url)
logging.debug('push task url %s' % url)
except:
traceback.print_exc()
logging.error('push task error')
def demo_worker_exception():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.CONTROLLER_QUEUE_NAME, 20480)
for i in range(1, 100):
task_queue.push(consts.constant_manager.CONTROLLER_QUEUE_NAME,
'asdasdas')
worker = Worker(consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue,
3)
worker.start()
while True:
print 'demo'
time.sleep(30)
pass
def scheduler_download_status_queue(download_file_status_json):
redis_queue = RedisMsgQueue()
redis_queue.addQueue(consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME, 1)
redis_queue.hash_set(consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME, download_file_status_json['hash_sign'],
from_json_to_string(download_file_status_json))
pass
def demo_redis():
redis_queue = RedisMsgQueue()
aa = redis_queue.hash_get_all(
consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME)
pass
def demo_queue():
redis_queue = RedisMsgQueue()
aa = redis_queue.hash_del(
consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME,
'748ffc2387967c2d0ec518516239ea42')
pass
def scheduler_controller_queue(url):
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.CONTROLLER_QUEUE_NAME, 1)
task_queue.push(consts.constant_manager.CONTROLLER_QUEUE_NAME, url)
logging.debug('push task url %s' % url)