def demo_controller(urls):
for url in urls:
scheduler_controller_queue(url)
task_queue = RedisMsgQueue()
controller_worker = Controller(
consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 1)
controller_worker.start()
controller_worker.wait()
task_queue.addQueue(consts.constant_manager.DOWNLOAD_QUEUE_NAME, 20480)
task_queue.addQueue('demo', 20480)
# downloader_worker = Downloader(consts.constant_manager.DOWNLOAD_QUEUE_NAME, task_queue, 1)
# downloader_worker.start()
# downloader_worker.wait()
downloader_worker = Downloader('demo', task_queue, 1)
downloader_worker.start()
downloader_worker.wait()
# task_queue = RedisMsgQueue()
# controller_worker = Controller(consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 1)
# controller_worker.start()
while True:
logging.info(
"=================> queue_size: %d",
task_queue.size(consts.constant_manager.CONTROLLER_QUEUE_NAME))
time.sleep(10)
pass
def scheduler_download_status_queue(download_file_status_json):
redis_queue = RedisMsgQueue()
redis_queue.addQueue(consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME, 1)
redis_queue.set_add(consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME,
from_json_to_string(download_file_status_json))
logging.debug('push download_file_status hash_sign %s' %
download_file_status_json['hash_sign'])
pass
def scheduler_controller_queue(url):
try:
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.CONTROLLER_QUEUE_NAME, 1)
task_queue.push(consts.constant_manager.CONTROLLER_QUEUE_NAME, url)
logging.debug('push task url %s' % url)
except:
traceback.print_exc()
logging.error('push task error')
def run():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.CONTROLLER_QUEUE_NAME, 20480)
controller_worker = Controller(consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 10)
controller_worker.start()
while True:
logging.info("=================> queue_size: %d", task_queue.size(consts.constant_manager.CONTROLLER_QUEUE_NAME))
time.sleep(10)
def run(queue_name):
task_queue = RedisMsgQueue()
task_queue.addQueue(queue_name, 20480)
downloader_worker = Downloader(queue_name, task_queue, 10)
downloader_worker.start()
while True:
logging.info("queue_name %s==========> queue_size: %d" %
(queue_name, task_queue.size(queue_name)))
time.sleep(10)
def scheduler_download_queue(download_file_json, priority=False):
redis_queue = RedisMsgQueue()
redis_queue.addQueue(consts.constant_manager.DOWNLOAD_QUEUE_NAME, 1)
if priority:
redis_queue.l_push(consts.constant_manager.DOWNLOAD_QUEUE_NAME, from_json_to_string(download_file_json))
else:
# todo:下载任务细粒度管理
redis_queue.push('demo', from_json_to_string(download_file_json))
logging.debug('push download_file download_url %s' % download_file_json['download_url'])
pass
def run():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.DB_SAVE_QUEUE_NAME, 20480)
db_save_worker = DBSaveWorker(consts.constant_manager.DB_SAVE_QUEUE_NAME,
task_queue, 3)
db_save_worker.start()
while True:
logging.info(
"=================> queue_size: %d",
task_queue.size(consts.constant_manager.DB_SAVE_QUEUE_NAME))
time.sleep(10)
def run():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.DOWNLOAD_QUEUE_NAME, 20480)
downloader_worker = Downloader(consts.constant_manager.DOWNLOAD_QUEUE_NAME,
task_queue, 3)
downloader_worker.start()
while True:
logging.info(
"=================> queue_size: %d",
task_queue.size(consts.constant_manager.DOWNLOAD_QUEUE_NAME))
time.sleep(10)
def demo_worker_exception():
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.CONTROLLER_QUEUE_NAME, 20480)
for i in range(1, 100):
task_queue.push(consts.constant_manager.CONTROLLER_QUEUE_NAME,
'asdasdas')
worker = Worker(consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue,
3)
worker.start()
while True:
print 'demo'
time.sleep(30)
pass
def demo_controller(urls):
for url in urls:
scheduler_controller_queue(url)
task_queue = RedisMsgQueue()
controller_worker = Controller(
consts.constant_manager.CONTROLLER_QUEUE_NAME, task_queue, 3)
controller_worker.start()
controller_worker.wait()
task_queue.addQueue(consts.constant_manager.DOWNLOAD_QUEUE_NAME, 20480)
downloader_worker = Downloader(
consts.constant_manager.DOWNLOAD_QUEUE_NAME, task_queue, 3)
downloader_worker.start()
downloader_worker.wait()
# task_queue.addQueue(consts.constant_manager.DB_SAVE_QUEUE_NAME, 20480)
# db_save_worker = DBSaveWorker(consts.constant_manager.DB_SAVE_QUEUE_NAME, task_queue, 3)
# db_save_worker.start()
# db_save_worker.wait()
pass
def scheduler_controller_queue(url):
task_queue = RedisMsgQueue()
task_queue.addQueue(consts.constant_manager.CONTROLLER_QUEUE_NAME, 1)
task_queue.push(consts.constant_manager.CONTROLLER_QUEUE_NAME, url)
logging.debug('push task url %s' % url)
def scheduler_download_status_queue(download_file_status_json):
redis_queue = RedisMsgQueue()
redis_queue.addQueue(consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME, 1)
redis_queue.hash_set(consts.constant_manager.DOWNLOAD_STATUS_QUEUE_NAME, download_file_status_json['hash_sign'],
from_json_to_string(download_file_status_json))
pass