def __init__(self, **kwargs): self._module_dict = {} self._redis_client = RedisClient(host=config_setting.queue_host, port=config_setting.queue_port, password=config_setting.queue_pwd) self._queue_list = [LOGIN_QUEUE,CTASK_QUEUE,UPDATE_QUEUE] self._task_queue = Queue() # self.init_modules() gevent.spawn(self._get_task) gevent.spawn(self._dispatch_task) gevent.sleep()
class CentralEngine(object): def __init__(self, **kwargs): self._module_dict = {} self._redis_client = RedisClient(host=config_setting.queue_host, port=config_setting.queue_port, password=config_setting.queue_pwd) self._queue_list = [LOGIN_QUEUE,CTASK_QUEUE,UPDATE_QUEUE] self._task_queue = Queue() # self.init_modules() gevent.spawn(self._get_task) gevent.spawn(self._dispatch_task) gevent.sleep() def init_modules(self): l = json.loads(moddules_info) for setting in l: self.load_modules(setting) def load_modules(self, setting): name = setting['name'] is_effective = setting['isEffective'] if is_effective == 0: return module_name = 'ultron.cluster.central.extern_modules.' + name + '.module' try: module = importlib.import_module(module_name) if 'Module' in dir(module): strategy_class = module.__getattribute__('Module') self._module_dict[name] = strategy_class(name,self._redis_client) except Exception as e: print('Failed to import module: %s:[%s]' % (name,str(e))) def _get_task(self): while True: for queue in self._queue_list: task_all = self._redis_client.hmgetall(queue) task_list = task_all[0] self._redis_client.hmdel(queue, task_list.keys()) for tid, task in task_list.items(): self._task_queue.put(json.loads(task)) gevent.sleep(.3) def _dispatch_task(self): while True: while not self._task_queue.empty(): task = self._task_queue.get() space_name = str(task.get('name')) if space_name in self._module_dict: self._module_dict[space_name].process_respone(task) gevent.sleep(.3)
def __init__(self, **kwargs): if ('host' in kwargs) and ('port' in kwargs) and ('pwd' in kwargs): self._host = kwargs['host'] self._port = kwargs['port'] self._port = kwargs['pwd'] else: self._host = config_setting.queue_host self._port = config_setting.queue_port self._pwd = config_setting.queue_pwd self._queue = 'ultron:cache' self._zip_level = 9 #创建redis连接 self._redis_client = RedisClient(host=self._host, port=self._port, password=self._pwd, db=6)
def __init__(self, **kwargs): self._module_dict = {} self._redis_client = RedisClient(host=config_setting.queue_host, port=config_setting.queue_port, password=config_setting.queue_pwd) self._secret_key = 'd6f89b09' self._wid = unique_machine self._queue_list = ['ultron:work:work_id:' + str(self._wid)] #生成token self._token = hashlib.sha1( (self._secret_key + self._wid.replace('-', '')).encode()).hexdigest() self._task_queue = Queue() # self.init_modules() gevent.spawn(self._get_task) gevent.spawn(self._dispatch_task) gevent.spawn(self._heart_tick) gevent.sleep()
class CacheData(object): def __init__(self, **kwargs): if ('host' in kwargs) and ('port' in kwargs) and ('pwd' in kwargs): self._host = kwargs['host'] self._port = kwargs['port'] self._port = kwargs['pwd'] else: self._host = config_setting.queue_host self._port = config_setting.queue_port self._pwd = config_setting.queue_pwd self._queue = 'ultron:cache' self._zip_level = 9 #创建redis连接 self._redis_client = RedisClient(host=self._host, port=self._port, password=self._pwd, db=6) def set_cache(self, session, key, values): cache_data_pb = ultron.proto.cache_data_pb2.CacheData() cache_data_pb.session = str(session).encode('raw_unicode_escape') cache_data_pb.key = str(key).encode('raw_unicode_escape') cache_data_pb.data_stream = zlib.compress( bytes(values, encoding='utf8'), self._zip_level) self._redis_client.hset( self._queue, str(key) + str(session), base64.b64encode(cache_data_pb.SerializeToString())) def get_cache(self, session, key, is_del=True): base64_values_str = self._redis_client.hget(self._queue, str(key) + str(session)) if is_del: self._redis_client.hdel(self._queue, str(key) + str(session)) zlilb_values = base64.b64decode(base64_values_str) cache_data_pb = ultron.proto.cache_data_pb2.CacheData() cache_data_pb.ParseFromString(zlilb_values) values = zlib.decompress(cache_data_pb.data_stream) return values
class WorkEngine(object): def __init__(self, **kwargs): self._module_dict = {} self._redis_client = RedisClient(host=config_setting.queue_host, port=config_setting.queue_port, password=config_setting.queue_pwd) self._secret_key = 'd6f89b09' self._wid = unique_machine self._queue_list = ['ultron:work:work_id:' + str(self._wid)] #生成token self._token = hashlib.sha1( (self._secret_key + self._wid.replace('-', '')).encode()).hexdigest() self._task_queue = Queue() # self.init_modules() gevent.spawn(self._get_task) gevent.spawn(self._dispatch_task) gevent.spawn(self._heart_tick) gevent.sleep() def init_modules(self): l = json.loads(moddules_info) for setting in l: self.load_modules(setting) def load_modules(self, setting): name = setting['name'] is_effective = setting['isEffective'] if is_effective == 0: return module_name = 'ultron.cluster.work.extern_modules.' + name + '.module' try: module = importlib.import_module(module_name) if 'Module' in dir(module): strategy_class = module.__getattribute__('Module') self._module_dict[name] = strategy_class( name, self._wid, self._token, self._redis_client) print('module %s loading' % (name)) if name == 'login': self._module_dict[name].login_master() except Exception as e: print('Failed to import module:%s:[%s]' % (name, str(e))) def _heart_tick(self): last_time = datetime.datetime.now() while True: now_time = datetime.datetime.now() if (now_time - last_time).seconds > 20: #发送心跳包 task = {'name': 'login', 'opcode': 'heart_tick'} self._module_dict[task['name']].process_respone(task) last_time = now_time gevent.sleep(.3) def _get_task(self): while True: for queue in self._queue_list: task_all = self._redis_client.hmgetall(queue) task_list = task_all[0] self._redis_client.hmdel(queue, task_list.keys()) for tid, task in task_list.items(): self._task_queue.put(json.loads(task)) gevent.sleep(.3) #用于处理各个节点登录 def _dispatch_task(self): while True: while not self._task_queue.empty(): task = self._task_queue.get() space_name = str(task.get('name')) if space_name in self._module_dict: self._module_dict[space_name].process_respone(task) gevent.sleep(.3)
# -*- coding: utf-8 -*- import sys import json import datetime import time sys.path.append('..') from ultron.utilities.redis.redis_client import RedisClient redis_client = RedisClient(host='47.95.193.202',port=6378,password='******') task_id = time.time() * 1000000 + datetime.datetime.now().microsecond task_info = {'name':'tasks','opcode':'shutoff_task','work_name':'polymeriza','dir_name':'./q7_task'} redis_client.hset('ultron:work:ctask',str(task_id),json.dumps(task_info))
import time import os import zlib import base64 import sys import pdb sys.path.append('..') from ultron.utilities.redis.redis_client import RedisClient def upload(redis_client, dir_name, file_list): upload_info = {} upload_info['name'] = 'packet' upload_info['opcode'] = 'upload_packet' upload_info['uid'] = 1000 upload_info['dir_name'] = dir_name task_id = int(time.time() * 1000000 + datetime.datetime.now().microsecond) file_info = [] for file_name in file_list: with open(os.path.join(dir_name, file_name) ,'r') as f: content = bytes(f.read(), encoding = "utf8") file_info.append({'file_name':file_name, 'content': str(base64.b64encode(zlib.compress(content)),encoding = "utf-8") }) upload_info['file_info'] = file_info pdb.set_trace() redis_client.hset('ultron:work:update',str(task_id),json.dumps(upload_info)) from ultron.utilities.redis.redis_client import RedisClient redis_client = RedisClient(host='47.95.193.202',port=6378,password='******') upload(redis_client, 'q7_task',['polymeriza.py','tasks.py'])