def __init__(self, **kwargs):
self._module_dict = {}
self._redis_client = RedisClient(host=config_setting.queue_host,
port=config_setting.queue_port,
password=config_setting.queue_pwd)
self._queue_list = [LOGIN_QUEUE,CTASK_QUEUE,UPDATE_QUEUE]
self._task_queue = Queue() #
self.init_modules()
gevent.spawn(self._get_task)
gevent.spawn(self._dispatch_task)
gevent.sleep()
class CentralEngine(object):
def __init__(self, **kwargs):
self._module_dict = {}
self._redis_client = RedisClient(host=config_setting.queue_host,
port=config_setting.queue_port,
password=config_setting.queue_pwd)
self._queue_list = [LOGIN_QUEUE,CTASK_QUEUE,UPDATE_QUEUE]
self._task_queue = Queue() #
self.init_modules()
gevent.spawn(self._get_task)
gevent.spawn(self._dispatch_task)
gevent.sleep()
def init_modules(self):
l = json.loads(moddules_info)
for setting in l:
self.load_modules(setting)
def load_modules(self, setting):
name = setting['name']
is_effective = setting['isEffective']
if is_effective == 0:
return
module_name = 'ultron.cluster.central.extern_modules.' + name + '.module'
try:
module = importlib.import_module(module_name)
if 'Module' in dir(module):
strategy_class = module.__getattribute__('Module')
self._module_dict[name] = strategy_class(name,self._redis_client)
except Exception as e:
print('Failed to import module: %s:[%s]' % (name,str(e)))
def _get_task(self):
while True:
for queue in self._queue_list:
task_all = self._redis_client.hmgetall(queue)
task_list = task_all[0]
self._redis_client.hmdel(queue, task_list.keys())
for tid, task in task_list.items():
self._task_queue.put(json.loads(task))
gevent.sleep(.3)
def _dispatch_task(self):
while True:
while not self._task_queue.empty():
task = self._task_queue.get()
space_name = str(task.get('name'))
if space_name in self._module_dict:
self._module_dict[space_name].process_respone(task)
gevent.sleep(.3)
def __init__(self, **kwargs):
if ('host' in kwargs) and ('port' in kwargs) and ('pwd' in kwargs):
self._host = kwargs['host']
self._port = kwargs['port']
self._port = kwargs['pwd']
else:
self._host = config_setting.queue_host
self._port = config_setting.queue_port
self._pwd = config_setting.queue_pwd
self._queue = 'ultron:cache'
self._zip_level = 9
#创建redis连接
self._redis_client = RedisClient(host=self._host,
port=self._port,
password=self._pwd,
db=6)
def __init__(self, **kwargs):
self._module_dict = {}
self._redis_client = RedisClient(host=config_setting.queue_host,
port=config_setting.queue_port,
password=config_setting.queue_pwd)
self._secret_key = 'd6f89b09'
self._wid = unique_machine
self._queue_list = ['ultron:work:work_id:' + str(self._wid)]
#生成token
self._token = hashlib.sha1(
(self._secret_key +
self._wid.replace('-', '')).encode()).hexdigest()
self._task_queue = Queue() #
self.init_modules()
gevent.spawn(self._get_task)
gevent.spawn(self._dispatch_task)
gevent.spawn(self._heart_tick)
gevent.sleep()
class CacheData(object):
def __init__(self, **kwargs):
if ('host' in kwargs) and ('port' in kwargs) and ('pwd' in kwargs):
self._host = kwargs['host']
self._port = kwargs['port']
self._port = kwargs['pwd']
else:
self._host = config_setting.queue_host
self._port = config_setting.queue_port
self._pwd = config_setting.queue_pwd
self._queue = 'ultron:cache'
self._zip_level = 9
#创建redis连接
self._redis_client = RedisClient(host=self._host,
port=self._port,
password=self._pwd,
db=6)
def set_cache(self, session, key, values):
cache_data_pb = ultron.proto.cache_data_pb2.CacheData()
cache_data_pb.session = str(session).encode('raw_unicode_escape')
cache_data_pb.key = str(key).encode('raw_unicode_escape')
cache_data_pb.data_stream = zlib.compress(
bytes(values, encoding='utf8'), self._zip_level)
self._redis_client.hset(
self._queue,
str(key) + str(session),
base64.b64encode(cache_data_pb.SerializeToString()))
def get_cache(self, session, key, is_del=True):
base64_values_str = self._redis_client.hget(self._queue,
str(key) + str(session))
if is_del:
self._redis_client.hdel(self._queue, str(key) + str(session))
zlilb_values = base64.b64decode(base64_values_str)
cache_data_pb = ultron.proto.cache_data_pb2.CacheData()
cache_data_pb.ParseFromString(zlilb_values)
values = zlib.decompress(cache_data_pb.data_stream)
return values
class WorkEngine(object):
def __init__(self, **kwargs):
self._module_dict = {}
self._redis_client = RedisClient(host=config_setting.queue_host,
port=config_setting.queue_port,
password=config_setting.queue_pwd)
self._secret_key = 'd6f89b09'
self._wid = unique_machine
self._queue_list = ['ultron:work:work_id:' + str(self._wid)]
#生成token
self._token = hashlib.sha1(
(self._secret_key +
self._wid.replace('-', '')).encode()).hexdigest()
self._task_queue = Queue() #
self.init_modules()
gevent.spawn(self._get_task)
gevent.spawn(self._dispatch_task)
gevent.spawn(self._heart_tick)
gevent.sleep()
def init_modules(self):
l = json.loads(moddules_info)
for setting in l:
self.load_modules(setting)
def load_modules(self, setting):
name = setting['name']
is_effective = setting['isEffective']
if is_effective == 0:
return
module_name = 'ultron.cluster.work.extern_modules.' + name + '.module'
try:
module = importlib.import_module(module_name)
if 'Module' in dir(module):
strategy_class = module.__getattribute__('Module')
self._module_dict[name] = strategy_class(
name, self._wid, self._token, self._redis_client)
print('module %s loading' % (name))
if name == 'login':
self._module_dict[name].login_master()
except Exception as e:
print('Failed to import module:%s:[%s]' % (name, str(e)))
def _heart_tick(self):
last_time = datetime.datetime.now()
while True:
now_time = datetime.datetime.now()
if (now_time - last_time).seconds > 20: #发送心跳包
task = {'name': 'login', 'opcode': 'heart_tick'}
self._module_dict[task['name']].process_respone(task)
last_time = now_time
gevent.sleep(.3)
def _get_task(self):
while True:
for queue in self._queue_list:
task_all = self._redis_client.hmgetall(queue)
task_list = task_all[0]
self._redis_client.hmdel(queue, task_list.keys())
for tid, task in task_list.items():
self._task_queue.put(json.loads(task))
gevent.sleep(.3)
#用于处理各个节点登录
def _dispatch_task(self):
while True:
while not self._task_queue.empty():
task = self._task_queue.get()
space_name = str(task.get('name'))
if space_name in self._module_dict:
self._module_dict[space_name].process_respone(task)
gevent.sleep(.3)
# -*- coding: utf-8 -*-
import sys
import json
import datetime
import time
sys.path.append('..')
from ultron.utilities.redis.redis_client import RedisClient
redis_client = RedisClient(host='47.95.193.202',port=6378,password='******')
task_id = time.time() * 1000000 + datetime.datetime.now().microsecond
task_info = {'name':'tasks','opcode':'shutoff_task','work_name':'polymeriza','dir_name':'./q7_task'}
redis_client.hset('ultron:work:ctask',str(task_id),json.dumps(task_info))
import time
import os
import zlib
import base64
import sys
import pdb
sys.path.append('..')
from ultron.utilities.redis.redis_client import RedisClient
def upload(redis_client, dir_name, file_list):
upload_info = {}
upload_info['name'] = 'packet'
upload_info['opcode'] = 'upload_packet'
upload_info['uid'] = 1000
upload_info['dir_name'] = dir_name
task_id = int(time.time() * 1000000 + datetime.datetime.now().microsecond)
file_info = []
for file_name in file_list:
with open(os.path.join(dir_name, file_name) ,'r') as f:
content = bytes(f.read(), encoding = "utf8")
file_info.append({'file_name':file_name,
'content': str(base64.b64encode(zlib.compress(content)),encoding = "utf-8") })
upload_info['file_info'] = file_info
pdb.set_trace()
redis_client.hset('ultron:work:update',str(task_id),json.dumps(upload_info))
from ultron.utilities.redis.redis_client import RedisClient
redis_client = RedisClient(host='47.95.193.202',port=6378,password='******')
upload(redis_client, 'q7_task',['polymeriza.py','tasks.py'])