コード例 #1
0
ファイル: task_multi_thread.py プロジェクト: limingmax/WFCode
    def load_task_class(self):
        for parent, dirnames, filenames in os.walk(
                relative_project_path('src')):
            for filename in filenames:
                if filename.endswith('.py'):
                    with open(os.path.join(parent, filename)) as f:
                        code = f.read()
                    if ('##multi_task' in code) and ('##hahaha' not in code):
                        package_name = parent[len(relative_project_path('src/')
                                                  ):].replace('/', '.')

                        module_name = package_name + '.' + filename[:filename.
                                                                    find('.')]
                        # print(module_name)
                        try:
                            # print('name:', module_name)
                            module = importlib.import_module(module_name)
                            class_list = dir(module)
                            for class_name in class_list:

                                try:
                                    taskclass = getattr(module, class_name)

                                    if issubclass(
                                            taskclass, TaskMultiTemplate
                                    ) and taskclass != TaskMultiTemplate:
                                        # print(class_name)
                                        print('add multi task:',
                                              module_name + '.' + class_name)
                                        run_instance = taskclass()
                                        self.task_dict[run_instance.task_name(
                                        )] = {
                                            'run_instance':
                                            run_instance,
                                            'logger':
                                            Logger(self.log_level,
                                                   run_instance.task_name(),
                                                   self.is_debug),
                                            'recv_task_redis':
                                            TaskRedis(run_instance.task_name(),
                                                      self.redis_host,
                                                      self.redis_port,
                                                      self.redis_timeout)
                                        }
                                    # print(class_name)
                                except:
                                    pass
                        except:
                            pass
コード例 #2
0
ファイル: myclass.py プロジェクト: limingmax/threshold
    def __init__(self):
        self.logger = Logger(Logger.INFO, 'myclass', True)
        threading.Thread.__init__(self)
        self.logger().info("=====开始初始化=====")
        self.data = data
        cf = configparser.ConfigParser()
        cf.read("python.ini")
        self.kafka_host = cf.get("kafka", "host")
        self.kafka_port = cf.getint("kafka", "port")
        self.hbase_host = cf.get("hbase", "host")
        self.hbase_port = cf.getint("hbase", "port")
        self.consumer_topic = cf.get("kafka", "consumer_topic")
        self.producer_topic = cf.get("kafka", "producer_topic")
        #	print self.consumer_topic
        #	print self.producer_topic
        print self.kafka_port
        print self.kafka_host
        # self.kafka_host = '192.168.212.71'  # kafka服务器地址
        # self.kafka_port = 9092  # kafka服务器端口
        # self.hbase_host='192.168.195.1'
        # self.hbase_port=9090
        # print(self.kafka_host, self.kafka_port, self.hbase_host, self.hbase_port,self.consumer_topic,self.producer_topic)

        # self.connection=happybase.Connection(self.hbase_host, self.hbase_port)
        self.producer = KafkaProducer(
            bootstrap_servers=[
                '{kafka_host}:{kafka_port}'.format(kafka_host=self.kafka_host,
                                                   kafka_port=self.kafka_port)
            ],
            value_serializer=lambda v: json.dumps(v).encode('utf-8'))
        self.consumer = KafkaConsumer(
            # 'k8s_MonitorData',
            self.consumer_topic,
            # 'ai_threshold_alert',
            group_id='mygroup1',
            enable_auto_commit=False,
            # auto_offset_reset='earliest',
            bootstrap_servers=[
                '{kafka_host}:{kafka_port}'.format(kafka_host=self.kafka_host,
                                                   kafka_port=self.kafka_port)
            ],
            # 一定要设置下列两个参数。官网里有参数的解释https://kafka-python.readthed...
            # 主要是这句话:“ If no heartbeats are received by the broker before the expiration of this session timeout, then the broker will remove this consumer from the group and initiate a rebalance.”
            session_timeout_ms=6000,
            heartbeat_interval_ms=2000)
        self.logger().info("====初始化完成====")
コード例 #3
0
ファイル: task-bak.py プロジェクト: limingmax/WFCode
    def __init__(self, task_name, model_class, model_config,
                 redis_host, redis_port, redis_timeout,
                 cuda, gpu_mem, allow_growth,
                 log_level, is_debug):

        cuda = str(cuda)
        self.task_name = task_name
        self.task_uname = task_name + cuda
        self.task_redis = TaskRedis(task_name, redis_host, redis_port, redis_timeout)
        self.task_redis.upload_data_convert(model_class)
        self.input_convert, self.output_convert = self.task_redis.create_data_convert()
        self.input_convert = self.input_convert()
        self.output_convert = self.output_convert()

        if cuda is not None:
            self.sess = TFSession(cuda, gpu_mem, allow_growth).get_sess()
        else:
            self.sess = None
        self.logger = Logger(log_level, self.task_uname, is_debug)
        self.model_class_instance = model_class(self.sess, self.task_redis,model_config)
        self.task_run = self.model_class_instance.run
        self.kill_last_task()
コード例 #4
0
        return aaaa


# print(12)
t = MyClass()
t.start()
# print(11)
cf = configparser.ConfigParser()
cf.read("python.ini")
redis_server = cf.get("register", "address")

host = cf.get("mysql", "host").encode()
port = int(cf.get("mysql", "port"))
user = cf.get("mysql", "user").encode()
passwd = cf.get("mysql", "passwd").encode()
db = cf.get("mysql", "db").encode()
charset = cf.get("mysql", "charset").encode()
minute = int(cf.get("mysql", "minute"))
Polling(data, minute, host, port, user, passwd, db, charset).run()

logger = Logger(Logger.INFO, 'example', True)
# 参数:日志级别   日志文件名   日志是否控制台输出

# print(redis_server)
# print(type(redis_server))
server = ThreadpoolServer(redis_server, [ExampleSvr], logger, 'JSON', 30, 30,
                          1)
#          消息系统地址    服务列表     日志     交互协议    线程数量

server.start()