def send(cls, request: SeeyaRequest, clazz: Serializable.__class__): try: client = GearmanClient(["localhost:4730"]) workload = request.to_json() response = client.submit_job(cls.QUEUE, workload, background=False) cls.log_conversation(request, response.result) return clazz.from_json(response.result) finally: client.shutdown()
class TaskController(object): """任务管理类,通过gearman提交任务,查看任务数量等""" def __init__(self): self.gm_client = GearmanClient( [config.GEARMAN_HOST + ':' + config.GEARMAN_PORT]) self.gm_admin_client = GearmanAdminClient( [config.GEARMAN_HOST + ':' + config.GEARMAN_PORT]) def task_list(self): return self.gm_admin_client.get_status() def add_task(self, name): data = {} # 后期扩展,任务附加数据 self.gm_client.submit_job(name, str(data), background=True)
class GearmanScheduler(Scheduler): def __init__(self, host, port, max_retries, poll_timeout): server = ":".join([host, str(port)]) self.connection = GearmanClient([server]) self.max_retries = max_retries self.poll_timeout = poll_timeout def close(self): self.connection.shutdown() def schedule(self, task_name, payload): response = self.connection.submit_job(task_name, json.dumps(payload), background=False, priority=PRIORITY_LOW, max_retries=self.max_retries, poll_timeout=self.poll_timeout) if response.timed_out: raise SchedulerTimedOut('Scheduler request timed out') return json.loads(response.result)
def __init__(self): self.gm_client = GearmanClient( [config.GEARMAN_HOST + ':' + config.GEARMAN_PORT]) self.gm_admin_client = GearmanAdminClient( [config.GEARMAN_HOST + ':' + config.GEARMAN_PORT])
# -*- coding: utf-8 -*- from python3_gearman import GearmanClient import time import types import json import types import msgpack import logging import logging import sys import os import multiprocessing import traceback cv_client = GearmanClient(['192.168.8.39:4731']) def get_cv_info(cv_id, corp_map=None, corp_client=None): global cv_client """ get the work experice, eg: corporation_name, position infomation """ skip = int(len(cv_id) / 20 + 0.5) num = 0 for i in range(0, skip + 1): req_dict = { 'header': { 'uid': '1', 'uname': '', 'version': '1',
def __init__(self, worker_name, worker_info): self.client = GearmanClient(worker_info) self.worker_name = worker_name
class JDFromGearman(object): def __init__(self, worker_name, worker_info): self.client = GearmanClient(worker_info) self.worker_name = worker_name def getJson(self, jid): msg = { 'header': { 'product_name': 'algo_rec_test', 'uid': '0x666', 'log_id': '0x666' }, 'request': { 'c': 'positions/logic_position', 'm': 'read', 'p': { 'ids': [int(jid)], 'selected': "" } } } #try: resp = self.client.submit_job(self.worker_name, json.dumps(msg)) res = json.loads(resp.result) #print(json.dumps(res, indent=4, ensure_ascii=False)) #res["response"]["results"][str(jid)]["jd_comment"] = json.loads(res["response"]["results"][str(jid)]["jd_comment"]) #print(json.dumps(res, ensure_ascii=False, indent="\t")) res = res["response"]["results"][str(jid)] return res def get(self, jid): msg = { 'header': { 'product_name': 'algo_rec_test', 'uid': '0x666', 'log_id': '0x666' }, 'request': { 'c': 'positions/logic_position', 'm': 'read', 'p': { 'ids': [int(jid)], 'selected': "" } } } #try: resp = self.client.submit_job(self.worker_name, json.dumps(msg)) res = json.loads(resp.result) print(json.dumps(res, indent=4, ensure_ascii=False)) #res["response"]["results"][str(jid)]["jd_comment"] = json.loads(res["response"]["results"][str(jid)]["jd_comment"]) #print(json.dumps(res, ensure_ascii=False, indent="\t")) res = res["response"]["results"][str(jid)] #print(json.dumps(res, ensure_ascii=False, indent="\t")) title = " ".join([ s for s in re.split(r"[ _::\-]", res["name"]) if not re.search(r"(公司$|集团$|^[0-9]+$)", s) ]) city = list(json.loads(res["city_ids"]).items()) city = city[0][1] if city else "" desc = res["description"] req = res["requirement"] try: comp = json.loads( res["jd_original_corporations"] )[0]["company_info"] if "jd_original_corporations" in res else {} ckwds = comp["keyword"] if "keyword" in comp else [] except Exception as e: ckwds = [] if not desc or not req: arr = sorted(re.split(r"(^|\n|\\n)[^ ]{2,5}[::]", desc + req), key=lambda x: len(x), reverse=True) if len(arr) > 1: desc = arr[0] req = arr[1] return title, city, desc, req, ckwds #except Exception as e: # pass return "", "", "", "", []
parsed_args = parser.parse_args() ############################ # Set up logging before we do any other argument parsing (so that we # can log problems with argument parsing). LOGGING_FORMAT = '%(asctime)-15s %(levelname)s - %(message)s' logging.basicConfig(format=LOGGING_FORMAT) LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} parsed_args.verbosity = min(parsed_args.verbosity, max(LOG_LEVELS)) logging.getLogger().setLevel(LOG_LEVELS[parsed_args.verbosity]) logging.debug("Creating Geaman Client...") gm_client = GearmanClient([ovdm.get_gearman_server()]) time.sleep(10) while True: CURRENT_SEC = 0 while True: t = time.gmtime() if CURRENT_SEC < t.tm_sec: CURRENT_SEC = t.tm_sec time.sleep(60-t.tm_sec) else: break collection_system_transfers = ovdm.get_active_collection_system_transfers()
def __init__(self, host, port, max_retries, poll_timeout): server = ":".join([host, str(port)]) self.connection = GearmanClient([server]) self.max_retries = max_retries self.poll_timeout = poll_timeout