def startRPCServer():
gm_worker = gearman.GearmanWorker([django_settings.GEARMAN_SERVER])
hostID = gethostname() + "_MCPServer"
gm_worker.set_client_id(hostID)
# The tasks registered in this worker should not block.
gm_worker.register_task(
"approveJob", job_approve_handler)
gm_worker.register_task(
"getJobsAwaitingApproval", job_awaiting_approval_handler)
gm_worker.register_task(
"packageCreate", package_create_handler)
gm_worker.register_task(
"approveTransferByPath", approve_transfer_by_path_handler)
gm_worker.register_task(
"approvePartialReingest", approve_partial_reingest_handler)
gm_worker.register_task(
"getProcessingConfigFields", get_processing_config_fields_handler)
failMaxSleep = 30
failSleep = 1
failSleepIncrementor = 2
while True:
try:
gm_worker.work()
except gearman.errors.ServerUnavailable:
time.sleep(failSleep)
if failSleep < failMaxSleep:
failSleep += failSleepIncrementor
def __init__(self):
#self.worker=CustomGearmanWorker(['sandbox:4730'],max_jobs=1)
self.worker = gearman.GearmanWorker(['sandbox:4730'])
self.perfdata = GearmanPerfData(
secret=
'Aloh9uibshojeF8oAhyo3eefGu5ohr3iDeek4ehamaM9eisoas6OoveiareQuo0i')
self.worker.register_task('perfdata', self.perfdata.getData)
def start_gearman_worker(supported_modules):
"""Setup a gearman client, for the thread."""
gm_worker = gearman.GearmanWorker([django_settings.GEARMAN_SERVER])
host_id = '{}_{}'.format(gethostname(), os.getpid())
gm_worker.set_client_id(host_id)
task_handler = partial(execute_command, supported_modules)
for client_script in supported_modules:
logger.info('Registering: %s', client_script)
gm_worker.register_task(client_script, task_handler)
fail_max_sleep = 30
fail_sleep = 1
fail_sleep_incrementor = 2
while True:
try:
gm_worker.work()
except gearman.errors.ServerUnavailable as inst:
logger.error(
'Gearman server is unavailable: %s. Retrying in %d'
' seconds.', inst.args, fail_sleep)
time.sleep(fail_sleep)
if fail_sleep < fail_max_sleep:
fail_sleep += fail_sleep_incrementor
except Exception as e:
# Generally execute_command should have caught and dealt with any
# errors gracefully, but we should never let an exception take down
# the whole process, so one last catch-all.
logger.exception(
'Unexpected error while handling gearman job: %s.'
' Retrying in %d seconds.', e, fail_sleep)
time.sleep(fail_sleep)
if fail_sleep < fail_max_sleep:
fail_sleep += fail_sleep_incrementor
def __init__(self, config):
self.config = config
self.worker = gearman.GearmanWorker([
'%s:%s' %
(self.config.get_gearman_host(), self.config.get_gearman_port())
])
self.worker.set_client_id(self.config.get_worker_name())
def dowork():
import os
gw = gearman.GearmanWorker(Servers)
gw.set_client_id('work_%s' % os.getpid()) #optional
gw.register_task(workfn, task_fans)
# Enter our work loop
gw.work()
def startRPCServer():
logger = logging.getLogger("archivematica")
logger.addHandler(
GroupWriteRotatingFileHandler(
"/var/log/archivematica/MCPServer/MCPServer.log",
maxBytes=4194304))
gm_worker = gearman.GearmanWorker(
[archivematicaMCP.config.get('MCPServer', 'GearmanServerWorker')])
hostID = gethostname() + "_MCPServer"
gm_worker.set_client_id(hostID)
gm_worker.register_task("approveJob", gearmanApproveJob)
gm_worker.register_task("getJobsAwaitingApproval",
gearmanGetJobsAwaitingApproval)
failMaxSleep = 30
failSleep = 1
failSleepIncrementor = 2
while True:
try:
gm_worker.work()
except gearman.errors.ServerUnavailable as inst:
#print >>sys.stderr, inst.args
#print >>sys.stderr, "Retrying in %d seconds." % (failSleep)
time.sleep(failSleep)
if failSleep < failMaxSleep:
failSleep += failSleepIncrementor
def __init__(self, redisClient, gearmanHost):
BasicGeodisWorker.__init__(self, redisClient)
self.worker = gearman.GearmanWorker([gearmanHost])
self.worker.register_task('geodis_lookup_city', self.lookup)
print "Registered worker"
def start_gearman(cls, gearman_servers):
"""
Connect to gearmand and fire up a worker
"""
gearman_worker = gearman.GearmanWorker(gearman_servers or ['localhost:4730'])
gearman_worker.register_task('ponyexpress', cls.from_gearman)
gearman_worker.work()
return True
def handle(self, *args, **options):
task_name = 'task_%s' % options['queue']
if task_name not in dir(self):
sys.exit(1)
task_func = lambda x, y: getattr(Command, task_name)(self, x, y)
gm_worker = gearman.GearmanWorker(settings.GEARMAN_JOB_SERVERS)
gm_worker.register_task(options['queue'], task_func)
workers_logger.info('Started worker with tasks: %s' % task_name)
gm_worker.work()
def run(self):
threading.Thread(target=self._shutdown_thread).start()
self.worker = gearman.GearmanWorker(self.rpc_server.host_list)
for service_name, (_, initializer) in six.iteritems(
self.rpc_server.service_mappings):
if initializer is not None:
initializer()
self.worker.register_task(service_name, self.gm_task_callback)
try:
self.worker.work(5.0)
except gearman.errors.ServerUnavailable:
pass
def handle(self, *args, **options):
gm_worker = gearman.GearmanWorker(settings.GEARMAN_JOB_SERVERS)
registered_tasks = []
for task_name in dir(self):
if task_name.startswith('task_'):
task_name = str(task_name)
t_name = task_name.replace('task_', '')
task_func = lambda i: (lambda x, y: getattr(Command, i)
(self, x, y))
gm_worker.register_task(t_name, task_func(task_name))
registered_tasks.append(t_name)
workers_logger.info('Started worker with tasks: %s' %
', '.join(registered_tasks))
gm_worker.work()
def __init__(self, task_limit=10):
with open('config.yml', 'r') as cnfg:
self.config = yaml.load(cnfg, Loader=yaml.FullLoader)
self._task_limit = task_limit
self._count_current_tasks = 0
self._current_tasks = []
self._timer = 0
self.gearman_client = gearman.GearmanClient(
[self.config.get('GJS_1').get('job_server_url')])
self.gearman_worker = gearman.GearmanWorker(
[self.config.get('GJS_1').get('job_server_url')])
self.gearman_worker.register_task(
self.config.get('worker_ids').get('async_governer'), self.governer)
ready_message = pyfiglet.figlet_format("Worker Ready!!")
print(ready_message)
def handle(self, *args, **options):
logger.info('Starting worker')
task_name = 'task_%s' % options['queue']
logger.info('Task: %s' % task_name)
if task_name not in dir(self):
logger.info("Wow.. That's crazy! Maybe try an existing queue?")
sys.exit(1)
task_func = lambda x, y: getattr(Command, task_name)(self, x, y)
logger.info('Initializing gm_worker')
gm_worker = gearman.GearmanWorker(settings.GEARMAN_JOB_SERVERS)
logger.info('Registering task %s, function %s' % (task_name, task_func))
gm_worker.register_task(options['queue'], task_func)
logger.info('Starting work')
gm_worker.work()
logger.info('Ended work')
def startRPCServer():
gm_worker = gearman.GearmanWorker([django_settings.GEARMAN_SERVER])
hostID = gethostname() + "_MCPServer"
gm_worker.set_client_id(hostID)
gm_worker.register_task("approveJob", gearmanApproveJob)
gm_worker.register_task("getJobsAwaitingApproval",
gearmanGetJobsAwaitingApproval)
failMaxSleep = 30
failSleep = 1
failSleepIncrementor = 2
while True:
try:
gm_worker.work()
except gearman.errors.ServerUnavailable:
time.sleep(failSleep)
if failSleep < failMaxSleep:
failSleep += failSleepIncrementor
def handle(self, *args, **options):
# N.B. don't take out the print statements as they're
# very very very very very very very very very very
# helpful in debugging supervisor+worker+gearman
self.write_stdout('Starting worker\n')
task_name = 'task_%s' % options['queue']
self.write_stdout('Task: %s\n' % task_name)
if task_name not in dir(self):
self.write_stdout("Wow.. That's crazy! Maybe try an existing queue?\n")
sys.exit(1)
task_func = lambda x, y: getattr(Command, task_name)(self, x, y)
self.write_stdout('Initializing gm_worker\n')
gm_worker = gearman.GearmanWorker(settings.GEARMAN_JOB_SERVERS)
self.write_stdout('Registering task %s, function %s\n' % (task_name, task_func))
gm_worker.register_task(options['queue'], task_func)
self.write_stdout('Starting work\n')
gm_worker.work()
self.write_stdout('Ended work\n')
def startRPCServer():
gm_worker = gearman.GearmanWorker(
[archivematicaMCP.config.get('MCPServer', 'GearmanServerWorker')])
hostID = gethostname() + "_MCPServer"
gm_worker.set_client_id(hostID)
gm_worker.register_task("approveJob", gearmanApproveJob)
gm_worker.register_task("getJobsAwaitingApproval",
gearmanGetJobsAwaitingApproval)
failMaxSleep = 30
failSleep = 1
failSleepIncrementor = 2
while True:
try:
gm_worker.work()
except gearman.errors.ServerUnavailable:
time.sleep(failSleep)
if failSleep < failMaxSleep:
failSleep += failSleepIncrementor
def startThread(threadNumber):
"""Setup a gearman client, for the thread."""
gm_worker = gearman.GearmanWorker(
[config.get('MCPClient', "MCPArchivematicaServer")])
hostID = gethostname() + "_" + threadNumber.__str__()
gm_worker.set_client_id(hostID)
for key in supportedModules.iterkeys():
printOutputLock.acquire()
print "registering:", '"' + key + '"'
printOutputLock.release()
gm_worker.register_task(key, executeCommand)
#load transoder jobs
sql = """SELECT CommandRelationships.pk
FROM CommandRelationships
JOIN Commands ON CommandRelationships.command = Commands.pk
JOIN CommandsSupportedBy ON Commands.supportedBy = CommandsSupportedBy.pk
WHERE CommandsSupportedBy.description = 'supported by default archivematica client';"""
rows = databaseInterface.queryAllSQL(sql)
if rows:
for row in rows:
CommandRelationshipsPK = row[0]
key = "transcoder_cr%s" % (CommandRelationshipsPK.__str__())
printOutputLock.acquire()
print "registering:", '"' + key + '"'
printOutputLock.release()
gm_worker.register_task(
key, transcoderNormalizer.executeCommandReleationship)
failMaxSleep = 30
failSleep = 1
failSleepIncrementor = 2
while True:
try:
gm_worker.work()
except gearman.errors.ServerUnavailable as inst:
print >> sys.stderr, inst.args
print >> sys.stderr, "Retrying in %d seconds." % (failSleep)
time.sleep(failSleep)
if failSleep < failMaxSleep:
failSleep += failSleepIncrementor
def handle(self, *args, **options):
# N.B. don't take out the print statements as they're
# very very very very very very very very very very
# helpful in debugging supervisor+worker+gearman
self.write_stdout('Initializing gm_worker\n')
gm_worker = gearman.GearmanWorker(settings.GEARMAN_JOB_SERVERS)
# Read all methods of the class and if it starts with 'task_' then
# register as a task on gearman
for task_name in dir(self):
if task_name.startswith('task_'):
task_name = str(task_name)
t_name = task_name.replace('task_', '')
self.write_stdout('Task: %s\n' % t_name)
task_func = lambda i: (lambda x, y: getattr(Command, i)
(self, x, y))
gm_worker.register_task(t_name, task_func(task_name))
self.write_stdout('Starting work\n')
gm_worker.work()
self.write_stdout('Ended work\n')
def startThread(threadNumber):
"""Setup a gearman client, for the thread."""
gm_worker = gearman.GearmanWorker(
[config.get('MCPClient', "MCPArchivematicaServer")])
hostID = gethostname() + "_" + threadNumber.__str__()
gm_worker.set_client_id(hostID)
for key in supportedModules.keys():
logger.info('Registering: %s', key)
gm_worker.register_task(key, executeCommand)
failMaxSleep = 30
failSleep = 1
failSleepIncrementor = 2
while True:
try:
gm_worker.work()
except gearman.errors.ServerUnavailable as inst:
logger.error(
'Gearman server is unavailable: %s. Retrying in %d seconds.',
inst.args, failSleep)
time.sleep(failSleep)
if failSleep < failMaxSleep:
failSleep += failSleepIncrementor
def startThread(threadNumber):
"""Setup a gearman client, for the thread."""
gm_worker = gearman.GearmanWorker([config.get('MCPClient', "MCPArchivematicaServer")])
hostID = gethostname() + "_" + threadNumber.__str__()
gm_worker.set_client_id(hostID)
for key in supportedModules.iterkeys():
printOutputLock.acquire()
print 'registering:"{}"'.format(key)
printOutputLock.release()
gm_worker.register_task(key, executeCommand)
failMaxSleep = 30
failSleep = 1
failSleepIncrementor = 2
while True:
try:
gm_worker.work()
except gearman.errors.ServerUnavailable as inst:
print >>sys.stderr, inst.args
print >>sys.stderr, "Retrying in %d seconds." % (failSleep)
time.sleep(failSleep)
if failSleep < failMaxSleep:
failSleep += failSleepIncrementor
import traceback
import torch
import numpy as np
from multiprocessing import Process, Queue, Pool, Manager
import os
from PIL import Image
from tool.utils import transform_extension_path
sys.path.append('/home/user/workspace/priv-0220/Pet-engine')
from core.semseg_priv_config import cfg_priv
sys.path.append(cfg_priv.PET_ROOT)
from modules import pet_engine
gm_worker = gearman.GearmanWorker(['localhost:4730'])
semseg_inference = None
q_file = Queue()
q_name = Queue()
def read_q(q_file, q_name):
while True:
if q_file.empty() and q_name.empty():
file = q_file.get(True)
name = q_name.get(True)
print('name -- {}'.format(name))
np.save(name, file)
import sys
import urllib2
#import urllib.request as urlreq
from bs4 import BeautifulSoup as bs
import requests
from multiprocessing import Pool
import os
import getopt
import threading
import wikipedia
import sys
import codecs
from StringIO import StringIO
gm_worker = gearman.GearmanWorker(['10.0.2.13:4730'])
def linklisting(search):
linklist = []
url = "http://www.google.com/search?"
payload = {'q': search}
try:
r = requests.get(url, payload, timeout=10)
#print(r.url)
print "--------------------------------".encode('utf-8')
print "\t", search, "\t".encode('utf-8')
print "--------------------------------".encode('utf-8')
# print "1".encode('utf-8')
if r.status_code == 200:
html = r.text
import gearman
gm_worker = gearman.GearmanWorker(['172.26.183.16:4735', '172.26.183.15:4735'])
def task_listener_reverse(gearman_worker, gearman_job):
print 'Reversing string: ' + gearman_job.data
return gearman_job.data[::-1]
# gm_worker.set_client_id is optional
# gm_worker.set_client_id('python-worker')
gm_worker.register_task('reverse', task_listener_reverse)
# Enter our work loop and call gm_worker.after_poll() after each time we timeout/see socket activity
gm_worker.work()
from collections import defaultdict
from elasticsearch import Elasticsearch
# 存储csv的目录
csv_path = '../data/csvs/'
# 定义gearman server
GEARMAN_SERVERS = ['127.0.0.1:4730']
ES_HOST = '127.0.0.1'
ES_PORT = 9200
# 建立链接
gearman_worker = gearman.GearmanWorker(GEARMAN_SERVERS)
# 调用分词器
def es_analyze_name(name):
# 建立链接
es = Elasticsearch([{'host': ES_HOST, 'port': ES_PORT }])
# 调用分词器
res = es.indices.analyze('customers',body={'analyzer':'ik_smart','text': name})
words = res['tokens']
# 解析分词结果
word_list = []
for word in words :
word_list.append(word['token'])
# 记录日志
print_log('create_live_activity',
'%s ==========================> Finished' % (data['event_id']))
return ''
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-env',
action='store',
dest='wxenv',
required=True,
help='Test|Stage|Production')
args = parser.parse_args(sys.argv[1:])
wxenv = args.wxenv
if wxenv not in ['Local', 'Test', 'Stage', 'Production', 'UnitTest']:
raise EnvironmentError('The environment variable (WXENV) is invalid ')
os.environ['WXENV'] = wxenv
sys.path.append(dirname(dirname(dirname(abspath(__file__)))))
from wanx import app
servers = app.config.get("GEARMAN_SERVERS")
gm_worker = gearman.GearmanWorker(servers)
gm_worker.set_client_id('create_live_activity')
gm_worker.register_task('create_live_activity', do_task)
try:
gm_worker.work()
except:
pass
gm_worker.unregister_task('create_live_activity')
import time
import json
from addtask import addtask
from lib.cmdLib import *
def gearmanwork(gearman_worker, gearman_job):
params = {"RET": "INTERNAL ERROR"}
try:
params = json.loads(gearman_job.data)
'''addtask(params['JenkinsURL'], params['JobName'], params['BuildParams'])'''
cmdstr = "python addtask.py " + params['JenkinsURL'] + " " + params[
'JobName'] + " '" + json.dumps(params['BuildParams']) + "'"
print cmdstr
status, output = cmd_execute(cmdstr)
except Exception as e:
print(e)
finally:
print json.dumps(params)
return json.dumps(params)
if __name__ == '__main__':
try:
worker = gearman.GearmanWorker(["127.0.0.1:8899"])
worker.register_task('gearmanwork', gearmanwork)
worker.work()
except Exception as e:
print(e)
def worker():
gm_worker = gearman.GearmanWorker(['localhost:4730'])
gm_worker.register_task('index', work)
gm_worker.work()
# coding=utf-8
from Base import *
from scrapy import Selector
import requests
import gearman
import pymongo
import uniout
import hashlib
import time
import re
JobWorker = gearman.GearmanWorker([app_config['job']['gearman']])
Mongo = pymongo.MongoClient(host=app_config['db']['mongo'],
socketTimeoutMS=None,
socketKeepAlive=True)
Collection = Mongo['wulo']
Database = Collection['data']
RAWDB = Collection['raw']
_hash = hashlib.sha224()
# res = requests.get('https://www.ptt.cc/bbs/StupidClown/M.1457268374.A.827.html')
# res = requests.get('https://www.ptt.cc/bbs/StupidClown/M.1457268828.A.363.html')
# res = requests.get('https://www.ptt.cc/bbs/StupidClown/M.1457273711.A.902.html')
res = requests.get('https://www.ptt.cc/bbs/LGBT_SEX/M.1456717141.A.C4D.html',
headers={'cookie': ';over18=1;'})
if res.status_code == 200:
_hash.update(res.url)
hexhash = _hash.hexdigest()
RAWDB.update_one(
import requests
import json
import time
import sys
import traceback
import cv2
import requests
from urllib.parse import quote
import gearman
import base64
import numpy as np
gm_worker = gearman.GearmanWorker(['127.0.0.1:4730'])
def task_listener_reverse(gearman_worker, gearman_job):
print("receve data")
s=gearman_job.data
img_s = json.loads(s)
d64 = base64.b64decode(img_s['imgs'])
nparr = np.fromstring(d64, np.uint8)
image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
cv2.imwrite('./imgs/woker_py.jpg', image)
# 2 维度数组
bbox = img_s['bbox']
print(bbox)
print("write img done")
#!/usr/bin/python
import gearman, json, hashlib, argparse
parser = argparse.ArgumentParser()
parser.add_argument('server', help='Address of server')
parser.add_argument('-p', '--port', help='Port at the server', default='4730')
args = parser.parse_args()
def crack(gearman_worker, gearman_job):
var = gearman_job.data
var = json.loads(var)
hash = var['hash']
words = var['words']
for i in words:
word = i.rstrip().lower()
hash_object = hashlib.sha256(word)
hex_dig = hash_object.hexdigest()
print 'Word:' + i + 'Hash:' + hex_dig + '\n'
if hex_dig == hash:
return str(word)
return ""
gm_worker = gearman.GearmanWorker([args.server + ':' + args.port])
gm_worker.register_task('crack', crack)
gm_worker.work()
