def test_aaa(self):
p2 = multiprocessing.Process(target = run)
p2.start()
# t1 = threading.Thread(target=run)
# t1.setDaemon(True)
# t1.start()
loadMoudle = LoadMoudleHolder.getLoadMoudle(PYTHON)
# clientDir = ConfigHolder.getConfig().getProperty(BASE,CLIENT_FILE_DIRECTORY)
# path = clientDir+"/"+jobId
# print path
# print job.fileName
path="/Users/hua/workspace/python/cabbage/client_file_path/123456789"
import sys
sys.path.append(path)
classes =loadMoudle.load(path,"test_task.py")
classes2 =loadMoudle.load("/Users/hua/workspace/python/cabbage/client_file_path/1234567890","test_task1.py")
time.sleep(5)
CabbageHolder.getCabbage().stop(hostName="celery@huamac")
print "end"
p2.join()
print "start"
# t2 = threading.Thread(target=run)
# t2.setDaemon(True)
# t2.start()
p2 = multiprocessing.Process(target = run)
p2.start()
p2.join()
def stopService(self):
if (CabbageHolder.getServerCabbage(self.work.brokerServer).workIsAlive(
self.work.hostName)):
CabbageHolder.getServerCabbage(self.work.brokerServer).stop(
self.work.hostName)
zope.event.notify(WorkStatusEvent(self.work.hostName, OFF_LINE))
def addBroberServerHandler(event):
if event and event.brokerServer:
brokerServer = event.brokerServer
Logger.info(log,"添加队列服务器【%s】,URI:【%s】"%( brokerServer.hostName,brokerServer.connectUri))
cabbage = Cabbage(hostName=brokerServer.hostName,broker=brokerServer.connectUri)
CabbageHolder.getServerCabbages()[brokerServer.hostName]= cabbage
Logger.debug(log,"添加队列服务器【%s】"% CabbageHolder.getServerCabbagesStr())
def test_update_audit_job_pass(self):
CabbageHolder.getCabbage()
# self.kazooClient = ZookeeperClientHolder.getClient()
# self.store=ZookeeperStore()
# self.jobId="123456789"
# self.kazooClient.addChildListener("/cabbage/jobs", jobChildWatch)
#
# #
# t1 = threading.Thread(target=start_server)
# t1.setDaemon(True)
# t1.start()
# # time.sleep(10)
# self.store.updateAuditStatus(self.jobId,JOB_AUTH_PASS)
# time.sleep(15)
# for i in range(10):
zope.event.notify(JobRunEvent("job-a696407a-20c3-4a07-a68f-260c42157ddb"))
def run(self):
registerClientEvent()
work = Work()
work.hostName = getHostName()
work.port = "1024"
work.status = "aa"
self.kazooClient = ZookeeperClientHolder.getClient()
self.store = ZookeeperStore()
self.kazooClient.addChildListener("/cabbage/jobs", jobChildWatch)
self.store.saveWork(work)
self.t1 = threading.Thread(target=run)
self.t1.setDaemon(True)
self.t1.start()
time.sleep(5)
self.app = CabbageHolder.getCabbage().getApp()
# clientDir = ConfigHolder.getConfig().getProperty(BASE,CLIENT_FILE_DIRECTORY)
# if os.path.exists(clientDir) is False:
# os.mkdir(clientDir)
time.sleep(10)
i = self.app.control.inspect()
result = i.stats()
print result
print i.registered_tasks()
print self.app.events.State().workers.items()
def serviceIsAlive(self):
if len(
CabbageHolder.getServerCabbage(self.work.brokerServer).ping(
self.work.hostName)) > 0:
return True
return False
def start():
#使用子进程，加载任务，类沙箱的概念
cabbage = CabbageHolder.getCabbage()
self._initJobs(cabbage)
t1 = threading.Thread(target=run, args=(cabbage, ))
t1.setDaemon(True)
t1.start()
t1.join()
def brokerServerWatch(children):
for brokerName in children:
if not CabbageHolder.getServerCabbages().has_key(brokerName):
store = storeFactory.getStore()
try:
brokerServer = store.getBrokerServer(brokerName)
zope.event.notify(AddBrokerServerEvent(brokerServer))
finally:
storeFactory.returnStroe(store)
def monitorBroberServerHandler(event):
if event and event.brokerServer:
brokerServer = event.brokerServer
cabbage = CabbageHolder.getServerCabbages().get(brokerServer.hostName)
def monitor(cabbage):
Logger.info(log, "添加监控【%s】,URI:【%s】"%( brokerServer.hostName,brokerServer.connectUri))
cabbage_monitor(cabbage.getApp())
Logger.info(log, "添加监控结束")
t1 = threading.Thread(target=monitor,args=(cabbage,))
t1.setDaemon(True)
t1.start()
def taskSucceeded(state, event, app):
taskId = event['uuid']
task = state.tasks.get(taskId)
# taskName = task.name if task and hasattr(task,'name') else None
jobId, taskName = _getJobIdAndTaskName(taskId)
queueTime = 0
runtime = 0
# state.
#FIXME 经常找不到TASK.name
#@TODO
log.debug("【%s】 TASK SUCCEEDED !" % (event['uuid']))
try:
# jobId= None
# if hasattr(task,'kwargs') and task.kwargs is not None and JOB_ID in task.kwargs:
# jobId= eval(str(task.kwargs))[JOB_ID]
#
# if jobId is None and taskName:
# jobId = TaskCacheHolder.getJobCache().get(taskName)
#
# if taskName is None or jobId is None:
# jobId,taskName = _getJobIdAndTaskName(taskId)
# print jobId,taskName
job = CacheHolder.getCache().get(jobId, JOBS)
result = AsyncResult(taskId,
app=CabbageHolder.getServerCabbage(
job.brokerServer).getApp())
if not isinstance(result.backend, DisabledBackend):
log.debug("【%s】 TASK SUCCEEDED result【%s】 !" %
(event['uuid'], result.result))
if result.result:
jobResults.addResult(jobId, result.result)
# # print task.started - task.received
# print queueTime
# print "task.started:%s"%task.started
# print "task.received:%s"%task.received
# print "task.runtime:%s" % event['runtime']
# print event
if task and task.started and task.received:
queueTime = task.started - task.received
runtime = event['runtime']
# with storeFactory.store() as store:
# store.deleteTaskId( taskId)
CabbageCounterHolder.getCabbageCounter().updateTaskSucceeded(
taskName, _getHostName(event), runtime, queueTime)
# raise Exception("test")
except Exception as e:
Logger.exception(log)
def setUp(self):
registerClientEvent()
registerServerEvent()
self.t1 = threading.Thread(target=run)
self.t1.setDaemon(True)
self.t1.start()
time.sleep(5)
time.sleep(5)
self.t2 = threading.Thread(target=start_server)
self.t2.setDaemon(True)
self.t2.start()
self.app = CabbageHolder.getCabbage().getApp()
def jobRemoveHandler(event):
try:
jobId = event.jobId
if JobCacheHolder.getJobCache().has_key(jobId):
jobRun = JobCacheHolder.getJobCache().get(jobId)
if jobRun : #停止运行TASK
jobRun.stop()
else:
job =CacheHolder.getCache().get(jobId, JOBS)
for taskName in job.tasks:
CabbageHolder.getServerCabbage(job.brokerServer).revokeByTaskName(taskName)
with storeFactory.store() as store:
store.updateJobStatus(jobId, JOB_DELETE)
#删除缓存让下一个task可以同名
tasks=CacheHolder.getCache().get(jobId, JOBS).tasks
for taskName in tasks:
if TaskCacheHolder.getJobCache().has_key(taskName):
TaskCacheHolder.getJobCache().remove(taskName)
CacheHolder.getCache().remove(jobId, JOBS)
except:
Logger.exception(log)
def setUp(self):
registerClientEvent()
work = Work()
work.hostName = getHostName()
work.port = "1024"
work.status = "aa"
self.kazooClient = ZookeeperClientHolder.getClient()
self.store = ZookeeperStore()
self.kazooClient.addChildListener("/cabbage/jobs", jobChildWatch)
self.store.saveWork(work)
self.t1 = threading.Thread(target=run)
self.t1.setDaemon(True)
self.t1.start()
time.sleep(5)
self.app = CabbageHolder.getCabbage().getApp()
def worker():
print "worker_pid:%d"% os.getpid()
app=CabbageHolder.getCabbage().getApp()
i = app.control.inspect()
result = i.stats()
print result
print i.registered_tasks()
loadMoudle = LoadMoudleHolder.getLoadMoudle(PYTHON)
import sys
path="/Users/hua/workspace/python/cabbage/client_file_path/123456789"
sys.path.append(path)
classes =loadMoudle.load(path,"test_main.py")
for clazz in classes:
obj = clazz[1]
if isCabbageMain(obj):
m = obj
m().run()
def startCelery(self):
self.cabbage = CabbageHolder.getCabbage()
def run():
CabbageHolder.getCabbage().start()
def start():
#使用子进程，加载任务，类沙箱的概念
self._initJobs()
t1 = threading.Thread(target=run)
t1.setDaemon(True)
t1.start()
t1.join()
p2 = multiprocessing.Process(target = start)
p2.start()
p2.join()
def jobWebWatch(children):
store = storeFactory.getStore()
try:
brokers={}
for jobId in children:
try:
job=store.getJob(jobId)#toreHolder.getStore().getJob(jobId)
if CacheHolder.getCache().hasKey(jobId, JOBS) is False:
CacheHolder.getCache().put(jobId, job,JOBS)
# kazooClient.addDataListener(parent+"/"+STATUS, jobRunStatusWatch)
#偷个懒，只要没有删除的全部放到ROUTER里面去
if job.status != JOB_DELETE:
brokerServer=job.brokerServer
routes={}
for taskName in job.tasks:
que=store.getQueue(job.brokerQueue)
routes[taskName]={'queue': que.queueName, 'routing_key': que.routingKey}
TaskCacheHolder.getJobCache().put(taskName,job.jobId)
if brokerServer in brokers:
brokers[brokerServer].update(routes)
else:
brokers[brokerServer] = routes
except Exception:
Logger.exception( log)
#偷个懒，只要没有删除的全部放到ROUTER里面去
for broker,routes in brokers.items():
brokerServer = store.getBrokerServer(broker)
#修复BUG，导致任务提交的celery队列里面去了
cabbage = Cabbage(hostName=brokerServer.hostName,broker=brokerServer.connectUri)
cabbage.app.conf.update(CELERY_ROUTES = routes)
CabbageHolder.getServerCabbages()[brokerServer.hostName] = cabbage
# CabbageHolder.getServerCabbages()[brokerServer.hostName].getApp().conf.update(CELERY_ROUTES = routes)
Logger.info(log,"更新队列服务器【%s】ROUTES【%s】"% (brokerServer.hostName,str(routes)))
except Exception:
Logger.exception( log)
finally:
storeFactory.returnStroe(store)
def setUp(self):
self.app = CabbageHolder.getCabbage().getApp()
self.t1 = threading.Thread(target=run)
self.t1.setDaemon(True)
self.t1.start()
time.sleep(2)
def brokerServerWatch(children):
for brokerName in children:
if not CabbageHolder.getServerCabbages().has_key(brokerName):
brokerServer = StoreHolder.getServerStore().getBrokerServer(brokerName)
zope.event.notify(AddBrokerServerEvent(brokerServer))
zope.event.notify(MonitorBrokerServerEvent(brokerServer))
def stopWorkService(self, work):
if (CabbageHolder.getServerCabbage(self.work.brokerServer).workIsAlive(
self.work.hostName)):
CabbageHolder.getServerCabbage(self.work.brokerServer).stop(
self.work.hostName)
def run():
CabbageHolder.getCabbage().start()
print "shutdown t1"
def test_start(self):
cabbage = CabbageHolder.getCabbage()
cabbage.start()
def run():
CabbageHolder.getCabbage().start()
def getApp(self):
return CabbageHolder.getServerCabbage(self.job.brokerServer).getApp()
def stop(self):
for taskName in self.job.tasks:
CabbageHolder.getServerCabbage(
self.job.brokerServer).revokeByTaskName(taskName)
self.isRun = False
def restart():
CabbageHolder.getCabbage().restart()
print "shutdown restart"
def run():
print "worker_run:%d" % os.getpid()
CabbageHolder.getCabbage().start()
print "shutdown t1"
