def run(self):
'''
Start process, the subclass must rewrite this function or 'handle' function
when all the father processes quits, then break to quit
'''
if self.log:
self.log = Log(self.name, toFile="plugin")
self.log.info("plugin {0} start, ins:{1}, outs:{2}".format(
self.name, self._ins, self._outs))
else:
self.log = False
while True:
try:
data = self.get()
#print "debug:", "plugin ", self.name, "getting", "ins<<<<<<<<", [str(x) for x in self._ins]
except QueueEmpty:
continue
except IOError:
break
except EOFError:
break
except PluginExit:
self.quit()
#print "debug:", "plugin ", self.name, "quit"
if self.log:
self.log.info("plugin {0} quit".format(self.name))
break
else:
#print "debug:", "plugin ", self.name, "got", data
self.handle(data)
finally:
time.sleep(self.timeout)
'''
For get Shardid from etcd
'''
from config import Log
from config import base_config
from urllib.parse import urljoin
from os import path
import requests
import json
import re
from multiprocessing import Pool
Log = Log()
class Etcd():
def __init__(self, project_config):
config = base_config(project_config)
self.etcd_host = config.get('ETCD_HOST')
self.etcd_root = config.get('ETCD_ROOT')
def get_data(self, multi=1):
base_url = urljoin(self.etcd_host,
path.join('v2/keys', self.etcd_root.strip('/')))
Log.debug('baseUrl: %s' % base_url)
try:
root_all = []
sub_keys = json.loads(requests.get(base_url).content)
m_parten = '%s/[0-9]{1,3}$' % self.etcd_root
for each_keys in sub_keys['node']['nodes']:
rst = self.mul_gid(m_parten, each_keys, multi=multi)
