class ZKStore:
    def __init__(self, hosts):
        self.zk = KazooClient(hosts=hosts)
        self.zk.add_listener(listener)
        self.zk.start()


    def isConnected(self):
        if __state__ == 1:
            return True
        return False


    def write(self, path, node, value):
        self.zk.ensure_path(path)
        if self.zk.exists(path+"/"+node):
           self.zk.set(path+"/"+node, value)
        else:
           self.zk.create(path + "/" + node, value)


    def read(self, path):
        if self.zk.exists(path):
            data, stat = self.zk.get(path)
            return data
        return None
Beispiel #2
0
class ShellTestCase(unittest.TestCase):
    """ base class for all tests """

    def setUp(self):
        """
        make sure that the prefix dir is empty
        """
        self.tests_path = os.getenv("ZKSHELL_PREFIX_DIR", "/tests")
        self.zk_host = os.getenv("ZKSHELL_ZK_HOST", "localhost:2181")
        self.username = os.getenv("ZKSHELL_USER", "user")
        self.password = os.getenv("ZKSHELL_PASSWD", "user")
        self.digested_password = os.getenv("ZKSHELL_DIGESTED_PASSWD", "F46PeTVYeItL6aAyygIVQ9OaaeY=")
        self.super_password = os.getenv("ZKSHELL_SUPER_PASSWD", "secret")
        self.scheme = os.getenv("ZKSHELL_AUTH_SCHEME", "digest")

        self.client = KazooClient(self.zk_host, 5)
        self.client.start()
        self.client.add_auth(self.scheme, self.auth_id)
        if self.client.exists(self.tests_path):
            self.client.delete(self.tests_path, recursive=True)
        self.client.create(self.tests_path, str.encode(""))

        self.output = StringIO()
        self.shell = Shell([self.zk_host], 5, self.output, setup_readline=False, async=False)

        # Create an empty test dir (needed for some tests)
        self.temp_dir = tempfile.mkdtemp()

    @property
    def auth_id(self):
        return "%s:%s" % (self.username, self.password)

    @property
    def auth_digest(self):
        return "%s:%s" % (self.username, self.digested_password)

    def tearDown(self):
        self.output = None
        self.shell = None

        if os.path.isdir(self.temp_dir):
            shutil.rmtree(self.temp_dir)

        if self.client.exists(self.tests_path):
            self.client.delete(self.tests_path, recursive=True)

        self.client.stop()

    ###
    # Helpers.
    ##

    def create_compressed(self, path, value):
        """
        ZK Shell doesn't support creating directly from a bytes array so we use a Kazoo client
        to create a znode with zlib compressed content.
        """
        compressed = zlib.compress(bytes(value, "utf-8") if PYTHON3 else value)
        self.client.create(path, compressed, makepath=True)
Beispiel #3
0
class dzk:
    def __init__(self):
        self.BasePath = "/my/"
        self.zk = KazooClient(hosts='x.24.79.51:2181,x.24.79.53:2181',retry_max_delay=2000)
        self.zk.start()
        self.zk.add_listener(self.listener)

    def listener(state):
        if state == KazooState.LOST:
            self.zk.start()
        elif state == KazooState.SUSPENDED:
            print "*******listener saw KazooState.LOST"
        else:
            print "*******listener saw KazooState.CONNECT"

    def getIpHost(self):
        self.myname  = socket.getfqdn(socket.gethostname())
        myip = socket.gethostbyname(self.myname)
        return  myip

    def register(self):
        ip = self.getIpHost()
        if ip:
            NODE = self.BasePath + ip
            print "register:",NODE
        else:
            print "[ERROR:] %s does not exist " %(NODE)
            sys.exit(2)

        if not self.zk.exists(NODE): 
            self.zk.ensure_path(NODE)

    def getData(self):
        ip = self.getIpHost()
        if ip:
            NODE = self.BasePath + ip
        else:
            print "[ERROR:] %s does not exist " %(NODE) 

        if self.zk.exists(NODE):
            data, stat = self.zk.get(NODE)
            print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))

    def monitor(self):
        pass
    
    def heartbeat(self):
        pass

    def role(self):
        pass
    
    def command(self):
        pass
Beispiel #4
0
class ZKClient():

    def __init__(self, hosts, auth_data=None):
        if auth_data:
            auth_data = [('digest', auth_data)] 
        self.zk = KazooClient(hosts=hosts, auth_data=auth_data)
        self.zk.start()

    def __del__(self):
        self.zk.stop()

    def exist(self, zoo_path):
        ret = self.zk.exists(zoo_path)
        if ret:
            print True
        else:
            print False
        return ret

    def mv(self, zoo_path_src, zoo_path_dst):
        if self.zk.exists(zoo_path_dst):
            print '%s exists! Please delete it manually'
            return False
        if self.cp(zoo_path_src, zoo_path_dst):
            self.rmr(zoo_path_src)
        else:
            self.rmr(zoo_path_dst)

    def cp(self, zoo_path_src, zoo_path_dst):
        if not self.zk.exists(zoo_path_src):
            print '%s does not exist!' % zoo_path_src
            return False
        src_name = filter(lambda x:x, zoo_path_src.split('/'))[-1]
        try:
            children = self.zk.get_children(zoo_path_dst)
            if children:
                zoo_path_dst = zoo_path_dst + '/' + src_name
        except NoNodeError, e:
            pass
        children = self.zk.get_children(zoo_path_src)
        if not children:
            buf = self.__get_zk_file(zoo_path_src)
            src_name = filter(lambda x:x, zoo_path_src.split('/'))[-1]
            try:
                dst_is_dir = self.zk.get_children(zoo_path_dst)
                if dst_is_dir:
                    dst_path = zoo_path_dst + '/' + src_name
                else:
                    dst_path = zoo_path_dst
                self.put_str(dst_path, buf)
            except NoNodeError, e:
                self.create_long(zoo_path_dst)
                self.put_str(zoo_path_dst, buf)
Beispiel #5
0
def main():
    '''
    A manual configuration file pusher for the crawlers. This will update
    Zookeeper with the contents of the file specified in the args.
    '''
    import argparse
    from kazoo.client import KazooClient

    parser = argparse.ArgumentParser(
            description="Crawler config file pusher to Zookeeper")
    parser.add_argument('-f', '--file', action='store', required=True,
                        help="The yaml file to use")
    parser.add_argument('-i', '--id', action='store', default="all",
                        help="The crawler id to use in zookeeper")
    parser.add_argument('-p', '--path', action='store',
                        default="/scrapy-cluster/crawler/",
                        help="The zookeeper path to use")
    parser.add_argument('-w', '--wipe', action='store_const', const=True,
                        help="Remove the current config")
    parser.add_argument('-z', '--zoo-keeper', action='store', required=True,
                        help="The Zookeeper connection <host>:<port>")
    args = vars(parser.parse_args())

    filename = args['file']
    id = args['id']
    wipe = args['wipe']
    zoo = args['zoo_keeper']
    path = args['path']

    zk = KazooClient(hosts=zoo)
    zk.start()

    # ensure path exists
    zk.ensure_path(path)

    bytes = open(filename, 'rb').read()

    if zk.exists(path):
        # push the conf file
        if not zk.exists(path + id) and not wipe:
            print "creaing conf node"
            zk.create(path + id, bytes)
        elif not wipe:
            print "updating conf file"
            zk.set(path + id, bytes)

        if wipe:
            zk.set(path + id, None)

    zk.stop()
def random_nodes(number_of_nodes_to_return, exclude=None, nodes=None):
    """ Selects a group of nodes from the pool of registered nodes

    Arguments:
        number_of_nodes_to_return: The total number of nodes to be returned
        to the caller of the function
        exclude (optional): A list of nodes that will be excluded from the
        results
        nodes (optional): A list of nodes to process, if None then the
        specified zookeeper will be contacted and the registered brokers used
    :return:
    """
    if not nodes:
        zk = KazooClient(hosts=zookeeper_connection_string, read_only=True)
        zk.start()
        try:
            if zk.exists('/brokers/ids'):
                ids = zk.get_children('/brokers/ids')
        finally:
            zk.stop()
    else:
        ids = nodes

    if exclude:
        ids = [x for x in ids if x not in exclude]
    return random.sample(ids, number_of_nodes_to_return)
Beispiel #7
0
def get_scn_from_zookeeper():
	try:
		print("Starting the process")
		zk = KazooClient(hosts='10.0.57.146:2181,10.0.57.145:2181,10.0.77.195:2181')
		print("Connection established")
		zk.start()
		sum_of_scn_num = 0
		if zk.exists("/fk_kafka_cluster1"):
			children_list = zk.get_children("/fk_kafka_cluster1/PROPERTYSTORE/")
			sorted_list = sorted(children_list)
			partion_num_scn_num_dict = {}
			for children in sorted_list:
				recv_data,stat = zk.get("/fk_kafka_cluster1/PROPERTYSTORE/"+str(children))
				data_dict = ast.literal_eval(recv_data)
				partition_num =  data_dict["id"]
				fields = ast.literal_eval(data_dict['simpleFields']['c'])
				scn_num = fields["windowScn"]
				sum_of_scn_num += scn_num
				partion_num_scn_num_dict[partition_num] = scn_num
			print "Data fetching from Zookeeper complete"
			avg_scn_num = (sum_of_scn_num/len(children_list))
			sorted_dict = sorted(partion_num_scn_num_dict.items(),key=operator.itemgetter(1))
			return avg_scn_num,sorted_dict 
		else:
			print("Node does not exist")
	except Exception as e:
		print("Exception occured!",str(e))
def init_zk(namespace=''):
    """Initialize Zookeeper Instance."""
    hosts = os.getenv('ZOOKEEPER_CONN_STRING', 'localhost:2181')

    timeout = 600
    timeout_treshold = time() + timeout

    while True:
        try:
            # check if namespace exists
            zk = KazooClient(hosts,
                             read_only=False)
            zk.start()
            if zk.exists(namespace) is None:
                zk.create(namespace)
            zk.stop
            zk.close
            zk = KazooClient(hosts + namespace,
                             read_only=False)
            zk.start()
            if type(zk) is not KazooClient:
                logging.error("can't connect to Zookeeper ...")
                exit(1)
            return zk
        except KazooTimeoutError:
            if time() > timeout_treshold:
                logging.error("can't connect to Zookeeper ...")
                exit(1)
            else:
                logging.warning("can't connect to Zookeeper, " +
                                "trying again since timeout " +
                                "is not reached yet ...")
Beispiel #9
0
class Store(object):
    def __init__(self,**kwargs):
        self.config = kwargs
        self.client = None

    def get_client(self):
        return self.client

    def open(self):
        self.client = KazooClient(**self.config)
        self.client.add_listener
        self.client.start()

    def close(self):
        self.client.stop()

    def read(self,path):
        return self.client.get(path)

    def write(self,path,value):
        base_path = os.path.dirname(path)
        self.client.ensure_path(base_path)
        self.client.create(path,value)

    def overwrite(self,path,value):
        self.client.set(path,value)

    def exists(self,path):
        return self.client.exists(path)
Beispiel #10
0
    def run(self):
        zk = KazooClient(hosts='%s:%d' % (self.options.host, self.options.port),
                         read_only=True, timeout=3)

        try:
            zk.start()

            options = vars(self.options)
            options.update({
                'system.hostname': socket.gethostname()
            })

            if self.options.regex:
                content, stats = zk.get(self.options.file)

                options['stats'] = stats

                m = re.search(self.options.regex, content, re.MULTILINE | re.DOTALL)

                if m:
                    options.update(m.groupdict())

                    self.ok(self.options.message.format(**options))
                else:
                    self.critical(self.options.message.format(**options))
            elif zk.exists(self.options.file):
                self.ok(self.options.message.format(**options))
            else:
                self.critical(self.options.message.format(**options))
        except Exception as ex:
            self.critical(ex)
        finally:
            zk.stop()
def validate_zookeeper_fault_tolerance():
  """
  Validate that we can still connect to zookeeper instance 2 to read the node
  """
  zk2 = KazooClient(hosts=str(runtime.get_active_config('zookeeper_host') + ':2182'))
  zk2.start()
  assert zk2.exists("/my/zookeeper_errorinjection/"), "zookeeper_errorinjection node not found"

  zk2.stop()
def zookeeper_ephemeral_node(name):
  zk = KazooClient(hosts=str(runtime.get_active_config('zookeeper_host') + ':2181'))
  zk.start()
  zk.create("/my/zookeeper_test/node1", b"process1 running", ephemeral=True)
  #At 10 validate that ephemeral node exist that is the process is still running
  time.sleep(10)
  assert zk.exists("/my/zookeeper_test/node1"), "process node is not found at 10 s when it is still running"

  time.sleep(20)
  zk.stop()
Beispiel #13
0
class CockRoach(object):
    def __init__(self, zkHost, stale_max_days=30, assume_yes=False, preview=False):
        self.ConsumerGroups = []
        self.zk_client = KazooClient(hosts=zkHost)
        self.zk_client.start()
        self.stale_max_days = stale_max_days
        self.assume_yes = assume_yes
        self.preview = preview
        if self.zk_client.exists("/consumers"):
            for cg_name in self.zk_client.get_children("/consumers"):
                self.ConsumerGroups.append(ConsumerGroup(cg_name, \
                                                         self.zk_client))

    def get_stale_cgroups(self, display):
        """
           get_stale_cgroups returns ConsumerGroups
           that were not used for stale_max_days
        """
        ret = []
        for consumergroup in self.ConsumerGroups:
            delta = datetime.now() - consumergroup.last_seen().mtime
            if delta.days > self.stale_max_days:
                if display:
                    print "Stale: %s" % (consumergroup)
                ret.append(consumergroup)
        return ret

    def delete_stale_cgroups(self):
        """ Delete consumer groups that are considered stale"""
        stale_cgroups = self.get_stale_cgroups(display=False)
        for stale_cg in stale_cgroups:
            print stale_cg
            if self.assume_yes is False:
                confirm = raw_input("Delete?")
            else:
                confirm = "Y"

            if confirm == "Y":
                self.delete_cgroup(stale_cg)

    def delete_cgroup(self, consumergroup):
        """Deletes a consumer Group"""
        print "Deleting %s" % (consumergroup.gid)
        if self.preview is False:
          self.zk_client.delete("/consumers/%s" % (consumergroup.gid), version=-1, recursive=True)
          print "executed"
        else:
            print "pass"


    def __str__(self):
        ret = ""
        for consumer in self.ConsumerGroups:
            ret += "%s" % (consumer)
        return ret
Beispiel #14
0
class Pool(object):
    def __init__(self, name):
        self.name = name
        self.zk = None
        self.path = "/carousel/{}".format(name)

        # Pool meta-data
        self.meta = None
        self.meta_stat = None

        self.resources = set()

    def on_resources_change(self, res):
        self.resources = set(res)

    def create(self, metadata={}):
        # Create the base pool path with metadata
        self.zk.create(self.path, str.encode(json.dumps(metadata)), makepath=True)

        for path in ["resources", "nodes", "leaders"]:
            self.zk.create(os.path.join(self.path, path))

        self.load()

    def connect(self, *hosts, timeout=4):
        self.zk = KazooClient(",".join(hosts), timeout=timeout)
        self.zk.start()

    def load(self):
        # Check whether the pool exists
        if not self.zk.exists(self.path):
            raise PoolException("Pool with name {} does not exist!".format(self.name))

        # Next load the pool meta-data
        self.meta, self.meta_stat = self.zk.get(self.path)
        self.meta = json.loads(self.meta.decode())

        # Finally, we need to keep track of all resources in this pool
        ChildrenWatch(self.zk, os.path.join(self.path, "resources"), self.on_resources_change)

    def disconnect(self):
        self.zk.stop()

    def get_node(self):
        return Node(self)

    def get_resource(self, name):
        return Resource(self, name)

    def __eq__(self, other):
        return other.name == self.name

    def test(self, wat):
        print(wat)
def validate_zookeeper_process_tracking():
  """
  Verify if process register node correctly with zookeeper and zookeeper deletes it when process terminates
  """
  zk = KazooClient(hosts=str(runtime.get_active_config('zookeeper_host') + ':2181'))
  zk.start()

  #At 60 validate that process has terminated by looking at the ephemeral node
  time.sleep(60)
  assert not zk.exists("/my/zookeeper_test/node1"), "process node  not found at 60 s when it should have terminated"

  zk.stop()
Beispiel #16
0
class ZookeeperContext:

    def __init__(self, zookeeper_url, test_id, node_id):
        self.assertNotNone(zookeeper_url)
        self.assertNotNone(test_id)
        self.assertNotNone(node_id)

        self.zookeeper_url = zookeeper_url
        self.test_id = test_id
        self.node_id = node_id
        self.zk_path = '/bolt/testing/%s/%s' % (self.test_id, self.node_id)
        logger.info("Initialized decorator: %s " % self.__str__())
        self.__connect_zookeeper()

    def assertNotNone(self,thing):
        if thing == None:
            logger.fatal("Invalid key in initializer")

    def __connect_zookeeper(self):
        try:
            logger.info('Attempting to connect to: %s' % self.zookeeper_url)
            self.zk = KazooClient(hosts=self.zookeeper_url)
            self.zk.start(timeout=10)
            logger.info('initialized zookeeper')
        except Exception as exception:
            logger.error('Failed to connect to zk')
            logger.fatal(exception)


    def __str__(self):
        return json.dumps(self, default=lambda o: o.__dict__, indent=4)

    def __del__(self):
        self.zk.stop()

    def publish(self, key, data):
        self.assertNotNone(key)
        self.assertNotNone(data)

        path = self.zk_path + "/" + key
        logger.info("Creating data in: %s" % path)
        try:
            bytes = pickle.dumps(data);
            if not self.zk.exists(path):
                logger.info("Path does not exist in zk, creating... %s" % path)
                self.zk.create(path, value=bytes,  makepath=True)
            else:
                self.zk.set(path, value=bytes)

        except Exception as exception:
            logger.error('Error setting data in ' % path)
            logger.fatal(exception)
Beispiel #17
0
def test_zip_response(pub_node, host = '127.0.0.1', port = 2181):
    zookeeper = KazooClient('%s:%s' % (host, port,))
    zookeeper.start()

    node_value = {
        'status'      : 'ok',
        'update_time' : time.time(),
    }

    if zookeeper.exists('/test/to_zip_result/%s' % pub_node) is not None:
        zookeeper.set('/test/to_zip_result/%s' % pub_node, json.dumps(node_value))
    else:
        print 'Not found node `/test/to_zip_result/%s`' % pub_node
Beispiel #18
0
def main():
    zk = KazooClient(hosts="127.0.0.1:2181", timeout=2.0)
    zk.add_listener(my_listener)
    zk.start()

    if zk.exists("/ELECTION") == None:
        zk.ensure_path("/ELECTION")

    c = 1
    node_pathes = []
    while c < 10:
        c += 1
        node_path = zk.create("/ELECTION/guid-n_", b"a value", ephemeral=True, sequence=True)
        node_pathes.append(node_path)

    my_path = random.choice(node_pathes)
    my_path = my_path.replace("/ELECTION/", "")
    # print "print my_path", my_path

    children = zk.get_children("/ELECTION/", watch=election_child_watcher)
    get_next_path = False

    prev_path = None
    for child_path in sorted(children):
        if child_path == my_path:
            break
        prev_path = child_path

        # I'm the leader
    if prev_path == None:
        print "OK I'm leader don't have to watch"
        return

        # fires twice, once on creation ignore

    @zk.DataWatch("/ELECTION/" + prev_path)
    def watch_node(data, stat):
        # only watch for first change
        if stat.version == 1:
            print ("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
            print "setting watch on " + prev_path
            print "my", my_path

    zk.set("/ELECTION/" + prev_path, b"some data")

    print "boom. watch triggered?"
    # time.sleep(10)
    print "bye"

    zk.stop()
Beispiel #19
0
def test_leave_abc_behind():
    obj = type('',(),{})
    obj.flg = False
    config=skelton.Config()
    config.check_health=lambda:3
    config.become_active=lambda:0
    config.become_standby_from_active=lambda:-1
    z = zha.ZHA(config)
    trigger_zha(z)
    zk = KazooClient(hosts="127.0.0.1:2181")
    zk.start()
    assert zk.exists("/zha-abc")
    zk.stop()
    time.sleep(10)
Beispiel #20
0
def reset(bpath, new_val):
    zk = KazooClient(hosts=args.zkhost)
    zk.start()
    if zk.exists(bpath):  # /offsets
        for child in zk.get_children(bpath):  # offsets/topic
            c = '{0}/{1}'.format(bpath, child)
            print('Topic: {0}'.format(c))
            for c2 in zk.get_children(c):  # offsets/topic/partition
                c2 = '{0}/{1}'.format(c, c2)
                print('Set {0} to {1}'.format(c2, new_val))
                zk.set(c2, new_val)
    else:
        print('Path <{0}> not exists'.format(bpath))
    zk.stop()
def number_of_nodes():
    """ Will return the total number of nodes registered in the cluster

    Return:
        int: Total count of nodes that are a part of the cluster
    """
    zk = KazooClient(hosts=zookeeper_connection_string, read_only=True)
    zk.start()

    try:
        if zk.exists('/brokers/ids'):
            ids = zk.get_children('/brokers/ids')
            return len(ids)
    finally:
        zk.stop()
Beispiel #22
0
def put_topology(topology, config=config):
  zk = KazooClient(**config)
  zk.start()
  
  for service, links in topology.iteritems():
    if zk.exists("/sdn/services/{name}".format(name=service)):
      zk.set("/sdn/services/{name}".format(name=service), json.dumps(links))
    else:
      zk.ensure_path("/sdn/services")
      zk.create("/sdn/services/{name}".format(name=service), json.dumps(links))
      
  ret_check = zk.get_children("/sdn/services")
  zk.stop()
  
  return ret_check
Beispiel #23
0
def test_init_servers(num, host = '127.0.0.1', port = 2181):
    zookeeper = KazooClient('%s:%s' % (host, port,))
    zookeeper.start()

    try:
        node = '/test/server_list'
        if zookeeper.exists(node) is None:
            zookeeper.create(node, json.dumps({'update_time' : time.time()}), makepath = True)
    except kazoo.exceptions.NodeExistsError:
        pass

    v = 1
    while v <= num:
        try:
            node = '/test/server_list/s%s' % v
            if zookeeper.exists(node) is None:
                zookeeper.create(node, json.dumps({
                    'update_time' : time.time(),
                    'server_name' : 's%s' % v,
                    'server_id'   : v,
                }), makepath = True)
        except kazoo.exceptions.NodeExistsError:
            pass
        v += 1
Beispiel #24
0
def test_pub_response(pub_node, server_list, host = '127.0.0.1', port = 2181):
    zookeeper = KazooClient('%s:%s' % (host, port,))
    zookeeper.start()

    for s in server_list:
        if isinstance(s, int) is True:
            continue

        try:
            node = '/test/to_pub_result/%s/%s' % (pub_node, s, )
            if zookeeper.exists(node) is None:
                zookeeper.create(node, json.dumps({
                    'update_time' : time.time(),
                    'status'      : 'ok'
                }), makepath = True)
        except kazoo.exceptions.NodeExistsError:
            pass
class ZkServiceRegister:
    def __init__(self, zk_address, zk_timeout):
        self.__zkClient = KazooClient(hosts=zk_address, timeout=zk_timeout, read_only=False)
        self.__zkListener = ZkServiceRegisterListener(self.__zkClient)
        self.__zkClient.add_listener(self.__zkListener)
        self.__zkClient.start()

    def register(self, path, host, port, weight=DEFAULT_HOST_WEIGHT):
        try:
            if not self.__zkClient.exists(path):
                self.__zkClient.ensure_path(path)
        except Exception, e:
            print e.message

        reg_path = path + '/' + host + ':' + str(port) + ':' + str(weight)
        if self.__zkClient.exists(reg_path):
            self.__zkClient.delete(reg_path)

        self.__zkClient.create(reg_path, value='', ephemeral=True)
Beispiel #26
0
class ConfigFlags(StorageBase):
    """ A configuration manager using ZooKeeper. For setting flags
    on all instances for a given product. This will default to
    using a locally stored cache if ZooKeeper fails to respond.

    Note: current live flags are limited to 1MB total.

    Set initial values in config.ini file as
        flags.foo = bar
    to set "foo" flag to value bar.

    """

    #TODO:
    # * Break flags into separate elements instead of single JSON?
    # * Add fake dict to hold settings if ZK not installed/avaliable.
    localFlags = {}
    version = None

    def __init__(self, config, **kw):
        try:
            if 'Configurator' in type(config).__name__:
                config = config.get_settings()
            conf = config.get('flags.zk.settings')
            if conf is not None:
                conf = dict(json.loads(conf))
                self.zk = KazooClient(conf)
            else:
                self.zk = KazooClient()
            # get a copy of the local flags.
            self.zk_path = config.get('flags.zk.path',
                                      '/general/config')
            self.zk.start()
            node = self.zk.exists(self.zk_path)
            if node is None:
                # Virgin install, set from the config values.
                self._init_zk(config)
            self.zk.add_listener(self._zk_listener)
            self._refreshCache(config=config)

        except Exception, e:
            warnings.warn("Could not connect to ZooKeeper %s" % repr(e))
Beispiel #27
0
def get_zookeeper_master_ip(zkurl, zkpath):
    logger.info("Connecting to:%s" % zkurl)
    zk = KazooClient(hosts=zkurl)
    ip = ""
    try:
        logger.debug("Starting zk connection")
        zk.start()
        if not zk.exists(zkpath):
            logger.error('Path on zk doesn\'t exist: ' + zkpath)
            return ip
        logger.debug("Serializing TopologyMetadata() from %s" % zkpath)
        data, stat = zk.get(zkpath + "/masterip")
        logger.debug("Status of 'getting' %s/masterip: %s" % (zkpath, str(stat)))
        ip = str(data)
    except Exception as e:
        logger.exception(e)
    finally:
        logger.debug("Closing zk connection")
        zk.stop()
    return ip
class ZkServiceProvider:
    def __init__(self, zk_address, zk_timeout, connection):
        self.__service_dict = {}
        self.__zk_address = zk_address
        self.__zkClient = KazooClient(hosts=zk_address, timeout=zk_timeout, read_only=True)
        self.__zkClient.start()
        self.__zkListener = ZkServiceProviderListener(self.__zkClient)
        self.__zkClient.add_listener(self.__zkListener)
        self.__connection = connection

    def register_service(self, service, zk_path, client_cls):
        self.__service_dict[service] = (zk_path, client_cls)
        result = self._register_watcher(service, zk_path, client_cls)
        return result

    def _register_watcher(self, service, zk_path, client_cls):
        @self.__zkClient.ChildrenWatch(zk_path)
        def child_changed(data):
            print '+++++++++++++++' + service + ' child changed.++++++++++++++++++'
            print data
            hosts = data
            self.__connection.update_service(service, hosts)

        isExists = self.__zkClient.exists(zk_path)
        if not isExists:
            return False

        try:
            hosts = self.__zkClient.get_children(zk_path)
        except NoNodeError:
            print 'no node for the path of ' + zk_path
            return False
        except:
            print 'other exceptions.'
            return False

        self.__connection.update_service(service, hosts)
        return True

    def stop(self):
        self.__zkClient.stop()
class Zookeeper(object):
    def __init__(self, host):
        self.zk = KazooClient(hosts=host, connection_retry=True, read_only=True)
        self.zk.start()

    def stop(self):
        self.zk.stop()

    def get_path_instance(self, path):
        self.hosts = [] # store server list
        if self.zk.exists(path):
            if self.zk.get_children(path) > 0:
                for node in self.zk.get_children(path):
                    node_path = path + '/' + node
                    data, stat = self.zk.get(node_path)
                    host = data.split(':')[0]
                    self.hosts.append(host)
        else:
            raise Exception("path is not exsits")

        return self.hosts
Beispiel #30
0
def main():
    (options, args) = parse_options(sys.argv[1:])

    data = options.znode_data_size * b"D"

    s = KazooClient(options.server)
    s.start()

    if s.exists(options.root_znode):
        children = s.get_children(options.root_znode)
        print("delete old entries: %d" % len(children))
        for child in children:
            s.delete("%s/%s" % (options.root_znode, child))
    else:
        s.create(options.root_znode, "kazoo root znode")

    evaluation(s, options.root_znode, data, options)

    s.stop()

    print("Performance test complete")
Beispiel #31
0
class Backend:
    def __init__(self):
        self._logger = logging.getLogger('gunicorn.error')

        self._zk = KazooClient(hosts=f'{os.getenv("ZOOKEEPER_HOST")}:2181')
        self._hdfsClient = HdfsClient(os.getenv("HADOOP_NAMENODE_HOST"))

        self._active = False
        self._target_id = None
        self._zk_node_path = None

        self._range = None

        self._trie = None

        scheduler = BackgroundScheduler(timezone="UTC")
        scheduler.add_job(self._attempt_to_join_any, 'interval', minutes=1)
        scheduler.start()

    def start(self):
        self._zk.start()
        datawatch_next_target = DataWatch(client=self._zk,
                                          path=ZK_NEXT_TARGET,
                                          func=self._on_next_target_changed)
        datawatch_current_target = DataWatch(
            client=self._zk,
            path=ZK_CURRENT_TARGET,
            func=self._on_current_target_changed)

    def stop(self):
        self._zk.stop()

    def top_phrases_for_prefix(self, prefix):
        if (not self._active):
            raise NodeInactiveError(
                "This backend node is not active. Consult zookeeper for the most recent active nodes"
            )
        return self._trie.top_phrases_for_prefix(prefix)

    def _on_next_target_changed(self, data, stat, event=None):
        self._logger.info("_on_next_target_changed Data is %s" % data)
        if (data is None):
            return

        current_target_id = self._zk.get(ZK_CURRENT_TARGET)[0].decode()
        next_target_id = data.decode()
        self._deactivate_if_not_used(current_target_id, next_target_id)

        success = self._attempt_to_join_target(next_target_id)

    def _on_current_target_changed(self, data, stat):
        self._logger.info("_on_current_target_changed Data is %s" % data)
        if (data is None):
            return

        current_target_id = data.decode()
        next_target_id = self._zk.get(ZK_NEXT_TARGET)[0].decode()
        self._deactivate_if_not_used(current_target_id, next_target_id)

    def _deactivate_if_not_used(self, current_target_id, next_target_id):
        if (self._active and self._target_id
                and current_target_id != self._target_id
                and next_target_id != self._target_id):
            self._logger.info(
                f'Deactivating {self._target_id}, {current_target_id}, {next_target_id}'
            )

            if (self._zk.exists(self._zk_node_path)):
                self._zk.delete(self._zk_node_path)

            self._active = False
            self._target_id = None
            self._trie = None
            self._zk_node_path = None

    def _attempt_to_join_any(self):
        self._logger.debug("Attempting to join any")
        if (self._active):
            return
        if (self._zk.exists(ZK_CURRENT_TARGET) is not None):
            target_id = self._zk.get(ZK_CURRENT_TARGET)[0].decode()
            if (self._attempt_to_join_target(target_id)):
                return
        if (self._zk.exists(ZK_NEXT_TARGET) is not None):
            target_id = self._zk.get(ZK_NEXT_TARGET)[0].decode()
            if (self._attempt_to_join_target(target_id)):
                return

    def _attempt_to_join_target(self, target_id):
        if (self._active):
            return False

        if (not target_id or
                self._zk.exists(f'/phrases/distributor/{target_id}') is None):
            return False

        self._logger.info(f'Attempting to join {target_id}')

        partitions = self._zk.get_children(
            f'/phrases/distributor/{target_id}/partitions')
        for partition in partitions:

            nodes_path = f'/phrases/distributor/{target_id}/partitions/{partition}/nodes/'

            self._zk.sync(nodes_path)
            nodes = self._zk.get_children(nodes_path)
            if (len(nodes) >= NUMBER_NODES_PER_PARTITION):
                self._logger.info(
                    f'Cannot join {nodes_path}, it has enough nodes {nodes}; current number of nodes {len(nodes)} >= {NUMBER_NODES_PER_PARTITION}'
                )
                continue  # No more nodes needed here

            created_node_path = self._zk.create(nodes_path,
                                                value=b'',
                                                ephemeral=True,
                                                sequence=True)
            self._zk_node_path = created_node_path

            try:
                created_node_name = created_node_path.split('/')[-1]

                self._zk.sync(nodes_path)
                nodes = self._zk.get_children(nodes_path)
                nodes.sort()

                if (nodes and int(created_node_name) > int(
                        nodes[0:NUMBER_NODES_PER_PARTITION][-1])):
                    # The node was not able to join the partition
                    self._logger.info(
                        f'Cannot join {nodes_path}, it has already filled up {nodes}; created_node_name: {created_node_name}'
                    )
                    self._zk.delete(created_node_path)
                    continue

                if (not self._load_trie_and_activate(target_id, partition)):
                    self._logger.error(
                        f'Error while loading and activating trie for {nodes_path}, target_id: {target_id}, partition: {partition}'
                    )
                    self._zk.delete(created_node_path)
                    continue

                # Finishes the initialization
                self._zk.set(created_node_path, socket.gethostname().encode())

                return True
            except:
                self._zk.delete(created_node_path)

        return False

    def _load_trie_and_activate(self, target_id, partition):
        trie_data_hdfs_path = f'/phrases/distributor/{target_id}/partitions/{partition}/trie_data_hdfs_path'
        trie = self._load_trie(self._zk.get(trie_data_hdfs_path)[0].decode())
        if (trie):
            self._trie = trie
            self._active = True
            self._target_id = target_id
            self._logger.info(
                f'Now ACTIVE and loaded trie for partition {partition} and target_id {target_id}'
            )
            return True
        else:
            return False

    def _load_trie(self, trie_hdfs_path):
        local_path = 'trie.dat'
        self._hdfsClient.download(trie_hdfs_path, local_path)
        with open(local_path, 'rb') as f:
            return pickle.load(f)
Beispiel #32
0
zk = KazooClient(hosts='zoo:2181')


def zk_listener(state):
    if (state == KazooState.LOST):
        logging.warning("Zookeeper connection lost")
    elif (state == KazooState.SUSPENDED):
        logging.warning("Zookeeper connection suspended")
    else:
        logging.info("Zookeeper connected")


zk.add_listener(zk_listener)
zk.start()

if (zk.exists("/Workers")):
    zk.delete("/Workers", recursive=True)


@zk.ChildrenWatch("/Workers/", send_event=True)
def watch_children(children, event):
    print("Children are now: %s" % children)
    if (event == None):
        pass
    elif (event.type is DELETED):
        print("Slave deleted")


timer_start_flag = False
timer_started_flag = False
read_request_count = 0
class Proxy:
    """Implementation of the proxy"""
    def __init__(self):
        # Use XPUB/XSUB to get multiple contents from different publishers
        self.context = zmq.Context()
        self.xsubsocket = self.context.socket(zmq.XSUB)
        self.xpubsocket = self.context.socket(zmq.XPUB)
        self.xpubsocket.setsockopt(zmq.XPUB_VERBOSE, 1)
        self.xpubsocket.send_multipart([b'\x01', b'10001'])
        # Now we are going to create a poller
        self.poller = zmq.Poller()
        self.poller.register(self.xsubsocket, zmq.POLLIN)
        self.poller.register(self.xpubsocket, zmq.POLLIN)

        self.global_url = 0
        self.global_port = 0
        self.newSub = False

        self.topic_info_queue = [
        ]  # the content queue for different topic (zipcode)
        self.topicInd = 0
        self.zip_list = [
        ]  # the ziplist to keep track with the zipcodes received

        self.zk_object = KazooClient(hosts='127.0.0.1:2181')
        self.zk_object.start()

        self.path = '/home/'

        znode1 = self.path + "broker1"
        if self.zk_object.exists(znode1):
            pass
        else:
            # Ensure a path, create if necessary
            self.zk_object.ensure_path(self.path)
            # Create a node with data
            self.zk_object.create(znode1, to_bytes("5555,5556"))
            # Print the version of a node and its data

        znode2 = self.path + "broker2"
        if self.zk_object.exists(znode2):
            pass
        else:
            # Ensure a path, create if necessary
            self.zk_object.ensure_path(self.path)
            # Create a node with data
            self.zk_object.create(znode2, to_bytes("5557,5558"))

        znode3 = self.path + "broker3"
        if self.zk_object.exists(znode3):
            pass
        else:
            # Ensure a path, create if necessaryto_bytes
            self.zk_object.ensure_path(self.path)
            # Create a node with data
            self.zk_object.create(znode3, to_bytes("5553,5554"))
        '''
        elect a leader and put that node under the path of /leader/ and send the port number to the publisher and subscriber
        '''
        self.election = self.zk_object.Election(self.path, "leader")
        leader_list = self.election.contenders()
        self.leader = leader_list[-1].encode(
            'latin-1')  # the leader here is a pair of address
        self.leader = str(self.leader)
        address = self.leader.split(",")
        tem = address[0]
        tem = tem.split("'")[1]
        address[0] = tem
        tem = address[1]
        tem = tem.split("'")[0]
        address[1] = tem
        pub_addr = "tcp://*:" + address[0]
        sub_addr = "tcp://*:" + address[1]
        print("Current elected broker: ", pub_addr + "," + sub_addr)
        self.xsubsocket.bind(pub_addr)
        self.xpubsocket.bind(sub_addr)

        self.watch_dir = self.path + self.leader  # use  Datawatch

        self.leader_path = "/leader/"
        self.leader_node = self.leader_path + "node"
        if self.zk_object.exists(self.leader_node):
            pass
        else:
            # Ensure a path, create if necessary
            self.zk_object.ensure_path(self.leader_path)
            # Create a node with data
            self.zk_object.create(self.leader_node, ephemeral=True)
        self.zk_object.set(self.leader_node, to_bytes(self.leader))

    def sendToSubscriber(self):

        events = dict(self.poller.poll(10000))
        if self.xsubsocket in events:
            msg = self.xsubsocket.recv_multipart()
            content = msg[0]
            content = str(content)
            content = content.split("'")[1]
            #print("content here is {}".format(content))
            zipcode, temperature, relhumidity, pub_time = content.split(
                " ")[:4]
            cur_msg = [zipcode, temperature, relhumidity, pub_time]
            if zipcode not in self.zip_list:  # a new topic just come from a new publisher
                self.zip_list.append(zipcode)

                topic_info = cur_msg
                self.topic_info_queue.append(topic_info)
            else:
                zipInd = self.zip_list.index(zipcode)
                self.topic_info_queue[zipInd] = cur_msg

            self.xpubsocket.send_string(
                "%i %i %i %i " % (int(cur_msg[0]), int(
                    cur_msg[1]), int(cur_msg[2]), int(cur_msg[3])))

        if self.xpubsocket in events:  # a subscriber comes here
            msg = self.xpubsocket.recv_multipart()
            self.xsubsocket.send_multipart(msg)

    def schedule(self):
        while True:

            @self.zk_object.DataWatch(self.watch_dir)
            def watch_node(data, stat, event):
                if event != None:
                    print(event.type)
                    if event.type == "DELETED":  # redo a election
                        self.election = self.zk_object.Election(
                            self.path, "leader")
                        leader_list = self.election.contenders()
                        self.leader = leader_list[-1].encode('latin-1')

                        self.zk_object.set(self.leader_node, self.leader)

                        self.xsubsocket.unbind(self.current_pub)
                        self.xpubsocket.unbind(self.current_sub)

                        address = self.leader.split(",")
                        pub_addr = "tcp://*:" + address[0]
                        sub_addr = "tcp://*:" + address[1]

                        self.xsubsocket.bind(pub_addr)
                        self.xpubsocket.bind(sub_addr)

            self.election.run(self.sendToSubscriber)

    def close(self):
        """ This method closes the PyZMQ socket. """
        self.xsubsocket.close(0)
        self.xpubsocket.close(0)
Beispiel #34
0
class zkHander(object):
    '''zookeeper系列操作'''
    def __init__(self):
        zk_hosts = GetConf().GetZKHosts()
        self.zk = KazooClient(hosts=zk_hosts)
        self.zk.start()

    def Get(self, path):
        value, stat = self.zk.get(path=path)
        return value

    def GetChildren(self, path):
        return self.zk.get_children(path=path)

    def Exists(self, path):
        return self.zk.exists(path=path)

    def Create(self, **kwargs):
        return self.zk.create(
            path=kwargs["path"],
            value=kwargs["value"],
            sequence=kwargs['seq'],
            makepath=(kwargs['mp'] if 'mp' in kwargs else False))

    def Set(self, path, value):
        return self.zk.set(path=path, value=value)

    def init_node(self, object):
        node_list = {
            'mysql': [
                'lock', 'white', 'meta/host', 'meta/group', 'meta/router',
                'online-list', 'master', 'task', 'haproxy', 'watch-down',
                'addition', 'addition/replication', 'addition/region',
                'readbinlog-status', 'slavedown', 'online-clients',
                'execute-gtid'
            ]
        }
        for server in node_list:
            for i in node_list[server]:
                node_path = '{}/{}'.format(server, i)
                state = self.zk.exists(path=node_path)
                if state is None:
                    object(node_path)

    def InitNode(self):
        '''初始化node'''
        @self.init_node
        def _init_node(node_path):
            self.zk.ensure_path(node_path)

    def SetReadBinlog(self, master_host, value):
        '''创建binlog信息,用于宕机记录读取的点'''
        _path = '/mysql/readbinlog-status/{}'.format(
            master_host.replace('.', '-'))
        self.Create(path=_path, value=value,
                    seq=False) if not self.Exists(_path) else None

    def SetExecuteGtid(self, master_host, value):
        '''创建gtid信息,用于切换记录执行到的gtid'''
        _path = '/mysql/execute-gtid/{}'.format(master_host.replace('.', '-'))
        self.Create(path=_path, value=value,
                    seq=False) if not self.Exists(_path) else None

    def CheckOnlineClient(self, host):
        '''检查mysql客户端程序是否在线'''
        _path = '{}/{}'.format(GetConf().GetOnlineClientPath(), Replace(host))
        return self.Exists(_path)

    def CreateDownTask(self, groupname, addition=None, region=None):
        task_path = '{}/{}'.format(GetConf().GetTaskPath(), groupname)
        if self.Exists(task_path) is None:
            time.sleep(random.uniform(0, 1))
            try:
                if addition:
                    task_path += '_' + region
                    self.Create(path=task_path,
                                value=str([groupname, region, 'append',
                                           'dow']),
                                seq=False)  #跨区域同步主机宕机
                else:
                    self.Create(path=task_path,
                                value=str([groupname, 'down']),
                                seq=False)
            except:
                Logging(msg='Existing outage task for  {}'.format(groupname),
                        level='info')
        else:
            Logging(msg='Existing outage task for  {}'.format(groupname),
                    level='info')

    def CreateWatch(self,
                    host,
                    addition=None,
                    region=None,
                    region_for_groupname=None):
        '''创建watch,触发时写入task节点'''
        online_host_path = GetConf().GetOnlinePath()
        _group_name = self.GetMeta(type='host', name=host)
        group_name = eval(
            _group_name)['group'] if _group_name else region_for_groupname
        online_state = self.zk.exists('{}/{}'.format(online_host_path, host))
        Logging(msg='master watch :{}'.format(host), level='info')
        if online_state is not None:

            @self.zk.DataWatch('{}/{}'.format(online_host_path, host))
            def my_func(data, stat):
                if data is None:
                    self.CreateDownTask(group_name,
                                        addition=addition,
                                        region=region)
                    Logging(msg='master({}) has been down!'.format(host),
                            level='error')
                    self.zk.stop()
                    sys.exit()
        else:
            _name = group_name + '_' + region if region else group_name
            Logging(msg="this master {} node not exists".format(host),
                    level='error')
            state = self.Exists('{}/{}'.format(GetConf().GetWatchDown(),
                                               _name))
            self.Create(path='{}/{}'.format(GetConf().GetWatchDown(), _name),
                        value="master not online",
                        seq=False) if state is None else None
            self.zk.stop()

    def __checklock(self, taskname):
        """循环检查lock是否存在,当不存在时对新master建立监听"""
        import time
        path = '{}/{}'.format(GetConf().GetLockPath(), taskname)
        while True:
            state = self.Exists(path)
            if state is None:
                return True
                break
            time.sleep(0.5)

    def CreateLockWatch(self, taskname):
        '''用于任务在其他节点执行,本节点对锁监听,当删除时获取新master启动watch'''
        path = '{}/{}'.format(GetConf().GetLockPath(), taskname)

        @self.zk.DataWatch(path)
        def my_func(data, stat):
            if data is None:
                masterhost = self.GetMasterMeta(taskname)
                self.CreateWatch(masterhost)
                sys.exit()

    def CreateChildrenWatch(self, path, func):
        '''任务列表监听'''
        state = self.zk.exists(path=path)
        if state is not None:

            @self.zk.ChildrenWatch(path)
            def my_func(data):
                if data:
                    data.sort()
                    func(data[0])

    def GetWhite(self, groupname):
        '''check 白名单'''
        white_list = self.GetChildren(path='/mysql/white')
        return True if groupname in white_list else False

    def GetMasterGroupHosts(self):
        '''获取当前master列表,启动检查是否有现有的master,如果有直接监控'''
        group_hosts_path = GetConf().GetMasterPath()
        master_list = self.GetChildren(group_hosts_path)
        return master_list if master_list is not None else False

    def GetAddHost(self):
        '''获取是否需要添加的集群,该节点下记录的为集群组名称,不能并发'''
        Lock_root = GetConf().GetLockPath()
        add_host_path = Lock_root + '/add-host'
        state = self.Exists(add_host_path)
        return self.Get(add_host_path) if state is not None else False

    def GetMeta(self, **kwargs):
        '''获取元数据信息'''
        if kwargs['type'] == 'group':
            node_path = '{}/{}'.format(GetConf().GetMetaGroup(),
                                       kwargs['name'])
            return self.Get(
                node_path) if self.Exists(node_path) != None else False
        elif kwargs['type'] == 'host':
            node_path = '{}/{}'.format(GetConf().GetMetaHost(), kwargs['name'])
            return self.Get(
                node_path) if self.Exists(node_path) != None else False
        else:
            Logging(msg="type is error ,only 【group ,host】", level='error')
            raise "type is error ,only 【group ,host】"

    def CreateMasterMeta(self, name, host):
        '''创建master信息节点,用于新集群加入'''
        node_path = '{}/{}'.format(GetConf().GetMasterPath(), name)
        return self.Create(
            path=node_path, value=host,
            seq=False) if self.Exists(node_path) is None else False

    def AlterMasterMeta(self, name, host):
        '''修改master信息节点,用于切换过后修改元数据'''
        node_path = '{}/{}'.format(GetConf().GetMasterPath(), name)
        return self.Set(node_path,
                        host) if self.Exists(node_path) is None else False

    def GetMasterMeta(self, groupname):
        '''获取master元数据信息'''
        node_path = '{}/{}'.format(GetConf().GetMasterPath(), groupname)
        return self.Get(node_path) if self.Exists(
            node_path) is not None else False

    def GetOnlineHostAll(self):
        '''获取在线的节点'''
        node_path = GetConf().GetOnlinePath()
        return self.GetChildren(node_path)

    def GetOnlineState(self, host):
        '''获取单个节点在线状态'''
        node_path = '{}/{}'.format(GetConf().GetOnlinePath(), host)
        return True if self.Exists(path=node_path) is not None else False

    def SetHaproxyMeta(self, group, reads, master, type=None):
        '''加入及master变动设置haproxy配置信息'''
        node_path = '{}/{}'.format(GetConf().GetHaproxy(), group)
        if type is None:
            meta_path = GetConf().GetMetaGroup()
            if reads is None:
                hosts = self.Get('{}/{}'.format(meta_path, group))
                reads = hosts.split(',')
            _reads = []
            for host in reads:
                host_meta = self.GetMeta(type='host', name=host)
                host_port = '%s:%d' % (host.replace(
                    '-', '.'), int(eval(host_meta)['port']))
                _reads.append(host_port)
            master_host_meta = self.GetMeta(type='host', name=master)
            _master = '%s:%d' % (master.replace(
                '-', '.'), int(eval(master_host_meta)['port']))

            value = '{"read":"%s","write":"%s"}' % (_reads, _master)
            state = self.Exists(node_path)
            if state is None:
                self.Create(path=node_path, value=value, seq=False)
            else:
                self.Set(path=node_path, value=value)
            return True
        else:
            value = '{"read":"%s","write":"%s"}' % (reads, master)
            self.Set(path=node_path, value=value)
            return True

    def GetTaskList(self):
        '''获取任务列表'''
        task_path = GetConf().GetTaskPath()
        task_list = self.GetChildren(path=task_path)
        return task_list if task_list is not None else False

    def GetTaskContent(self, taskname):
        '''获取任务内容'''
        task_path = GetConf().GetTaskPath()
        task_value = self.Get('{}/{}'.format(task_path, taskname))
        return task_value if task_value is not None else False

    def DeleteTask(self, taskname):
        '''删除任务'''
        task_path = GetConf().GetTaskPath()
        self.zk.delete('{}/{}'.format(task_path, taskname)) if self.Exists(
            '{}/{}'.format(task_path, taskname)) else None

    def DeleteWhite(self, groupname):
        node_path = GetConf().GetWhitePath()
        self.zk.delete('{}/{}'.format(node_path, groupname)) if self.Exists(
            '{}/{}'.format(node_path, groupname)) else None

    def DeleteWatchDown(self, groupname):
        '''清除心跳丢失的节点'''
        watch_down_path = GetConf().GetWatchDown()
        self.zk.delete('{}/{}'.format(
            watch_down_path, groupname)) if self.Exists('{}/{}'.format(
                watch_down_path, groupname)) else None

    def SetMasterHost(self, groupname, masterhost):
        '''写入master信息'''
        masterhost_path = '{}/{}'.format(GetConf().GetMasterPath(), groupname)
        self.Set(masterhost_path, masterhost) if self.Exists(
            masterhost_path) is not None else self.Create(
                path=masterhost_path, value=masterhost, seq=False)

    def OnlineExists(self, host):
        '''判断是否在线'''
        path = GetConf().GetOnlinePath()
        return True if self.Exists('{}/{}'.format(path, host)) else False

    def GetRouter(self, groupname):
        '''获取路由所在'''
        path = GetConf().GetRouter()
        return self.Get('{}/{}'.format(path, groupname)) if self.Exists(
            '{}/{}'.format(path, groupname)) else False

    def SetWatchDown(self, groupname, value):
        path = GetConf().GetWatchDown()
        self.Create(path='{}/{}_send'.format(path, groupname),
                    value=value,
                    seq=False) if self.Exists('{}/{}_send'.format(
                        path, groupname)) is None else None

    def SetLockTask(self, taskname):
        '''创建锁文件,用户多点运行server控制任务'''
        path = GetConf().GetLockPath()
        if self.Exists(path='{}/{}'.format(path, taskname)) is None:
            try:
                time.sleep(random.uniform(0, 1))
                self.zk.create(path='{}/{}'.format(path, taskname),
                               value=b'',
                               ephemeral=False)
                return True
            except Exception, e:
                Logging(msg=traceback.format_exc(), level='error')
                return False
        else:
Beispiel #35
0
        children = zk.get("/worker/master", watch=master_function)
 
#zk.delete("/worker/slave", recursive=True)
#zk.delete("/worker", recursive=True)

cmd = "cat /proc/self/cgroup | grep 'docker' | sed 's/^.*\///' | tail -n1"
cid = subprocess.check_output(cmd,shell=True)
cid = cid.decode("utf-8")
cid=cid[0:12]
client2 = docker.APIClient()
pid = client2.inspect_container(cid)['State']['Pid']
print("---MASTER PID", pid)
print("---CID", cid)

zk.ensure_path("/worker")
if zk.exists("/worker/master"):
    print("Master exists")
else:
    data1 = "I am master CID : "+cid+" PID : "+str(pid)
    data1 = data1.encode()
    zk.create("/worker/master", data1,ephemeral = True)

children = zk.get("/worker/master", watch=master_function)

app = Flask(__name__)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+str(pid)+'.db'
db = SQLAlchemy(app)


class user_details(db.Model):
from kazoo.client import KazooClient
import logging
logging.basicConfig()

zk = KazooClient(hosts='zookeeper:2181')
zk.start()

zk_path = "/kafka/replicator/test_kafka/checkpoint"

if zk.exists(zk_path):
    data, stat = zk.get(zk_path)
    print(data)
Beispiel #37
0
class Zookeeper:
    def __init__(self, hosts, max_merge_seq):
        print('create a zookeeper object')
        self.zk = ""
        self.IsConn = False
        self.Hosts = hosts
        self.MAX_MERGE_FILE_SEQUENCE = max_merge_seq
        self.filename = ''
        self.pattern = ''
        self.process_path = ''

    def connect(self):
        """
        connect to zookeeper
        :return:zookeeper object
        """
        print('try connect to zookeeper')
        self.zk = KazooClient(self.Hosts)
        try:
            self.zk.start()
        except Exception as e:
            print("connect zookeeper failed, err:%s" % e)
            sys.exit()
        self.IsConn = True
        print('connect zookeeper success')
        return self.zk

    def get_node(self, node_path):
        """
        获取空闲的process_id
        :return: process_id
        """
        self.connect()
        self.process_path = node_path
        node_list = []
        if not (self.zk.exists(node_path)):
            logging.error('zookeeper process node path: %s not exist' %
                          node_path)
            sys.exit()
        childs = self.zk.get_children(node_path)
        # len = 0
        p1 = re.compile(r"^process")
        for c in childs:
            if re.findall(p1, c):
                node_list.append(c)
        node_list = sorted(node_list)
        if len(node_list) <= 0:
            print("no process id in zookeeper process path")
            sys.exit()
        get_times = 0
        while 1:
            for node in node_list:
                lock_flag = False
                node_name = '%s/%s' % (node_path, node)
                n_child = self.zk.get_children(node_name)
                if len(n_child) > 0:
                    for n in n_child:
                        if n == 'lock':
                            lock_flag = True
                if lock_flag:
                    continue
                lock_node = "%s/%s" % (node_name, 'lock')
                self.zk.create(lock_node, ephemeral=True)
                # process_id = ''.join(node.split('_')[1:])
                print('get process_id :%s from zookeeper ' % node)
                return node
            get_times += 1
            print("no free process id in zookeeper")
            if get_times >= 3:
                print(
                    "get process id faild three times, please check zookeeper process id, exit"
                )
                sys.exit()

    def lock(self, lock):
        """
        lock the free node
        :param lock:
        :return:
        """
        self.zk.create(lock, ephemeral=True)

    def check_exists(self, node_path):
        return self.zk.exists(node_path)

    def get_config(self, config_path, config_node):
        """
        generate config files based on node's information
        :param config_path:
        :param config_node:
        :return:
        """
        data, stat = self.zk.get(config_node)
        with open(config_path + "config.ini", 'w') as f:
            f.writelines(data.decode())

    def get_node_value(self, zk_node):
        """
        获取zookeeper的节点信息
        :param zk_node:
        :return: data:node的value
                 stat:node的状态信息
        """
        data, stat = self.zk.get(zk_node)
        return data, stat

    def set_node_value(self, zk_node, data):
        """
        设置zookeeper节点的value
        :param zk_node:
        :param data:
        :return:
        """
        return self.zk.set(zk_node, value=data)

    def delete_node(self, zk_node):
        """
        删除某一节点
        :param zk_node:
        :return:
        """
        self.zk.delete(zk_node)

    def create_node(self, node, flag=False):
        """
        创建zookeeper节点
        :param node:
        :param flag:
        :return:
        """
        try:
            self.zk.create(node, ephemeral=flag)
        except Exception as e:
            logging.info("create zookeeper node:%s failed, err:%s" % (node, e))
            print(node, e)
            return False
        return True

    def cp(self, src, dest):
        """
        copy the local file to zookeeper
        :param src:local file
        :param dest:zookeeper node
        :return:
        """
        if not os.path.isfile(src):
            print("%s: `%s': Local file does not exist" % ('cp', src))
            sys.exit()

        file_size = os.path.getsize(src)
        if file_size > 1048576:
            print("%s: `%s': Local file maximum limit of 1M" % ('cp', src))
            sys.exit()

        self.connect()
        if self.zk.exists(dest):
            print("%s: `%s': Zookeeper exists" % ('cp', dest))
            sys.exit()

        with open(src, 'rb') as file:
            data = file.read()

        self.zk.create(dest)
        self.zk.set(dest, value=data)

    def zk_get_merge_fn(self, process_path, work_node, cur_seq, filename_pool):
        """
        获取filename_pool下的序号,记录redo
        :param process_path
        :param work_node
        :param cur_seq:
        :param filename_pool:
        :return: zk_seq:
                       0: 返回0代表未到合并时间点
                       1: 返回1代表没有抢占到filename_pool
                       next_child:返回获取到的filename_pool节点
        """
        if not self.zk.exists(filename_pool):
            logging.error('no filename_pool in zookeeper')
            sys.exit()
        childs = self.zk.get_children(filename_pool)
        if not childs:
            logging.error('the zookeeper filename_pool is empty')
            sys.exit()
        # zk_fn_seq = childs[0]
        childs = sorted(childs)
        redo_info = []
        for child in childs:
            file_date, zk_seq, prov = child.split('.')
            zk_fs = ("%s%s" % (file_date, zk_seq))
            zk_fs = re.sub("[A-Za-z.]", "", zk_fs)
            if int(zk_fs) > int(cur_seq):
                logging.info('zk_seq:%s > cur_seq:%s, wait...' %
                             (zk_fs, cur_seq))
                return 0
            zk_seq = int(zk_seq) + 1
            if zk_seq > self.MAX_MERGE_FILE_SEQUENCE:
                zk_seq = 0
                file_date = datetime.datetime.strptime(file_date, '%Y%m%d')
                next_time = file_date + datetime.timedelta(days=1)
                file_date = ('%s%02d%02d' %
                             (next_time.year, next_time.month, next_time.day))
            zk_seq = "%03d" % zk_seq
            next_child = '%s.%s.%s' % (file_date, zk_seq, prov)
            # 创建一次事务,删除旧的序号并创建新的序号,保证原子性
            transaction_request = self.zk.transaction()
            transaction_request.delete("%s/%s" % (filename_pool, child))
            transaction_request.create("%s/%s" % (filename_pool, next_child))
            redo_seq = ",".join([file_date, zk_seq, prov])
            redo_info.append("filenamepool:" + redo_seq)
            redo_node = process_path + "/" + work_node + "/" + "redo"
            self.create_node(redo_node)
            self.set_node_value(redo_node, ";".join(redo_info).encode("utf-8"))
            results = transaction_request.commit()
            if results[0] is True and results[1] == (
                    "%s/%s" % (filename_pool, next_child)):
                return next_child
            else:
                continue
        return 1
    # 为了方便观察而设置的阻塞时间
    sleep = 1 if len(sys.argv) < 2 else int(sys.argv[1])
    # 是否使用锁 "yes"/"no"
    is_lock = "yes" if len(sys.argv) > 2 and sys.argv[2] == "yes" else "no"

    # zookeeper 地址
    hosts = "127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183"
    # 在这个节点上做自增操作
    path = "/data"

    # 启动 kazoo
    zk = KazooClient(hosts=hosts)
    zk.start()

    # 初始化数据节点
    exists = zk.exists(path)
    if exists is None:
        zk.create(path, b"0")

    # 执行次数
    times = 0
    while True:
        times += 1
        work(zk, path, sleep, is_lock, times)

        if times > 10 // sleep:
            break

    print("该节点当前的值:", zk.get(path)[0])
    zk.stop()
Beispiel #39
0
app.config['ZK_HOST'] = environ.get('ZK_HOST')

STORAGE_HOST = []
for i in range(4):
    STORAGE_HOST.append(environ.get('STORAGE_HOST_' + str(i)))

MAX_RETRIES = 3

db = MongoEngine(app)
jwt = JWTManager(app)

zk_app = KazooClient(hosts=app.config['ZK_HOST'])
zk_app.start()
zk_app.ensure_path("/app")

if not zk_app.exists('/app/1'):
    zk_app.create('/app/1', b"5000", ephemeral=True)


def zk_get_storage_children(zk_client):
    retries = 0
    if not zk_client.exists('/storage'):
        return []

    while len(
            zk_client.get_children('/storage')) < 2 and retries < MAX_RETRIES:
        retries += 1
        time.sleep(1)

    # if retries == MAX_RETRIES:
    #     return []
Beispiel #40
0
class ValidatorDetector:
    def __init__(self):
        # 获取Client实例。
        self.zk = KazooClient(
            hosts="192.168.21.145:2181,192.168.21.144:2181,192.168.21.141:2181",
            timeout=2)

        # 设置节点监听和数值监听。
        self.my_watch1 = ChildrenWatch(
            client=self.zk, path="/",
            func=self.validator_watcher_fun)  # func= 设置监听函数,这个监听会一直有效。
        self.my_watch2 = DataWatch(client=self.zk,
                                   path="/app7",
                                   func=self.data_watch_fun)

        # 初始化连接到zookeeper
        self.zk.start()

    def validator_watcher_fun(self, children):
        """
        监听函数
        :param children: 返回的字节点
        :return:
        """
        print("The children now are:", children)
        # self.zk.get_children("/", )

    def data_watch_fun(self, data, stat, event):
        """监听函数"""
        print("The Node data is Change", data, stat, event)

    def create_node(self):
        """创建节点"""
        self.zk.create('/app-python', b'python_create_node', makepath=True)

    def is_exist(self, path):
        """判断节点是否存在"""
        print("Yes") if (self.zk.exists(path) is not None) else print("No")

        # if self.zk.exists(path) is not None:
        #     print("Yes")
        # else:
        #     print("No")

    def getData(self, path):
        """获取节点上的数据"""
        try:
            ret = self.zk.get(path)  # 返回一个元组,第一个元素是value,第二个是Stat
            print("%s : " % path, ret)
        except:
            print("%s : 无数据 " % path)

    def setData(self, path, value):
        """更新数据"""
        try:
            # 传入的value必须转为byte[]
            self.zk.set(path, bytes(value, encoding='utf8'))
            ret = self.zk.get(path)
            print("更新数据成功。", str(ret[0], encoding='utf8'))
        except Exception as ex:
            print("更新数据失败。", str(ex))

    def delete(self, path):
        """删除节点"""
        try:
            if self.zk.delete(path):
                print("删除 %s 成功" % path)
        except Exception as ex:
            print("删除 %s 失败!" % path, str(ex))
Beispiel #41
0
status = 0
write_stat = 0
write_ret=dict()




#SETTING UP INITIAL ZOOKEEPER STRUCTURE
mpath = '/master'
spath = '/slaves'
sspath = '/slaves/slave'

zk.delete(mpath, recursive=True)
zk.delete(spath, recursive=True)

if zk.exists(mpath):
    print("We have a master.")
else:
   zk.create(mpath,b'')

if zk.exists(spath):
    print("We have slaves.")
else:
    zk.create(spath, b'')

mdata = ''
for key in master_id:
    mdata = key+'pid'+str(master_id[key])

sdata='All the slaves.'.encode()
Beispiel #42
0
class ZkClient:
    def __init__(self, hosts, ports, on_failure_action=None):
        server = ''
        for host, port in zip(hosts, ports):
            server = server + '%s:%s,' % (host, port)

        server = server[:len(server) - 1]
        self.zk = KazooClient(server)
        if on_failure_action is not None:
            if on_failure_action == 'RECONN':
                self.zk.add_listener(self.__zk_status_listener__)
        self.__service_node = ""
        self.zk.start()

    def get_children(self, node):
        if self.zk.exists(node):
            return self.zk.get_children(node)

    def create_node(self, node_name, data):
        self.zk.ensure_path(node_name)

    def update_node(self, node_name, data):
        self.zk.set(node_name, data)

    def delete_node(self, node_name):
        self.zk.delete(node_name)

    def read_data(self, node):
        if self.zk.exists(node):
            return self.zk.get(node)

    def register_service(self, service_type, service_id):
        """
        向Zookeeper注册远程服务
        :param service_type:    服务类型
        :param service_id:      服务ID
        :return:
        """
        path = ConstData.ServicePath % (service_type, service_id)
        self.__service_node = path
        self.zk.create(path,
                       str(ServiceStatusEnum.Idle).encode("UTF-8"),
                       ephemeral=True,
                       makepath=True)

    def create_task(self, major_task_id, sub_task_id):
        todopath = ConstData.TodoTaskPath % (major_task_id, sub_task_id)
        self.zk.create(todopath, makepath=True)

    def require_task(self, major_task_id, sub_task_id, service_id) -> bool:
        '''
        向zk请求一个任务
        首先向特定的任务类型节点下请求一个临时节点,如果没有报错,则表示任务请求成功,否则任务请求失败
        service_id指定了执行任务的服务
        '''
        todo_path = ConstData.TodoTaskPath % (major_task_id, sub_task_id)
        path = ConstData.InProgressPath % (major_task_id, sub_task_id)
        result = False
        try:
            # 首先确认待办中存在对应的节点
            if self.zk.exists(todo_path):
                self.zk.create(path,
                               service_id.encode('UTF-8'),
                               ephemeral=True,
                               makepath=True)
                try:
                    self.zk.set(self.__service_node,
                                str(ServiceStatusEnum.Running).encode("UTF-8"))
                    result = True
                except:
                    # 如果更改状态出错,应该放弃该任务
                    self.zk.delete(path)
                    result = False
            # 任务节点不存在,则任务已经被执行了
            else:
                result = False
        except Exception as e:
            print(e)
            # 出现异常,任务请求失败
            result = False
        return result

    def finish_task(self, major_task_id, task_id):
        todo_path = ConstData.TodoTaskPath % (major_task_id, task_id)
        path = ConstData.InProgressPath % (major_task_id, task_id)
        self.zk.delete(todo_path)
        self.zk.delete(path)

    def task_execute_error(self, task_type, task_id):
        path = ConstData.InProgressPath % (task_type, task_id)
        self.zk.delete(path)

    def __zk_status_listener__(self, state):
        if state == KazooState.LOST:
            self.zk.start()

    def start_service(self):
        self.zk.set(self.__service_node,
                    str(ServiceStatusEnum.Idle).encode("utf-8"))

    def running_service(self):
        self.zk.set(self.__service_node,
                    str(ServiceStatusEnum.Running).encode("utf-8"))

    def stop_service(self):
        # 停止服务其实只是把节点状态设置为了服务下线
        self.zk.set(self.__service_node,
                    str(ServiceStatusEnum.Offline).encode("utf-8"))

    def idle_service(self):
        self.start_service()
Beispiel #43
0
zk = KazooClient(hosts='zoo:2181', timeout=1.0)
zk.start(timeout=1)

cmd = "cat /proc/self/cgroup | grep 'docker' | sed 's/^.*\///' | tail -n1"
cid = subprocess.check_output(cmd, shell=True)
cid = cid.decode("utf-8")
cid = cid[0:12]

client2 = docker.APIClient()
pid = client2.inspect_container(cid)['State']['Pid']
print("---PID", pid)
print("---CID", cid)

zk.ensure_path("/worker")

if zk.exists("/worker/slave"):
    print("Slave exists")
else:
    zk.create("/worker/slave", b"hi")

if zk.exists("/worker/master"):
    present = 0
    print("Master exists")
    ms = "/worker/master"
    data, stat = zk.get(ms)
    data = data.decode("utf-8")
    ind = data.find('PID')
    pid_master = data[ind + 5:len(data) + 1]
    pid_master = int(pid_master)
    print("HELLO FROM MASTERKK - ", pid_master)
    data, stat = zk.get("/worker/master")
Beispiel #44
0
log_dir = '/home/shweta/workspace/research/dag-placement/log/msr'
processing_intervals = [1, 5, 10, 15, 20, 25, 30]
vertex_bbb = 'bbb-a702'
collector_bbb = 'bbb-ed97'

for proc in processing_intervals:
    for sel in [.5, 1]:
        for rate in range(1, 21):
            print('Running experiment for proc:%d, sel:%.1f and rate:%d\n' %
                  (proc, sel, rate))

            #start zk
            zk = KazooClient(zk_connector)
            zk.start()
            #set runid
            if not zk.exists('/dom%d/runid' % (domain_id)):
                zk.ensure_path('/dom%d/runid' % (domain_id))
            zk.set('/dom%d/runid' % (domain_id), b'%d' % (rate))

            #make sub-dirs
            if not os.path.exists('%s/p%d/s%.1f/r%d' %
                                  (log_dir, proc, sel, rate)):
                os.makedirs('%s/p%d/s%.1f/r%d' % (log_dir, proc, sel, rate))
            if not os.path.exists('%s/p%d/s%.1f/r%d/graphs' %
                                  (log_dir, proc, sel, rate)):
                os.makedirs('%s/p%d/s%.1f/r%d/graphs' %
                            (log_dir, proc, sel, rate))

            with open(
                    '%s/p%d/s%.1f/r%d/graphs/g1.txt' %
                (log_dir, proc, sel, rate), 'w') as f:
Beispiel #45
0
class ZookeeperClient(object):
    def __init__(self, hosts=None, read_only=True):
        self.hosts = hosts if hosts else ENSEMBLE.get()
        self.read_only = read_only

        hdfs = cluster.get_hdfs()

        if hdfs is None:
            raise ZookeeperConfigurationException(
                'No [hdfs] configured in hue.ini.')

        if hdfs.security_enabled:
            self.sasl_server_principal = PRINCIPAL_NAME.get()
        else:
            self.sasl_server_principal = None

        self.zk = KazooClient(hosts=self.hosts,
                              read_only=self.read_only,
                              sasl_server_principal=self.sasl_server_principal)

    def start(self):
        """Start the zookeeper session."""
        self.zk.start()

    def stop(self):
        """Stop the zookeeper session, but leaves the socket open."""
        self.zk.stop()

    def close(self):
        """Closes a stopped zookeeper socket."""
        self.zk.close()

    def get_children_data(self, namespace):
        children = self.zk.get_children(namespace)

        children_data = []

        for node in children:
            data, stat = self.zk.get("%s/%s" % (namespace, node))
            children_data.append(data)

        return children_data

    def path_exists(self, namespace):
        return self.zk.exists(namespace) is not None

    def set(self, path, value, version=-1):
        return self.zk.set(path, value, version)

    def copy_path(self, namespace, filepath):
        if self.read_only:
            raise ReadOnlyClientException(
                'Cannot execute copy_path when read_only is set to True.')

        self.zk.ensure_path(namespace)
        for dir, subdirs, files in os.walk(filepath):
            path = dir.replace(filepath, '').strip('/')
            if path:
                node_path = '%s/%s' % (namespace, path)
                self.zk.create(path=node_path, value='', makepath=True)
            for filename in files:
                node_path = '%s/%s/%s' % (namespace, path, filename)
                with open(os.path.join(dir, filename), 'r') as f:
                    file_content = f.read()
                    self.zk.create(path=node_path,
                                   value=file_content,
                                   makepath=True)

    def delete_path(self, namespace):
        if self.read_only:
            raise ReadOnlyClientException(
                'Cannot execute delete_path when read_only is set to True.')

        self.zk.delete(namespace, recursive=True)

    def __enter__(self):
        """Start a zookeeper session and return a `with` context."""
        self.zk.start()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        """Stops and closes zookeeper session at the end of the `with` context."""
        try:
            self.stop()
        finally:
            self.close()
Beispiel #46
0
class WebWindow(object):
	def setupUi(self, Window):
		Window.setObjectName("Window")
		self.centralwidget = QtWidgets.QWidget(Window)
		self.centralwidget.setObjectName("centralwidget")
		self.verticalLayout = QtWidgets.QVBoxLayout(self.centralwidget)
		self.verticalLayout.setContentsMargins(0, 0, 0, 0)
		self.verticalLayout.setObjectName("verticalLayout")
		self.zk = None
		self.webView = QtWebKitWidgets.QWebView(self.centralwidget)
		self.webView.setObjectName("webView")
		self.webView.setRenderHint(QPainter.Antialiasing, True)
		self.webView.setRenderHint(QPainter.TextAntialiasing, True)
		self.webView.setRenderHint(QPainter.SmoothPixmapTransform, True)
		self.webView.setRenderHint(QPainter.HighQualityAntialiasing, True)
		self.webView.setPage(WebPage())
		frame = self.webView.page().mainFrame()
		frame.javaScriptWindowObjectCleared.connect(self.initJsComm)
		self.verticalLayout.addWidget(self.webView)
		Window.setCentralWidget(self.centralwidget)
		self.retranslateUi(Window)
		QtCore.QMetaObject.connectSlotsByName(Window)
		self.loadDefaultAcl()
	def retranslateUi(self, Window):
		_translate = QtCore.QCoreApplication.translate
		Window.setWindowTitle(_translate("Window", "Zookeeper GUI"))
	def loadLocalFile(self, filename):
		localdir = os.path.abspath(sys.path[0])
		if os.path.isfile(localdir):
			localdir = os.path.dirname(localdir)
		self.webView.setUrl( QtCore.QUrl.fromLocalFile(localdir+'/'+filename) )
	def getCfgVar(self, name):
		localdir = os.path.abspath(sys.path[0])
		if os.path.isfile(localdir):
			localdir = os.path.dirname(localdir)
		cfg = {}
		obj = None
		try:
			obj = open(localdir+'/cfg.json','r')
			cfg = json.loads(obj.read())
		except Exception as e:
			logging.info(str(e))
		finally:
			if obj is not None:
				obj.close()
		if name in cfg:
			return str(cfg[name])
		return ''
	def setCfgVar(self, name, value):
		localdir = os.path.abspath(sys.path[0])
		if os.path.isfile(localdir):
			localdir = os.path.dirname(localdir)
		cfg = {}
		obj = None
		try:
			obj = open(localdir+'/cfg.json','r')
			cfg = json.loads(obj.read())
		except Exception as e:
			pass
		finally:
			if obj is not None:
				obj.close()
		cfg[name] = value
		obj = None
		try:
			obj = open(localdir+'/cfg.json','w')
			obj.truncate()
			obj.write(json.dumps(cfg))
		except Exception as e:
			logging.info(str(e))
		finally:
			if obj is not None:
				obj.close()
	def makeDigestCred(self, user, plain_pass):
		m = hashlib.sha1( bytes(user,'utf8') + b':' + bytes(plain_pass,'utf8') ).digest()
		return user+':'+base64.b64encode(m).strip().decode('utf8')
	def initJsComm(self):
		frame = self.webView.page().mainFrame()
		frame.addToJavaScriptWindowObject('py',self)
	@pyqtSlot(str)
	def jsSetWinTitle(self, title):
		_translate = QtCore.QCoreApplication.translate
		self.setWindowTitle(_translate("Window", title))
	@pyqtSlot(str, result=str)
	def jsCheckYaml(self, s):
		try:
			a = yaml.load(s)
		except Exception as e:
			return str(e)
		if a is None:
			return 'Failed'
		return ''
	@pyqtSlot(str,result=str)
	def jsGetCfg(self, name):
		return self.getCfgVar(name)
	@pyqtSlot(str,str)
	def jsSetCfg(self, name, value):
		self.setCfgVar(name, value)
	def loadDefaultAcl(self):
		self.updateDefaultAclCache( self.getCfgVar('defaultacl') )
	def updateDefaultAclCache(self, list_str):
		if list_str is not None and len(list_str)>0:
			cache = json.loads(list_str)
			self.default_acl_plain = []
			self.default_acl = []
			for one in cache:
				if(one['scheme']=='world'):
					self.default_acl_plain.append(one)
					acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('world', 'anyone') )
					self.default_acl.append(acl)
				elif(one['scheme']=='digest'):
					self.default_acl_plain.append(one)
					if 'id' in one:
						acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', one['id']) )
					else:
						acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', self.makeDigestCred(one['user'],one['pass'])) )
					self.default_acl.append(acl)
				elif(one['scheme']=='ip'):
					self.default_acl_plain.append(one)
					acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('ip', one['ip']) )
					self.default_acl.append(acl)
		else:
			self.default_acl_plain = []
			self.default_acl = None
	@pyqtSlot(str,result=str)
	def jsSetDefaultAcl(self, list_str):
		self.updateDefaultAclCache(list_str)
		self.setCfgVar('defaultacl', json.dumps(self.default_acl_plain))
	@pyqtSlot(result=str)
	def jsGetDefaultAcl(self):
		return json.dumps(self.default_acl_plain)
	@pyqtSlot()
	def jsGetZk(self):
		return self.zk
	@pyqtSlot(result=int)
	def jsZkIsConnected(self):
		if self.zk is not None:
			return int(self.zk.state=='CONNECTED')
		return 0
	@pyqtSlot(str, str, result=str)
	def jsZkConnect(self,host, auth_list_str):
		try:
			if self.zk is not None:
				#self.zk.remove_listener(self.onZkStateChange)
				self.zk.stop()
				self.zk.close()
			self.zk = KazooClient(hosts=host)
			#self.zk.add_listener(self.onZkStateChange)
			self.zk.start(15)
			auth_list = json.loads(auth_list_str)
			for one in auth_list:
				cred = self.makeDigestCred(one['user'], one['pass'])
				self.zk.add_auth('digest', one['user']+':'+one['pass'])
		except Exception as e:
			logging.error("jsZkConnect, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return str(e)+', traceback='+str(strlist)
		return ''
	#def onZkStateChange(self,state):
	#	frame = self.webView.page().mainFrame()
	#	frame.evaluateJavaScript("onPyZkStateChange('"+state+"')")
	@pyqtSlot(str, result=QVariant)
	def jsZkGetChildren(self, path):
		try:
			logging.info("jsZkGetChildren, path="+path)
			children = self.zk.get_children(path)
		except NoNodeError:
			logging.error("jsZkGetChildren, NoNodeError")
			return QVariant({"err":"node not exists"})
		except Exception as e:
			logging.error("jsZkGetChildren, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return QVariant({"err":str(e)+', traceback='+str(strlist)})
		return QVariant({"err":"", "children":children})
	@pyqtSlot(str, result=QVariant)
	def jsZkGet(self, path):
		try:
			logging.info("jsZkGet, path="+path)
			ret = self.zk.get(path)
		except NoNodeError:
			logging.error("jsZkGet, NoNodeError")
			return QVariant({"err":"node not exists"})
		except Exception as e:
			logging.error("jsZkGet, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return QVariant({"err":str(e)+', traceback='+str(strlist)})
		ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ret[1].ctime/1000))
		mtime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ret[1].mtime/1000))
		stat = {'ctime':ctime,'mtime':mtime,'version':ret[1].version}
		data = ''
		if ret[0] is not None:
			data = ret[0].decode('utf8')
		else:
			logging.info('jsZkGet data None, path='+path)
		return QVariant({"err":"", "data":data, "stat":QVariant(stat)})
	@pyqtSlot(str, str, int, result=str)
	def jsZkSet(self, path, data, ver):
		try:
			logging.info("jsZkSet, path="+path+',ver='+str(ver))
			self.zk.set(path, bytes(data, 'utf8'),ver)
		except NoNodeError as e:
			logging.error("jsZkSet, NoNodeError")
			return "node not exists"
		except BadVersionError as e:
			logging.error("jsZkSet, BadVersionError")
			return "bad version"
		except Exception as e:
			logging.error("jsZkSet, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return str(e)+', traceback='+str(strlist)
		return ''
	@pyqtSlot(str, result=QVariant)
	def jsZkGetAcl(self, path):
		try:
			logging.info("jsZkGetAcl, path="+path)
			ret = self.zk.get_acls(path)
		except NoNodeError as e:
			logging.error("jsZkGetAcl, NoNodeError")
			return QVariant({"err":"node not exists"})
		except Exception as e:
			logging.error("jsZkGetAcl, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return QVariant({"err":str(e)+', traceback='+str(strlist)})
		lst = []
		for acl in ret[0]:
			dacl = {"perm":acl.perms,'scheme':acl.id.scheme,'id':acl.id.id}
			lst.append(QVariant(dacl))
		stat = {'ctime':ret[1].ctime,'mtime':ret[1].mtime,'version':ret[1].version}
		return QVariant({"err":"", "acl_list":QVariant(lst), "stat":QVariant(stat)})
	@pyqtSlot(str, str, result=str)
	def jsZkSetAcl(self, path, list_str):
		try:
			acl_list = None
			if list_str is not None and len(list_str)>0:
				cache = json.loads(list_str)
				acl_list = []
				for one in cache:
					if(one['scheme']=='world'):
						acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('world', 'anyone') )
						acl_list.append(acl)
					elif(one['scheme']=='digest'):
						if 'id' in one:
							acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', one['id']) )
						else:
							acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('digest', self.makeDigestCred(one['user'],one['pass'])) )
						acl_list.append(acl)
					elif(one['scheme']=='ip'):
						acl = kazoo.security.ACL( one['perm'], kazoo.security.Id('ip', one['ip']) )
						acl_list.append(acl)
			self.zk.set_acls(path, acl_list)
		except NoNodeError as e:
			logging.error("jsZkSetAcl, NoNodeError")
			return "node not exists"
		except InvalidACLError as e:
			logging.error("jsZkSetAcl, InvalidACLError")
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return "invalid acl, traceback: "+str(strlist)
		except BadVersionError as e:
			logging.error("jsZkSetAcl, BadVersionError")
			return "bad version error"
		except Exception as e:
			logging.error("jsZkSetAcl, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return str(e)+', traceback='+str(strlist)
		return ''
	@pyqtSlot(str,str,int,int,result=str)
	def jsZkCreate(self, path, data, ephem, seq):
		try:
			logging.info("jsZkCreate, path="+path)
			self.zk.create(path=path, value=bytes(data,'utf8'), ephemeral=bool(ephem), sequence=bool(seq))
			if self.default_acl is not None and len(self.default_acl)>0:
				self.zk.set_acls(path, self.default_acl)
		except NoNodeError as e:
			logging.error("jsZkCreate, NoNodeError")
			return "node not exists"
		except NodeExistsError as e:
			logging.error("jsZkCreate, NodeExistsError")
			return "node already exists"
		except NoChildrenForEphemeralsError as e:
			logging.error("jsZkCreate, NoChildrenForEphemeralsError")
			return "ephemeral node can not have child"
		except Exception as e:
			logging.error("jsZkCreate, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return str(e)+', traceback='+str(strlist)
		return ''
	@pyqtSlot(str, int, int, result=str)
	def jsZkDelete(self, path, ver, recursive):
		try:
			logging.info("jsZkDelete, path="+path+',ver='+str(ver)+', recursive='+str(recursive))
			self.zk.delete(path, ver, bool(recursive))
		except NoNodeError as e:
			logging.error("jsZkDelete, NoNodeError")
			return "node not exists"
		except BadVersionError as e:
			logging.error("jsZkDelete, BadVersionError")
			return "bad version"
		except NotEmptyError as e:
			logging.error("jsZkDelete, NotEmptyError")
			return "node not empty"
		except Exception as e:
			logging.error("jsZkDelete, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return str(e)+', traceback='+str(strlist)
		return ''
	@pyqtSlot(str, str, int, int, result=str)
	def jsZkCopy(self, dest_path, ori_path, max_depth, children_only):
		logging.info("jsZkCopy, dest_path="+dest_path+", ori_path="+ori_path+", children_only="+str(children_only))
		#copy node first
		if children_only==0:
			try:
				ori_data = self.zk.get(ori_path)
				if self.zk.exists(dest_path) is None:
					self.zk.create(dest_path, ori_data[0], acl=self.default_acl)
				else:
					self.zk.set(dest_path, ori_data[0])
			except NoNodeError as e:
				logging.error("jsZkCopy, node, NoNodeError, ori_path="+ori_path+', dest_path='+dest_path)
				return "node not exists"
			except Exception as e:
				logging.error("jsZkCopy, "+str(e))
				t,v,tb = sys.exc_info()
				strlist = traceback.format_tb(tb)
				return str(e)+', traceback='+str(strlist)
		#copy children
		path = ''
		try:
			max_depth -= 1
			path = ori_path
			ori_children = self.zk.get_children(ori_path)
			for child in ori_children:
				path = ori_path+'/'+child
				ret = self.jsZkCopy(dest_path+'/'+child, ori_path+'/'+child, max_depth, 0)
				if isinstance(ret, QVariant):
					return ret
				elif len(ret)>0:
					return ret
		except NoNodeError as e:
			logging.error("jsZkCopy, child, NoNodeError")
			return "node not exists, path="+path
		except Exception as e:
			logging.error("jsZkCopy, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return str(e)+', traceback='+str(strlist)
		return ''
	'''
	@pyqtSlot(str, str, int, result=str)
	def jsZkCopyChildren(self, dest_path, ori_path, max_depth):
		path = ''
		try:
			max_depth -= 1;
			logging.info("jsZkCopyChildren, dest_path="+dest_path+", ori_path="+ori_path)
			path = ori_path
			ori_children = self.zk.get_children(ori_path)
			path = dest_path
			dest_children = self.zk.get_children(dest_path)
			for child in ori_children:
				if child in dest_children:
					return 'child ['+child+'] is found in both path'
			for child in ori_children:
				path = ori_path+'/'+child
				data = self.zk.get(path)[0]
				path = dest_path+'/'+child
				self.zk.create(path, data, acl=self.default_acl)
				if max_depth>0:
					ret = self.jsZkCopyChildren(dest_path+'/'+child, ori_path+'/'+child, max_depth)
					if len(ret)>0:
						return ret
		except NoNodeError as e:
			logging.info("jsZkCopyChildren, NoNodeError")
			return "node not exists, path="+path
		except ZookeeperError as e:
			logging.info("jsZkCopyChildren, ZookeeperError")
			return str(e)+', path='+path
		return ''
	'''
	@pyqtSlot(str, int, result=str)
	def jsZkDeleteChildren(self, main_path, max_depth):
		path = ''
		try:
			max_depth -= 1
			path = main_path
			logging.info("jsZkDeleteChildren, path="+main_path)
			children = self.zk.get_children(path)
			for child in children:
				path = main_path+'/'+child
				if max_depth>0:
					ret = self.jsZkDeleteChildren(path, max_depth)
					if len(ret)>0:
						return ret
				self.zk.delete(path)
		except NoNodeError as e:
			logging.error("jsZkDeleteChildren, NoNodeError")
			return "node not exists, path="+path
		except Exception as e:
			logging.error("jsZkDeleteChildren, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return str(e)+', traceback='+str(strlist)
		return ''
	@pyqtSlot(str, str, int, int, result=str)
	def jsZkExport(self, local_dir, main_path, max_depth, without_acl):
		path = ''
		try:
			max_depth -= 1
			path = main_path
			logging.info("jsZkExport, path="+main_path+' to local dir '+local_dir)
			data = self.zk.get(main_path)
			p = pathlib.Path(local_dir)
			if not p.exists():
				p.mkdir(parents=True)
			elif not p.is_dir():
				return 'local '+local_dir+' exists, but not a directory'
			for child in p.iterdir():
				return 'local path '+local_dir+' is not empty, clear it first'
			p = pathlib.Path(local_dir+'/____data')
			p.touch()
			obj = open(str(p),'wb')
			try:
				if data[0] is not None:
					obj.write(data[0])
			finally:
				obj.close()
			if not without_acl:
				ret = self.zk.get_acls(path)
				lst = []
				if ret is not None:
					for acl in ret[0]:
						lst.append( {"perm":acl.perms,'scheme':acl.id.scheme,'id':acl.id.id} )
				p = pathlib.Path(local_dir+'/____acl')
				p.touch()
				obj = open(str(p),'w')
				try:
					obj.write(json.dumps(lst))
				finally:
					obj.close()
			children = self.zk.get_children(path)
			if children is not None:
				for child in children:
					if child=='zookeeper':
						continue
					path = main_path+'/'+child
					if max_depth>0:
						ret = self.jsZkExport(local_dir+'/'+child, path, max_depth, without_acl)
						if len(ret)>0:
							return ret
		except NoNodeError as e:
			logging.error("jsZkExport, NoNodeError")
			return "node not exists, path="+path
		except Exception as e:
			logging.error("jsZkExport, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return str(e)+', traceback='+str(strlist)
		return ''
	@pyqtSlot(str, str, int, int, result=str)
	def jsZkImport(self, local_dir, main_path, max_depth, without_acl):
		path = ''
		try:
			max_depth -= 1
			path = main_path
			logging.info("jsZkImport, path="+main_path+' from local dir '+local_dir)
			obj = open(local_dir+'/____data', 'rb')
			if self.zk.exists(path) is None:
				self.zk.create(path, obj.read(), acl=self.default_acl)
			else:
				self.zk.set(path, obj.read())
			if not without_acl:
				obj = open(local_dir+'/____acl', 'r')
				acl_list = None
				list_str = obj.read()
				if list_str is not None and len(list_str)>0:
					cache = json.loads(list_str)
					acl_list = []
					for one in cache:
						acl = kazoo.security.ACL( one['perm'], kazoo.security.Id(one['scheme'], one['id']) )
						acl_list.append(acl)
					self.zk.set_acls(path, acl_list)
			p = pathlib.Path(local_dir)
			for child in p.iterdir():
				if not child.is_dir():
					continue
				if child.name=='zookeeper':
					continue
				ret = self.jsZkImport(str(child), path+'/'+child.name, max_depth, without_acl)
				if len(ret)>0:
					return ret
		except NoNodeError as e:
			logging.error("jsZkImport, NoNodeError")
			return "node not exists, path="+path
		except Exception as e:
			logging.error("jsZkImport, "+str(e))
			t,v,tb = sys.exc_info()
			strlist = traceback.format_tb(tb)
			return str(e)+', traceback='+str(strlist)
		return ''
Beispiel #47
0
            # 发布精确任务
            # zk
            elif jdata[0]['Agreement'] == '5':
                urls = jdata[0]['Content'].split(', ')
                print 'urls', urls
                url = str(urls[0])
                keyword = urls[1]
                priority = int(urls[2])
                task_type = info_zk[url][1] + '_' + keyword

                ready_list = self.get_ready_node(priority)
                for i in range(priority):
                    zk.set("/command/" + ready_list[i], value=task_type)
                    node_map[ready_list[i]][2] += 1  # task_num++
                if zk.exists("/signal/" + task_type) != None:
                    temp_list = zk.get_children("/signal/" + task_type)
                    for i in temp_list:
                        zk.delete('/signal/' + task_type + '/' + str(i))
                else:
                    zk.create("/signal/" + task_type)

                zk.create("/signal/" + task_type + '/start')
                for ready_node in ready_list:
                    sql = "insert into task values('" + url + "', '" + keyword + "','" + info[
                        url][1] + "'," + int(ready_node) + ",now(),'5')"
                    nsql = "update info set tasknum=tasknum+1 where id='" + ready_node + "'"
                    # 发布任务后在amount表中插入20条空数据,防止出现时间倒序
                    tsql = "insert into amount values('" + url + "','" + keyword + "',0,'" + info[
                        url][1] + "',now())"
                    for i in range(20):
Beispiel #48
0
class KafkaMonitor(object):
    def __init__(self,
                 queue,
                 kf_ip_port='localhost',
                 zk_ip_port='localhost',
                 sleep_time=10):
        # 连接 kafka
        self.kafka_hosts = kf_ip_port
        self.broker = SimpleClient(hosts=self.kafka_hosts)
        # 连接zookeeper
        self.zookeepers_hosts = zk_ip_port
        self.zk = KazooClient(hosts=self.zookeepers_hosts, read_only=True)
        # 数据存放
        self.queue = queue
        # 时间间隔
        self.sleep_time = sleep_time - 1

    def get_group(self):
        """获取zookeepers下的group"""
        group_name = self.zk.get_children('/consumers')
        return group_name

    def get_topics(self, group):
        """group下的topic"""
        try:
            topics = self.zk.get_children("/consumers/%s/owners" % group)
            return topics
        except Exception as e:
            logging.error('get group topics failed! %s' % e)

    def get_logsize(self, topic):
        return get_logsize(self.broker, topic)

    def get_partition_offset(self, group, topic, partition):
        path = "/consumers/%s/offsets/%s/%s" % (group, topic, partition)

        if self.zk.exists(path):
            data, stat = self.zk.get(path)
            return data
        else:
            logger.error('get group:%s topic:%s partition:%s offset failed!' %
                         (group, topic, partition))
            return None

    def get_offset(self, group, topic):
        """topic下消费量"""
        try:
            offset = {}

            # 获取 partitions
            path = "/consumers/%s/offsets/%s" % (group, topic)
            partitions = sorted(self.zk.get_children(path))

            for partition in partitions:
                data = self.get_partition_offset(group, topic, partition)
                offset[int(partition)] = int(data)

            return offset

        except Exception as e:
            logger.error('get group:%s topic:%s offset failed! %s' %
                         (group, topic, e))
            return None

    def get_group_data(self, group, topics):
        """获取一个 group 的所有 topic 的数据"""
        try:
            group_data = {}
            for topic in topics:

                # 获取 topic log_size 值
                log_size = self.get_logsize(topic)
                # 获取 topic offset 值
                offset = self.get_offset(group, topic)

                # topic 内的数据字典
                topic_data = dict(topic_name=topic,
                                  logsize=log_size,
                                  offset=offset,
                                  group_name=group)
                key = '%s_%s' % (group, topic)
                group_data[key] = topic_data

            return group_data

        except Exception as e:
            logger.error('get group:%s offset failed! %s' % (group, e))
            return None

    def get_data(self):
        try:
            # 获取 groups_name 列表
            groups_name = self.get_group()
            # 定义本次循环 data 字典
            data = {}

            for group in groups_name:

                # 获取 group 内的 topics 列表
                topics = self.get_topics(group)
                if not topics:
                    continue

                # 获取 group 数据列表
                group_data = self.get_group_data(group, topics)
                if group_data:
                    # group 数据加入到本次循环的data 字典
                    data = dict(data, **group_data)

            return data

        except Exception as e:
            logger.error('get offset failed! %s' % e)
            time.sleep(3)

    def run(self):
        self.zk.start()
        while True:
            t0 = time.clock()
            data = self.get_data()
            self.queue.put(data)
            # 执行间隔时间
            t = time.clock() - t0

            # 睡眠时间减去执行时间 保证间隔时间相等
            sleep_time = self.sleep_time - t
            if sleep_time < 0:
                sleep_time = 0

            time.sleep(sleep_time)
        # Handle being disconnected from Zookeeper
        logging.warning("suspended")
    else:
        # Handle being connected/reconnected to Zookeeper
        logging.info("zookeeper is connected")


zk.add_listener(my_listener)
try:
    zk.start()
except:
    time.sleep(10)
finally:
    zk.start()

if (zk.exists("/Workers")):
    zk.delete("/Workers", recursive=True)

if (zk.exists("/Slave")):
    zk.delete("/Slave", recursive=True)

zk.ensure_path("/Slave/")


@zk.ChildrenWatch("/Workers/", send_event=True)
def watch(children, event):
    print("in zookeeper event")
    if event == None:
        pass
    elif event.type is DELETED:
        print("slave deleted")
Beispiel #50
0
class ZookeeperAgent2(object):
    def __init__(self,
                 ip=None,
                 port=None,
                 serviceType=None,
                 path=None,
                 monitorPort=None):
        self._ip = ip  #'10.66.170.3'
        self._port = port  #'22182'
        self._serviceType = serviceType  #'/workers'
        self._path = path  #'/workers/10.66.170.3'
        self._monitorPort = monitorPort  #'
        self.zk_node = self._path + self._monitorPort
        try:
            self.zk = KazooClient(hosts=self._ip + ':' + self._port,
                                  timeout=500,
                                  read_only=True)
            self.zk.add_listener(self.conn_state_watcher)
            self.zk.start()
        except Exception as ex:
            self.init_ret = False
            self.err_str = "Create KazooClient failed! Exception: %s" % str(ex)
            logging.error(self.err_str)

    def conn_state_watcher(self, state):
        if state == KazooState.CONNECTED:
            print("Now connected")

            if self.zk_node is None:
                print("create method has not been called")
                return
            # info_keeper = InfoKeeper(self)
            # info_keeper.start()
        elif state == KazooState.LOST:
            print("Now lost")
        else:
            print("Now suspended")

    def main(self):

        if self.zk.exists(self._path):
            print(self._path + " already exists")
        else:
            self.zk.create(self._path)

        while True:
            time.sleep(10)
            self.check_port(self._ip, int(self._monitorPort))

    def check_port(self, ip, port):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.settimeout(5)
        host = (ip, port)
        print(host)
        r = s.connect_ex(host)
        if r == 0:
            if self.zk.exists(self.zk_node):
                self.zk.delete(self.zk_node)
            self.zk.create(self.zk_node, b"ocr/30%", ephemeral=True)
            #print(self.zk.commit())
        else:
            self.zk.delete(self.zk_node)
        s.close()
        return 1
class Broker_Driver ():

	# constructor
	def __init__ (self, args):
		self.name = args.name
		self.zkIPAddr = args.zkIPAddr  # ZK server IP address
		self.zkPort = args.zkPort # ZK server port num
		self.cond = args.cond # used as barrier condition
		self.ppath = args.zkPpath # refers to the parent znode path
		self.topicPath = args.topicPath # refers to the parent znode path
		self.brokerAddressPort = args.port  # indicating if barrier has reached
		self.zk = None  # session handle to the server
		
    # override the run method which is invoked by start
	def init_broker (self):
		# It is here that we connect with the zookeeper server
		# and see if the condition has been met
		try:

			# Instantiate zookeeper client object
			# right now only one host; it could be the ensemble
			hosts = self.zkIPAddr + str (":") + str (self.zkPort)
			self.zk = KazooClient (hosts)

		except:
			print("Unexpected error in ClientApp::run", sys.exc_info()[0])
			raise

	def run_broker(self):
		try:
			# First, open connection to zookeeper
			self.zk.start ()


			if (self.zk.exists (self.ppath)) :
				# if parent path already exists, set first_flat 'False'
				first_flag=False
			else:
				# if parent path does not exist, create the Parent Path and set first_flag 'True'
				# if first_flag is True then delete the tables' contents(publisher, subscriber)
				print("Create the znode Path: "+self.ppath)
				self.zk.create (self.ppath, value=self.name.encode())
				first_flag=True

			# set the broker_path so that children node can be created
			broker_path = self.ppath + str("/") + "guid-n_"
			broker_path = self.zk.create (broker_path, value=self.name.encode(), ephemeral=True, sequence=True)

			
			# Topic Path Creation if not exists
			if ( self.zk.exists (self.topicPath)) :
				print("Path("+self.topicPath+") already exists")
			else :
				# if topic path does not exist, create the Topic Path
				print("Create the Topic Path: "+self.topicPath)
				self.zk.create (self.topicPath, value=b"topic")
			


    
		except:
			print("Unexpected error in ClientApp::run", sys.exc_info()[0])
			raise

		# Get the Host name and IP of the server
		host_name = socket.gethostname() 
		host_ip = socket.gethostbyname(host_name) 

		search_list = []


		mydb = db_connect()
		mycursor = mydb.cursor()

		#print("First_flag: "+str(first_flag))
		# Table Initialization if first_flag is True
		if first_flag == True:
			#print("delete execution")
			mycursor.execute("DELETE FROM publisher")
			mycursor.execute("DELETE FROM subscriber")
			mycursor.execute("DELETE FROM brokerrecord")
			mydb.commit()
		


		# Prepare our context and sockets
		context = zmq.Context()
		frontend = context.socket(zmq.ROUTER)
		bindAddress = "tcp://*:"+self.brokerAddressPort
		frontend.bind(bindAddress)

		brokeraddress = "tcp://"+host_ip+":"+self.brokerAddressPort

		# store the BrokerAddress to the children node
		self.zk.set (broker_path, str(brokeraddress).encode ())
		brokeraddress = "tcp://"+host_ip+":"+self.brokerAddressPort
		print("Broker Zookeeper path: "+broker_path)
		print("Broker Address: "+brokeraddress)


		# Initialize poll set
		poller = zmq.Poller()
		poller.register(frontend, zmq.POLLIN)

		
		# Initialize message_type Variable
		# message_type
		# '1' : publisher Registration(to Mysql DB Table-publisher)
		# '2' : Subscriber Registration(to Mysql DB Table-subscriber)
		# '3' : Publish message through Broker
		message_type = "0"

		# Switch messages between sockets
		while True:
			print("\nWaiting for the message to be received..")
			socks = dict(poller.poll())
			
			if socks.get(frontend) == zmq.POLLIN:
				message = frontend.recv_multipart()
				
				message_type = message[2].decode()
				socket_id = message[3].decode()
				topic = message[4].decode()

				print("\n\n\nReceived msg_type: "+message_type+", socket_id: "+socket_id+", topic: "+topic)
				
				if message_type == "1" : # request to register the publisher
					append_publisher(topic, socket_id)
					# DB Insert to the table 'publisher'
					sql = "INSERT INTO publisher VALUES (%s, %s)"
					val = (topic, socket_id)
					
					mycursor.execute(sql, val)
					mydb.commit()
					print("\nInsert into publisher("+topic+","+socket_id+")")
					#print "SQL execution: "+sql
					
					# Register the publisher's topic to the Zookeeper
					# set the topic_path so that children node can be created
					topic_path = self.topicPath + str("/") + topic

					
					if (self.zk.exists (topic_path)) :
						# if topic already exists then add new 
						print("Topic("+topic_path+") already exists");
					else:
						# if parent path does not exist, create the Parent Path and set first_flag 'True'
						# if first_flag is True then delete the tables' contents(publisher, subscriber)
						print("Create the topic Path: "+topic_path)
						self.zk.create (topic_path, value=topic.encode())

					# set the broker_path so that children node can be created

					topic_path = topic_path + str("/") + topic+"_"					
					topic_path = self.zk.create (topic_path, value=socket_id.encode(), ephemeral=True, sequence=True)


					# send bakc the reply
					#frontend.send_multipart([socket_id.encode(),b"",b"Thank you 1"])
					# return the created topic_path
					frontend.send_multipart([socket_id.encode(),b"",topic_path.encode()])
				elif message_type == "2" :  # request to register the subscriber
					append_subscriber(topic, socket_id)
					# DB Insert to the table 'subscriber'
					mycursor = mydb.cursor()
					sql = "INSERT INTO subscriber VALUES (%s, %s)"
					val = (topic, socket_id)
					mycursor.execute(sql, val)

					mydb.commit()
					print("\nInsert into subscriber("+topic+","+socket_id+")")
					# send bakc the reply
					frontend.send_multipart([socket_id.encode(),b"",b"Thank you 2"])
				elif message_type == "3" :
					#search_list = find_brute(subscriber, topic)
					contents = str(message[5])
					print("\nBroker publishing the message("+contents+") to the subscribers")
					mycursor = mydb.cursor()
					sql = "SELECT * FROM subscriber WHERE topic='"+topic+"'"
					#print(sql)

					mycursor.execute(sql)
					myresult = mycursor.fetchall()
					
					#print "Len of myresult"+str(len(myresult))
					if len(myresult)!=0 : # Add the searched DB result to the search_list
						for x in myresult:
							search_list.append([x[0],x[1]])

						# send the return Messages to the subscribers
						for i in range(len(search_list)):
							return_socket_id = "Wait"+search_list[i][1] #Make the socket_id by adding "Wait" to the front
							#print("return topic: "+topic+", return socket: "+return_socket_id)
							frontend.send_multipart([return_socket_id.encode(),b"",contents.encode()])
							frontend.send_multipart([socket_id.encode(),b"",b"Thank you 3"])
						
						# Initialize the search_list
						search_list = []
					else:
						print("\nNo matched subscriber to receive the contents")
						frontend.send_multipart([socket_id.encode(),b"",b"Thank you 3"])	
Beispiel #52
0
class zookeeper:
    def __init__(self, address):
        self.address = address
        self.zk = KazooClient(address)
        self.zk.start()

        self.arcus_cache_map = {}
        self.arcus_node_map = {}

        self.force = False
        self.meta = ('', None)
        self.meta_mtime = None

    def __repr__(self):
        repr = '[ZooKeeper: %s] %s, %s' % (self.address, self.meta[0],
                                           str(self.meta[1]))

        for code, cache in self.arcus_cache_map.items():
            repr = '%s\n\n%s' % (repr, cache)

        return repr

    def set_force(self):
        self.force = True

    def zk_read(self, path):
        data, stat = self.zk.get(path)
        children = self.zk.get_children(path)
        return data, stat, children

    def zk_children(self, path, watch=None):
        if watch != None:
            return self.zk.get_children(path, watch=watch)
        else:
            return self.zk.get_children(path)

    def zk_children_if_exists(self, path, watch=None):
        if self.zk_exists(path) == False:
            return []

        return self.zk_children(path, watch)

    def zk_exists(self, path):
        if self.zk.exists(path) == None:
            return False

        return True

    def zk_create(self, path, value):
        try:
            self.zk.create(path, bytes(value, 'utf-8'))
        except NodeExistsError:
            if self.force == False:
                raise NodeExistsError

    def zk_delete(self, path):
        try:
            self.zk.delete(path)
        except NoNodeError:
            if self.force == False:
                raise NoNodeError

    def zk_delete_tree(self, path):
        try:
            self.zk.delete(path, recursive=True)
        except NoNodeError:
            if self.force == False:
                raise NoNodeError

    def zk_update(self, path, value):
        try:
            self.zk.set(path, bytes(value, 'utf-8'))
        except NoNodeError:
            if self.force == False:
                raise NoNodeError

    def get_arcus_cache_list(self):
        children = self.zk_children_if_exists('/arcus/cache_list/')
        children += self.zk_children_if_exists('/arcus_repl/cache_list/')

        return children

    def get_arcus_node_of_code(self, code, server):
        # repl case
        children = self.zk_children_if_exists('/arcus_repl/cache_list/' + code)
        children += self.zk_children_if_exists('/arcus/cache_list/' + code)
        ret = []
        for child in children:
            tmp = child.split('^', 2)  # remove repl info
            if len(tmp) == 3:
                child = tmp[2]

            addr, name = child.split('-', 1)
            ip, port = addr.split(':', 1)

            if server != '' and (server != ip and server != name):
                continue  # skip this

            node = arcus_node(ip, port)
            node.name = name
            ret.append(node)

        return ret

    def get_arcus_node_of_server(self, addr):
        ip = socket.gethostbyname(addr)

        children = self.zk_children_if_exists(
            '/arcus_repl/cache_server_mapping/')
        children += self.zk_children_if_exists('/arcus/cache_server_mapping/')
        ret = []
        for child in children:
            l = len(ip)
            if child[:l] == ip:
                code = self.zk_children_if_exists(
                    '/arcus_repl/cache_server_mapping/' + child)
                if len(code) == 0:
                    code = self.zk_children_if_exists(
                        '/arcus/cache_server_mapping/' + child)

                code = code[0]

                tmp = code.split('^')  # remove repl info
                code = tmp[0]

                try:
                    ip, port = child.split(':')
                except ValueError:
                    print('No port defined in cache_server_mapping: %s' %
                          child)
                    continue

                node = arcus_node(ip, port)
                node.code = code
                ret.append(node)

        return ret

    def _get_arcus_node(self, child, results):
        code = self.zk_children_if_exists('/arcus_repl/cache_server_mapping/' +
                                          child)
        if len(code) == 0:
            code = self.zk_children_if_exists('/arcus/cache_server_mapping/' +
                                              child)

        if len(code) == 0:
            print('no childrens in cache_server_mapping error: %s' % child)
            print(code)
            return

        code = code[0]

        tmp = code.split('^')  # remove repl info
        code = tmp[0]

        try:
            ip, port = child.split(':')
        except ValueError:
            print('No port defined in cache_server_mapping: %s' % child)
            ip = child
            port = '0'

        node = arcus_node(ip, port)
        node.code = code
        results.append(node)

    def get_arcus_node_all(self):
        children = self.zk_children_if_exists(
            '/arcus_repl/cache_server_mapping/')
        children += self.zk_children_if_exists('/arcus/cache_server_mapping/')

        ret = []
        threads = []

        #print(children)
        for child in children:
            th = threading.Thread(target=self._get_arcus_node,
                                  args=(child, ret))
            th.start()
            threads.append(th)

        for th in threads:
            th.join()

        return ret

    def _get_arcus_meta(self, child, results):
        data, stat, children = self.zk_read('/arcus/meta/' + child)
        results[child] = [data.decode('utf-8'), stat]

    def get_arcus_meta_all(self):
        if self.zk.exists('/arcus/meta') == None:
            self.zk.create('/arcus/meta', b'arcus meta info')

        children = self.zk.get_children('/arcus/meta')
        print('# children')
        print(children)

        threads = []
        ret = {}

        #print(children)
        for child in children:
            th = threading.Thread(target=self._get_arcus_meta,
                                  args=(child, ret))
            th.start()
            threads.append(th)

        for th in threads:
            th.join()

        return ret

    def _match_code_and_nodes(self, code, cache, meta):
        #repl case
        children = self.zk_children_if_exists('/arcus_repl/cache_list/' + code)
        children += self.zk_children_if_exists('/arcus/cache_list/' + code)
        for child in children:
            tmp = child.split('^', 2)  # remove repl info
            if len(tmp) == 3:
                child = tmp[2]

            addr, name = child.split('-')
            try:
                node = self.arcus_node_map[addr]
            except KeyError:
                print('%s of %s is not defined in cache_server_mapping' %
                      (addr, code))
                ip, port = addr.split(':')
                node = arcus_node(ip, port)
                node.noport = True

            node.active = True
            cache.active_node.append(node)

        for node in cache.node:
            if node.active == False:
                cache.dead_node.append(node)

        if code in meta:
            cache.meta = meta[code]

    def load_all(self):
        codes = self.get_arcus_cache_list()
        for code in codes:
            cache = arcus_cache(self.address, code)
            self.arcus_cache_map[code] = cache

        print('# get_arcus_node_all()')
        nodes = self.get_arcus_node_all()
        print('# done')

        for node in nodes:
            self.arcus_node_map[node.ip + ":" + node.port] = node
            self.arcus_cache_map[node.code].node.append(node)

        # meta info
        print('# get_arcus_meta_all()')
        meta = self.get_arcus_meta_all()
        print('# done')

        print('# match code & nodes')
        threads = []

        for code, cache in self.arcus_cache_map.items():
            th = threading.Thread(target=self._match_code_and_nodes,
                                  args=(code, cache, meta))
            th.start()
            threads.append(th)

        for th in threads:
            th.join()

        print('#done')

        if 'zookeeper' in meta:
            self.meta = meta['zookeeper']

    def _callback(self, event):
        child_list = self.zk.get_children(event.path)
        cloud = os.path.basename(event.path)
        cache = self.arcus_cache_map[cloud]

        event_list = {'created': [], 'deleted': []}
        current = {}
        print('##### active node')
        print(cache.active_node)

        children = []
        for child in child_list:
            addr = child.split('-')[0]
            children.append(addr)

        print('#### children')
        print(children)

        for node in cache.active_node:
            current[node.ip + ':' + node.port] = True

        print('##### current')
        print(current)

        for node in cache.active_node:
            addr = node.ip + ':' + node.port
            if addr not in children:
                event_list['deleted'].append(addr)
                cache.active_node.remove(node)

        for child in children:
            if child not in current:
                event_list['created'].append(child)
                ip, port = child.split(':')
                node = arcus_node(ip, port)
                cache.active_node.append(node)

        print('####### result')
        print(cache.active_node)

        self.callback(event, event_list)
        children = self.zk.get_children(event.path, watch=self._callback)

    def watch(self, callback):
        self.callback = callback
        for code, cache in self.arcus_cache_map.items():
            children = self.zk_children_if_exists('/arcus/cache_list/' + code,
                                                  watch=self._callback)
            children += self.zk_children_if_exists('/arcus_repl/cache_list/' +
                                                   code,
                                                   watch=self._callback)
Beispiel #53
0
	alphadict={"@": "atr" , "?":"qm" , "#":"hash" ,"$":"dollar" , "%":"perc"}	
	from socket import *
	serverPort = 45786
	serverSocket = socket(AF_INET,SOCK_STREAM) 
	serverSocket.bind(('',serverPort)) 
	serverSocket.listen(1)
	temp_var1=[]
	port_list=[]
	name_list=[]
	port_list1=[]
	name_list1=[]
	lis_mixed=[]
	list_mixed=[]
	change=" "
	bk_dict={}
	while zk.exists("/masternode") or zk.exists("/masternode1") :
		if zk.exists("/specnode") :
			zk.set("/specnode","45786 special")
		else :	
			zk.create("/specnode", "45786 special", ephemeral=True)
			if zk.exists("/zspec") and change==" " :
				t=zk.get("/zspec")
				bk_dict=ast.literal_eval(t[0])
				change="!!!"
				if len(bk_dict)>=len(alphadict):
					alphadict=bk_dict 
					print(alphadict)
				else :
					pass

		connectionSocket, addr = serverSocket.accept() 
Beispiel #54
0
class ZooKeeper(object):
    '''
    Class implementing the ZooKeeper interface.

    This class uses the facade design pattern to keep common interaction
    with the ZooKeeper API simple and consistent for the caller, and
    limits coupling between objects. It allows for more complex interactions
    by providing direct access to the client connection when needed (though
    that is discouraged). It also provides for a convenient entry point for
    testing only ZooKeeper interactions.
    '''

    log = logging.getLogger("zuul.zk.ZooKeeper")

    REQUEST_ROOT = '/nodepool/requests'
    REQUEST_LOCK_ROOT = "/nodepool/requests-lock"
    NODE_ROOT = '/nodepool/nodes'

    # Log zookeeper retry every 10 seconds
    retry_log_rate = 10

    def __init__(self):
        '''
        Initialize the ZooKeeper object.
        '''
        self.client = None
        self._became_lost = False
        self._last_retry_log = 0

    def _dictToStr(self, data):
        return json.dumps(data).encode('utf8')

    def _strToDict(self, data):
        return json.loads(data.decode('utf8'))

    def _connection_listener(self, state):
        '''
        Listener method for Kazoo connection state changes.

        .. warning:: This method must not block.
        '''
        if state == KazooState.LOST:
            self.log.debug("ZooKeeper connection: LOST")
            self._became_lost = True
        elif state == KazooState.SUSPENDED:
            self.log.debug("ZooKeeper connection: SUSPENDED")
        else:
            self.log.debug("ZooKeeper connection: CONNECTED")

    @property
    def connected(self):
        return self.client.state == KazooState.CONNECTED

    @property
    def suspended(self):
        return self.client.state == KazooState.SUSPENDED

    @property
    def lost(self):
        return self.client.state == KazooState.LOST

    @property
    def didLoseConnection(self):
        return self._became_lost

    def resetLostFlag(self):
        self._became_lost = False

    def logConnectionRetryEvent(self):
        now = time.monotonic()
        if now - self._last_retry_log >= self.retry_log_rate:
            self.log.warning("Retrying zookeeper connection")
            self._last_retry_log = now

    def connect(self, hosts, read_only=False, timeout=10.0):
        '''
        Establish a connection with ZooKeeper cluster.

        Convenience method if a pre-existing ZooKeeper connection is not
        supplied to the ZooKeeper object at instantiation time.

        :param str hosts: Comma-separated list of hosts to connect to (e.g.
            127.0.0.1:2181,127.0.0.1:2182,[::1]:2183).
        :param bool read_only: If True, establishes a read-only connection.
        :param float timeout: The ZooKeeper session timeout, in
            seconds (default: 10.0).
        '''
        if self.client is None:
            self.client = KazooClient(hosts=hosts,
                                      read_only=read_only,
                                      timeout=timeout)
            self.client.add_listener(self._connection_listener)
            # Manually retry initial connection attempt
            while True:
                try:
                    self.client.start(1)
                    break
                except KazooTimeoutError:
                    self.logConnectionRetryEvent()

    def disconnect(self):
        '''
        Close the ZooKeeper cluster connection.

        You should call this method if you used connect() to establish a
        cluster connection.
        '''
        if self.client is not None and self.client.connected:
            self.client.stop()
            self.client.close()
            self.client = None

    def resetHosts(self, hosts):
        '''
        Reset the ZooKeeper cluster connection host list.

        :param str hosts: Comma-separated list of hosts to connect to (e.g.
            127.0.0.1:2181,127.0.0.1:2182,[::1]:2183).
        '''
        if self.client is not None:
            self.client.set_hosts(hosts=hosts)

    def submitNodeRequest(self, node_request, watcher):
        '''
        Submit a request for nodes to Nodepool.

        :param NodeRequest node_request: A NodeRequest with the
            contents of the request.

        :param callable watcher: A callable object that will be
            invoked each time the request is updated.  It is called
            with two arguments: (node_request, deleted) where
            node_request is the same argument passed to this method,
            and deleted is a boolean which is True if the node no
            longer exists (notably, this will happen on disconnection
            from ZooKeeper).  The watcher should return False when
            further updates are no longer necessary.
        '''
        node_request.created_time = time.time()
        data = node_request.toDict()

        path = '{}/{:0>3}-'.format(self.REQUEST_ROOT, node_request.priority)
        path = self.client.create(path,
                                  self._dictToStr(data),
                                  makepath=True,
                                  sequence=True,
                                  ephemeral=True)
        reqid = path.split("/")[-1]
        node_request.id = reqid

        def callback(data, stat):
            if data:
                self.updateNodeRequest(node_request, data)
            deleted = (data is None)  # data *are* none
            return watcher(node_request, deleted)

        self.client.DataWatch(path, callback)

    def deleteNodeRequest(self, node_request):
        '''
        Delete a request for nodes.

        :param NodeRequest node_request: A NodeRequest with the
            contents of the request.
        '''

        path = '%s/%s' % (self.REQUEST_ROOT, node_request.id)
        try:
            self.client.delete(path)
        except kze.NoNodeError:
            pass

    def nodeRequestExists(self, node_request):
        '''
        See if a NodeRequest exists in ZooKeeper.

        :param NodeRequest node_request: A NodeRequest to verify.

        :returns: True if the request exists, False otherwise.
        '''
        path = '%s/%s' % (self.REQUEST_ROOT, node_request.id)
        if self.client.exists(path):
            return True
        return False

    def storeNodeRequest(self, node_request):
        '''Store the node request.

        The request is expected to already exist and is updated in its
        entirety.

        :param NodeRequest node_request: The request to update.
        '''

        path = '%s/%s' % (self.REQUEST_ROOT, node_request.id)
        self.client.set(path, self._dictToStr(node_request.toDict()))

    def updateNodeRequest(self, node_request, data=None):
        '''Refresh an existing node request.

        :param NodeRequest node_request: The request to update.
        :param dict data: The data to use; query ZK if absent.
        '''
        if data is None:
            path = '%s/%s' % (self.REQUEST_ROOT, node_request.id)
            data, stat = self.client.get(path)
        data = self._strToDict(data)
        request_nodes = list(node_request.nodeset.getNodes())
        for i, nodeid in enumerate(data.get('nodes', [])):
            request_nodes[i].id = nodeid
            self.updateNode(request_nodes[i])
        node_request.updateFromDict(data)

    def storeNode(self, node):
        '''Store the node.

        The node is expected to already exist and is updated in its
        entirety.

        :param Node node: The node to update.
        '''

        path = '%s/%s' % (self.NODE_ROOT, node.id)
        self.client.set(path, self._dictToStr(node.toDict()))

    def updateNode(self, node):
        '''Refresh an existing node.

        :param Node node: The node to update.
        '''

        node_path = '%s/%s' % (self.NODE_ROOT, node.id)
        node_data, node_stat = self.client.get(node_path)
        node_data = self._strToDict(node_data)
        node.updateFromDict(node_data)

    def lockNode(self, node, blocking=True, timeout=None):
        '''
        Lock a node.

        This should be called as soon as a request is fulfilled and
        the lock held for as long as the node is in-use.  It can be
        used by nodepool to detect if Zuul has gone offline and the
        node should be reclaimed.

        :param Node node: The node which should be locked.
        '''

        lock_path = '%s/%s/lock' % (self.NODE_ROOT, node.id)
        try:
            lock = Lock(self.client, lock_path)
            have_lock = lock.acquire(blocking, timeout)
        except kze.LockTimeout:
            raise LockException("Timeout trying to acquire lock %s" %
                                lock_path)

        # If we aren't blocking, it's possible we didn't get the lock
        # because someone else has it.
        if not have_lock:
            raise LockException("Did not get lock on %s" % lock_path)

        node.lock = lock

    def unlockNode(self, node):
        '''
        Unlock a node.

        The node must already have been locked.

        :param Node node: The node which should be unlocked.
        '''

        if node.lock is None:
            raise LockException("Node %s does not hold a lock" % (node, ))
        node.lock.release()
        node.lock = None

    def lockNodeRequest(self, request, blocking=True, timeout=None):
        '''
        Lock a node request.

        This will set the `lock` attribute of the request object when the
        lock is successfully acquired.

        :param NodeRequest request: The request to lock.
        :param bool blocking: Whether or not to block on trying to
            acquire the lock
        :param int timeout: When blocking, how long to wait for the lock
            to get acquired. None, the default, waits forever.

        :raises: TimeoutException if we failed to acquire the lock when
            blocking with a timeout. ZKLockException if we are not blocking
            and could not get the lock, or a lock is already held.
        '''

        path = "%s/%s" % (self.REQUEST_LOCK_ROOT, request.id)
        try:
            lock = Lock(self.client, path)
            have_lock = lock.acquire(blocking, timeout)
        except kze.LockTimeout:
            raise LockException("Timeout trying to acquire lock %s" % path)
        except kze.NoNodeError:
            have_lock = False
            self.log.error("Request not found for locking: %s", request)

        # If we aren't blocking, it's possible we didn't get the lock
        # because someone else has it.
        if not have_lock:
            raise LockException("Did not get lock on %s" % path)

        request.lock = lock
        self.updateNodeRequest(request)

    def unlockNodeRequest(self, request):
        '''
        Unlock a node request.

        The request must already have been locked.

        :param NodeRequest request: The request to unlock.

        :raises: ZKLockException if the request is not currently locked.
        '''
        if request.lock is None:
            raise LockException("Request %s does not hold a lock" % request)
        request.lock.release()
        request.lock = None

    def heldNodeCount(self, autohold_key):
        '''
        Count the number of nodes being held for the given tenant/project/job.

        :param set autohold_key: A set with the tenant/project/job names.
        '''
        identifier = " ".join(autohold_key)
        try:
            nodes = self.client.get_children(self.NODE_ROOT)
        except kze.NoNodeError:
            return 0

        count = 0
        for nodeid in nodes:
            node_path = '%s/%s' % (self.NODE_ROOT, nodeid)
            try:
                node_data, node_stat = self.client.get(node_path)
            except kze.NoNodeError:
                # Node got removed on us. Just ignore.
                continue

            if not node_data:
                self.log.warning("Node ID %s has no data", nodeid)
                continue
            node_data = self._strToDict(node_data)
            if (node_data['state'] == zuul.model.STATE_HOLD
                    and node_data.get('hold_job') == identifier):
                count += 1
        return count

    # Copy of nodepool/zk.py begins here
    NODE_ROOT = "/nodepool/nodes"
    LAUNCHER_ROOT = "/nodepool/launchers"

    def _bytesToDict(self, data):
        return json.loads(data.decode('utf8'))

    def _launcherPath(self, launcher):
        return "%s/%s" % (self.LAUNCHER_ROOT, launcher)

    def _nodePath(self, node):
        return "%s/%s" % (self.NODE_ROOT, node)

    def getRegisteredLaunchers(self):
        '''
        Get a list of all launchers that have registered with ZooKeeper.

        :returns: A list of Launcher objects, or empty list if none are found.
        '''
        try:
            launcher_ids = self.client.get_children(self.LAUNCHER_ROOT)
        except kze.NoNodeError:
            return []

        objs = []
        for launcher in launcher_ids:
            path = self._launcherPath(launcher)
            try:
                data, _ = self.client.get(path)
            except kze.NoNodeError:
                # launcher disappeared
                continue

            objs.append(Launcher.fromDict(self._bytesToDict(data)))
        return objs

    def getNodes(self):
        '''
        Get the current list of all nodes.

        :returns: A list of nodes.
        '''
        try:
            return self.client.get_children(self.NODE_ROOT)
        except kze.NoNodeError:
            return []

    def getNode(self, node):
        '''
        Get the data for a specific node.

        :param str node: The node ID.

        :returns: The node data, or None if the node was not found.
        '''
        path = self._nodePath(node)
        try:
            data, stat = self.client.get(path)
        except kze.NoNodeError:
            return None
        if not data:
            return None

        d = self._bytesToDict(data)
        d['id'] = node
        return d

    def nodeIterator(self):
        '''
        Utility generator method for iterating through all nodes.
        '''
        for node_id in self.getNodes():
            node = self.getNode(node_id)
            if node:
                yield node
Beispiel #55
0
        ]
        cdeploy.migrator.main()
        sys.argv = argv_bak
    print "setting up zookeeper"
    wait_for('zookeeper', 2181, 30)

    from kazoo.client import KazooClient

    SERVICE_ENDPOINTS = CONFIG['host_ports']
    print 'connecting to zookeeper'
    zk = KazooClient(hosts='zookeeper:2181')
    zk.start()
    for service, paths in SERVICE_ENDPOINTS.items():
        for path in paths:
            full_path = '/services/cluster_local/' + service + '/' + path
            if not zk.exists(full_path):
                zk.create(full_path, makepath=True)
    zk.stop()
    print 'setup complete'
elif cmd == 'START':
    if 'mysql' in DEPENDS_ON:
        wait_for('mysql', 3306, 30)
    wait_for('zookeeper', 2181, 30)
    if 'cassandra' in DEPENDS_ON:
        wait_for('cassandra', 9042, 30)
    run_py('-m', NAME, '--nodaemon', '--flagfile=/home/app/development-flags')
elif cmd == 'START_DEBUG':
    # TODO: argparse/dedupe
    if 'mysql' in DEPENDS_ON:
        wait_for('mysql', 3306, 30)
    wait_for('zookeeper', 2181, 30)
Beispiel #56
0
class KazooCommandProxy():
    def __init__(self, module):
        self.module = module
        self.zk = KazooClient(module.params['hosts'])

    def absent(self):
        return self._absent(self.module.params['name'])

    def exists(self, znode):
        return self.zk.exists(znode)

    def list(self):
        children = self.zk.get_children(self.module.params['name'])
        return True, {
            'count': len(children),
            'items': children,
            'msg': 'Retrieved znodes in path.',
            'znode': self.module.params['name']
        }

    def present(self):
        return self._present(self.module.params['name'],
                             self.module.params['value'])

    def get(self):
        return self._get(self.module.params['name'])

    def shutdown(self):
        self.zk.stop()
        self.zk.close()

    def start(self):
        self.zk.start()

    def wait(self):
        return self._wait(self.module.params['name'],
                          self.module.params['timeout'])

    def _absent(self, znode):
        if self.exists(znode):
            self.zk.delete(znode, recursive=self.module.params['recursive'])
            return True, {'changed': True, 'msg': 'The znode was deleted.'}
        else:
            return True, {'changed': False, 'msg': 'The znode does not exist.'}

    def _get(self, path):
        if self.exists(path):
            value, zstat = self.zk.get(path)
            stat_dict = {}
            for i in dir(zstat):
                if not i.startswith('_'):
                    attr = getattr(zstat, i)
                    if isinstance(attr, (int, str)):
                        stat_dict[i] = attr
            result = True, {
                'msg': 'The node was retrieved.',
                'znode': path,
                'value': value,
                'stat': stat_dict
            }
        else:
            result = False, {'msg': 'The requested node does not exist.'}

        return result

    def _present(self, path, value):
        if self.exists(path):
            (current_value, zstat) = self.zk.get(path)
            if value != current_value:
                self.zk.set(path, value)
                return True, {
                    'changed': True,
                    'msg': 'Updated the znode value.',
                    'znode': path,
                    'value': value
                }
            else:
                return True, {
                    'changed': False,
                    'msg': 'No changes were necessary.',
                    'znode': path,
                    'value': value
                }
        else:
            self.zk.create(path, value, makepath=True)
            return True, {
                'changed': True,
                'msg': 'Created a new znode.',
                'znode': path,
                'value': value
            }

    def _wait(self, path, timeout, interval=5):
        lim = time.time() + timeout

        while time.time() < lim:
            if self.exists(path):
                return True, {
                    'msg': 'The node appeared before the configured timeout.',
                    'znode': path,
                    'timeout': timeout
                }
            else:
                time.sleep(interval)

        return False, {
            'msg': 'The node did not appear before the operation timed out.',
            'timeout': timeout,
            'znode': path
        }
Beispiel #57
0
# -*- coding: utf-8 -*-
import time
import glob
import sys
import os
from kazoo.client import KazooClient
import logging
logging.basicConfig()

if __name__ == "__main__":
    print("begin.")
    zk = KazooClient(hosts='172.10.3.111:2181')
    zk.start()
    mypath = "/my/favorite"
    #zk.ensure_path(mypath)
    if zk.exists(mypath):
        print(mypath, "exist")
        data, stat = zk.get(mypath)
        print("Version: %s, data: %s" % (stat.version, data.decode("utf-8")))
    else:
        print(mypath, "not exist")
    zk.stop()
    print("exit.")
Beispiel #58
0
def do_operate(arglist):
    def print_usage():
        print 'Usage:'
        print '\t%s add --target Cluster1' % sys.argv[0]
        print '\t%s delete --target Cluster1' % sys.argv[0]
        print '\t%s root --target Cluster1' % sys.argv[0]
        print ''

    def parse_op_args(arglist):
        global target
        try:
            if len(arglist) <= 0:
                print_usage()
                sys.exit(0)

            opts, args = getopt.getopt(arglist, "", ["help", "target="])

            for opt, arg in opts:
                if opt in ("--target"):
                    target = str(arg)
                elif opt in ("--help"):
                    print_usage()
                    sys.exit(0)
                else:
                    log.error('Invalid arg: %s' % opt)
                    print_usage()
                    sys.exit(-1)

            if not target:
                log.error("target host/cluster must be set with --target")
                print_usage()
                sys.exit(-1)

        except getopt.GetoptError as err:
            log.error("%s" % err)
            print_usage()
            sys.exit(-1)

    if len(arglist) <= 0:
        print_usage()
        sys.exit(0)

    subcmd = arglist[0]
    parse_op_args(arglist[1:])
    log.info('Running operate %s target = %s' % (subcmd, target))

    zk_client = None
    try:
        clusternode = target
        conf_ret = read_conf(clusternode)
        if conf_ret != 0:
            raise Exception('Invalid target param, clusternode:%s!' %
                            clusternode)
        # create node
        zk_client = KazooClient(hosts=zk_servers)
        zk_client.start()

        zk_node = zk_path + clusternode
        exist_flag = zk_client.exists(zk_node)

        if subcmd == 'root':
            zk_node = zk_path
            subcmd = 'delete'

        if subcmd == 'add':
            if not exist_flag:
                zk_client.ensure_path(zk_node)
            for host in cluster_iplist:
                new_node = zk_node + '/' + host
                if not zk_client.exists(new_node):
                    zk_client.create(new_node, b'')
            # check host node
            children = zk_client.get_children(zk_node)
            if len(children) != len(cluster_iplist):
                raise Exception('Create cluster:%s node failed!' % clusternode)
            log.info('create cluster:%s node success!' % clusternode)
        elif subcmd == 'delete':
            if exist_flag:
                zk_client.delete(zk_node, recursive=True)
                log.info('Delete cluster:%s node success!' % clusternode)
                # if is_check:  # TODO
                # zk_client.delete(zk_path,recursive=True)
                # log.info('Delete zk_path:%s node success!' % clusternode)
            else:
                log.info('cluster:%s node not exist!' % clusternode)
        else:
            raise Exception('Invalid op command %s' % subcmd)

    finally:
        if zk_client:
            zk_client.stop()
Beispiel #59
0
class QACBox(object):

    def __init__(self, type, name):
        self._Type = type
        self._Name = "%s%s" % (type, str(name))
        self._VName = HOST['name']
        self._Path = "/%s/%s/%s" % (self._Type, self._VName, self._Name)
        self._ZKHost = ZKSERVERS['hosts']
        self._IP = HOST['ip']
        self._Port = HOST['port']
        self._ID = int(name)
        self._ZK = None
        self._Chatbot = None
        self._Conn = None
        self._KGname = None
        logger.debug('create a %s box named %s in VM %s.' % (self._Type, self._Name, self._VName))

    def connectZK(self):
        self._ZK = KazooClient(hosts=self._ZKHost)
        logger.info('%s is connecting ZK server.' % self._Path)

    def getType(self):
        return self._Type

    def getName(self):
        return self._Name

    def getVName(self):
        return self._VName

    def setZK(self,zk):
        self._ZK = zk

    def getZK(self):
        return self._ZK

    def startZK(self):
        self._ZK.start()
        logger.debug('start one connection with ZK server by a %s box named %s in VM %s' % (self._Type, self._Name, self._VName))

    def stopZK(self):
        self._ZK.stop()
        logger.info('stop connection with ZK server by a %s box named %s in VM %s' % (self._Type, self._Name, self._VName))

    def InitialBBOXNode(self):
        address = "{\"Target\":\"Null\",\"Add\":\"%s:%s/%s\",\"status\":\"0\",\"update_time\":\"%f\"}" % (self._IP, self._Port,self._ID,time.time())
        address = address.encode('utf-8')
        if self._ZK.exists(self._Path):
            self._ZK.delete(self._Path, recursive=True)
        self._ZK.create(self._Path, address, None, ephemeral=False, sequence=False, makepath=True)
        logger.info('create a B box node: %s, data: %s' % (self._Path, address.decode("utf-8")))
        vmknode = "/%s/%s" % (HOST['name'], CBOX['Bk'])
        if self._ZK.exists(vmknode):
            self._ZK.delete(vmknode, recursive=True)
        self._ZK.create(vmknode, None, None, ephemeral=False, sequence=False, makepath=True)
        logger.info('create a VM/k node: %s, not data. ' % vmknode)

    def updateselfZKBBox(self, status):
        # updating 'oneabox' B tpye node data in ZK Server
        #oneabox_path = "/%s/%s/%s" % (self._Type, self._VName, oneabox)
        oneabox_path = self._Path
        data, _ = self._ZK.get(oneabox_path)
        oneabox_address = "{\"Target\":\"%s\",\"Add\":\"%s\",\"status\":\"%s\",\"update_time\":\"%f\"}" % (eval(data.decode("utf-8"))['Target'], eval(data.decode("utf-8"))['Add'],str(status),time.time())
        oneabox_address = oneabox_address.encode('utf-8')
        self._ZK.set(oneabox_path, oneabox_address)
        logger.info('success update B Box node %s with data %s.' % (oneabox_path, oneabox_address))

    def updateselfZKBBoxTarget(self, target, status):
        # updating 'oneabox' B tpye node data in ZK Server
        #oneabox_path = "/%s/%s/%s" % (self._Type, self._VName, oneabox)
        oneabox_path = self._Path
        data, _ = self._ZK.get(oneabox_path)
        oneabox_address = "{\"Target\":\"%s\",\"Add\":\"%s\",\"status\":\"%s\",\"update_time\":\"%f\"}" % (target, eval(data.decode("utf-8"))['Add'],str(status),time.time())
        oneabox_address = oneabox_address.encode('utf-8')
        self._ZK.set(oneabox_path, oneabox_address)
        logger.info('success update B')

    def stop(self):
        self.stopZK()

    def initcbot(self, kbname, onlyread=True):
        try:
            self._Chatbot = ChatBot(self._Name,
            storage_adapter=CHATTERBOT['storage_adapter'],
            # logic_adapters=[{
				# 	 'import_path': 'chatterbot.logic.BestMatch'
				#  }],
            filters=['chatterbot.filters.RepetitiveResponseFilter'],
            database_uri=KGDATABASES['database_uri'],
            database='ai_%s' % kbname,
            read_only=onlyread,)
            self._KGname = kbname
        except Exception:
            logger.info('Failure to initialize Chatterbot.', exc_info=True)

    def get_response(self, inputstatement):
        return self._Chatbot.get_response(inputstatement)

    def get_similar(self):
        # Note that get_similar must be called after get_response called, or return None
        return self._Chatbot.similar

    # def preprocess(self,sentence):
    #     if ISFENCI:
    #         return fenci.seg_sentence(sentence)
    #     else:
    #         return sentence

    def preprocess(self, sentence, companyid=None):
        if ISFENCI:
            if ISSYMS:
                return fenci.symp_sentence(sentence, companyid)
            else:
                return sentence
        else:
            return sentence
Beispiel #60
0
# -*- coding: utf-8 -*-
######################################################################
#                                                                    #
# 创建时间:2019年09月16日                                           #
# 创 建 者:wyl                                                      #
# 功能:写入hive测试脚本                                              #
#                                                                    #
######################################################################

import sys, time, datetime
import traceback
from kazoo.client import KazooClient
import hdfs

nameNodeA = [b'phmnn1.bigdata.com', 'phmnn1.bigdata.com']
nameNodeB = [b'phmnn2.bigdata.com', 'phmnn2.bigdata.com']
zooQuorum = '10.73.95.19:2181,10.73.95.20:2181,10.73.95.21:2181'
path = '/hadoop-ha/PHMBIGDATA/ActiveBreadCrumb'

nameNodeADevelop = [b'spark.bigdevelop.com', 'spark.bigdevelop.com']
nameNodeBDevelop = [b'nnredis.bigdevelop.com', 'nnredis.bigdevelop.com']
zooQuorumDevelop = 'datanode1.bigdevelop.com:2181,schedule.bigdevelop.com:2181,datanode2.bigdevelop.com:2181'
pathDevelop = '/hadoop-ha/hanamenode/ActiveBreadCrumb'

zk = KazooClient(hosts=zooQuorum)
zk.start()
if zk.exists(path):
    data = zk.get(path)
    print(data)