def readRegister(self, address, cache=False, timeout=2):
     """Return the value of register at [address] location
     if timeout is elapsed, NoneValue is returned
     """
     if not isinstance(address, int):
         raise TypeError("Address must be an integer")
     if int(address) < 0 or int(address) > 255:
         raise ValueError("Address is out of range [0, 255]")
     if address in self.regs.keys() and cache:
         if not self.regs[address] == None:
             return self.regs[address]
     self.regs[address] = None
     self.vscp.sendSimpleEvent(
         vscp_class=constant.VSCP_CLASS1_PROTOCOL,
         vscp_type=constant.VSCP_TYPE_PROTOCOL_READ_REGISTER,
         vscp_data=[self.id, address],
     )
     try:
         timer = Timeout(timeout, self.__timeout)
         while self.regs[address] == None:
             time.sleep(0.02)
         timer.cancel()
         return self.regs[address]
     except VSCPException:
         return None
Esempio n. 2
0
def execute_with_timeout(*args, **kwargs):
    time = kwargs.get('timeout', 30)

    def cb_timeout():
        msg = (_("Time out after waiting"
                 " %(time)s seconds when running proc: %(args)s"
                 " %(kwargs)s") % {'time': time, 'args': args,
                                   'kwargs': kwargs})
        LOG.error(msg)
        raise exception.ProcessExecutionError(msg)

    timeout = Timeout(time)
    try:
        return execute(*args, **kwargs)
    except Timeout as t:
        if t is not timeout:
            LOG.error("Timeout reached but not from our timeout. This is bad!")
            raise
        else:
            msg = (_("Time out after waiting "
                     "%(time)s seconds when running proc: %(args)s"
                     " %(kwargs)s") % {'time': time, 'args': args,
                                       'kwargs': kwargs})
            LOG.error(msg)
            raise exception.ProcessExecutionError(msg)
    finally:
        timeout.cancel()
 def writeRegister(self, address, value, timeout=2):
     """Return True if value is correctly written
     if timeout is elapsed, False is returned
     """
     if not isinstance(address, int):
         raise TypeError("Address must be an integer")
     if int(address) < 0 or int(address) > 255:
         raise ValueError("Address is out of range [0, 255]")
     if not isinstance(value, int):
         raise TypeError("Value must be an integer")
     if int(value) < 0 or int(value) > 255:
         raise ValueError("Value is out of range [0, 255]")
     self.regs[address] = None
     self.vscp.sendSimpleEvent(
         vscp_class=constant.VSCP_CLASS1_PROTOCOL,
         vscp_type=constant.VSCP_TYPE_PROTOCOL_WRITE_REGISTER,
         vscp_data=[self.id, address, value],
     )
     try:
         timer = Timeout(timeout, self.__timeout)
         self.regs[address] = None
         while self.regs[address] == None:
             time.sleep(0.02)
         timer.cancel()
         return self.regs[address] == value
     except VSCPException:
         return False
Esempio n. 4
0
    def grow_cluster(self, context, cluster_id, new_instance_ids):

        def _grow_cluster():
            LOG.debug("begin grow_cluster for Vertica cluster %s" % cluster_id)

            db_instances = DBInstance.find_all(cluster_id=cluster_id,
                                               deleted=False).all()

            instance_ids = [db_instance.id for db_instance in db_instances]

            # Wait for new cluster members to get to cluster-ready status.
            if not self._all_instances_ready(new_instance_ids, cluster_id):
                return

            new_insts = [Instance.load(context, instance_id)
                         for instance_id in new_instance_ids]

            existing_instances = [Instance.load(context, instance_id)
                                  for instance_id
                                  in instance_ids
                                  if instance_id not in new_instance_ids]

            existing_guests = [self.get_guest(i) for i in existing_instances]
            new_guests = [self.get_guest(i) for i in new_insts]
            all_guests = new_guests + existing_guests

            authorized_users_without_password = ['root', 'dbadmin']
            new_ips = [self.get_ip(instance) for instance in new_insts]

            for user in authorized_users_without_password:
                pub_key = [guest.get_public_keys(user) for guest in all_guests]
                for guest in all_guests:
                    guest.authorize_public_keys(user, pub_key)

            for db_instance in db_instances:
                if db_instance['type'] == 'master':
                    LOG.debug("Found 'master' instance, calling grow on guest")
                    master_instance = Instance.load(context,
                                                    db_instance.id)
                    self.get_guest(master_instance).grow_cluster(new_ips)
                    break

            for guest in new_guests:
                guest.cluster_complete()

        timeout = Timeout(CONF.cluster_usage_timeout)

        try:
            _grow_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for growing cluster."))
            self.update_statuses_on_failure(cluster_id)
        except Exception:
            LOG.exception(_("Error growing cluster %s.") % cluster_id)
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()
Esempio n. 5
0
    def piper(self, in_sock, out_sock, out_addr, onkill):
        "Worker thread for data reading"
        try:
            timeout = Timeout(self.transmission_timeout_seconds)
            try:
                while True:
                    written = in_sock.recv(32768)
                    if not written:
                        try:
                            out_sock.shutdown(socket.SHUT_WR)
                        except socket.error:
                            self.threads[onkill].kill()
                        break
                    try:
                        out_sock.sendall(written)
                    except socket.error:
                        pass
                    self.data_handled += len(written)
            finally:
                timeout.cancel()
        except greenlet.GreenletExit:
            return
        except Timeout:
            # This one prevents only from closing connection without any data nor status code returned
            # from mantrid when no data was received from backend.
            # When it happens, nginx reports 'upstream prematurely closed connection' and returns 500,
            # and want to have our custom error page to know when it happens. 

            if onkill == "stoc" and self.data_handled == 0:
                out_sock.sendall("HTTP/1.0 594 Backend timeout\r\nConnection: close\r\nContent-length: 0\r\n\r\n")
            logging.warn("Timeout serving request to backend %s of %s", self.backend, self.host)
            return
Esempio n. 6
0
    def shrink_cluster(self, context, cluster_id, instance_ids):
        LOG.debug("begin shrink_cluster for MongoDB cluster %s" % cluster_id)

        def _shrink_cluster():
            def all_instances_marked_deleted():
                non_deleted_instances = DBInstance.find_all(
                    cluster_id=cluster_id, deleted=False).all()
                non_deleted_ids = [db_instance.id for db_instance
                                   in non_deleted_instances]
                return not bool(
                    set(instance_ids).intersection(set(non_deleted_ids))
                )
            try:
                utils.poll_until(all_instances_marked_deleted,
                                 sleep_time=2,
                                 time_out=CONF.cluster_delete_time_out)
            except PollTimeOut:
                LOG.error(_("timeout for instances to be marked as deleted."))
                return

        cluster_usage_timeout = CONF.cluster_usage_timeout
        timeout = Timeout(cluster_usage_timeout)
        try:
            _shrink_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("timeout for shrinking cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("end shrink_cluster for MongoDB cluster %s" % self.id)
Esempio n. 7
0
    def rengine_side(self, appid, token, uri):
        """ Handle rengine (client) GET requests """
        if not self.rengine_authorization_ok(appid, token):
            LOGGER.info('Rengine content request authorization fails')
            abort(401, 'Authorization failed')

        evt = Event()
        request_id = str(uuid4())
        self.request_id_events[request_id] = evt

        headers = ["%s: %s" % (header, val) for (header, val) in request.headers.items()]
        packet = ScpPacket.make_sfkcontent(uri, request_id, headers)
        try:
            self._send(packet, appid)
        except Exception as e:
            abort(500, str(e))

        LOGGER.debug("uri %s expected" % uri)
        timeout = Timeout(TIMEOUT)
        try:
            resp = evt.wait()
        except Timeout:
            del self.request_id_events[request_id]
            abort(504, 'Gateway Timeout')
        finally:
            timeout.cancel()

        LOGGER.debug("uri %s got" % uri)
        
        return resp
Esempio n. 8
0
def execute_with_timeout(*args, **kwargs):
    time = kwargs.pop('timeout', 30)
    log_output_on_error = kwargs.pop('log_output_on_error', False)

    timeout = Timeout(time)
    try:
        return execute(*args, **kwargs)
    except exception.ProcessExecutionError as e:
        if log_output_on_error:
            LOG.error(
                _("Command '%(cmd)s' failed. %(description)s "
                  "Exit code: %(exit_code)s\nstderr: %(stderr)s\n"
                  "stdout: %(stdout)s") %
                {'cmd': e.cmd, 'description': e.description or '',
                 'exit_code': e.exit_code, 'stderr': e.stderr,
                 'stdout': e.stdout})
        raise
    except Timeout as t:
        if t is not timeout:
            LOG.error(_("Got a timeout but not the one expected."))
            raise
        else:
            msg = (_("Time out after waiting "
                     "%(time)s seconds when running proc: %(args)s"
                     " %(kwargs)s.") % {'time': time, 'args': args,
                                        'kwargs': kwargs})
            LOG.error(msg)
            raise exception.ProcessExecutionError(msg)
    finally:
        timeout.cancel()
Esempio n. 9
0
    def create_cluster(self, context, cluster_id):
        LOG.debug("Begin create_cluster for id: %s." % cluster_id)

        def _create_cluster():

            # Fetch instances by cluster_id against instances table.
            db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
            instance_ids = [db_instance.id for db_instance in db_instances]

            # Wait for cluster members to get to cluster-ready status.
            if not self._all_instances_ready(instance_ids, cluster_id):
                return

            LOG.debug("All members ready, proceeding for cluster setup.")
            instances = [Instance.load(context, instance_id) for instance_id
                         in instance_ids]

            # Connect nodes to the first node
            guests = [self.get_guest(instance) for instance in instances]
            try:
                cluster_head = instances[0]
                cluster_head_port = '6379'
                cluster_head_ip = self.get_ip(cluster_head)
                for guest in guests[1:]:
                    guest.cluster_meet(cluster_head_ip, cluster_head_port)

                num_nodes = len(instances)
                total_slots = 16384
                slots_per_node = total_slots / num_nodes
                leftover_slots = total_slots % num_nodes
                first_slot = 0
                for guest in guests:
                    last_slot = first_slot + slots_per_node
                    if leftover_slots > 0:
                        leftover_slots -= 1
                    else:
                        last_slot -= 1
                    guest.cluster_addslots(first_slot, last_slot)
                    first_slot = last_slot + 1

                for guest in guests:
                    guest.cluster_complete()
            except Exception:
                LOG.exception(_("Error creating cluster."))
                self.update_statuses_on_failure(cluster_id)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _create_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for building cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End create_cluster for id: %s." % cluster_id)
Esempio n. 10
0
 def request(self, address, path, method='GET', params=None, headers={}):
     t = Timeout(2, RuntimeError("Timeout trying to send request."))
     try:
         conn = httplib.HTTPConnection("%s:%s" % address)
         conn.request(method, path, params, headers)
     finally:
         t.cancel()
     return conn.getresponse()
Esempio n. 11
0
    def create_cluster(self, context, cluster_id):
        LOG.debug("Begin create_cluster for id: %s." % cluster_id)

        def _create_cluster():

            # Fetch instances by cluster_id against instances table.
            db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
            instance_ids = [db_instance.id for db_instance in db_instances]

            # Wait for cluster members to get to cluster-ready status.
            if not self._all_instances_ready(instance_ids, cluster_id):
                return

            LOG.debug("All members ready, proceeding for cluster setup.")
            instances = [Instance.load(context, instance_id) for instance_id
                         in instance_ids]

            member_ips = [self.get_ip(instance) for instance in instances]
            guests = [self.get_guest(instance) for instance in instances]

            # Users to be configured for password-less SSH.
            authorized_users_without_password = ['root', 'dbadmin']

            # Configuring password-less SSH for cluster members.
            # Strategy for setting up SSH:
            # get public keys for user from member-instances in cluster,
            # combine them, finally push it back to all instances,
            # and member instances add them to authorized keys.
            LOG.debug("Configuring password-less SSH on cluster members.")
            try:
                for user in authorized_users_without_password:
                    pub_key = [guest.get_public_keys(user) for guest in guests]
                    for guest in guests:
                        guest.authorize_public_keys(user, pub_key)

                LOG.debug("Installing cluster with members: %s." % member_ips)
                guests[0].install_cluster(member_ips)

                LOG.debug("Finalizing cluster configuration.")
                for guest in guests:
                    guest.cluster_complete()
            except Exception:
                LOG.exception(_("Error creating cluster."))
                self.update_statuses_on_failure(cluster_id)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _create_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for building cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End create_cluster for id: %s." % cluster_id)
Esempio n. 12
0
 def wrapper(*args,**kwargs):
     timeout = Timeout(time, exception)
     try:
         func(*args,**kwargs)
     except exception as e:
         if reraise:
             raise e
     else:
         timeout.cancel()
Esempio n. 13
0
 def update_data(self):
     print "updating data"
     timeout = Timeout(10)
     try:
         self.parser = KMLParser()
         temp = self.parse_data()
         timeout.cancel()
         self.bikeways = temp
         self.update_database()
     except:
         print "couldn't get data"
Esempio n. 14
0
 def _write_with_timeout(self, writer, chunk):
     timeout = Timeout(self.timeout)
     try:
         writer.write(chunk)
     except Timeout as t:
         if t is timeout:
             writer.close()
             raise t
     except Exception as e:
         raise e
     finally:
         timeout.cancel()
Esempio n. 15
0
    def add_shard_cluster(self, context, cluster_id, shard_id,
                          replica_set_name):

        LOG.debug("begin add_shard_cluster for cluster %s shard %s"
                  % (cluster_id, shard_id))

        def _add_shard_cluster():

            db_instances = DBInstance.find_all(cluster_id=cluster_id,
                                               shard_id=shard_id).all()
            instance_ids = [db_instance.id for db_instance in db_instances]
            LOG.debug("instances in shard %s: %s" % (shard_id,
                                                     instance_ids))
            if not self._all_instances_ready(instance_ids, cluster_id,
                                             shard_id):
                return

            members = [Instance.load(context, instance_id)
                       for instance_id in instance_ids]

            if not self._create_replica_set(members, cluster_id, shard_id):
                return

            db_query_routers = DBInstance.find_all(cluster_id=cluster_id,
                                                   type='query_router',
                                                   deleted=False).all()
            query_routers = [Instance.load(context, db_query_router.id)
                             for db_query_router in db_query_routers]

            if not self._create_shard(query_routers, replica_set_name,
                                      members, cluster_id, shard_id):
                return

            for member in members:
                self.get_guest(member).cluster_complete()

        cluster_usage_timeout = CONF.cluster_usage_timeout
        timeout = Timeout(cluster_usage_timeout)
        try:
            _add_shard_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("timeout for building shard."))
            self.update_statuses_on_failure(cluster_id, shard_id)
        finally:
            timeout.cancel()

        LOG.debug("end add_shard_cluster for cluster %s shard %s"
                  % (cluster_id, shard_id))
Esempio n. 16
0
    def put(self, item, block=True, timeout=None):
        """Put an item into the queue.

        If optional arg *block* is true and *timeout* is ``None`` (the default),
        block if necessary until a free slot is available. If *timeout* is
        a positive number, it blocks at most *timeout* seconds and raises
        the :class:`Full` exception if no free slot was available within that time.
        Otherwise (*block* is false), put an item on the queue if a free slot
        is immediately available, else raise the :class:`Full` exception (*timeout*
        is ignored in that case).
        """
        if self.maxsize is None or self.qsize() < self.maxsize:
            # there's a free slot, put an item right away
            self._put(item)
            if self.getters:
                self._schedule_unlock()
        elif not block and get_hub().greenlet is getcurrent():
            # we're in the mainloop, so we cannot wait; we can switch() to other greenlets though
            # find a getter and deliver an item to it
            while self.getters:
                getter = self.getters.pop()
                if getter:
                    self._put(item)
                    item = self._get()
                    getter.switch(item)
                    return
            raise Full
        elif block:
            waiter = ItemWaiter(item, block)
            self.putters.add(waiter)
            timeout = Timeout(timeout, Full)
            try:
                if self.getters:
                    self._schedule_unlock()
                result = waiter.wait()
                assert result is waiter, "Invalid switch into Queue.put: %r" % (result, )
                if waiter.item is not _NONE:
                    self._put(item)
            finally:
                timeout.cancel()
                self.putters.discard(waiter)
        elif self.getters:
            waiter = ItemWaiter(item, block)
            self.putters.add(waiter)
            self._schedule_unlock()
            result = waiter.wait()
            assert result is waiter, "Invalid switch into Queue.put: %r" % (result, )
            if waiter.item is not _NONE:
                raise Full
        else:
            raise Full
Esempio n. 17
0
 def recv_events(self, timeout_msecs):
     events = []
     timeout = Timeout(timeout_msecs / 1000.0)
     try:
         while True:
             event = self._event_q.get()
             events.append(event)
     except Timeout:
         pass
     except Exception as e:
         raise e
     finally:
         timeout.cancel()
     return events
Esempio n. 18
0
    def _check_health(self):
        logging.debug("Checking health of %s", self)
        try:
            timeout = Timeout(self.healthcheck_timeout_seconds)
            try:
                socket = eventlet.connect((self.host, self.port))
            finally:
                timeout.cancel()

            logging.debug("%s is alive, making sure it is not blacklisted", self)
            self.blacklisted = False
            socket.close()
        except:
            logging.debug("%s seems dead, will check again later", self)
Esempio n. 19
0
    def create_cluster(self, context, cluster_id):
        LOG.debug("Begin create_cluster for id: %s." % cluster_id)

        def _create_cluster():
            cluster_node_ids = self.find_cluster_node_ids(cluster_id)

            # Wait for cluster nodes to get to cluster-ready status.
            LOG.debug("Waiting for all nodes to become ready.")
            if not self._all_instances_ready(cluster_node_ids, cluster_id):
                return

            cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids)
            coordinator = self._get_coordinator_node(cluster_nodes)

            LOG.debug("Initializing the cluster on node '%s'."
                      % coordinator['ip'])

            # start with the coordinator as it will have all the required
            # services.
            guest_node_info = self.build_guest_node_info([coordinator])
            # now add all the other nodes so we can get a list of all services
            # needed to calculate the memory allocation properly.
            add_node_info = [node for node in cluster_nodes
                             if node != coordinator]
            guest_node_info.extend(self.build_guest_node_info(add_node_info))
            coordinator['guest'].initialize_cluster(guest_node_info)

            self._add_nodes(coordinator, add_node_info)
            coordinator['guest'].cluster_complete()
            LOG.debug("Cluster create finished successfully.")

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            with EndNotification(context, cluster_id=cluster_id):
                _create_cluster()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for building cluster."))
            self.update_statuses_on_failure(cluster_id)
        except Exception:
            LOG.exception(_("Error creating cluster."))
            self.update_statuses_on_failure(cluster_id)
            raise
        finally:
            self.reset_task()
            timeout.cancel()

        LOG.debug("End create_cluster for id: %s." % cluster_id)
Esempio n. 20
0
    def shrink_cluster(self, context, cluster_id, instance_ids):
        def _shrink_cluster():
            db_instances = DBInstance.find_all(cluster_id=cluster_id,
                                               deleted=False).all()

            all_instance_ids = [db_instance.id for db_instance in db_instances]

            remove_instances = [Instance.load(context, instance_id)
                                for instance_id in instance_ids]

            left_instances = [Instance.load(context, instance_id)
                              for instance_id
                              in all_instance_ids
                              if instance_id not in instance_ids]

            remove_member_ips = [self.get_ip(instance)
                                 for instance in remove_instances]

            k = VerticaCluster.k_safety(len(left_instances))

            for db_instance in db_instances:
                if db_instance['type'] == 'master':
                    master_instance = Instance.load(context,
                                                    db_instance.id)
                    if self.get_ip(master_instance) in remove_member_ips:
                        raise RuntimeError(_("Cannot remove master instance!"))
                    LOG.debug(_("Marking cluster k-safety: %s") % k)
                    self.get_guest(master_instance).mark_design_ksafe(k)
                    self.get_guest(master_instance).shrink_cluster(
                        remove_member_ips)
                    break

            for r in remove_instances:
                Instance.delete(r)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _shrink_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise
            LOG.exception(_("Timeout for shrinking cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("end shrink_cluster for Vertica cluster id %s" % self.id)
Esempio n. 21
0
    def grow_cluster(self, context, cluster_id, new_instance_ids):
        LOG.debug("Begin grow_cluster for id: %s." % cluster_id)

        def _grow_cluster():

            db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
            cluster_head = next(Instance.load(context, db_inst.id)
                                for db_inst in db_instances
                                if db_inst.id not in new_instance_ids)
            if not cluster_head:
                raise TroveError("Unable to determine existing Redis cluster "
                                 "member")

            (cluster_head_ip, cluster_head_port) = (
                self.get_guest(cluster_head).get_node_ip())

            # Wait for cluster members to get to cluster-ready status.
            if not self._all_instances_ready(new_instance_ids, cluster_id):
                return

            LOG.debug("All members ready, proceeding for cluster setup.")
            new_insts = [Instance.load(context, instance_id)
                         for instance_id in new_instance_ids]
            new_guests = map(self.get_guest, new_insts)

            # Connect nodes to the cluster head
            for guest in new_guests:
                guest.cluster_meet(cluster_head_ip, cluster_head_port)

            for guest in new_guests:
                guest.cluster_complete()

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _grow_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for growing cluster."))
            self.update_statuses_on_failure(cluster_id)
        except Exception:
            LOG.exception(_("Error growing cluster %s.") % cluster_id)
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End grow_cluster for id: %s." % cluster_id)
Esempio n. 22
0
    def grow_cluster(self, context, cluster_id, new_instance_ids):
        LOG.debug("Begin grow_cluster for id: %s." % cluster_id)

        def _grow_cluster():
            # Wait for new nodes to get to cluster-ready status.
            LOG.debug("Waiting for new nodes to become ready.")
            if not self._all_instances_ready(new_instance_ids, cluster_id):
                return

            new_instances = [Instance.load(context, instance_id)
                             for instance_id in new_instance_ids]
            added_nodes = [self.build_node_info(instance)
                           for instance in new_instances]

            LOG.debug("All nodes ready, proceeding with cluster setup.")

            cluster_node_ids = self.find_cluster_node_ids(cluster_id)
            cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids)

            # Rebalance the cluster via one of the existing nodes.
            # Clients can continue to store and retrieve information and
            # do not need to be aware that a rebalance operation is taking
            # place.
            # The new nodes are marked active only if the rebalancing
            # completes.
            try:
                coordinator = cluster_nodes[0]
                self._add_nodes(coordinator, added_nodes)
                LOG.debug("Cluster configuration finished successfully.")
            except Exception:
                LOG.exception(_("Error growing cluster."))
                self.update_statuses_on_failure(cluster_id)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _grow_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for growing cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End grow_cluster for id: %s." % cluster_id)
Esempio n. 23
0
    def downloadDM(self, timeout=2, cached=False):
        self.DM = None
        self.vscp.sendSimpleEvent(
            vscp_class=constant.VSCP_CLASS1_PROTOCOL,
            vscp_type=constant.VSCP_TYPE_PROTOCOL_GET_MATRIX_INFO,
            vscp_data=[self.nodeId],
        )
        timer = Timeout(timeout, self.__timeout)
        try:
            while self.DM == None:
                time.sleep(0.02)
            timer.cancel()
            for i in range(self.DM["size"]):
                self.DM["rows"][i] = self.getRowOfDM(i)

        except:
            return constant.VSCP_ERROR_ERROR
Esempio n. 24
0
    def grow_cluster(self, context, cluster_id, new_instance_ids):
        LOG.debug("Begin grow_cluster for id: %s." % cluster_id)

        def _grow_cluster():
            # Wait for new nodes to get to cluster-ready status.
            LOG.debug("Waiting for new nodes to become ready.")
            if not self._all_instances_ready(new_instance_ids, cluster_id):
                return

            new_instances = [Instance.load(context, instance_id)
                             for instance_id in new_instance_ids]
            add_node_info = [self.build_node_info(instance)
                             for instance in new_instances]

            LOG.debug("All nodes ready, proceeding with cluster setup.")

            cluster_node_ids = self.find_cluster_node_ids(cluster_id)
            cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids)

            old_node_info = [node for node in cluster_nodes
                             if node['id'] not in new_instance_ids]

            # Rebalance the cluster via one of the existing nodes.
            # Clients can continue to store and retrieve information and
            # do not need to be aware that a rebalance operation is taking
            # place.
            coordinator = old_node_info[0]
            self._add_nodes(coordinator, add_node_info)
            LOG.debug("Cluster grow finished successfully.")

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            with EndNotification(context, cluster_id=cluster_id):
                _grow_cluster()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for growing cluster."))
        except Exception:
            LOG.exception(_("Error growing cluster."))
            raise
        finally:
            self.reset_task()
            timeout.cancel()

        LOG.debug("End grow_cluster for id: %s." % cluster_id)
Esempio n. 25
0
 def recv_events(self, timeout_msecs):
     """
     Let explorer receive events
     :param timeout_msecs: int
     :return:
     """
     events = []
     timeout = Timeout(timeout_msecs / 1000.0)
     try:
         while True:
             event = self._event_q.get()
             events.append(event)
     except Timeout:
         pass
     except Exception as e:
         raise e
     finally:
         timeout.cancel()
     return events
Esempio n. 26
0
    def get(self, block=True, timeout=None):
        """Remove and return an item from the queue.

        If optional args *block* is true and *timeout* is ``None`` (the default),
        block if necessary until an item is available. If *timeout* is a positive number,
        it blocks at most *timeout* seconds and raises the :class:`Empty` exception
        if no item was available within that time. Otherwise (*block* is false), return
        an item if one is immediately available, else raise the :class:`Empty` exception
        (*timeout* is ignored in that case).
        """
        if self.qsize():
            if self.putters:
                self._schedule_unlock()
            return self._get()
        elif not block and get_hub().greenlet is getcurrent():
            # special case to make get_nowait() runnable in the mainloop greenlet
            # there are no items in the queue; try to fix the situation by unlocking putters
            while self.putters:
                putter = self.putters.pop()
                if putter:
                    putter.switch(putter)
                    if self.qsize():
                        return self._get()
            raise Empty
        elif block:
            waiter = Waiter()
            timeout = Timeout(timeout, Empty)
            try:
                self.getters.add(waiter)
                if self.putters:
                    self._schedule_unlock()
                try:
                    return waiter.wait()
                except:
                    self._schedule_unlock()
                    raise
            finally:
                self.getters.discard(waiter)
                timeout.cancel()
        else:
            raise Empty
Esempio n. 27
0
    def wait(self):
        do_wait = True
        up = 0
        to = Timeout(self.sandbox_wait_timeout)
        try:
            while do_wait == True:
                rc = self.ping()
                if (rc != 1):
                    time.sleep(self.sandbox_ping_interval)
                    continue
                else:
                    to.cancel()
                    do_wait = False
                    up = 1
        except Timeout as t:
            self.logger.info("wait for sandbox %s timedout" % self.account)
            do_wait = False
        finally:
            to.cancel()

        return up
Esempio n. 28
0
def execute_with_timeout(*args, **kwargs):
    time = kwargs.get('timeout', 30)

    def cb_timeout():
        msg = _("Time out after waiting"
                " %(time)s seconds when running proc: %(args)s"
                " %(kwargs)s") % locals()
        raise exception.ProcessExecutionError(msg)

    timeout = Timeout(time)
    try:
        return execute(*args, **kwargs)
    except Timeout as t:
        if t is not timeout:
            raise
        else:
            msg = _("Time out after waiting "
                    "%(time)s seconds when running proc: %(args)s"
                    " %(kwargs)s") % locals()
            raise exception.ProcessExecutionError(msg)
    finally:
        timeout.cancel()
Esempio n. 29
0
    def handle(self, sock, read_data, path, headers):
        request_id = headers.get("X-Request-Id", "-")
        for attempt in range(self.attempts):
            if attempt > 0:
                logging.warn("[%s] Retrying connection for host %s", request_id, self.host)

            backend = self.select_backend()
            try:
                timeout = Timeout(self.connection_timeout_seconds)
                try:
                    server_sock = eventlet.connect((backend.host, backend.port))
                finally:
                    timeout.cancel()

                backend.add_connection()
                break
            except socket.error:
                logging.error("[%s] Proxy socket error on connect() to %s of %s", request_id, backend, self.host)
                self.blacklist(backend)
                eventlet.sleep(self.delay)
                continue
            except:
                logging.warn("[%s] Proxy timeout on connect() to %s of %s", request_id, backend, self.host)
                self.blacklist(backend)
                eventlet.sleep(self.delay)
                continue

        # Function to help track data usage
        def send_onwards(data):
            server_sock.sendall(data)
            return len(data)

        try:
            size = send_onwards(read_data)
            size += SocketMelder(sock, server_sock, backend, self.host).run()
        except socket.error, e:
            if e.errno != errno.EPIPE:
                raise
Esempio n. 30
0
def execute_with_timeout(*args, **kwargs):
    time = kwargs.pop("timeout", 30)
    log_output_on_error = kwargs.pop("log_output_on_error", False)

    timeout = Timeout(time)
    try:
        return execute(*args, **kwargs)
    except exception.ProcessExecutionError as e:
        if log_output_on_error:
            LOG.error(
                _(
                    "Command '%(cmd)s' failed. %(description)s "
                    "Exit code: %(exit_code)s\nstderr: %(stderr)s\n"
                    "stdout: %(stdout)s"
                )
                % {
                    "cmd": e.cmd,
                    "description": e.description or "",
                    "exit_code": e.exit_code,
                    "stderr": e.stderr,
                    "stdout": e.stdout,
                }
            )
        raise
    except Timeout as t:
        if t is not timeout:
            LOG.error(_("Got a timeout but not the one expected."))
            raise
        else:
            msg = _("Time out after waiting " "%(time)s seconds when running proc: %(args)s" " %(kwargs)s.") % {
                "time": time,
                "args": args,
                "kwargs": kwargs,
            }
            LOG.error(msg)
            raise exception.ProcessExecutionError(msg)
    finally:
        timeout.cancel()
Esempio n. 31
0
def execute_task(task, headers, args):
    """ Executes a task to a url with the given args.

  Args:
    task: A celery Task instance.
    headers: A dictionary of headers for the task.
    args: A dictionary of arguments for the request.
          Contains the task body.
  Returns:
    The status code of the task fetch upon success.
  Raises:
    The current function to retry.
  """
    start_time = datetime.datetime.utcnow()

    content_length = len(args['body'])

    loggable_args = {
        key: args[key]
        for key in args if key not in ['task_name', 'body', 'payload']
    }
    loggable_args['body_length'] = content_length
    logger.info('Running {}\n'
                'Headers: {}\n'
                'Args: {}'.format(args['task_name'], headers, loggable_args))
    url = urlparse(args['url'])

    timeout = EventletTimeout(MAX_TASK_DURATION)
    try:
        redirects_left = 1
        while True:
            urlpath = url.path
            if url.query:
                urlpath += "?" + url.query

            method = args['method']
            if args['expires'] <= datetime.datetime.now():
                # We do this check because the expires attribute in
                # celery is not passed to retried tasks. This is a
                # documented bug in celery.
                logger.error(
                    "Task %s with id %s has expired with expiration date %s" %
                    (args['task_name'], task.request.id, args['expires']))
                celery.control.revoke(task.request.id)

                item = TaskName.get_by_key_name(args['task_name'])
                if item is not None:
                    item.state = TASK_STATES.EXPIRED
                    item.endtime = datetime.datetime.now()
                    db.put(item)

                return

            if (args['max_retries'] != 0
                    and task.request.retries >= args['max_retries']):
                logger.error(
                    "Task %s with id %s has exceeded retries: %s" %
                    (args['task_name'], task.request.id, args['max_retries']))
                celery.control.revoke(task.request.id)

                item = TaskName.get_by_key_name(args['task_name'])
                if item is not None:
                    item.state = TASK_STATES.FAILED
                    item.endtime = datetime.datetime.now()
                    db.put(item)

                return

            # Targets do not get X-Forwarded-Proto from nginx, they use haproxy port.
            headers['X-Forwarded-Proto'] = url.scheme
            if url.scheme == 'http':
                connection = httplib.HTTPConnection(remote_host, url.port)
            elif url.scheme == 'https':
                connection = httplib.HTTPSConnection(remote_host, url.port)
            else:
                logger.error("Task %s tried to use url scheme %s, "
                             "which is not supported." %
                             (args['task_name'], url.scheme))

            skip_host = False
            if 'host' in headers or 'Host' in headers:
                skip_host = True

            skip_accept_encoding = False
            if 'accept-encoding' in headers or 'Accept-Encoding' in headers:
                skip_accept_encoding = True

            connection.putrequest(method,
                                  urlpath,
                                  skip_host=skip_host,
                                  skip_accept_encoding=skip_accept_encoding)

            # Update the task headers
            headers['X-AppEngine-TaskRetryCount'] = str(task.request.retries)
            headers['X-AppEngine-TaskExecutionCount'] = str(
                task.request.retries)

            for header in headers:
                connection.putheader(header, headers[header])

            if 'content-type' not in headers or 'Content-Type' not in headers:
                if url.query:
                    connection.putheader('content-type',
                                         'application/octet-stream')
                else:
                    connection.putheader('content-type',
                                         'application/x-www-form-urlencoded')

            connection.putheader("Content-Length", str(content_length))

            retries = int(task.request.retries) + 1
            wait_time = get_wait_time(retries, args)

            try:
                connection.endheaders()
                if args["body"]:
                    connection.send(args['body'])

                response = connection.getresponse()
                response.read()
                response.close()
            except (BadStatusLine, SocketError):
                logger.warning(
                    '{task} failed before receiving response. It will retry in {wait} '
                    'seconds.'.format(task=args['task_name'], wait=wait_time))
                raise task.retry(countdown=wait_time)

            if 200 <= response.status < 300:
                # Task successful.
                item = TaskName.get_by_key_name(args['task_name'])
                if item is not None:
                    item.state = TASK_STATES.SUCCESS
                    item.endtime = datetime.datetime.now()
                    db.put(item)

                time_elapsed = datetime.datetime.utcnow() - start_time
                logger.info(
                  '{task} received status {status} from {url} [time elapsed: {te}]'. \
                  format(task=args['task_name'], status=response.status,
                         url=url, te=str(time_elapsed)))
                return response.status
            elif response.status == 302:
                redirect_url = response.getheader('Location')
                logger.info(
                    "Task %s asked us to redirect to %s, so retrying there." %
                    (args['task_name'], redirect_url))
                url = urlparse(redirect_url)
                if redirects_left == 0:
                    raise task.retry(countdown=wait_time)
                redirects_left -= 1
            else:
                message = ('Received a {status} for {task}. '
                           'Retrying in {wait} secs.'.format(
                               status=response.status,
                               task=args['task_name'],
                               wait=wait_time))
                logger.warning(message)
                raise task.retry(countdown=wait_time)
    except EventletTimeout as thrown_timeout:
        if thrown_timeout != timeout:
            raise

        logger.exception('Task {} timed out. Retrying.'.format(
            args['task_name']))
        # This could probably be calculated, but for now, just retry immediately.
        raise task.retry(countdown=0)
    finally:
        timeout.cancel()
Esempio n. 32
0
    def grow_cluster(self, context, cluster_id, new_instance_ids):
        LOG.debug("Begin Galera grow_cluster for id: %s.", cluster_id)

        def _grow_cluster():

            db_instances = DBInstance.find_all(cluster_id=cluster_id,
                                               deleted=False).all()
            existing_instances = [
                Instance.load(context, db_inst.id) for db_inst in db_instances
                if db_inst.id not in new_instance_ids
            ]
            if not existing_instances:
                raise TroveError(
                    _("Unable to determine existing cluster "
                      "member(s)"))

            # get list of ips of existing cluster members
            existing_cluster_ips = [
                self.get_ip(instance) for instance in existing_instances
            ]
            existing_instance_guests = [
                self.get_guest(instance) for instance in existing_instances
            ]

            # get the cluster context to setup new members
            cluster_context = existing_instance_guests[0].get_cluster_context()

            # Wait for cluster members to get to cluster-ready status.
            if not self._all_instances_ready(new_instance_ids, cluster_id):
                raise TroveError(
                    _("Instances in cluster did not report "
                      "ACTIVE"))

            LOG.debug("All members ready, proceeding for cluster setup.")

            # Get the new instances to join the cluster
            new_instances = [
                Instance.load(context, instance_id)
                for instance_id in new_instance_ids
            ]
            new_cluster_ips = [
                self.get_ip(instance) for instance in new_instances
            ]
            for instance in new_instances:
                guest = self.get_guest(instance)

                guest.reset_admin_password(cluster_context['admin_password'])

                # render the conf.d/cluster.cnf configuration
                cluster_configuration = self._render_cluster_config(
                    context, instance, ",".join(existing_cluster_ips),
                    cluster_context['cluster_name'],
                    cluster_context['replication_user'])

                # push the cluster config and bootstrap the first instance
                bootstrap = False
                guest.install_cluster(cluster_context['replication_user'],
                                      cluster_configuration, bootstrap)

            self._check_cluster_for_root(context, existing_instances,
                                         new_instances)

            # apply the new config to all instances
            for instance in existing_instances + new_instances:
                guest = self.get_guest(instance)
                # render the conf.d/cluster.cnf configuration
                cluster_configuration = self._render_cluster_config(
                    context, instance,
                    ",".join(existing_cluster_ips + new_cluster_ips),
                    cluster_context['cluster_name'],
                    cluster_context['replication_user'])
                guest.write_cluster_configuration_overrides(
                    cluster_configuration)

            for instance in new_instances:
                guest = self.get_guest(instance)
                guest.cluster_complete()

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _grow_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for growing cluster."))
            self.update_statuses_on_failure(
                cluster_id, status=inst_tasks.InstanceTasks.GROWING_ERROR)
        except Exception:
            LOG.exception(_("Error growing cluster %s."), cluster_id)
            self.update_statuses_on_failure(
                cluster_id, status=inst_tasks.InstanceTasks.GROWING_ERROR)
        finally:
            timeout.cancel()

        LOG.debug("End grow_cluster for id: %s.", cluster_id)
Esempio n. 33
0
    def create_cluster(self, context, cluster_id):
        LOG.debug("Begin create_cluster for id: %s." % cluster_id)

        def _create_cluster():

            # Fetch instances by cluster_id against instances table.
            db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
            instance_ids = [db_instance.id for db_instance in db_instances]

            # Wait for cluster members to get to cluster-ready status.
            if not self._all_instances_ready(instance_ids, cluster_id):
                return

            LOG.debug("All members ready, proceeding for cluster setup.")
            instances = [
                Instance.load(context, instance_id)
                for instance_id in instance_ids
            ]

            member_ips = [self.get_ip(instance) for instance in instances]
            guests = [self.get_guest(instance) for instance in instances]

            # Users to be configured for password-less SSH.
            authorized_users_without_password = ['root', 'dbadmin']

            # Configuring password-less SSH for cluster members.
            # Strategy for setting up SSH:
            # get public keys for user from member-instances in cluster,
            # combine them, finally push it back to all instances,
            # and member instances add them to authorized keys.
            LOG.debug("Configuring password-less SSH on cluster members.")
            try:
                for user in authorized_users_without_password:
                    pub_key = [guest.get_public_keys(user) for guest in guests]
                    for guest in guests:
                        guest.authorize_public_keys(user, pub_key)

                LOG.debug("Installing cluster with members: %s." % member_ips)
                for db_instance in db_instances:
                    if db_instance['type'] == 'master':
                        master_instance = Instance.load(
                            context, db_instance.id)
                        self.get_guest(master_instance).install_cluster(
                            member_ips)
                        break

                LOG.debug("Finalizing cluster configuration.")
                for guest in guests:
                    guest.cluster_complete()
            except Exception:
                LOG.exception(_("Error creating cluster."))
                self.update_statuses_on_failure(cluster_id)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _create_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for building cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End create_cluster for id: %s." % cluster_id)
Esempio n. 34
0
    def shrink_cluster(self, context, cluster_id, removal_instance_ids):
        LOG.debug("Begin Galera shrink_cluster for id: %s.", cluster_id)

        def _shrink_cluster():
            removal_instances = [
                Instance.load(context, instance_id)
                for instance_id in removal_instance_ids
            ]
            for instance in removal_instances:
                Instance.delete(instance)

            # wait for instances to be deleted
            def all_instances_marked_deleted():
                non_deleted_instances = DBInstance.find_all(
                    cluster_id=cluster_id, deleted=False).all()
                non_deleted_ids = [
                    db_instance.id for db_instance in non_deleted_instances
                ]
                return not bool(
                    set(removal_instance_ids).intersection(
                        set(non_deleted_ids)))

            try:
                LOG.info(_("Deleting instances (%s)"), removal_instance_ids)
                utils.poll_until(all_instances_marked_deleted,
                                 sleep_time=2,
                                 time_out=CONF.cluster_delete_time_out)
            except PollTimeOut:
                LOG.error(_("timeout for instances to be marked as deleted."))
                return

            db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
            leftover_instances = [
                Instance.load(context, db_inst.id) for db_inst in db_instances
                if db_inst.id not in removal_instance_ids
            ]
            leftover_cluster_ips = [
                self.get_ip(instance) for instance in leftover_instances
            ]

            # Get config changes for left over instances
            rnd_cluster_guest = self.get_guest(leftover_instances[0])
            cluster_context = rnd_cluster_guest.get_cluster_context()

            # apply the new config to all leftover instances
            for instance in leftover_instances:
                guest = self.get_guest(instance)
                # render the conf.d/cluster.cnf configuration
                cluster_configuration = self._render_cluster_config(
                    context, instance, ",".join(leftover_cluster_ips),
                    cluster_context['cluster_name'],
                    cluster_context['replication_user'])
                guest.write_cluster_configuration_overrides(
                    cluster_configuration)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _shrink_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for shrinking cluster."))
            self.update_statuses_on_failure(
                cluster_id, status=inst_tasks.InstanceTasks.SHRINKING_ERROR)
        except Exception:
            LOG.exception(_("Error shrinking cluster %s."), cluster_id)
            self.update_statuses_on_failure(
                cluster_id, status=inst_tasks.InstanceTasks.SHRINKING_ERROR)
        finally:
            timeout.cancel()

        LOG.debug("End shrink_cluster for id: %s.", cluster_id)
Esempio n. 35
0
    def grow_cluster(self, context, cluster_id, new_instance_ids):
        def _grow_cluster():
            LOG.debug("begin grow_cluster for Vertica cluster %s" % cluster_id)

            db_instances = DBInstance.find_all(cluster_id=cluster_id,
                                               deleted=False).all()

            instance_ids = [db_instance.id for db_instance in db_instances]

            # Wait for new cluster members to get to cluster-ready status.
            if not self._all_instances_ready(new_instance_ids, cluster_id):
                return

            new_insts = [
                Instance.load(context, instance_id)
                for instance_id in new_instance_ids
            ]

            existing_instances = [
                Instance.load(context, instance_id)
                for instance_id in instance_ids
                if instance_id not in new_instance_ids
            ]

            existing_guests = [self.get_guest(i) for i in existing_instances]
            new_guests = [self.get_guest(i) for i in new_insts]
            all_guests = new_guests + existing_guests

            authorized_users_without_password = ['root', 'dbadmin']
            new_ips = [self.get_ip(instance) for instance in new_insts]

            for user in authorized_users_without_password:
                pub_key = [guest.get_public_keys(user) for guest in all_guests]
                for guest in all_guests:
                    guest.authorize_public_keys(user, pub_key)

            for db_instance in db_instances:
                if db_instance['type'] == 'master':
                    LOG.debug("Found 'master' instance, calling grow on guest")
                    master_instance = Instance.load(context, db_instance.id)
                    self.get_guest(master_instance).grow_cluster(new_ips)
                    break

            for guest in new_guests:
                guest.cluster_complete()

        timeout = Timeout(CONF.cluster_usage_timeout)

        try:
            _grow_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for growing cluster."))
            self.update_statuses_on_failure(cluster_id)
        except Exception:
            LOG.exception(_("Error growing cluster %s.") % cluster_id)
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()
Esempio n. 36
0
    def shrink_cluster(self, context, cluster_id, removal_ids):
        LOG.debug("Begin shrink_cluster for id: %s.", cluster_id)

        def _shrink_cluster():
            cluster_node_ids = self.find_cluster_node_ids(cluster_id)
            cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids)

            removed_nodes = CassandraClusterTasks.load_cluster_nodes(
                context, removal_ids)

            LOG.debug("All nodes ready, proceeding with cluster setup.")

            # Update the list of seeds on remaining nodes if necessary.
            # Once all nodes are configured, decommission the removed nodes.
            # Cassandra will stream data from decommissioned nodes to the
            # remaining ones.
            try:

                # All nodes should have the same seeds.
                # We retrieve current seeds from the first node.
                test_node = self.load_cluster_nodes(context,
                                                    cluster_node_ids[:1])[0]
                current_seeds = test_node['guest'].get_seeds()
                # The seeds will have to be updated on all remaining instances
                # if any of the seed nodes is going to be removed.
                update_seeds = any(node['ip'] in current_seeds
                                   for node in removed_nodes)

                LOG.debug("Decommissioning removed nodes.")
                for node in removed_nodes:
                    node['guest'].node_decommission()
                    node['instance'].update_db(cluster_id=None)

                # Recompute the seed nodes based on the updated cluster
                # geometry if any of the existing seed nodes was removed.
                if update_seeds:
                    LOG.debug("Updating seeds on the remaining nodes.")
                    cluster_nodes = self.load_cluster_nodes(
                        context, cluster_node_ids)

                    remaining_nodes = [
                        node for node in cluster_nodes
                        if node['id'] not in removal_ids
                    ]
                    seeds = self.choose_seed_nodes(remaining_nodes)
                    LOG.debug("Selected seed nodes: %s", seeds)
                    for node in remaining_nodes:
                        LOG.debug("Configuring node: %s.", node['id'])
                        node['guest'].set_seeds(seeds)

                # Wait for the removed nodes to go SHUTDOWN.
                LOG.debug("Waiting for all decommissioned nodes to shutdown.")
                if not self._all_instances_shutdown(removal_ids, cluster_id):
                    # Now detached, failed nodes will stay available
                    # in the list of standalone instances.
                    return

                # Delete decommissioned instances only when the cluster is in a
                # consistent state.
                LOG.debug("Deleting decommissioned instances.")
                for node in removed_nodes:
                    Instance.delete(node['instance'])

                LOG.debug("Cluster configuration finished successfully.")
            except Exception:
                LOG.exception(_("Error shrinking cluster."))
                self.update_statuses_on_failure(cluster_id)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _shrink_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for shrinking cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End shrink_cluster for id: %s.", cluster_id)
Esempio n. 37
0
    def create_cluster(self, context, cluster_id):
        LOG.debug("Begin create_cluster for id: %s." % cluster_id)

        def _create_cluster():

            # Fetch instances by cluster_id against instances table.
            db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
            instance_ids = [db_instance.id for db_instance in db_instances]

            # Wait for cluster members to get to cluster-ready status.
            if not self._all_instances_ready(instance_ids, cluster_id):
                return

            LOG.debug("All members ready, proceeding for cluster setup.")
            instances = [
                Instance.load(context, instance_id)
                for instance_id in instance_ids
            ]

            # Connect nodes to the first node
            guests = [self.get_guest(instance) for instance in instances]
            try:
                cluster_head = instances[0]
                cluster_head_port = '6379'
                cluster_head_ip = self.get_ip(cluster_head)
                for guest in guests[1:]:
                    guest.cluster_meet(cluster_head_ip, cluster_head_port)

                num_nodes = len(instances)
                total_slots = 16384
                slots_per_node = total_slots / num_nodes
                leftover_slots = total_slots % num_nodes
                first_slot = 0
                for guest in guests:
                    last_slot = first_slot + slots_per_node
                    if leftover_slots > 0:
                        leftover_slots -= 1
                    else:
                        last_slot -= 1
                    guest.cluster_addslots(first_slot, last_slot)
                    first_slot = last_slot + 1

                for guest in guests:
                    guest.cluster_complete()
            except Exception:
                LOG.exception(_("Error creating cluster."))
                self.update_statuses_on_failure(cluster_id)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _create_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for building cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End create_cluster for id: %s." % cluster_id)
Esempio n. 38
0
    def grow_cluster(self, context, cluster_id, new_instance_ids):
        LOG.debug("Begin grow_cluster for id: %s." % cluster_id)

        def _grow_cluster():
            # Wait for new nodes to get to cluster-ready status.
            LOG.debug("Waiting for new nodes to become ready.")
            if not self._all_instances_ready(new_instance_ids, cluster_id):
                return

            new_instances = [Instance.load(context, instance_id)
                             for instance_id in new_instance_ids]
            added_nodes = [self.build_node_info(instance)
                           for instance in new_instances]

            LOG.debug("All nodes ready, proceeding with cluster setup.")

            cluster_node_ids = self.find_cluster_node_ids(cluster_id)
            cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids)

            # Recompute the seed nodes based on the updated cluster geometry.
            seeds = self.choose_seed_nodes(cluster_nodes)

            # Configure each cluster node with the updated list of seeds.
            # Since we are adding to an existing cluster, ensure that the
            # new nodes have auto-bootstrapping enabled.
            # Start the added nodes.
            try:
                LOG.debug("Selected seed nodes: %s" % seeds)

                # Update the seeds on all nodes.
                # Also retrieve the superuser password from one previously
                # existing node.
                admin_creds = None
                for node in cluster_nodes:
                    LOG.debug("Configuring node: %s." % node['id'])
                    node['guest'].set_seeds(seeds)
                    if (admin_creds is None) and (node not in added_nodes):
                        admin_creds = node['guest'].get_admin_credentials()

                # Start any seeds from the added nodes first.
                LOG.debug("Starting new seed nodes.")
                for node in added_nodes:
                    if node['ip'] in seeds:
                        node['guest'].set_auto_bootstrap(True)
                        node['guest'].store_admin_credentials(admin_creds)
                        node['guest'].restart()
                        node['guest'].cluster_complete()

                LOG.debug("All new seeds running, starting the remaining of "
                          "added nodes.")
                for node in added_nodes:
                    if node['ip'] not in seeds:
                        node['guest'].set_auto_bootstrap(True)
                        node['guest'].store_admin_credentials(admin_creds)
                        node['guest'].restart()
                        node['guest'].cluster_complete()

                # Run nodetool cleanup on each of the previously existing nodes
                # to remove the keys that no longer belong to those nodes.
                # Wait for cleanup to complete on one node before running
                # it on the next node.
                LOG.debug("Cleaning up orphan data on old cluster nodes.")
                for node in cluster_nodes:
                    if node not in added_nodes:
                        nid = node['id']
                        node['guest'].node_cleanup_begin()
                        node['guest'].node_cleanup()
                        LOG.debug("Waiting for node to finish its "
                                  "cleanup: %s" % nid)
                        if not self._all_instances_running([nid], cluster_id):
                            LOG.warning(_("Node did not complete cleanup "
                                          "successfully: %s") % nid)

                LOG.debug("Cluster configuration finished successfully.")
            except Exception:
                LOG.exception(_("Error growing cluster."))
                self.update_statuses_on_failure(cluster_id)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _grow_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for growing cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End grow_cluster for id: %s." % cluster_id)
Esempio n. 39
0
    def grow_cluster(self, context, cluster_id, instance_ids):
        LOG.debug("begin grow_cluster for MongoDB cluster %s", cluster_id)

        def _grow_cluster():
            new_instances = [db_instance for db_instance in self.db_instances
                             if db_instance.id in instance_ids]
            new_members = [db_instance for db_instance in new_instances
                           if db_instance.type == 'member']
            new_query_routers = [db_instance for db_instance in new_instances
                                 if db_instance.type == 'query_router']
            instances = []
            if new_members:
                shard_ids = set([db_instance.shard_id for db_instance
                                 in new_members])
                query_router_id = self._get_running_query_router_id()
                if not query_router_id:
                    return
                for shard_id in shard_ids:
                    LOG.debug('growing cluster by adding shard %(shard_id)s '
                              'on query router %(router_id)s',
                              {'shard_id': shard_id,
                               'router_id': query_router_id})
                    member_ids = [db_instance.id for db_instance in new_members
                                  if db_instance.shard_id == shard_id]
                    if not self._all_instances_ready(
                        member_ids, cluster_id, shard_id
                    ):
                        return
                    members = [Instance.load(context, member_id)
                               for member_id in member_ids]
                    query_router = Instance.load(context, query_router_id)
                    if not self._create_shard(query_router, members):
                        return
                    instances.extend(members)
            if new_query_routers:
                query_router_ids = [db_instance.id for db_instance
                                    in new_query_routers]
                config_servers_ids = [db_instance.id for db_instance
                                      in self.db_instances
                                      if db_instance.type == 'config_server']
                LOG.debug('growing cluster by adding query routers '
                          '%(router)s, with config servers %(server)s',
                          {'router': query_router_ids,
                           'server': config_servers_ids})
                if not self._all_instances_ready(
                    query_router_ids, cluster_id
                ):
                    return
                query_routers = [Instance.load(context, instance_id)
                                 for instance_id in query_router_ids]
                config_servers_ips = [
                    self.get_ip(Instance.load(context, config_server_id))
                    for config_server_id in config_servers_ids
                ]
                if not self._add_query_routers(
                        query_routers, config_servers_ips,
                        admin_password=self.get_cluster_admin_password(context)
                ):
                    return
                instances.extend(query_routers)
            for instance in instances:
                self.get_guest(instance).cluster_complete()

        cluster_usage_timeout = CONF.cluster_usage_timeout
        timeout = Timeout(cluster_usage_timeout)
        try:
            _grow_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("timeout for growing cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("end grow_cluster for MongoDB cluster %s", self.id)
Esempio n. 40
0
    def create_cluster(self, context, cluster_id):
        LOG.debug("Begin create_cluster for id: %s.", cluster_id)

        def _create_cluster():
            # Fetch instances by cluster_id against instances table.
            db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
            instance_ids = [db_instance.id for db_instance in db_instances]

            LOG.debug("Waiting for instances to get to cluster-ready status.")
            # Wait for cluster members to get to cluster-ready status.
            if not self._all_instances_ready(instance_ids, cluster_id):
                raise TroveError(
                    _("Instances in cluster did not report "
                      "ACTIVE"))

            LOG.debug("All members ready, proceeding for cluster setup.")
            instances = [
                Instance.load(context, instance_id)
                for instance_id in instance_ids
            ]

            cluster_ips = [self.get_ip(instance) for instance in instances]
            instance_guests = [
                self.get_guest(instance) for instance in instances
            ]

            # Create replication user and password for synchronizing the
            # galera cluster
            replication_user = {
                "name": self.CLUSTER_REPLICATION_USER,
                "password": utils.generate_random_password(),
            }

            # Galera cluster name must be unique and be shorter than a full
            # uuid string so we remove the hyphens and chop it off. It was
            # recommended to be 16 chars or less.
            # (this is not currently documented on Galera docs)
            cluster_name = utils.generate_uuid().replace("-", "")[:16]

            LOG.debug("Configuring cluster configuration.")
            try:
                # Set the admin password for all the instances because the
                # password in the my.cnf will be wrong after the joiner
                # instances syncs with the donor instance.
                admin_password = str(utils.generate_random_password())
                for guest in instance_guests:
                    guest.reset_admin_password(admin_password)

                bootstrap = True
                for instance in instances:
                    guest = self.get_guest(instance)

                    # render the conf.d/cluster.cnf configuration
                    cluster_configuration = self._render_cluster_config(
                        context, instance, ",".join(cluster_ips), cluster_name,
                        replication_user)

                    # push the cluster config and bootstrap the first instance
                    guest.install_cluster(replication_user,
                                          cluster_configuration, bootstrap)
                    bootstrap = False

                LOG.debug("Finalizing cluster configuration.")
                for guest in instance_guests:
                    guest.cluster_complete()
            except Exception:
                LOG.exception(_("Error creating cluster."))
                self.update_statuses_on_failure(cluster_id)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _create_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for building cluster."))
            self.update_statuses_on_failure(cluster_id)
        except TroveError:
            LOG.exception(_("Error creating cluster %s."), cluster_id)
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End create_cluster for id: %s.", cluster_id)
Esempio n. 41
0
    def shrink_cluster(self, context, cluster_id, removal_ids):
        """Shrink a K2hdkc Cluster."""
        LOG.debug(
            "Begins shrink_cluster for %s. removal_ids:{}".format(removal_ids),
            cluster_id)

        # 1. validates args
        if context is None:
            LOG.error("no context")
            return
        if cluster_id is None:
            LOG.error("no cluster_id")
            return
        if removal_ids is None:
            LOG.error("no removal_ids")
            return

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            # 2. Retrieves db_instances from the database
            db_instances = DBInstance.find_all(cluster_id=cluster_id,
                                               deleted=False).all()
            # 3. Retrieves instance ids from the db_instances
            instance_ids = [db_instance.id for db_instance in db_instances]
            # 4. Checks if instances are running
            if not self._all_instances_running(instance_ids, cluster_id):
                LOG.error("instances are not ready yet")
                return
            # 4. Loads instances
            instances = [
                Instance.load(context, instance_id)
                for instance_id in removal_ids
            ]
            LOG.debug("len(instances) {}".format(len(instances)))

            # 5. Instances GuestAgent class
            # 6.2. Checks if removing instances are
            # if not self._all_instances_shutdown(removal_ids, cluster_id):
            #    LOG.error("removing instances are not shutdown yet")
            #    return
            # 7. Calls cluster_complete endpoint of K2hdkcGuestAgent
            LOG.debug(
                "Calling cluster_complete as a final hook to each node in the cluster"
            )
            for instance in instances:
                self.get_guest(instance).cluster_complete()
            # 8. delete node from OpenStack
            LOG.debug("delete node from OpenStack")
            for instance in instances:
                Instance.delete(instance)
            # 9. reset the current cluster task status to None
            LOG.debug("reset cluster task to None")
            self.reset_task()
        except Timeout:
            # Note adminstrators should reset task via CLI in this case.
            if Timeout is not timeout:
                raise  # not my timeout
            LOG.exception("Timeout for shrink cluster.")
            self.update_statuses_on_failure(
                cluster_id, status=inst_tasks.InstanceTasks.SHRINKING_ERROR)
        finally:
            timeout.cancel()

        LOG.debug("Completed shrink_cluster for %s.", cluster_id)
Esempio n. 42
0
class ResultContext(object):
    """
    ResultContext
    =============

    The ResultContext class is a Python context manager used in the
    automatic collection of captured output and exception handling.
    It is instantiated by DTestResult.accumulate()
    """

    def __init__(self, result, ctx, excs):
        """
        Initialize a ResultContext associated with the given
        ``result``.  The context will handle messages for the part of
        the test run given by ``ctx`` (may be PRE, TEST, or POST).
        The exceptions listed in the ``excs`` tuple are expected; if
        no exceptions are expected, pass ``excs`` as None.
        """

        # Save the basic information
        self.result = result
        self.ctx = ctx
        self.excs = excs

        # There's no timeout...
        self.timeout = None

    def __enter__(self):
        """
        Begin the context handling.  Clears out any captured data and
        initializes any timeouts defined for the test.
        """

        # Clear the captured values for this thread
        capture.retrieve()

        # If test should be timed, set up the timeout
        if self.result._test._timeout:
            self.timeout = Timeout(self.result._test._timeout,
                                   AssertionError("Timed out after %s "
                                                  "seconds" %
                                                  self.result._test._timeout))

    def __exit__(self, exc_type, exc_value, tb):
        """
        Ends context handling.  Cancels any pending timeouts,
        retrieves output data and exceptions, and determines the final
        result of the test.  A DTestMessage object is initialized if
        necessary.
        """

        # Cancel the timeout if one is pending
        if self.timeout is not None:
            self.timeout.cancel()
            self.timeout = None

        # Get the output and clean up
        captured = capture.retrieve()

        # If this was the test, determine a result
        if self.ctx in (PRE, TEST):
            self.result._set_result(self, exc_type, exc_value, tb)

        # Generate a message, if necessary
        if captured or exc_type or exc_value or tb:
            self.result._storemsg(self, captured, exc_type, exc_value, tb)

        # We handled the exception
        return True
Esempio n. 43
0
    def create_cluster(self, context, cluster_id):
        LOG.debug("begin create_cluster for id: %s" % cluster_id)

        def _create_cluster():

            # fetch instances by cluster_id against instances table
            db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
            instance_ids = [db_instance.id for db_instance in db_instances]
            LOG.debug("instances in cluster %s: %s" % (cluster_id,
                                                       instance_ids))

            if not self._all_instances_ready(instance_ids, cluster_id):
                return

            LOG.debug("all instances in cluster %s ready." % cluster_id)

            instances = [Instance.load(context, instance_id) for instance_id
                         in instance_ids]

            # filter query routers in instances into a new list: query_routers
            query_routers = [instance for instance in instances if
                             instance.type == 'query_router']
            LOG.debug("query routers: %s" %
                      [instance.id for instance in query_routers])
            # filter config servers in instances into new list: config_servers
            config_servers = [instance for instance in instances if
                              instance.type == 'config_server']
            LOG.debug("config servers: %s" %
                      [instance.id for instance in config_servers])
            # filter members (non router/configsvr) into a new list: members
            members = [instance for instance in instances if
                       instance.type == 'member']
            LOG.debug("members: %s" %
                      [instance.id for instance in members])

            # for config_server in config_servers, append ip/hostname to
            # "config_server_hosts", then
            # peel off the replica-set name and ip/hostname from 'x'
            config_server_ips = [self.get_ip(instance)
                                 for instance in config_servers]
            LOG.debug("config server ips: %s" % config_server_ips)

            # Give the query routers the configsvr ips to connect to.
            # Create the admin user on the query routers.
            # The first will create the user, and the others will just reset
            # the password to the same value.
            LOG.debug("calling add_config_servers on, and sending admin user "
                      "password to, query_routers")
            try:
                admin_created = False
                admin_password = utils.generate_random_password()
                for query_router in query_routers:
                    guest = self.get_guest(query_router)
                    guest.add_config_servers(config_server_ips)
                    if admin_created:
                        guest.store_admin_password(admin_password)
                    else:
                        guest.create_admin_user(admin_password)
                        admin_created = True
            except Exception:
                LOG.exception(_("error adding config servers"))
                self.update_statuses_on_failure(cluster_id)
                return

            if not self._create_replica_set(members, cluster_id):
                return

            replica_set_name = "rs1"
            if not self._create_shard(query_routers, replica_set_name,
                                      members, cluster_id):
                return
            # call to start checking status
            for instance in instances:
                self.get_guest(instance).cluster_complete()

        cluster_usage_timeout = CONF.cluster_usage_timeout
        timeout = Timeout(cluster_usage_timeout)
        try:
            _create_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("timeout for building cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("end create_cluster for id: %s" % cluster_id)
Esempio n. 44
0
    def create_cluster(self, context, cluster_id):
        LOG.debug("Begin create_cluster for id: %s.", cluster_id)

        def _create_cluster():
            cluster_node_ids = self.find_cluster_node_ids(cluster_id)

            # Wait for cluster nodes to get to cluster-ready status.
            LOG.debug("Waiting for all nodes to become ready.")
            if not self._all_instances_ready(cluster_node_ids, cluster_id):
                return

            cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids)

            LOG.debug("All nodes ready, proceeding with cluster setup.")
            seeds = self.choose_seed_nodes(cluster_nodes)

            # Configure each cluster node with the list of seeds.
            # Once all nodes are configured, start the seed nodes one at a time
            # followed by the rest of the nodes.
            try:
                LOG.debug("Selected seed nodes: %s", seeds)

                for node in cluster_nodes:
                    LOG.debug("Configuring node: %s.", node['id'])
                    node['guest'].set_seeds(seeds)
                    node['guest'].set_auto_bootstrap(False)

                LOG.debug("Starting seed nodes.")
                for node in cluster_nodes:
                    if node['ip'] in seeds:
                        node['guest'].restart()
                        node['guest'].set_auto_bootstrap(True)

                LOG.debug("All seeds running, starting remaining nodes.")
                for node in cluster_nodes:
                    if node['ip'] not in seeds:
                        node['guest'].restart()
                        node['guest'].set_auto_bootstrap(True)

                # Create the in-database user via the first node. The remaining
                # nodes will replicate in-database changes automatically.
                # Only update the local authentication file on the other nodes.
                LOG.debug("Securing the cluster.")
                key = utils.generate_random_password()
                admin_creds = None
                for node in cluster_nodes:
                    if admin_creds is None:
                        admin_creds = node['guest'].cluster_secure(key)
                    else:
                        node['guest'].store_admin_credentials(admin_creds)
                    node['guest'].cluster_complete()

                LOG.debug("Cluster configuration finished successfully.")
            except Exception:
                LOG.exception(_("Error creating cluster."))
                self.update_statuses_on_failure(cluster_id)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _create_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for building cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End create_cluster for id: %s.", cluster_id)
def main():
    # set up headless proxy
    headless_proxy = "127.0.0.1:3128"
    proxy = Proxy({
        'proxyType': ProxyType.MANUAL,
        'httpProxy': headless_proxy,
        'ftpProxy': headless_proxy,
        'sslProxy': headless_proxy,
        'noProxy': ''
    })

    chrome_options = webdriver.ChromeOptions()

    # disable images to speed up the page loading
    prefs = {"profile.managed_default_content_settings.images": 2}
    chrome_options.add_experimental_option("prefs", prefs)

    capabilities = dict(DesiredCapabilities.CHROME)
    proxy.add_to_capabilities(capabilities)

    url = 'https://dhp.virginiainteractive.org/Lookup/Index'

    driver = webdriver.Chrome(
        executable_path='/Users/markelle/Desktop/chromedriver')

    states = list(pd.read_csv("states.txt", header=None)[0])
    tables = []

    # repeat the full scrape for each state
    for state in states:
        driver.get(url)
        driver.implicitly_wait(10)

        # select the desired occupation and state from dropdowns
        driver.find_element_by_xpath(
            "//select[@id='OccupationId']/option[text()='Licensed Massage Therapist']"
        ).click()
        driver.find_element_by_xpath("//select[@id='State']/option[text()='" +
                                     state + "']").click()
        driver.find_elements_by_name("submitBtn")[2].click()

        # handle the case when the result page is empty (selenium gets stuck)
        timeout = Timeout(2, NoSuchElementException)
        try:
            rows = [x for x in driver.find_elements_by_tag_name('tr')[2:]]
        except NoSuchElementException:
            rows = []
        finally:
            timeout.cancel()

        # extract the text from each row and add to table
        for row in rows:
            cells = [x.text for x in row.find_elements_by_tag_name('td')]
            tables.append(cells)

        next_ = check_for_mult_pages(driver)

        # continue scraping while there are more pages
        while next_ != None:
            next_.click()
            rows = [x for x in driver.find_elements_by_tag_name('tr')[2:]]
            for row in rows:
                cells = [x.text for x in row.find_elements_by_tag_name('td')]
                tables.append(cells)
            next_ = check_for_mult_pages(driver)

    # export to csv
    pd.DataFrame(tables).to_csv("FULL_data.csv")
Esempio n. 46
0
for ln in rows:
    timeout = Timeout(6)
    try:
        print(ln)
        domain = ln[1]
        print(domain)
        try:
            r = requests.get('http://' + domain + '/license.txt')
        except:
            pass
        found = r.content.find(b'WordPress') + 1
        try:
            r = requests.get('https://' + domain + '/license.txt')
        except:
            pass
        found += r.content.find(b'WordPress') + 1
        print(found)
        if found > 0:
            cursor.execute(
                "replace into domains (domain, wpcheck) values (?, ?)",
                (domain, 1))
        else:
            cursor.execute(
                "replace into domains (domain, wpcheck) values (?, ?)",
                (domain, 0))

        connection.commit()

    finally:
        timeout.cancel()
Esempio n. 47
0
    def grow_cluster(self, context, cluster_id, new_instance_ids):
        LOG.debug("Begin grow_cluster for id: %s.", cluster_id)

        def _grow_cluster():
            # Wait for new nodes to get to cluster-ready status.
            LOG.debug("Waiting for new nodes to become ready.")
            if not self._all_instances_ready(new_instance_ids, cluster_id):
                return

            new_instances = [
                Instance.load(context, instance_id)
                for instance_id in new_instance_ids
            ]
            added_nodes = [
                self.build_node_info(instance) for instance in new_instances
            ]

            LOG.debug("All nodes ready, proceeding with cluster setup.")

            cluster_node_ids = self.find_cluster_node_ids(cluster_id)
            cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids)

            old_nodes = [
                node for node in cluster_nodes
                if node['id'] not in new_instance_ids
            ]

            try:

                # All nodes should have the same seeds and credentials.
                # Retrieve the information from the first node.
                test_node = old_nodes[0]
                current_seeds = test_node['guest'].get_seeds()
                admin_creds = test_node['guest'].get_admin_credentials()

                # Bootstrap new nodes.
                # Seed nodes do not bootstrap. Current running nodes
                # must be used as seeds during the process.
                # Since we are adding to an existing cluster, ensure that the
                # new nodes have auto-bootstrapping enabled.
                # Start the added nodes.
                LOG.debug("Starting new nodes.")
                for node in added_nodes:
                    node['guest'].set_auto_bootstrap(True)
                    node['guest'].set_seeds(current_seeds)
                    node['guest'].store_admin_credentials(admin_creds)
                    node['guest'].restart()
                    node['guest'].cluster_complete()

                # Recompute the seed nodes based on the updated cluster
                # geometry.
                seeds = self.choose_seed_nodes(cluster_nodes)

                # Configure each cluster node with the updated list of seeds.
                LOG.debug("Updating all nodes with new seeds: %s", seeds)
                for node in cluster_nodes:
                    node['guest'].set_seeds(seeds)

                # Run nodetool cleanup on each of the previously existing nodes
                # to remove the keys that no longer belong to those nodes.
                # Wait for cleanup to complete on one node before running
                # it on the next node.
                LOG.debug("Cleaning up orphan data on old cluster nodes.")
                for node in old_nodes:
                    nid = node['id']
                    node['guest'].node_cleanup_begin()
                    node['guest'].node_cleanup()
                    LOG.debug("Waiting for node to finish its "
                              "cleanup: %s", nid)
                    if not self._all_instances_running([nid], cluster_id):
                        LOG.warning(
                            _("Node did not complete cleanup "
                              "successfully: %s"), nid)

                LOG.debug("Cluster configuration finished successfully.")
            except Exception:
                LOG.exception(_("Error growing cluster."))
                self.update_statuses_on_failure(cluster_id)

        timeout = Timeout(CONF.cluster_usage_timeout)
        try:
            _grow_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("Timeout for growing cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("End grow_cluster for id: %s.", cluster_id)
Esempio n. 48
0
    def create_cluster(self, context, cluster_id):
        LOG.debug("begin create_cluster for id: %s", cluster_id)

        def _create_cluster():

            # fetch instances by cluster_id against instances table
            db_instances = DBInstance.find_all(cluster_id=cluster_id).all()
            instance_ids = [db_instance.id for db_instance in db_instances]
            LOG.debug("instances in cluster %(cluster_id)s: %(instance_ids)s",
                      {'cluster_id': cluster_id, 'instance_ids': instance_ids})

            if not self._all_instances_ready(instance_ids, cluster_id):
                return

            LOG.debug("all instances in cluster %s ready.", cluster_id)

            instances = [Instance.load(context, instance_id) for instance_id
                         in instance_ids]

            # filter query routers in instances into a new list: query_routers
            query_routers = [instance for instance in instances if
                             instance.type == 'query_router']
            LOG.debug("query routers: %s",
                      [instance.id for instance in query_routers])
            # filter config servers in instances into new list: config_servers
            config_servers = [instance for instance in instances if
                              instance.type == 'config_server']
            LOG.debug("config servers: %s",
                      [instance.id for instance in config_servers])
            # filter members (non router/configsvr) into a new list: members
            members = [instance for instance in instances if
                       instance.type == 'member']
            LOG.debug("members: %s",
                      [instance.id for instance in members])

            # for config_server in config_servers, append ip/hostname to
            # "config_server_hosts", then
            # peel off the replica-set name and ip/hostname from 'x'
            config_server_ips = [self.get_ip(instance)
                                 for instance in config_servers]
            LOG.debug("config server ips: %s", config_server_ips)

            if not self._add_query_routers(query_routers,
                                           config_server_ips):
                return

            if not self._create_shard(query_routers[0], members):
                return

            # call to start checking status
            for instance in instances:
                self.get_guest(instance).cluster_complete()

        cluster_usage_timeout = CONF.cluster_usage_timeout
        timeout = Timeout(cluster_usage_timeout)
        try:
            _create_cluster()
            self.reset_task()
        except Timeout as t:
            if t is not timeout:
                raise  # not my timeout
            LOG.exception(_("timeout for building cluster."))
            self.update_statuses_on_failure(cluster_id)
        finally:
            timeout.cancel()

        LOG.debug("end create_cluster for id: %s", cluster_id)