def readRegister(self, address, cache=False, timeout=2): """Return the value of register at [address] location if timeout is elapsed, NoneValue is returned """ if not isinstance(address, int): raise TypeError("Address must be an integer") if int(address) < 0 or int(address) > 255: raise ValueError("Address is out of range [0, 255]") if address in self.regs.keys() and cache: if not self.regs[address] == None: return self.regs[address] self.regs[address] = None self.vscp.sendSimpleEvent( vscp_class=constant.VSCP_CLASS1_PROTOCOL, vscp_type=constant.VSCP_TYPE_PROTOCOL_READ_REGISTER, vscp_data=[self.id, address], ) try: timer = Timeout(timeout, self.__timeout) while self.regs[address] == None: time.sleep(0.02) timer.cancel() return self.regs[address] except VSCPException: return None
def execute_with_timeout(*args, **kwargs): time = kwargs.get('timeout', 30) def cb_timeout(): msg = (_("Time out after waiting" " %(time)s seconds when running proc: %(args)s" " %(kwargs)s") % {'time': time, 'args': args, 'kwargs': kwargs}) LOG.error(msg) raise exception.ProcessExecutionError(msg) timeout = Timeout(time) try: return execute(*args, **kwargs) except Timeout as t: if t is not timeout: LOG.error("Timeout reached but not from our timeout. This is bad!") raise else: msg = (_("Time out after waiting " "%(time)s seconds when running proc: %(args)s" " %(kwargs)s") % {'time': time, 'args': args, 'kwargs': kwargs}) LOG.error(msg) raise exception.ProcessExecutionError(msg) finally: timeout.cancel()
def writeRegister(self, address, value, timeout=2): """Return True if value is correctly written if timeout is elapsed, False is returned """ if not isinstance(address, int): raise TypeError("Address must be an integer") if int(address) < 0 or int(address) > 255: raise ValueError("Address is out of range [0, 255]") if not isinstance(value, int): raise TypeError("Value must be an integer") if int(value) < 0 or int(value) > 255: raise ValueError("Value is out of range [0, 255]") self.regs[address] = None self.vscp.sendSimpleEvent( vscp_class=constant.VSCP_CLASS1_PROTOCOL, vscp_type=constant.VSCP_TYPE_PROTOCOL_WRITE_REGISTER, vscp_data=[self.id, address, value], ) try: timer = Timeout(timeout, self.__timeout) self.regs[address] = None while self.regs[address] == None: time.sleep(0.02) timer.cancel() return self.regs[address] == value except VSCPException: return False
def grow_cluster(self, context, cluster_id, new_instance_ids): def _grow_cluster(): LOG.debug("begin grow_cluster for Vertica cluster %s" % cluster_id) db_instances = DBInstance.find_all(cluster_id=cluster_id, deleted=False).all() instance_ids = [db_instance.id for db_instance in db_instances] # Wait for new cluster members to get to cluster-ready status. if not self._all_instances_ready(new_instance_ids, cluster_id): return new_insts = [Instance.load(context, instance_id) for instance_id in new_instance_ids] existing_instances = [Instance.load(context, instance_id) for instance_id in instance_ids if instance_id not in new_instance_ids] existing_guests = [self.get_guest(i) for i in existing_instances] new_guests = [self.get_guest(i) for i in new_insts] all_guests = new_guests + existing_guests authorized_users_without_password = ['root', 'dbadmin'] new_ips = [self.get_ip(instance) for instance in new_insts] for user in authorized_users_without_password: pub_key = [guest.get_public_keys(user) for guest in all_guests] for guest in all_guests: guest.authorize_public_keys(user, pub_key) for db_instance in db_instances: if db_instance['type'] == 'master': LOG.debug("Found 'master' instance, calling grow on guest") master_instance = Instance.load(context, db_instance.id) self.get_guest(master_instance).grow_cluster(new_ips) break for guest in new_guests: guest.cluster_complete() timeout = Timeout(CONF.cluster_usage_timeout) try: _grow_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for growing cluster.")) self.update_statuses_on_failure(cluster_id) except Exception: LOG.exception(_("Error growing cluster %s.") % cluster_id) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel()
def piper(self, in_sock, out_sock, out_addr, onkill): "Worker thread for data reading" try: timeout = Timeout(self.transmission_timeout_seconds) try: while True: written = in_sock.recv(32768) if not written: try: out_sock.shutdown(socket.SHUT_WR) except socket.error: self.threads[onkill].kill() break try: out_sock.sendall(written) except socket.error: pass self.data_handled += len(written) finally: timeout.cancel() except greenlet.GreenletExit: return except Timeout: # This one prevents only from closing connection without any data nor status code returned # from mantrid when no data was received from backend. # When it happens, nginx reports 'upstream prematurely closed connection' and returns 500, # and want to have our custom error page to know when it happens. if onkill == "stoc" and self.data_handled == 0: out_sock.sendall("HTTP/1.0 594 Backend timeout\r\nConnection: close\r\nContent-length: 0\r\n\r\n") logging.warn("Timeout serving request to backend %s of %s", self.backend, self.host) return
def shrink_cluster(self, context, cluster_id, instance_ids): LOG.debug("begin shrink_cluster for MongoDB cluster %s" % cluster_id) def _shrink_cluster(): def all_instances_marked_deleted(): non_deleted_instances = DBInstance.find_all( cluster_id=cluster_id, deleted=False).all() non_deleted_ids = [db_instance.id for db_instance in non_deleted_instances] return not bool( set(instance_ids).intersection(set(non_deleted_ids)) ) try: utils.poll_until(all_instances_marked_deleted, sleep_time=2, time_out=CONF.cluster_delete_time_out) except PollTimeOut: LOG.error(_("timeout for instances to be marked as deleted.")) return cluster_usage_timeout = CONF.cluster_usage_timeout timeout = Timeout(cluster_usage_timeout) try: _shrink_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("timeout for shrinking cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("end shrink_cluster for MongoDB cluster %s" % self.id)
def rengine_side(self, appid, token, uri): """ Handle rengine (client) GET requests """ if not self.rengine_authorization_ok(appid, token): LOGGER.info('Rengine content request authorization fails') abort(401, 'Authorization failed') evt = Event() request_id = str(uuid4()) self.request_id_events[request_id] = evt headers = ["%s: %s" % (header, val) for (header, val) in request.headers.items()] packet = ScpPacket.make_sfkcontent(uri, request_id, headers) try: self._send(packet, appid) except Exception as e: abort(500, str(e)) LOGGER.debug("uri %s expected" % uri) timeout = Timeout(TIMEOUT) try: resp = evt.wait() except Timeout: del self.request_id_events[request_id] abort(504, 'Gateway Timeout') finally: timeout.cancel() LOGGER.debug("uri %s got" % uri) return resp
def execute_with_timeout(*args, **kwargs): time = kwargs.pop('timeout', 30) log_output_on_error = kwargs.pop('log_output_on_error', False) timeout = Timeout(time) try: return execute(*args, **kwargs) except exception.ProcessExecutionError as e: if log_output_on_error: LOG.error( _("Command '%(cmd)s' failed. %(description)s " "Exit code: %(exit_code)s\nstderr: %(stderr)s\n" "stdout: %(stdout)s") % {'cmd': e.cmd, 'description': e.description or '', 'exit_code': e.exit_code, 'stderr': e.stderr, 'stdout': e.stdout}) raise except Timeout as t: if t is not timeout: LOG.error(_("Got a timeout but not the one expected.")) raise else: msg = (_("Time out after waiting " "%(time)s seconds when running proc: %(args)s" " %(kwargs)s.") % {'time': time, 'args': args, 'kwargs': kwargs}) LOG.error(msg) raise exception.ProcessExecutionError(msg) finally: timeout.cancel()
def create_cluster(self, context, cluster_id): LOG.debug("Begin create_cluster for id: %s." % cluster_id) def _create_cluster(): # Fetch instances by cluster_id against instances table. db_instances = DBInstance.find_all(cluster_id=cluster_id).all() instance_ids = [db_instance.id for db_instance in db_instances] # Wait for cluster members to get to cluster-ready status. if not self._all_instances_ready(instance_ids, cluster_id): return LOG.debug("All members ready, proceeding for cluster setup.") instances = [Instance.load(context, instance_id) for instance_id in instance_ids] # Connect nodes to the first node guests = [self.get_guest(instance) for instance in instances] try: cluster_head = instances[0] cluster_head_port = '6379' cluster_head_ip = self.get_ip(cluster_head) for guest in guests[1:]: guest.cluster_meet(cluster_head_ip, cluster_head_port) num_nodes = len(instances) total_slots = 16384 slots_per_node = total_slots / num_nodes leftover_slots = total_slots % num_nodes first_slot = 0 for guest in guests: last_slot = first_slot + slots_per_node if leftover_slots > 0: leftover_slots -= 1 else: last_slot -= 1 guest.cluster_addslots(first_slot, last_slot) first_slot = last_slot + 1 for guest in guests: guest.cluster_complete() except Exception: LOG.exception(_("Error creating cluster.")) self.update_statuses_on_failure(cluster_id) timeout = Timeout(CONF.cluster_usage_timeout) try: _create_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for building cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End create_cluster for id: %s." % cluster_id)
def request(self, address, path, method='GET', params=None, headers={}): t = Timeout(2, RuntimeError("Timeout trying to send request.")) try: conn = httplib.HTTPConnection("%s:%s" % address) conn.request(method, path, params, headers) finally: t.cancel() return conn.getresponse()
def create_cluster(self, context, cluster_id): LOG.debug("Begin create_cluster for id: %s." % cluster_id) def _create_cluster(): # Fetch instances by cluster_id against instances table. db_instances = DBInstance.find_all(cluster_id=cluster_id).all() instance_ids = [db_instance.id for db_instance in db_instances] # Wait for cluster members to get to cluster-ready status. if not self._all_instances_ready(instance_ids, cluster_id): return LOG.debug("All members ready, proceeding for cluster setup.") instances = [Instance.load(context, instance_id) for instance_id in instance_ids] member_ips = [self.get_ip(instance) for instance in instances] guests = [self.get_guest(instance) for instance in instances] # Users to be configured for password-less SSH. authorized_users_without_password = ['root', 'dbadmin'] # Configuring password-less SSH for cluster members. # Strategy for setting up SSH: # get public keys for user from member-instances in cluster, # combine them, finally push it back to all instances, # and member instances add them to authorized keys. LOG.debug("Configuring password-less SSH on cluster members.") try: for user in authorized_users_without_password: pub_key = [guest.get_public_keys(user) for guest in guests] for guest in guests: guest.authorize_public_keys(user, pub_key) LOG.debug("Installing cluster with members: %s." % member_ips) guests[0].install_cluster(member_ips) LOG.debug("Finalizing cluster configuration.") for guest in guests: guest.cluster_complete() except Exception: LOG.exception(_("Error creating cluster.")) self.update_statuses_on_failure(cluster_id) timeout = Timeout(CONF.cluster_usage_timeout) try: _create_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for building cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End create_cluster for id: %s." % cluster_id)
def wrapper(*args,**kwargs): timeout = Timeout(time, exception) try: func(*args,**kwargs) except exception as e: if reraise: raise e else: timeout.cancel()
def update_data(self): print "updating data" timeout = Timeout(10) try: self.parser = KMLParser() temp = self.parse_data() timeout.cancel() self.bikeways = temp self.update_database() except: print "couldn't get data"
def _write_with_timeout(self, writer, chunk): timeout = Timeout(self.timeout) try: writer.write(chunk) except Timeout as t: if t is timeout: writer.close() raise t except Exception as e: raise e finally: timeout.cancel()
def add_shard_cluster(self, context, cluster_id, shard_id, replica_set_name): LOG.debug("begin add_shard_cluster for cluster %s shard %s" % (cluster_id, shard_id)) def _add_shard_cluster(): db_instances = DBInstance.find_all(cluster_id=cluster_id, shard_id=shard_id).all() instance_ids = [db_instance.id for db_instance in db_instances] LOG.debug("instances in shard %s: %s" % (shard_id, instance_ids)) if not self._all_instances_ready(instance_ids, cluster_id, shard_id): return members = [Instance.load(context, instance_id) for instance_id in instance_ids] if not self._create_replica_set(members, cluster_id, shard_id): return db_query_routers = DBInstance.find_all(cluster_id=cluster_id, type='query_router', deleted=False).all() query_routers = [Instance.load(context, db_query_router.id) for db_query_router in db_query_routers] if not self._create_shard(query_routers, replica_set_name, members, cluster_id, shard_id): return for member in members: self.get_guest(member).cluster_complete() cluster_usage_timeout = CONF.cluster_usage_timeout timeout = Timeout(cluster_usage_timeout) try: _add_shard_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("timeout for building shard.")) self.update_statuses_on_failure(cluster_id, shard_id) finally: timeout.cancel() LOG.debug("end add_shard_cluster for cluster %s shard %s" % (cluster_id, shard_id))
def put(self, item, block=True, timeout=None): """Put an item into the queue. If optional arg *block* is true and *timeout* is ``None`` (the default), block if necessary until a free slot is available. If *timeout* is a positive number, it blocks at most *timeout* seconds and raises the :class:`Full` exception if no free slot was available within that time. Otherwise (*block* is false), put an item on the queue if a free slot is immediately available, else raise the :class:`Full` exception (*timeout* is ignored in that case). """ if self.maxsize is None or self.qsize() < self.maxsize: # there's a free slot, put an item right away self._put(item) if self.getters: self._schedule_unlock() elif not block and get_hub().greenlet is getcurrent(): # we're in the mainloop, so we cannot wait; we can switch() to other greenlets though # find a getter and deliver an item to it while self.getters: getter = self.getters.pop() if getter: self._put(item) item = self._get() getter.switch(item) return raise Full elif block: waiter = ItemWaiter(item, block) self.putters.add(waiter) timeout = Timeout(timeout, Full) try: if self.getters: self._schedule_unlock() result = waiter.wait() assert result is waiter, "Invalid switch into Queue.put: %r" % (result, ) if waiter.item is not _NONE: self._put(item) finally: timeout.cancel() self.putters.discard(waiter) elif self.getters: waiter = ItemWaiter(item, block) self.putters.add(waiter) self._schedule_unlock() result = waiter.wait() assert result is waiter, "Invalid switch into Queue.put: %r" % (result, ) if waiter.item is not _NONE: raise Full else: raise Full
def recv_events(self, timeout_msecs): events = [] timeout = Timeout(timeout_msecs / 1000.0) try: while True: event = self._event_q.get() events.append(event) except Timeout: pass except Exception as e: raise e finally: timeout.cancel() return events
def _check_health(self): logging.debug("Checking health of %s", self) try: timeout = Timeout(self.healthcheck_timeout_seconds) try: socket = eventlet.connect((self.host, self.port)) finally: timeout.cancel() logging.debug("%s is alive, making sure it is not blacklisted", self) self.blacklisted = False socket.close() except: logging.debug("%s seems dead, will check again later", self)
def create_cluster(self, context, cluster_id): LOG.debug("Begin create_cluster for id: %s." % cluster_id) def _create_cluster(): cluster_node_ids = self.find_cluster_node_ids(cluster_id) # Wait for cluster nodes to get to cluster-ready status. LOG.debug("Waiting for all nodes to become ready.") if not self._all_instances_ready(cluster_node_ids, cluster_id): return cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids) coordinator = self._get_coordinator_node(cluster_nodes) LOG.debug("Initializing the cluster on node '%s'." % coordinator['ip']) # start with the coordinator as it will have all the required # services. guest_node_info = self.build_guest_node_info([coordinator]) # now add all the other nodes so we can get a list of all services # needed to calculate the memory allocation properly. add_node_info = [node for node in cluster_nodes if node != coordinator] guest_node_info.extend(self.build_guest_node_info(add_node_info)) coordinator['guest'].initialize_cluster(guest_node_info) self._add_nodes(coordinator, add_node_info) coordinator['guest'].cluster_complete() LOG.debug("Cluster create finished successfully.") timeout = Timeout(CONF.cluster_usage_timeout) try: with EndNotification(context, cluster_id=cluster_id): _create_cluster() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for building cluster.")) self.update_statuses_on_failure(cluster_id) except Exception: LOG.exception(_("Error creating cluster.")) self.update_statuses_on_failure(cluster_id) raise finally: self.reset_task() timeout.cancel() LOG.debug("End create_cluster for id: %s." % cluster_id)
def shrink_cluster(self, context, cluster_id, instance_ids): def _shrink_cluster(): db_instances = DBInstance.find_all(cluster_id=cluster_id, deleted=False).all() all_instance_ids = [db_instance.id for db_instance in db_instances] remove_instances = [Instance.load(context, instance_id) for instance_id in instance_ids] left_instances = [Instance.load(context, instance_id) for instance_id in all_instance_ids if instance_id not in instance_ids] remove_member_ips = [self.get_ip(instance) for instance in remove_instances] k = VerticaCluster.k_safety(len(left_instances)) for db_instance in db_instances: if db_instance['type'] == 'master': master_instance = Instance.load(context, db_instance.id) if self.get_ip(master_instance) in remove_member_ips: raise RuntimeError(_("Cannot remove master instance!")) LOG.debug(_("Marking cluster k-safety: %s") % k) self.get_guest(master_instance).mark_design_ksafe(k) self.get_guest(master_instance).shrink_cluster( remove_member_ips) break for r in remove_instances: Instance.delete(r) timeout = Timeout(CONF.cluster_usage_timeout) try: _shrink_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise LOG.exception(_("Timeout for shrinking cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("end shrink_cluster for Vertica cluster id %s" % self.id)
def grow_cluster(self, context, cluster_id, new_instance_ids): LOG.debug("Begin grow_cluster for id: %s." % cluster_id) def _grow_cluster(): db_instances = DBInstance.find_all(cluster_id=cluster_id).all() cluster_head = next(Instance.load(context, db_inst.id) for db_inst in db_instances if db_inst.id not in new_instance_ids) if not cluster_head: raise TroveError("Unable to determine existing Redis cluster " "member") (cluster_head_ip, cluster_head_port) = ( self.get_guest(cluster_head).get_node_ip()) # Wait for cluster members to get to cluster-ready status. if not self._all_instances_ready(new_instance_ids, cluster_id): return LOG.debug("All members ready, proceeding for cluster setup.") new_insts = [Instance.load(context, instance_id) for instance_id in new_instance_ids] new_guests = map(self.get_guest, new_insts) # Connect nodes to the cluster head for guest in new_guests: guest.cluster_meet(cluster_head_ip, cluster_head_port) for guest in new_guests: guest.cluster_complete() timeout = Timeout(CONF.cluster_usage_timeout) try: _grow_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for growing cluster.")) self.update_statuses_on_failure(cluster_id) except Exception: LOG.exception(_("Error growing cluster %s.") % cluster_id) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End grow_cluster for id: %s." % cluster_id)
def grow_cluster(self, context, cluster_id, new_instance_ids): LOG.debug("Begin grow_cluster for id: %s." % cluster_id) def _grow_cluster(): # Wait for new nodes to get to cluster-ready status. LOG.debug("Waiting for new nodes to become ready.") if not self._all_instances_ready(new_instance_ids, cluster_id): return new_instances = [Instance.load(context, instance_id) for instance_id in new_instance_ids] added_nodes = [self.build_node_info(instance) for instance in new_instances] LOG.debug("All nodes ready, proceeding with cluster setup.") cluster_node_ids = self.find_cluster_node_ids(cluster_id) cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids) # Rebalance the cluster via one of the existing nodes. # Clients can continue to store and retrieve information and # do not need to be aware that a rebalance operation is taking # place. # The new nodes are marked active only if the rebalancing # completes. try: coordinator = cluster_nodes[0] self._add_nodes(coordinator, added_nodes) LOG.debug("Cluster configuration finished successfully.") except Exception: LOG.exception(_("Error growing cluster.")) self.update_statuses_on_failure(cluster_id) timeout = Timeout(CONF.cluster_usage_timeout) try: _grow_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for growing cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End grow_cluster for id: %s." % cluster_id)
def downloadDM(self, timeout=2, cached=False): self.DM = None self.vscp.sendSimpleEvent( vscp_class=constant.VSCP_CLASS1_PROTOCOL, vscp_type=constant.VSCP_TYPE_PROTOCOL_GET_MATRIX_INFO, vscp_data=[self.nodeId], ) timer = Timeout(timeout, self.__timeout) try: while self.DM == None: time.sleep(0.02) timer.cancel() for i in range(self.DM["size"]): self.DM["rows"][i] = self.getRowOfDM(i) except: return constant.VSCP_ERROR_ERROR
def grow_cluster(self, context, cluster_id, new_instance_ids): LOG.debug("Begin grow_cluster for id: %s." % cluster_id) def _grow_cluster(): # Wait for new nodes to get to cluster-ready status. LOG.debug("Waiting for new nodes to become ready.") if not self._all_instances_ready(new_instance_ids, cluster_id): return new_instances = [Instance.load(context, instance_id) for instance_id in new_instance_ids] add_node_info = [self.build_node_info(instance) for instance in new_instances] LOG.debug("All nodes ready, proceeding with cluster setup.") cluster_node_ids = self.find_cluster_node_ids(cluster_id) cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids) old_node_info = [node for node in cluster_nodes if node['id'] not in new_instance_ids] # Rebalance the cluster via one of the existing nodes. # Clients can continue to store and retrieve information and # do not need to be aware that a rebalance operation is taking # place. coordinator = old_node_info[0] self._add_nodes(coordinator, add_node_info) LOG.debug("Cluster grow finished successfully.") timeout = Timeout(CONF.cluster_usage_timeout) try: with EndNotification(context, cluster_id=cluster_id): _grow_cluster() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for growing cluster.")) except Exception: LOG.exception(_("Error growing cluster.")) raise finally: self.reset_task() timeout.cancel() LOG.debug("End grow_cluster for id: %s." % cluster_id)
def recv_events(self, timeout_msecs): """ Let explorer receive events :param timeout_msecs: int :return: """ events = [] timeout = Timeout(timeout_msecs / 1000.0) try: while True: event = self._event_q.get() events.append(event) except Timeout: pass except Exception as e: raise e finally: timeout.cancel() return events
def get(self, block=True, timeout=None): """Remove and return an item from the queue. If optional args *block* is true and *timeout* is ``None`` (the default), block if necessary until an item is available. If *timeout* is a positive number, it blocks at most *timeout* seconds and raises the :class:`Empty` exception if no item was available within that time. Otherwise (*block* is false), return an item if one is immediately available, else raise the :class:`Empty` exception (*timeout* is ignored in that case). """ if self.qsize(): if self.putters: self._schedule_unlock() return self._get() elif not block and get_hub().greenlet is getcurrent(): # special case to make get_nowait() runnable in the mainloop greenlet # there are no items in the queue; try to fix the situation by unlocking putters while self.putters: putter = self.putters.pop() if putter: putter.switch(putter) if self.qsize(): return self._get() raise Empty elif block: waiter = Waiter() timeout = Timeout(timeout, Empty) try: self.getters.add(waiter) if self.putters: self._schedule_unlock() try: return waiter.wait() except: self._schedule_unlock() raise finally: self.getters.discard(waiter) timeout.cancel() else: raise Empty
def wait(self): do_wait = True up = 0 to = Timeout(self.sandbox_wait_timeout) try: while do_wait == True: rc = self.ping() if (rc != 1): time.sleep(self.sandbox_ping_interval) continue else: to.cancel() do_wait = False up = 1 except Timeout as t: self.logger.info("wait for sandbox %s timedout" % self.account) do_wait = False finally: to.cancel() return up
def execute_with_timeout(*args, **kwargs): time = kwargs.get('timeout', 30) def cb_timeout(): msg = _("Time out after waiting" " %(time)s seconds when running proc: %(args)s" " %(kwargs)s") % locals() raise exception.ProcessExecutionError(msg) timeout = Timeout(time) try: return execute(*args, **kwargs) except Timeout as t: if t is not timeout: raise else: msg = _("Time out after waiting " "%(time)s seconds when running proc: %(args)s" " %(kwargs)s") % locals() raise exception.ProcessExecutionError(msg) finally: timeout.cancel()
def handle(self, sock, read_data, path, headers): request_id = headers.get("X-Request-Id", "-") for attempt in range(self.attempts): if attempt > 0: logging.warn("[%s] Retrying connection for host %s", request_id, self.host) backend = self.select_backend() try: timeout = Timeout(self.connection_timeout_seconds) try: server_sock = eventlet.connect((backend.host, backend.port)) finally: timeout.cancel() backend.add_connection() break except socket.error: logging.error("[%s] Proxy socket error on connect() to %s of %s", request_id, backend, self.host) self.blacklist(backend) eventlet.sleep(self.delay) continue except: logging.warn("[%s] Proxy timeout on connect() to %s of %s", request_id, backend, self.host) self.blacklist(backend) eventlet.sleep(self.delay) continue # Function to help track data usage def send_onwards(data): server_sock.sendall(data) return len(data) try: size = send_onwards(read_data) size += SocketMelder(sock, server_sock, backend, self.host).run() except socket.error, e: if e.errno != errno.EPIPE: raise
def execute_with_timeout(*args, **kwargs): time = kwargs.pop("timeout", 30) log_output_on_error = kwargs.pop("log_output_on_error", False) timeout = Timeout(time) try: return execute(*args, **kwargs) except exception.ProcessExecutionError as e: if log_output_on_error: LOG.error( _( "Command '%(cmd)s' failed. %(description)s " "Exit code: %(exit_code)s\nstderr: %(stderr)s\n" "stdout: %(stdout)s" ) % { "cmd": e.cmd, "description": e.description or "", "exit_code": e.exit_code, "stderr": e.stderr, "stdout": e.stdout, } ) raise except Timeout as t: if t is not timeout: LOG.error(_("Got a timeout but not the one expected.")) raise else: msg = _("Time out after waiting " "%(time)s seconds when running proc: %(args)s" " %(kwargs)s.") % { "time": time, "args": args, "kwargs": kwargs, } LOG.error(msg) raise exception.ProcessExecutionError(msg) finally: timeout.cancel()
def execute_task(task, headers, args): """ Executes a task to a url with the given args. Args: task: A celery Task instance. headers: A dictionary of headers for the task. args: A dictionary of arguments for the request. Contains the task body. Returns: The status code of the task fetch upon success. Raises: The current function to retry. """ start_time = datetime.datetime.utcnow() content_length = len(args['body']) loggable_args = { key: args[key] for key in args if key not in ['task_name', 'body', 'payload'] } loggable_args['body_length'] = content_length logger.info('Running {}\n' 'Headers: {}\n' 'Args: {}'.format(args['task_name'], headers, loggable_args)) url = urlparse(args['url']) timeout = EventletTimeout(MAX_TASK_DURATION) try: redirects_left = 1 while True: urlpath = url.path if url.query: urlpath += "?" + url.query method = args['method'] if args['expires'] <= datetime.datetime.now(): # We do this check because the expires attribute in # celery is not passed to retried tasks. This is a # documented bug in celery. logger.error( "Task %s with id %s has expired with expiration date %s" % (args['task_name'], task.request.id, args['expires'])) celery.control.revoke(task.request.id) item = TaskName.get_by_key_name(args['task_name']) if item is not None: item.state = TASK_STATES.EXPIRED item.endtime = datetime.datetime.now() db.put(item) return if (args['max_retries'] != 0 and task.request.retries >= args['max_retries']): logger.error( "Task %s with id %s has exceeded retries: %s" % (args['task_name'], task.request.id, args['max_retries'])) celery.control.revoke(task.request.id) item = TaskName.get_by_key_name(args['task_name']) if item is not None: item.state = TASK_STATES.FAILED item.endtime = datetime.datetime.now() db.put(item) return # Targets do not get X-Forwarded-Proto from nginx, they use haproxy port. headers['X-Forwarded-Proto'] = url.scheme if url.scheme == 'http': connection = httplib.HTTPConnection(remote_host, url.port) elif url.scheme == 'https': connection = httplib.HTTPSConnection(remote_host, url.port) else: logger.error("Task %s tried to use url scheme %s, " "which is not supported." % (args['task_name'], url.scheme)) skip_host = False if 'host' in headers or 'Host' in headers: skip_host = True skip_accept_encoding = False if 'accept-encoding' in headers or 'Accept-Encoding' in headers: skip_accept_encoding = True connection.putrequest(method, urlpath, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding) # Update the task headers headers['X-AppEngine-TaskRetryCount'] = str(task.request.retries) headers['X-AppEngine-TaskExecutionCount'] = str( task.request.retries) for header in headers: connection.putheader(header, headers[header]) if 'content-type' not in headers or 'Content-Type' not in headers: if url.query: connection.putheader('content-type', 'application/octet-stream') else: connection.putheader('content-type', 'application/x-www-form-urlencoded') connection.putheader("Content-Length", str(content_length)) retries = int(task.request.retries) + 1 wait_time = get_wait_time(retries, args) try: connection.endheaders() if args["body"]: connection.send(args['body']) response = connection.getresponse() response.read() response.close() except (BadStatusLine, SocketError): logger.warning( '{task} failed before receiving response. It will retry in {wait} ' 'seconds.'.format(task=args['task_name'], wait=wait_time)) raise task.retry(countdown=wait_time) if 200 <= response.status < 300: # Task successful. item = TaskName.get_by_key_name(args['task_name']) if item is not None: item.state = TASK_STATES.SUCCESS item.endtime = datetime.datetime.now() db.put(item) time_elapsed = datetime.datetime.utcnow() - start_time logger.info( '{task} received status {status} from {url} [time elapsed: {te}]'. \ format(task=args['task_name'], status=response.status, url=url, te=str(time_elapsed))) return response.status elif response.status == 302: redirect_url = response.getheader('Location') logger.info( "Task %s asked us to redirect to %s, so retrying there." % (args['task_name'], redirect_url)) url = urlparse(redirect_url) if redirects_left == 0: raise task.retry(countdown=wait_time) redirects_left -= 1 else: message = ('Received a {status} for {task}. ' 'Retrying in {wait} secs.'.format( status=response.status, task=args['task_name'], wait=wait_time)) logger.warning(message) raise task.retry(countdown=wait_time) except EventletTimeout as thrown_timeout: if thrown_timeout != timeout: raise logger.exception('Task {} timed out. Retrying.'.format( args['task_name'])) # This could probably be calculated, but for now, just retry immediately. raise task.retry(countdown=0) finally: timeout.cancel()
def grow_cluster(self, context, cluster_id, new_instance_ids): LOG.debug("Begin Galera grow_cluster for id: %s.", cluster_id) def _grow_cluster(): db_instances = DBInstance.find_all(cluster_id=cluster_id, deleted=False).all() existing_instances = [ Instance.load(context, db_inst.id) for db_inst in db_instances if db_inst.id not in new_instance_ids ] if not existing_instances: raise TroveError( _("Unable to determine existing cluster " "member(s)")) # get list of ips of existing cluster members existing_cluster_ips = [ self.get_ip(instance) for instance in existing_instances ] existing_instance_guests = [ self.get_guest(instance) for instance in existing_instances ] # get the cluster context to setup new members cluster_context = existing_instance_guests[0].get_cluster_context() # Wait for cluster members to get to cluster-ready status. if not self._all_instances_ready(new_instance_ids, cluster_id): raise TroveError( _("Instances in cluster did not report " "ACTIVE")) LOG.debug("All members ready, proceeding for cluster setup.") # Get the new instances to join the cluster new_instances = [ Instance.load(context, instance_id) for instance_id in new_instance_ids ] new_cluster_ips = [ self.get_ip(instance) for instance in new_instances ] for instance in new_instances: guest = self.get_guest(instance) guest.reset_admin_password(cluster_context['admin_password']) # render the conf.d/cluster.cnf configuration cluster_configuration = self._render_cluster_config( context, instance, ",".join(existing_cluster_ips), cluster_context['cluster_name'], cluster_context['replication_user']) # push the cluster config and bootstrap the first instance bootstrap = False guest.install_cluster(cluster_context['replication_user'], cluster_configuration, bootstrap) self._check_cluster_for_root(context, existing_instances, new_instances) # apply the new config to all instances for instance in existing_instances + new_instances: guest = self.get_guest(instance) # render the conf.d/cluster.cnf configuration cluster_configuration = self._render_cluster_config( context, instance, ",".join(existing_cluster_ips + new_cluster_ips), cluster_context['cluster_name'], cluster_context['replication_user']) guest.write_cluster_configuration_overrides( cluster_configuration) for instance in new_instances: guest = self.get_guest(instance) guest.cluster_complete() timeout = Timeout(CONF.cluster_usage_timeout) try: _grow_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for growing cluster.")) self.update_statuses_on_failure( cluster_id, status=inst_tasks.InstanceTasks.GROWING_ERROR) except Exception: LOG.exception(_("Error growing cluster %s."), cluster_id) self.update_statuses_on_failure( cluster_id, status=inst_tasks.InstanceTasks.GROWING_ERROR) finally: timeout.cancel() LOG.debug("End grow_cluster for id: %s.", cluster_id)
def create_cluster(self, context, cluster_id): LOG.debug("Begin create_cluster for id: %s." % cluster_id) def _create_cluster(): # Fetch instances by cluster_id against instances table. db_instances = DBInstance.find_all(cluster_id=cluster_id).all() instance_ids = [db_instance.id for db_instance in db_instances] # Wait for cluster members to get to cluster-ready status. if not self._all_instances_ready(instance_ids, cluster_id): return LOG.debug("All members ready, proceeding for cluster setup.") instances = [ Instance.load(context, instance_id) for instance_id in instance_ids ] member_ips = [self.get_ip(instance) for instance in instances] guests = [self.get_guest(instance) for instance in instances] # Users to be configured for password-less SSH. authorized_users_without_password = ['root', 'dbadmin'] # Configuring password-less SSH for cluster members. # Strategy for setting up SSH: # get public keys for user from member-instances in cluster, # combine them, finally push it back to all instances, # and member instances add them to authorized keys. LOG.debug("Configuring password-less SSH on cluster members.") try: for user in authorized_users_without_password: pub_key = [guest.get_public_keys(user) for guest in guests] for guest in guests: guest.authorize_public_keys(user, pub_key) LOG.debug("Installing cluster with members: %s." % member_ips) for db_instance in db_instances: if db_instance['type'] == 'master': master_instance = Instance.load( context, db_instance.id) self.get_guest(master_instance).install_cluster( member_ips) break LOG.debug("Finalizing cluster configuration.") for guest in guests: guest.cluster_complete() except Exception: LOG.exception(_("Error creating cluster.")) self.update_statuses_on_failure(cluster_id) timeout = Timeout(CONF.cluster_usage_timeout) try: _create_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for building cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End create_cluster for id: %s." % cluster_id)
def shrink_cluster(self, context, cluster_id, removal_instance_ids): LOG.debug("Begin Galera shrink_cluster for id: %s.", cluster_id) def _shrink_cluster(): removal_instances = [ Instance.load(context, instance_id) for instance_id in removal_instance_ids ] for instance in removal_instances: Instance.delete(instance) # wait for instances to be deleted def all_instances_marked_deleted(): non_deleted_instances = DBInstance.find_all( cluster_id=cluster_id, deleted=False).all() non_deleted_ids = [ db_instance.id for db_instance in non_deleted_instances ] return not bool( set(removal_instance_ids).intersection( set(non_deleted_ids))) try: LOG.info(_("Deleting instances (%s)"), removal_instance_ids) utils.poll_until(all_instances_marked_deleted, sleep_time=2, time_out=CONF.cluster_delete_time_out) except PollTimeOut: LOG.error(_("timeout for instances to be marked as deleted.")) return db_instances = DBInstance.find_all(cluster_id=cluster_id).all() leftover_instances = [ Instance.load(context, db_inst.id) for db_inst in db_instances if db_inst.id not in removal_instance_ids ] leftover_cluster_ips = [ self.get_ip(instance) for instance in leftover_instances ] # Get config changes for left over instances rnd_cluster_guest = self.get_guest(leftover_instances[0]) cluster_context = rnd_cluster_guest.get_cluster_context() # apply the new config to all leftover instances for instance in leftover_instances: guest = self.get_guest(instance) # render the conf.d/cluster.cnf configuration cluster_configuration = self._render_cluster_config( context, instance, ",".join(leftover_cluster_ips), cluster_context['cluster_name'], cluster_context['replication_user']) guest.write_cluster_configuration_overrides( cluster_configuration) timeout = Timeout(CONF.cluster_usage_timeout) try: _shrink_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for shrinking cluster.")) self.update_statuses_on_failure( cluster_id, status=inst_tasks.InstanceTasks.SHRINKING_ERROR) except Exception: LOG.exception(_("Error shrinking cluster %s."), cluster_id) self.update_statuses_on_failure( cluster_id, status=inst_tasks.InstanceTasks.SHRINKING_ERROR) finally: timeout.cancel() LOG.debug("End shrink_cluster for id: %s.", cluster_id)
def grow_cluster(self, context, cluster_id, new_instance_ids): def _grow_cluster(): LOG.debug("begin grow_cluster for Vertica cluster %s" % cluster_id) db_instances = DBInstance.find_all(cluster_id=cluster_id, deleted=False).all() instance_ids = [db_instance.id for db_instance in db_instances] # Wait for new cluster members to get to cluster-ready status. if not self._all_instances_ready(new_instance_ids, cluster_id): return new_insts = [ Instance.load(context, instance_id) for instance_id in new_instance_ids ] existing_instances = [ Instance.load(context, instance_id) for instance_id in instance_ids if instance_id not in new_instance_ids ] existing_guests = [self.get_guest(i) for i in existing_instances] new_guests = [self.get_guest(i) for i in new_insts] all_guests = new_guests + existing_guests authorized_users_without_password = ['root', 'dbadmin'] new_ips = [self.get_ip(instance) for instance in new_insts] for user in authorized_users_without_password: pub_key = [guest.get_public_keys(user) for guest in all_guests] for guest in all_guests: guest.authorize_public_keys(user, pub_key) for db_instance in db_instances: if db_instance['type'] == 'master': LOG.debug("Found 'master' instance, calling grow on guest") master_instance = Instance.load(context, db_instance.id) self.get_guest(master_instance).grow_cluster(new_ips) break for guest in new_guests: guest.cluster_complete() timeout = Timeout(CONF.cluster_usage_timeout) try: _grow_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for growing cluster.")) self.update_statuses_on_failure(cluster_id) except Exception: LOG.exception(_("Error growing cluster %s.") % cluster_id) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel()
def shrink_cluster(self, context, cluster_id, removal_ids): LOG.debug("Begin shrink_cluster for id: %s.", cluster_id) def _shrink_cluster(): cluster_node_ids = self.find_cluster_node_ids(cluster_id) cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids) removed_nodes = CassandraClusterTasks.load_cluster_nodes( context, removal_ids) LOG.debug("All nodes ready, proceeding with cluster setup.") # Update the list of seeds on remaining nodes if necessary. # Once all nodes are configured, decommission the removed nodes. # Cassandra will stream data from decommissioned nodes to the # remaining ones. try: # All nodes should have the same seeds. # We retrieve current seeds from the first node. test_node = self.load_cluster_nodes(context, cluster_node_ids[:1])[0] current_seeds = test_node['guest'].get_seeds() # The seeds will have to be updated on all remaining instances # if any of the seed nodes is going to be removed. update_seeds = any(node['ip'] in current_seeds for node in removed_nodes) LOG.debug("Decommissioning removed nodes.") for node in removed_nodes: node['guest'].node_decommission() node['instance'].update_db(cluster_id=None) # Recompute the seed nodes based on the updated cluster # geometry if any of the existing seed nodes was removed. if update_seeds: LOG.debug("Updating seeds on the remaining nodes.") cluster_nodes = self.load_cluster_nodes( context, cluster_node_ids) remaining_nodes = [ node for node in cluster_nodes if node['id'] not in removal_ids ] seeds = self.choose_seed_nodes(remaining_nodes) LOG.debug("Selected seed nodes: %s", seeds) for node in remaining_nodes: LOG.debug("Configuring node: %s.", node['id']) node['guest'].set_seeds(seeds) # Wait for the removed nodes to go SHUTDOWN. LOG.debug("Waiting for all decommissioned nodes to shutdown.") if not self._all_instances_shutdown(removal_ids, cluster_id): # Now detached, failed nodes will stay available # in the list of standalone instances. return # Delete decommissioned instances only when the cluster is in a # consistent state. LOG.debug("Deleting decommissioned instances.") for node in removed_nodes: Instance.delete(node['instance']) LOG.debug("Cluster configuration finished successfully.") except Exception: LOG.exception(_("Error shrinking cluster.")) self.update_statuses_on_failure(cluster_id) timeout = Timeout(CONF.cluster_usage_timeout) try: _shrink_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for shrinking cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End shrink_cluster for id: %s.", cluster_id)
def create_cluster(self, context, cluster_id): LOG.debug("Begin create_cluster for id: %s." % cluster_id) def _create_cluster(): # Fetch instances by cluster_id against instances table. db_instances = DBInstance.find_all(cluster_id=cluster_id).all() instance_ids = [db_instance.id for db_instance in db_instances] # Wait for cluster members to get to cluster-ready status. if not self._all_instances_ready(instance_ids, cluster_id): return LOG.debug("All members ready, proceeding for cluster setup.") instances = [ Instance.load(context, instance_id) for instance_id in instance_ids ] # Connect nodes to the first node guests = [self.get_guest(instance) for instance in instances] try: cluster_head = instances[0] cluster_head_port = '6379' cluster_head_ip = self.get_ip(cluster_head) for guest in guests[1:]: guest.cluster_meet(cluster_head_ip, cluster_head_port) num_nodes = len(instances) total_slots = 16384 slots_per_node = total_slots / num_nodes leftover_slots = total_slots % num_nodes first_slot = 0 for guest in guests: last_slot = first_slot + slots_per_node if leftover_slots > 0: leftover_slots -= 1 else: last_slot -= 1 guest.cluster_addslots(first_slot, last_slot) first_slot = last_slot + 1 for guest in guests: guest.cluster_complete() except Exception: LOG.exception(_("Error creating cluster.")) self.update_statuses_on_failure(cluster_id) timeout = Timeout(CONF.cluster_usage_timeout) try: _create_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for building cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End create_cluster for id: %s." % cluster_id)
def grow_cluster(self, context, cluster_id, new_instance_ids): LOG.debug("Begin grow_cluster for id: %s." % cluster_id) def _grow_cluster(): # Wait for new nodes to get to cluster-ready status. LOG.debug("Waiting for new nodes to become ready.") if not self._all_instances_ready(new_instance_ids, cluster_id): return new_instances = [Instance.load(context, instance_id) for instance_id in new_instance_ids] added_nodes = [self.build_node_info(instance) for instance in new_instances] LOG.debug("All nodes ready, proceeding with cluster setup.") cluster_node_ids = self.find_cluster_node_ids(cluster_id) cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids) # Recompute the seed nodes based on the updated cluster geometry. seeds = self.choose_seed_nodes(cluster_nodes) # Configure each cluster node with the updated list of seeds. # Since we are adding to an existing cluster, ensure that the # new nodes have auto-bootstrapping enabled. # Start the added nodes. try: LOG.debug("Selected seed nodes: %s" % seeds) # Update the seeds on all nodes. # Also retrieve the superuser password from one previously # existing node. admin_creds = None for node in cluster_nodes: LOG.debug("Configuring node: %s." % node['id']) node['guest'].set_seeds(seeds) if (admin_creds is None) and (node not in added_nodes): admin_creds = node['guest'].get_admin_credentials() # Start any seeds from the added nodes first. LOG.debug("Starting new seed nodes.") for node in added_nodes: if node['ip'] in seeds: node['guest'].set_auto_bootstrap(True) node['guest'].store_admin_credentials(admin_creds) node['guest'].restart() node['guest'].cluster_complete() LOG.debug("All new seeds running, starting the remaining of " "added nodes.") for node in added_nodes: if node['ip'] not in seeds: node['guest'].set_auto_bootstrap(True) node['guest'].store_admin_credentials(admin_creds) node['guest'].restart() node['guest'].cluster_complete() # Run nodetool cleanup on each of the previously existing nodes # to remove the keys that no longer belong to those nodes. # Wait for cleanup to complete on one node before running # it on the next node. LOG.debug("Cleaning up orphan data on old cluster nodes.") for node in cluster_nodes: if node not in added_nodes: nid = node['id'] node['guest'].node_cleanup_begin() node['guest'].node_cleanup() LOG.debug("Waiting for node to finish its " "cleanup: %s" % nid) if not self._all_instances_running([nid], cluster_id): LOG.warning(_("Node did not complete cleanup " "successfully: %s") % nid) LOG.debug("Cluster configuration finished successfully.") except Exception: LOG.exception(_("Error growing cluster.")) self.update_statuses_on_failure(cluster_id) timeout = Timeout(CONF.cluster_usage_timeout) try: _grow_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for growing cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End grow_cluster for id: %s." % cluster_id)
def grow_cluster(self, context, cluster_id, instance_ids): LOG.debug("begin grow_cluster for MongoDB cluster %s", cluster_id) def _grow_cluster(): new_instances = [db_instance for db_instance in self.db_instances if db_instance.id in instance_ids] new_members = [db_instance for db_instance in new_instances if db_instance.type == 'member'] new_query_routers = [db_instance for db_instance in new_instances if db_instance.type == 'query_router'] instances = [] if new_members: shard_ids = set([db_instance.shard_id for db_instance in new_members]) query_router_id = self._get_running_query_router_id() if not query_router_id: return for shard_id in shard_ids: LOG.debug('growing cluster by adding shard %(shard_id)s ' 'on query router %(router_id)s', {'shard_id': shard_id, 'router_id': query_router_id}) member_ids = [db_instance.id for db_instance in new_members if db_instance.shard_id == shard_id] if not self._all_instances_ready( member_ids, cluster_id, shard_id ): return members = [Instance.load(context, member_id) for member_id in member_ids] query_router = Instance.load(context, query_router_id) if not self._create_shard(query_router, members): return instances.extend(members) if new_query_routers: query_router_ids = [db_instance.id for db_instance in new_query_routers] config_servers_ids = [db_instance.id for db_instance in self.db_instances if db_instance.type == 'config_server'] LOG.debug('growing cluster by adding query routers ' '%(router)s, with config servers %(server)s', {'router': query_router_ids, 'server': config_servers_ids}) if not self._all_instances_ready( query_router_ids, cluster_id ): return query_routers = [Instance.load(context, instance_id) for instance_id in query_router_ids] config_servers_ips = [ self.get_ip(Instance.load(context, config_server_id)) for config_server_id in config_servers_ids ] if not self._add_query_routers( query_routers, config_servers_ips, admin_password=self.get_cluster_admin_password(context) ): return instances.extend(query_routers) for instance in instances: self.get_guest(instance).cluster_complete() cluster_usage_timeout = CONF.cluster_usage_timeout timeout = Timeout(cluster_usage_timeout) try: _grow_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("timeout for growing cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("end grow_cluster for MongoDB cluster %s", self.id)
def create_cluster(self, context, cluster_id): LOG.debug("Begin create_cluster for id: %s.", cluster_id) def _create_cluster(): # Fetch instances by cluster_id against instances table. db_instances = DBInstance.find_all(cluster_id=cluster_id).all() instance_ids = [db_instance.id for db_instance in db_instances] LOG.debug("Waiting for instances to get to cluster-ready status.") # Wait for cluster members to get to cluster-ready status. if not self._all_instances_ready(instance_ids, cluster_id): raise TroveError( _("Instances in cluster did not report " "ACTIVE")) LOG.debug("All members ready, proceeding for cluster setup.") instances = [ Instance.load(context, instance_id) for instance_id in instance_ids ] cluster_ips = [self.get_ip(instance) for instance in instances] instance_guests = [ self.get_guest(instance) for instance in instances ] # Create replication user and password for synchronizing the # galera cluster replication_user = { "name": self.CLUSTER_REPLICATION_USER, "password": utils.generate_random_password(), } # Galera cluster name must be unique and be shorter than a full # uuid string so we remove the hyphens and chop it off. It was # recommended to be 16 chars or less. # (this is not currently documented on Galera docs) cluster_name = utils.generate_uuid().replace("-", "")[:16] LOG.debug("Configuring cluster configuration.") try: # Set the admin password for all the instances because the # password in the my.cnf will be wrong after the joiner # instances syncs with the donor instance. admin_password = str(utils.generate_random_password()) for guest in instance_guests: guest.reset_admin_password(admin_password) bootstrap = True for instance in instances: guest = self.get_guest(instance) # render the conf.d/cluster.cnf configuration cluster_configuration = self._render_cluster_config( context, instance, ",".join(cluster_ips), cluster_name, replication_user) # push the cluster config and bootstrap the first instance guest.install_cluster(replication_user, cluster_configuration, bootstrap) bootstrap = False LOG.debug("Finalizing cluster configuration.") for guest in instance_guests: guest.cluster_complete() except Exception: LOG.exception(_("Error creating cluster.")) self.update_statuses_on_failure(cluster_id) timeout = Timeout(CONF.cluster_usage_timeout) try: _create_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for building cluster.")) self.update_statuses_on_failure(cluster_id) except TroveError: LOG.exception(_("Error creating cluster %s."), cluster_id) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End create_cluster for id: %s.", cluster_id)
def shrink_cluster(self, context, cluster_id, removal_ids): """Shrink a K2hdkc Cluster.""" LOG.debug( "Begins shrink_cluster for %s. removal_ids:{}".format(removal_ids), cluster_id) # 1. validates args if context is None: LOG.error("no context") return if cluster_id is None: LOG.error("no cluster_id") return if removal_ids is None: LOG.error("no removal_ids") return timeout = Timeout(CONF.cluster_usage_timeout) try: # 2. Retrieves db_instances from the database db_instances = DBInstance.find_all(cluster_id=cluster_id, deleted=False).all() # 3. Retrieves instance ids from the db_instances instance_ids = [db_instance.id for db_instance in db_instances] # 4. Checks if instances are running if not self._all_instances_running(instance_ids, cluster_id): LOG.error("instances are not ready yet") return # 4. Loads instances instances = [ Instance.load(context, instance_id) for instance_id in removal_ids ] LOG.debug("len(instances) {}".format(len(instances))) # 5. Instances GuestAgent class # 6.2. Checks if removing instances are # if not self._all_instances_shutdown(removal_ids, cluster_id): # LOG.error("removing instances are not shutdown yet") # return # 7. Calls cluster_complete endpoint of K2hdkcGuestAgent LOG.debug( "Calling cluster_complete as a final hook to each node in the cluster" ) for instance in instances: self.get_guest(instance).cluster_complete() # 8. delete node from OpenStack LOG.debug("delete node from OpenStack") for instance in instances: Instance.delete(instance) # 9. reset the current cluster task status to None LOG.debug("reset cluster task to None") self.reset_task() except Timeout: # Note adminstrators should reset task via CLI in this case. if Timeout is not timeout: raise # not my timeout LOG.exception("Timeout for shrink cluster.") self.update_statuses_on_failure( cluster_id, status=inst_tasks.InstanceTasks.SHRINKING_ERROR) finally: timeout.cancel() LOG.debug("Completed shrink_cluster for %s.", cluster_id)
class ResultContext(object): """ ResultContext ============= The ResultContext class is a Python context manager used in the automatic collection of captured output and exception handling. It is instantiated by DTestResult.accumulate() """ def __init__(self, result, ctx, excs): """ Initialize a ResultContext associated with the given ``result``. The context will handle messages for the part of the test run given by ``ctx`` (may be PRE, TEST, or POST). The exceptions listed in the ``excs`` tuple are expected; if no exceptions are expected, pass ``excs`` as None. """ # Save the basic information self.result = result self.ctx = ctx self.excs = excs # There's no timeout... self.timeout = None def __enter__(self): """ Begin the context handling. Clears out any captured data and initializes any timeouts defined for the test. """ # Clear the captured values for this thread capture.retrieve() # If test should be timed, set up the timeout if self.result._test._timeout: self.timeout = Timeout(self.result._test._timeout, AssertionError("Timed out after %s " "seconds" % self.result._test._timeout)) def __exit__(self, exc_type, exc_value, tb): """ Ends context handling. Cancels any pending timeouts, retrieves output data and exceptions, and determines the final result of the test. A DTestMessage object is initialized if necessary. """ # Cancel the timeout if one is pending if self.timeout is not None: self.timeout.cancel() self.timeout = None # Get the output and clean up captured = capture.retrieve() # If this was the test, determine a result if self.ctx in (PRE, TEST): self.result._set_result(self, exc_type, exc_value, tb) # Generate a message, if necessary if captured or exc_type or exc_value or tb: self.result._storemsg(self, captured, exc_type, exc_value, tb) # We handled the exception return True
def create_cluster(self, context, cluster_id): LOG.debug("begin create_cluster for id: %s" % cluster_id) def _create_cluster(): # fetch instances by cluster_id against instances table db_instances = DBInstance.find_all(cluster_id=cluster_id).all() instance_ids = [db_instance.id for db_instance in db_instances] LOG.debug("instances in cluster %s: %s" % (cluster_id, instance_ids)) if not self._all_instances_ready(instance_ids, cluster_id): return LOG.debug("all instances in cluster %s ready." % cluster_id) instances = [Instance.load(context, instance_id) for instance_id in instance_ids] # filter query routers in instances into a new list: query_routers query_routers = [instance for instance in instances if instance.type == 'query_router'] LOG.debug("query routers: %s" % [instance.id for instance in query_routers]) # filter config servers in instances into new list: config_servers config_servers = [instance for instance in instances if instance.type == 'config_server'] LOG.debug("config servers: %s" % [instance.id for instance in config_servers]) # filter members (non router/configsvr) into a new list: members members = [instance for instance in instances if instance.type == 'member'] LOG.debug("members: %s" % [instance.id for instance in members]) # for config_server in config_servers, append ip/hostname to # "config_server_hosts", then # peel off the replica-set name and ip/hostname from 'x' config_server_ips = [self.get_ip(instance) for instance in config_servers] LOG.debug("config server ips: %s" % config_server_ips) # Give the query routers the configsvr ips to connect to. # Create the admin user on the query routers. # The first will create the user, and the others will just reset # the password to the same value. LOG.debug("calling add_config_servers on, and sending admin user " "password to, query_routers") try: admin_created = False admin_password = utils.generate_random_password() for query_router in query_routers: guest = self.get_guest(query_router) guest.add_config_servers(config_server_ips) if admin_created: guest.store_admin_password(admin_password) else: guest.create_admin_user(admin_password) admin_created = True except Exception: LOG.exception(_("error adding config servers")) self.update_statuses_on_failure(cluster_id) return if not self._create_replica_set(members, cluster_id): return replica_set_name = "rs1" if not self._create_shard(query_routers, replica_set_name, members, cluster_id): return # call to start checking status for instance in instances: self.get_guest(instance).cluster_complete() cluster_usage_timeout = CONF.cluster_usage_timeout timeout = Timeout(cluster_usage_timeout) try: _create_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("timeout for building cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("end create_cluster for id: %s" % cluster_id)
def create_cluster(self, context, cluster_id): LOG.debug("Begin create_cluster for id: %s.", cluster_id) def _create_cluster(): cluster_node_ids = self.find_cluster_node_ids(cluster_id) # Wait for cluster nodes to get to cluster-ready status. LOG.debug("Waiting for all nodes to become ready.") if not self._all_instances_ready(cluster_node_ids, cluster_id): return cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids) LOG.debug("All nodes ready, proceeding with cluster setup.") seeds = self.choose_seed_nodes(cluster_nodes) # Configure each cluster node with the list of seeds. # Once all nodes are configured, start the seed nodes one at a time # followed by the rest of the nodes. try: LOG.debug("Selected seed nodes: %s", seeds) for node in cluster_nodes: LOG.debug("Configuring node: %s.", node['id']) node['guest'].set_seeds(seeds) node['guest'].set_auto_bootstrap(False) LOG.debug("Starting seed nodes.") for node in cluster_nodes: if node['ip'] in seeds: node['guest'].restart() node['guest'].set_auto_bootstrap(True) LOG.debug("All seeds running, starting remaining nodes.") for node in cluster_nodes: if node['ip'] not in seeds: node['guest'].restart() node['guest'].set_auto_bootstrap(True) # Create the in-database user via the first node. The remaining # nodes will replicate in-database changes automatically. # Only update the local authentication file on the other nodes. LOG.debug("Securing the cluster.") key = utils.generate_random_password() admin_creds = None for node in cluster_nodes: if admin_creds is None: admin_creds = node['guest'].cluster_secure(key) else: node['guest'].store_admin_credentials(admin_creds) node['guest'].cluster_complete() LOG.debug("Cluster configuration finished successfully.") except Exception: LOG.exception(_("Error creating cluster.")) self.update_statuses_on_failure(cluster_id) timeout = Timeout(CONF.cluster_usage_timeout) try: _create_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for building cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End create_cluster for id: %s.", cluster_id)
def main(): # set up headless proxy headless_proxy = "127.0.0.1:3128" proxy = Proxy({ 'proxyType': ProxyType.MANUAL, 'httpProxy': headless_proxy, 'ftpProxy': headless_proxy, 'sslProxy': headless_proxy, 'noProxy': '' }) chrome_options = webdriver.ChromeOptions() # disable images to speed up the page loading prefs = {"profile.managed_default_content_settings.images": 2} chrome_options.add_experimental_option("prefs", prefs) capabilities = dict(DesiredCapabilities.CHROME) proxy.add_to_capabilities(capabilities) url = 'https://dhp.virginiainteractive.org/Lookup/Index' driver = webdriver.Chrome( executable_path='/Users/markelle/Desktop/chromedriver') states = list(pd.read_csv("states.txt", header=None)[0]) tables = [] # repeat the full scrape for each state for state in states: driver.get(url) driver.implicitly_wait(10) # select the desired occupation and state from dropdowns driver.find_element_by_xpath( "//select[@id='OccupationId']/option[text()='Licensed Massage Therapist']" ).click() driver.find_element_by_xpath("//select[@id='State']/option[text()='" + state + "']").click() driver.find_elements_by_name("submitBtn")[2].click() # handle the case when the result page is empty (selenium gets stuck) timeout = Timeout(2, NoSuchElementException) try: rows = [x for x in driver.find_elements_by_tag_name('tr')[2:]] except NoSuchElementException: rows = [] finally: timeout.cancel() # extract the text from each row and add to table for row in rows: cells = [x.text for x in row.find_elements_by_tag_name('td')] tables.append(cells) next_ = check_for_mult_pages(driver) # continue scraping while there are more pages while next_ != None: next_.click() rows = [x for x in driver.find_elements_by_tag_name('tr')[2:]] for row in rows: cells = [x.text for x in row.find_elements_by_tag_name('td')] tables.append(cells) next_ = check_for_mult_pages(driver) # export to csv pd.DataFrame(tables).to_csv("FULL_data.csv")
for ln in rows: timeout = Timeout(6) try: print(ln) domain = ln[1] print(domain) try: r = requests.get('http://' + domain + '/license.txt') except: pass found = r.content.find(b'WordPress') + 1 try: r = requests.get('https://' + domain + '/license.txt') except: pass found += r.content.find(b'WordPress') + 1 print(found) if found > 0: cursor.execute( "replace into domains (domain, wpcheck) values (?, ?)", (domain, 1)) else: cursor.execute( "replace into domains (domain, wpcheck) values (?, ?)", (domain, 0)) connection.commit() finally: timeout.cancel()
def grow_cluster(self, context, cluster_id, new_instance_ids): LOG.debug("Begin grow_cluster for id: %s.", cluster_id) def _grow_cluster(): # Wait for new nodes to get to cluster-ready status. LOG.debug("Waiting for new nodes to become ready.") if not self._all_instances_ready(new_instance_ids, cluster_id): return new_instances = [ Instance.load(context, instance_id) for instance_id in new_instance_ids ] added_nodes = [ self.build_node_info(instance) for instance in new_instances ] LOG.debug("All nodes ready, proceeding with cluster setup.") cluster_node_ids = self.find_cluster_node_ids(cluster_id) cluster_nodes = self.load_cluster_nodes(context, cluster_node_ids) old_nodes = [ node for node in cluster_nodes if node['id'] not in new_instance_ids ] try: # All nodes should have the same seeds and credentials. # Retrieve the information from the first node. test_node = old_nodes[0] current_seeds = test_node['guest'].get_seeds() admin_creds = test_node['guest'].get_admin_credentials() # Bootstrap new nodes. # Seed nodes do not bootstrap. Current running nodes # must be used as seeds during the process. # Since we are adding to an existing cluster, ensure that the # new nodes have auto-bootstrapping enabled. # Start the added nodes. LOG.debug("Starting new nodes.") for node in added_nodes: node['guest'].set_auto_bootstrap(True) node['guest'].set_seeds(current_seeds) node['guest'].store_admin_credentials(admin_creds) node['guest'].restart() node['guest'].cluster_complete() # Recompute the seed nodes based on the updated cluster # geometry. seeds = self.choose_seed_nodes(cluster_nodes) # Configure each cluster node with the updated list of seeds. LOG.debug("Updating all nodes with new seeds: %s", seeds) for node in cluster_nodes: node['guest'].set_seeds(seeds) # Run nodetool cleanup on each of the previously existing nodes # to remove the keys that no longer belong to those nodes. # Wait for cleanup to complete on one node before running # it on the next node. LOG.debug("Cleaning up orphan data on old cluster nodes.") for node in old_nodes: nid = node['id'] node['guest'].node_cleanup_begin() node['guest'].node_cleanup() LOG.debug("Waiting for node to finish its " "cleanup: %s", nid) if not self._all_instances_running([nid], cluster_id): LOG.warning( _("Node did not complete cleanup " "successfully: %s"), nid) LOG.debug("Cluster configuration finished successfully.") except Exception: LOG.exception(_("Error growing cluster.")) self.update_statuses_on_failure(cluster_id) timeout = Timeout(CONF.cluster_usage_timeout) try: _grow_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("Timeout for growing cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("End grow_cluster for id: %s.", cluster_id)
def create_cluster(self, context, cluster_id): LOG.debug("begin create_cluster for id: %s", cluster_id) def _create_cluster(): # fetch instances by cluster_id against instances table db_instances = DBInstance.find_all(cluster_id=cluster_id).all() instance_ids = [db_instance.id for db_instance in db_instances] LOG.debug("instances in cluster %(cluster_id)s: %(instance_ids)s", {'cluster_id': cluster_id, 'instance_ids': instance_ids}) if not self._all_instances_ready(instance_ids, cluster_id): return LOG.debug("all instances in cluster %s ready.", cluster_id) instances = [Instance.load(context, instance_id) for instance_id in instance_ids] # filter query routers in instances into a new list: query_routers query_routers = [instance for instance in instances if instance.type == 'query_router'] LOG.debug("query routers: %s", [instance.id for instance in query_routers]) # filter config servers in instances into new list: config_servers config_servers = [instance for instance in instances if instance.type == 'config_server'] LOG.debug("config servers: %s", [instance.id for instance in config_servers]) # filter members (non router/configsvr) into a new list: members members = [instance for instance in instances if instance.type == 'member'] LOG.debug("members: %s", [instance.id for instance in members]) # for config_server in config_servers, append ip/hostname to # "config_server_hosts", then # peel off the replica-set name and ip/hostname from 'x' config_server_ips = [self.get_ip(instance) for instance in config_servers] LOG.debug("config server ips: %s", config_server_ips) if not self._add_query_routers(query_routers, config_server_ips): return if not self._create_shard(query_routers[0], members): return # call to start checking status for instance in instances: self.get_guest(instance).cluster_complete() cluster_usage_timeout = CONF.cluster_usage_timeout timeout = Timeout(cluster_usage_timeout) try: _create_cluster() self.reset_task() except Timeout as t: if t is not timeout: raise # not my timeout LOG.exception(_("timeout for building cluster.")) self.update_statuses_on_failure(cluster_id) finally: timeout.cancel() LOG.debug("end create_cluster for id: %s", cluster_id)