def fcn_wrapper(*args, **kargs): with Timeout(seconds, TimeoutException): return func(*args, **kargs)
def test_api_call_timeout(self): self.call_context.call.side_effect = Timeout() self.assertRaises(exception.GuestTimeout, self.api.restart)
def wait(self, timeout=None): with Timeout(timeout): return self.ready.wait()
def direct_get_object(node, part, account, container, obj, conn_timeout=5, response_timeout=15, resp_chunk_size=None, headers={}): """ Get object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param resp_chunk_size: if defined, chunk size of data to read. :param headers: dict to be passed into HTTPConnection headers :returns: a tuple of (response headers, the object's contents) The response headers will be a dict and all header names will be lowercase. """ path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'GET', path, headers=headers) with Timeout(response_timeout): resp = conn.getresponse() if resp.status < 200 or resp.status >= 300: resp.read() raise ClientException( 'Object server %s:%s direct GET %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) if resp_chunk_size: def _object_body(): buf = resp.read(resp_chunk_size) while buf: yield buf buf = resp.read(resp_chunk_size) object_body = _object_body() else: object_body = resp.read() resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value return resp_headers, object_body
def direct_get_account(node, part, account, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15): """ Get listings directly from the account server. :param node: node dictionary from the ring :param part: partition the account is on :param account: account name :param marker: marker query :param limit: query limit :param prefix: prefix query :param delimeter: delimeter for the query :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :returns: a tuple of (response headers, a list of containers) The response headers will be a dict and all header names will be lowercase. """ path = '/' + account qs = 'format=json' if marker: qs += '&marker=%s' % quote(marker) if limit: qs += '&limit=%d' % limit if prefix: qs += '&prefix=%s' % quote(prefix) if delimiter: qs += '&delimiter=%s' % quote(delimiter) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'GET', path, query_string='format=json') with Timeout(response_timeout): resp = conn.getresponse() if resp.status < 200 or resp.status >= 300: resp.read() raise ClientException( 'Account server %s:%s direct GET %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value if resp.status == 204: resp.read() return resp_headers, [] return resp_headers, json_loads(resp.read())
def fake_one_audit_pass(reported): raise Timeout()
def getresponse(self): if kwargs.get('raise_exc'): raise Exception('test') if kwargs.get('raise_timeout_exc'): raise Timeout() return self
def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice, policy, logger_thread_locals=None): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in :param policy: the associated BaseStoragePolicy instance :param logger_thread_locals: The thread local values to be set on the self.logger to retain transaction logging information. """ if logger_thread_locals: self.logger.thread_locals = logger_thread_locals headers_out['user-agent'] = 'object-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error( _('ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), { 'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice }) except (Exception, Timeout): self.logger.exception( _('ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)' ), { 'ip': ip, 'port': port, 'dev': contdevice }) data = { 'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out } timestamp = headers_out['x-timestamp'] self._diskfile_router[policy].pickle_async_update( objdevice, account, container, obj, data, timestamp, policy)
def container_update(self, op, account, container, obj, request, headers_out, objdevice, policy): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request object driving the update :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in :param policy: the BaseStoragePolicy instance """ headers_in = request.headers conthosts = [ h.strip() for h in headers_in.get('X-Container-Host', '').split(',') ] contdevices = [ d.strip() for d in headers_in.get('X-Container-Device', '').split(',') ] contpartition = headers_in.get('X-Container-Partition', '') if len(conthosts) != len(contdevices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error( _('ERROR Container update failed: different ' 'numbers of hosts and devices in request: ' '"%s" vs "%s"') % (headers_in.get('X-Container-Host', ''), headers_in.get('X-Container-Device', ''))) return if contpartition: updates = zip(conthosts, contdevices) else: updates = [] headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-') headers_out['referer'] = request.as_referer() headers_out['X-Backend-Storage-Policy-Index'] = int(policy) update_greenthreads = [] for conthost, contdevice in updates: gt = spawn(self.async_update, op, account, container, obj, conthost, contpartition, contdevice, headers_out, objdevice, policy, logger_thread_locals=self.logger.thread_locals) update_greenthreads.append(gt) # Wait a little bit to see if the container updates are successful. # If we immediately return after firing off the greenthread above, then # we're more likely to confuse the end-user who does a listing right # after getting a successful response to the object create. The # `container_update_timeout` bounds the length of time we wait so that # one slow container server doesn't make the entire request lag. try: with Timeout(self.container_update_timeout): for gt in update_greenthreads: gt.wait() except Timeout: # updates didn't go through, log it and return self.logger.debug( 'Container update timeout (%.4fs) waiting for %s', self.container_update_timeout, updates)
def update(self, job): """ High-level method that replicates a single partition. :param job: a dict containing info about the partition to be replicated """ self.replication_count += 1 self.logger.increment('partition.update.count.%s' % (job['device'], )) headers = dict(self.default_headers) headers['X-Backend-Storage-Policy-Index'] = int(job['policy']) target_devs_info = set() failure_devs_info = set() begin = time.time() df_mgr = self._df_router[job['policy']] try: hashed, local_hash = tpool_reraise(df_mgr._get_hashes, job['device'], job['partition'], job['policy'], do_listdir=_do_listdir( int(job['partition']), self.replication_cycle)) self.suffix_hash += hashed self.logger.update_stats('suffix.hashes', hashed) attempts_left = len(job['nodes']) synced_remote_regions = set() random.shuffle(job['nodes']) nodes = itertools.chain( job['nodes'], job['policy'].object_ring.get_more_nodes( int(job['partition']))) while attempts_left > 0: # If this throws StopIteration it will be caught way below node = next(nodes) target_devs_info.add((node['replication_ip'], node['device'])) attempts_left -= 1 # if we have already synced to this remote region, # don't sync again on this replication pass if node['region'] in synced_remote_regions: continue try: with Timeout(self.http_timeout): resp = http_connect(node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '', headers=headers).getresponse() if resp.status == HTTP_INSUFFICIENT_STORAGE: self.logger.error( _('%(replication_ip)s/%(device)s ' 'responded as unmounted'), node) attempts_left += 1 failure_devs_info.add( (node['replication_ip'], node['device'])) continue if resp.status != HTTP_OK: self.logger.error( _("Invalid response %(resp)s " "from %(ip)s"), { 'resp': resp.status, 'ip': node['replication_ip'] }) failure_devs_info.add( (node['replication_ip'], node['device'])) continue remote_hash = pickle.loads(resp.read()) del resp suffixes = [ suffix for suffix in local_hash if local_hash[suffix] != remote_hash.get(suffix, -1) ] if not suffixes: self.stats['hashmatch'] += 1 continue hashed, recalc_hash = tpool_reraise(df_mgr._get_hashes, job['device'], job['partition'], job['policy'], recalculate=suffixes) self.logger.update_stats('suffix.hashes', hashed) local_hash = recalc_hash suffixes = [ suffix for suffix in local_hash if local_hash[suffix] != remote_hash.get(suffix, -1) ] self.stats['rsync'] += 1 success, _junk = self.sync(node, job, suffixes) with Timeout(self.http_timeout): conn = http_connect(node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '/' + '-'.join(suffixes), headers=headers) conn.getresponse().read() if not success: failure_devs_info.add( (node['replication_ip'], node['device'])) # add only remote region when replicate succeeded if success and node['region'] != job['region']: synced_remote_regions.add(node['region']) self.suffix_sync += len(suffixes) self.logger.update_stats('suffix.syncs', len(suffixes)) except (Exception, Timeout): failure_devs_info.add( (node['replication_ip'], node['device'])) self.logger.exception( _("Error syncing with node: %s") % node) self.suffix_count += len(local_hash) except StopIteration: self.logger.error( 'Ran out of handoffs while replicating ' 'partition %s of policy %d', job['partition'], int(job['policy'])) except (Exception, Timeout): failure_devs_info.update(target_devs_info) self.logger.exception(_("Error syncing partition")) finally: self._add_failure_stats(failure_devs_info) self.stats['success'] += len(target_devs_info - failure_devs_info) self.partition_times.append(time.time() - begin) self.logger.timing_since('partition.update.timing', begin)
def replicate(self, override_devices=None, override_partitions=None, override_policies=None): """Run a replication pass""" self.start = time.time() self.suffix_count = 0 self.suffix_sync = 0 self.suffix_hash = 0 self.replication_count = 0 self.last_replication_count = -1 self.replication_cycle = (self.replication_cycle + 1) % 10 self.partition_times = [] self.my_replication_ips = self._get_my_replication_ips() self.all_devs_info = set() self.handoffs_remaining = 0 stats = eventlet.spawn(self.heartbeat) lockup_detector = eventlet.spawn(self.detect_lockups) eventlet.sleep() # Give spawns a cycle current_nodes = None try: self.run_pool = GreenPool(size=self.concurrency) jobs = self.collect_jobs(override_devices=override_devices, override_partitions=override_partitions, override_policies=override_policies) for job in jobs: current_nodes = job['nodes'] dev_path = check_drive(self.devices_dir, job['device'], self.mount_check) if not dev_path: self._add_failure_stats([(failure_dev['replication_ip'], failure_dev['device']) for failure_dev in job['nodes']]) self.logger.warning(_('%s is not mounted'), job['device']) continue if self.handoffs_first and not job['delete']: # in handoffs first mode, we won't process primary # partitions until rebalance was successful! if self.handoffs_remaining: self.logger.warning( _("Handoffs first mode still has handoffs " "remaining. Aborting current " "replication pass.")) break if not self.check_ring(job['policy'].object_ring): self.logger.info( _("Ring change detected. Aborting " "current replication pass.")) return try: if isfile(job['path']): # Clean up any (probably zero-byte) files where a # partition should be. self.logger.warning( 'Removing partition directory ' 'which was a file: %s', job['path']) os.remove(job['path']) continue except OSError: continue if job['delete']: self.run_pool.spawn(self.update_deleted, job) else: self.run_pool.spawn(self.update, job) current_nodes = None with Timeout(self.lockup_timeout): self.run_pool.waitall() except (Exception, Timeout): if current_nodes: self._add_failure_stats([(failure_dev['replication_ip'], failure_dev['device']) for failure_dev in current_nodes]) else: self._add_failure_stats(self.all_devs_info) self.logger.exception(_("Exception in top-level replication loop")) self.kill_coros() finally: stats.kill() lockup_detector.kill() self.stats_line() self.stats['attempted'] = self.replication_count
def update_deleted(self, job): """ High-level method that replicates a single partition that doesn't belong on this node. :param job: a dict containing info about the partition to be replicated """ def tpool_get_suffixes(path): return [ suff for suff in os.listdir(path) if len(suff) == 3 and isdir(join(path, suff)) ] self.replication_count += 1 self.logger.increment('partition.delete.count.%s' % (job['device'], )) headers = dict(self.default_headers) headers['X-Backend-Storage-Policy-Index'] = int(job['policy']) failure_devs_info = set() begin = time.time() handoff_partition_deleted = False try: responses = [] suffixes = tpool.execute(tpool_get_suffixes, job['path']) synced_remote_regions = {} delete_objs = None if suffixes: for node in job['nodes']: self.stats['rsync'] += 1 kwargs = {} if node['region'] in synced_remote_regions and \ self.conf.get('sync_method', 'rsync') == 'ssync': kwargs['remote_check_objs'] = \ synced_remote_regions[node['region']] # candidates is a dict(hash=>timestamp) of objects # for deletion success, candidates = self.sync(node, job, suffixes, **kwargs) if success: with Timeout(self.http_timeout): conn = http_connect(node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '/' + '-'.join(suffixes), headers=headers) conn.getresponse().read() if node['region'] != job['region']: synced_remote_regions[node['region']] = viewkeys( candidates) else: failure_devs_info.add( (node['replication_ip'], node['device'])) responses.append(success) for cand_objs in synced_remote_regions.values(): if delete_objs is None: delete_objs = cand_objs else: delete_objs = delete_objs & cand_objs if self.handoff_delete: # delete handoff if we have had handoff_delete successes delete_handoff = len([resp for resp in responses if resp]) >= \ self.handoff_delete else: # delete handoff if all syncs were successful delete_handoff = len(responses) == len(job['nodes']) and \ all(responses) if delete_handoff: self.stats['remove'] += 1 if (self.conf.get('sync_method', 'rsync') == 'ssync' and delete_objs is not None): self.logger.info(_("Removing %s objects"), len(delete_objs)) _junk, error_paths = self.delete_handoff_objs( job, delete_objs) # if replication works for a hand-off device and it failed, # the remote devices which are target of the replication # from the hand-off device will be marked. Because cleanup # after replication failed means replicator needs to # replicate again with the same info. if error_paths: failure_devs_info.update([ (failure_dev['replication_ip'], failure_dev['device']) for failure_dev in job['nodes'] ]) else: self.delete_partition(job['path']) handoff_partition_deleted = True elif not suffixes: self.delete_partition(job['path']) handoff_partition_deleted = True except (Exception, Timeout): self.logger.exception(_("Error syncing handoff partition")) self._add_failure_stats(failure_devs_info) finally: target_devs_info = set([(target_dev['replication_ip'], target_dev['device']) for target_dev in job['nodes']]) self.stats['success'] += len(target_devs_info - failure_devs_info) if not handoff_partition_deleted: self.handoffs_remaining += 1 self.partition_times.append(time.time() - begin) self.logger.timing_since('partition.delete.timing', begin)
def new_func(*args, **kwargs): try: with Timeout(second): return func(*args, **kwargs) except Timeout: logging.exception('{0} too long'.format(func.__name__))
def GETorHEAD_base(self, req, server_type, partition, nodes, path, attempts): """ Base handler for HTTP GET or HEAD requests. :param req: swob.Request object :param server_type: server type :param partition: partition :param nodes: nodes :param path: path for the request :param attempts: number of attempts to try :returns: swob.Response object """ statuses = [] reasons = [] bodies = [] sources = [] newest = config_true_value(req.headers.get('x-newest', 'f')) nodes = iter(nodes) while len(statuses) < attempts: try: node = nodes.next() except StopIteration: break if self.error_limited(node): continue try: with ConnectionTimeout(self.app.conn_timeout): headers = dict(req.headers) headers['Connection'] = 'close' conn = http_connect(node['ip'], node['port'], node['device'], partition, req.method, path, headers=headers, query_string=req.query_string) with Timeout(self.app.node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn except (Exception, Timeout): self.exception_occurred( node, server_type, _('Trying to %(method)s %(path)s') % { 'method': req.method, 'path': req.path }) continue if self.is_good_source(possible_source): # 404 if we know we don't have a synced copy if not float(possible_source.getheader('X-PUT-Timestamp', 1)): statuses.append(HTTP_NOT_FOUND) reasons.append('') bodies.append('') self.close_swift_conn(possible_source) else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append('') sources.append(possible_source) if not newest: # one good source is enough break else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append(possible_source.read()) if possible_source.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) elif is_server_error(possible_source.status): self.error_occurred( node, _('ERROR %(status)d %(body)s ' 'From %(type)s Server') % { 'status': possible_source.status, 'body': bodies[-1][:1024], 'type': server_type }) if sources: sources.sort(key=source_key) source = sources.pop() for src in sources: self.close_swift_conn(src) res = Response(request=req, conditional_response=True) if req.method == 'GET' and \ source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): res.app_iter = self._make_app_iter(node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn res.status = source.status update_headers(res, source.getheaders()) if not res.environ: res.environ = {} res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') res.accept_ranges = 'bytes' res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') return res return self.best_response(req, statuses, reasons, bodies, '%s %s' % (server_type, req.method))
def container_info(self, account, container, account_autocreate=False): """ Get container information and thusly verify container existance. This will also make a call to account_info to verify that the account exists. :param account: account name for the container :param container: container name to look up :returns: tuple of (container partition, container nodes, container read acl, container write acl, container sync key) or (None, None, None, None, None) if the container does not exist """ partition, nodes = self.app.container_ring.get_nodes( account, container) path = '/%s/%s' % (account, container) if self.app.memcache: cache_key = get_container_memcache_key(account, container) cache_value = self.app.memcache.get(cache_key) if isinstance(cache_value, dict): status = cache_value['status'] read_acl = cache_value['read_acl'] write_acl = cache_value['write_acl'] sync_key = cache_value.get('sync_key') versions = cache_value.get('versions') if status == HTTP_OK: return partition, nodes, read_acl, write_acl, sync_key, \ versions elif status == HTTP_NOT_FOUND: return None, None, None, None, None, None if not self.account_info(account, autocreate=account_autocreate)[1]: return None, None, None, None, None, None result_code = 0 read_acl = None write_acl = None sync_key = None container_size = None versions = None attempts_left = len(nodes) headers = {'x-trans-id': self.trans_id, 'Connection': 'close'} iternodes = self.iter_nodes(partition, nodes, self.app.container_ring) while attempts_left > 0: try: node = iternodes.next() except StopIteration: break attempts_left -= 1 try: with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], partition, 'HEAD', path, headers) with Timeout(self.app.node_timeout): resp = conn.getresponse() body = resp.read() if is_success(resp.status): result_code = HTTP_OK read_acl = resp.getheader('x-container-read') write_acl = resp.getheader('x-container-write') sync_key = resp.getheader('x-container-sync-key') container_size = \ resp.getheader('X-Container-Object-Count') versions = resp.getheader('x-versions-location') break elif resp.status == HTTP_NOT_FOUND: if result_code == 0: result_code = HTTP_NOT_FOUND elif result_code != HTTP_NOT_FOUND: result_code = -1 elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) continue else: result_code = -1 except (Exception, Timeout): self.exception_occurred( node, _('Container'), _('Trying to get container info for %s') % path) if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND): if result_code == HTTP_OK: cache_timeout = self.app.recheck_container_existence else: cache_timeout = self.app.recheck_container_existence * 0.1 self.app.memcache.set(cache_key, { 'status': result_code, 'read_acl': read_acl, 'write_acl': write_acl, 'sync_key': sync_key, 'container_size': container_size, 'versions': versions }, timeout=cache_timeout) if result_code == HTTP_OK: return partition, nodes, read_acl, write_acl, sync_key, versions return None, None, None, None, None, None
def direct_put_object(node, part, account, container, name, contents, content_length=None, etag=None, content_type=None, headers=None, conn_timeout=5, response_timeout=15, chunk_size=65535): """ Put object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param name: object name :param contents: an iterable or string to read object data from :param content_length: value to send as content-length header :param etag: etag of contents :param content_type: value to send as content-type header :param headers: additional headers to include in the request :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param chunk_size: if defined, chunk size of data to send. :returns: etag from the server response :raises ClientException: HTTP PUT request failed """ path = '/%s/%s/%s' % (account, container, name) if headers is None: headers = {} if etag: headers['ETag'] = etag.strip('"') if content_length is not None: headers['Content-Length'] = str(content_length) else: for n, v in headers.items(): if n.lower() == 'content-length': content_length = int(v) if content_type is not None: headers['Content-Type'] = content_type else: headers['Content-Type'] = 'application/octet-stream' if not contents: headers['Content-Length'] = '0' if isinstance(contents, basestring): contents = [contents] #Incase the caller want to insert an object with specific age add_ts = 'X-Timestamp' not in headers if content_length is None: headers['Transfer-Encoding'] = 'chunked' with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', path, headers=gen_headers(headers, add_ts)) contents_f = FileLikeIter(contents) if content_length is None: chunk = contents_f.read(chunk_size) while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = contents_f.read(chunk_size) conn.send('0\r\n\r\n') else: left = content_length while left > 0: size = chunk_size if size > left: size = left chunk = contents_f.read(size) if not chunk: break conn.send(chunk) left -= len(chunk) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise DirectClientException('Object', 'PUT', node, part, path, resp) return resp.getheader('etag').strip('"')
def GETorHEAD_base(self, req, server_type, partition, nodes, path, attempts): """ Base handler for HTTP GET or HEAD requests. :param req: webob.Request object :param server_type: server type :param partition: partition :param nodes: nodes :param path: path for the request :param attempts: number of attempts to try :returns: webob.Response object """ statuses = [] reasons = [] bodies = [] source = None sources = [] newest = req.headers.get('x-newest', 'f').lower() in TRUE_VALUES nodes = iter(nodes) while len(statuses) < attempts: try: node = nodes.next() except StopIteration: break if self.error_limited(node): continue try: with ConnectionTimeout(self.app.conn_timeout): headers = dict(req.headers) headers['Connection'] = 'close' conn = http_connect(node['ip'], node['port'], node['device'], partition, req.method, path, headers=headers, query_string=req.query_string) with Timeout(self.app.node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn except (Exception, Timeout): self.exception_occurred( node, server_type, _('Trying to %(method)s %(path)s') % { 'method': req.method, 'path': req.path }) continue if possible_source.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) continue if is_success(possible_source.status) or \ is_redirection(possible_source.status): # 404 if we know we don't have a synced copy if not float(possible_source.getheader('X-PUT-Timestamp', 1)): statuses.append(HTTP_NOT_FOUND) reasons.append('') bodies.append('') possible_source.read() continue if newest: if sources: ts = float( source.getheader('x-put-timestamp') or source.getheader('x-timestamp') or 0) pts = float( possible_source.getheader('x-put-timestamp') or possible_source.getheader('x-timestamp') or 0) if pts > ts: sources.insert(0, possible_source) else: sources.append(possible_source) else: sources.insert(0, possible_source) source = sources[0] statuses.append(source.status) reasons.append(source.reason) bodies.append('') continue else: source = possible_source break statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append(possible_source.read()) if is_server_error(possible_source.status): self.error_occurred(node, _('ERROR %(status)d %(body)s ' \ 'From %(type)s Server') % {'status': possible_source.status, 'body': bodies[-1][:1024], 'type': server_type}) if source: if req.method == 'GET' and \ source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): if newest: # we need to close all hanging swift_conns sources.pop(0) for src in sources: self.close_swift_conn(src) res = Response(request=req, conditional_response=True) res.app_iter = self._make_app_iter(node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn update_headers(res, source.getheaders()) # Used by container sync feature if res.environ is None: res.environ = dict() res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') update_headers(res, {'accept-ranges': 'bytes'}) res.status = source.status res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') return res elif is_success(source.status) or is_redirection(source.status): res = status_map[source.status](request=req) update_headers(res, source.getheaders()) # Used by container sync feature if res.environ is None: res.environ = dict() res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') update_headers(res, {'accept-ranges': 'bytes'}) res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') return res return self.best_response(req, statuses, reasons, bodies, '%s %s' % (server_type, req.method))
def container_info(self, account, container, account_autocreate=False): """ Get container information and thusly verify container existance. This will also make a call to account_info to verify that the account exists. :param account: account name for the container :param container: container name to look up :returns: dict containing at least container partition ('partition'), container nodes ('containers'), container read acl ('read_acl'), container write acl ('write_acl'), and container sync key ('sync_key'). Values are set to None if the container does not exist. """ part, nodes = self.app.container_ring.get_nodes(account, container) path = '/%s/%s' % (account, container) container_info = { 'status': 0, 'read_acl': None, 'write_acl': None, 'sync_key': None, 'count': None, 'bytes': None, 'versions': None, 'partition': None, 'nodes': None } if self.app.memcache: cache_key = get_container_memcache_key(account, container) cache_value = self.app.memcache.get(cache_key) if isinstance(cache_value, dict): if 'container_size' in cache_value: cache_value['count'] = cache_value['container_size'] if is_success(cache_value['status']): container_info.update(cache_value) container_info['partition'] = part container_info['nodes'] = nodes return container_info if not self.account_info(account, autocreate=account_autocreate)[1]: return container_info attempts_left = len(nodes) headers = {'x-trans-id': self.trans_id, 'Connection': 'close'} for node in self.iter_nodes(part, nodes, self.app.container_ring): try: with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'HEAD', path, headers) with Timeout(self.app.node_timeout): resp = conn.getresponse() body = resp.read() if is_success(resp.status): container_info.update({ 'status': HTTP_OK, 'read_acl': resp.getheader('x-container-read'), 'write_acl': resp.getheader('x-container-write'), 'sync_key': resp.getheader('x-container-sync-key'), 'count': resp.getheader('x-container-object-count'), 'bytes': resp.getheader('x-container-bytes-used'), 'versions': resp.getheader('x-versions-location') }) break elif resp.status == HTTP_NOT_FOUND: container_info['status'] = HTTP_NOT_FOUND else: container_info['status'] = -1 if resp.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) except (Exception, Timeout): self.exception_occurred( node, _('Container'), _('Trying to get container info for %s') % path) attempts_left -= 1 if attempts_left <= 0: break if self.app.memcache: if container_info['status'] == HTTP_OK: self.app.memcache.set( cache_key, container_info, timeout=self.app.recheck_container_existence) elif container_info['status'] == HTTP_NOT_FOUND: self.app.memcache.set( cache_key, container_info, timeout=self.app.recheck_container_existence * 0.1) if container_info['status'] == HTTP_OK: container_info['partition'] = part container_info['nodes'] = nodes return container_info
def fake_get_hashes(*args, **kwargs): self.get_hash_count += 1 if self.get_hash_count == 3: # raise timeout on last call to get hashes raise Timeout() return 2, {'abc': 'def'}
def update_deleted(self, job): """ High-level method that replicates a single partition that doesn't belong on this node. :param job: a dict containing info about the partition to be replicated """ def tpool_get_suffixes(path): return [ suff for suff in os.listdir(path) if len(suff) == 3 and isdir(join(path, suff)) ] self.replication_count += 1 self.logger.increment('partition.delete.count.%s' % (job['device'], )) self.headers['X-Backend-Storage-Policy-Index'] = int(job['policy']) begin = time.time() try: responses = [] suffixes = tpool.execute(tpool_get_suffixes, job['path']) synced_remote_regions = {} delete_objs = None if suffixes: for node in job['nodes']: kwargs = {} if node['region'] in synced_remote_regions and \ self.conf.get('sync_method', 'rsync') == 'ssync': kwargs['remote_check_objs'] = \ synced_remote_regions[node['region']] # candidates is a dict(hash=>timestamp) of objects # for deletion success, candidates = self.sync(node, job, suffixes, **kwargs) if success: with Timeout(self.http_timeout): conn = http_connect(node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '/' + '-'.join(suffixes), headers=self.headers) conn.getresponse().read() if node['region'] != job['region']: synced_remote_regions[node['region']] = \ candidates.keys() responses.append(success) for region, cand_objs in synced_remote_regions.items(): if delete_objs is None: delete_objs = cand_objs else: delete_objs = delete_objs.intersection(cand_objs) if self.handoff_delete: # delete handoff if we have had handoff_delete successes delete_handoff = len([resp for resp in responses if resp]) >= \ self.handoff_delete else: # delete handoff if all syncs were successful delete_handoff = len(responses) == len(job['nodes']) and \ all(responses) if delete_handoff: if (self.conf.get('sync_method', 'rsync') == 'ssync' and delete_objs is not None): self.logger.info(_("Removing %s objects"), len(delete_objs)) self.delete_handoff_objs(job, delete_objs) else: self.delete_partition(job['path']) elif not suffixes: self.delete_partition(job['path']) except (Exception, Timeout): self.logger.exception(_("Error syncing handoff partition")) finally: self.partition_times.append(time.time() - begin) self.logger.timing_since('partition.delete.timing', begin)
def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out['user-agent'] = 'obj-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error( _('ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), { 'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice }) except (Exception, Timeout): self.logger.exception( _('ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)' ), { 'ip': ip, 'port': port, 'dev': contdevice }) async_dir = os.path.join(self.devices, objdevice, ASYNCDIR) ohash = hash_path(account, container, obj) self.logger.increment('async_pendings') self.threadpools[objdevice].run_in_thread( write_pickle, { 'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out }, os.path.join( async_dir, ohash[-3:], ohash + '-' + normalize_timestamp(headers_out['x-timestamp'])), os.path.join(self.devices, objdevice, 'tmp'))
def update(self, job): """ High-level method that replicates a single partition. :param job: a dict containing info about the partition to be replicated """ self.replication_count += 1 self.logger.increment('partition.update.count.%s' % (job['device'], )) begin = time.time() try: hashed, local_hash = tpool_reraise( get_hashes, job['path'], do_listdir=(self.replication_count % 10) == 0, reclaim_age=self.reclaim_age) self.suffix_hash += hashed self.logger.update_stats('suffix.hashes', hashed) attempts_left = len(job['nodes']) nodes = itertools.chain( job['nodes'], self.object_ring.get_more_nodes(int(job['partition']))) while attempts_left > 0: # If this throws StopIterator it will be caught way below node = next(nodes) attempts_left -= 1 try: with Timeout(self.http_timeout): resp = http_connect( node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '', headers=self.headers).getresponse() if resp.status == HTTP_INSUFFICIENT_STORAGE: self.logger.error( _('%(ip)s/%(device)s responded' ' as unmounted'), node) attempts_left += 1 continue if resp.status != HTTP_OK: self.logger.error( _("Invalid response %(resp)s " "from %(ip)s"), { 'resp': resp.status, 'ip': node['replication_ip'] }) continue remote_hash = pickle.loads(resp.read()) del resp suffixes = [ suffix for suffix in local_hash if local_hash[suffix] != remote_hash.get(suffix, -1) ] if not suffixes: continue hashed, recalc_hash = tpool_reraise( get_hashes, job['path'], recalculate=suffixes, reclaim_age=self.reclaim_age) self.logger.update_stats('suffix.hashes', hashed) local_hash = recalc_hash suffixes = [ suffix for suffix in local_hash if local_hash[suffix] != remote_hash.get(suffix, -1) ] self.rsync(node, job, suffixes) with Timeout(self.http_timeout): conn = http_connect(node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '/' + '-'.join(suffixes), headers=self.headers) conn.getresponse().read() self.suffix_sync += len(suffixes) self.logger.update_stats('suffix.syncs', len(suffixes)) except (Exception, Timeout): self.logger.exception( _("Error syncing with node: %s") % node) self.suffix_count += len(local_hash) except (Exception, Timeout): self.logger.exception(_("Error syncing partition")) finally: self.partition_times.append(time.time() - begin) self.logger.timing_since('partition.update.timing', begin)
def direct_put_object(node, part, account, container, name, contents, content_length=None, etag=None, content_type=None, headers=None, conn_timeout=5, response_timeout=15, resp_chunk_size=None): """ Put object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param name: object name :param contents: a string to read object data from :param content_length: value to send as content-length header :param etag: etag of contents :param content_type: value to send as content-type header :param headers: additional headers to include in the request :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param chunk_size: if defined, chunk size of data to send. :returns: etag from the server response """ # TODO: Add chunked puts path = '/%s/%s/%s' % (account, container, name) if headers is None: headers = {} if etag: headers['ETag'] = etag.strip('"') if content_length is not None: headers['Content-Length'] = str(content_length) if content_type is not None: headers['Content-Type'] = content_type else: headers['Content-Type'] = 'application/octet-stream' if not contents: headers['Content-Length'] = '0' headers['X-Timestamp'] = normalize_timestamp(time()) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', path, headers=headers) conn.send(contents) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException( 'Object server %s:%s direct PUT %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) return resp.getheader('etag').strip('"')
def __call__(self, req): obj = None container = None try: (version, account, container, obj) = \ split_path(req.path_info, 3, 4, True) except ValueError: # not an object request pass # Sets webhook metadata if 'x-webhook' in req.headers and req.method == 'PUT': # Check if webhook-auth is provided before creating webhook if 'x-webhook-auth' in req.headers: # translate user's request header to sysmeta # Add callback URL (webhook) to container System Metadata req.headers[SYSMETA_WEBHOOK] = req.headers['x-webhook'] self.logger.info( 'webhook.py: Adding webhook to container "%s": %s' % (container, req.headers['x-webhook'])) # Add auth credentials to system metadata req.headers[SYSMETA_WEBHOOK_AUTH] = \ req.headers['x-webhook-auth'] self.logger.info( 'webhook.py: Adding webhook credentials saved successfully' ) else: # Do we fail since there is no authentication? self.logger.info('webhook.py: ' 'ERROR: no webhook credentials provided; ' 'Webhook not created') else: # Webhook NOT in header pass # Removes webhook metadata if 'x-remove-webhook' in req.headers: # empty value will tombstone sysmeta req.headers[SYSMETA_WEBHOOK] = '' req.headers[SYSMETA_WEBHOOK_AUTH] = '' self.logger.info( 'webhook.py: Removing webhook from container "%s"' % container) ################### Webhook-Get creation and delete code ########################## # Sets webhook metadata if 'x-webhook-get' in req.headers and req.method == 'PUT': # Check if webhook-auth is provided before creating webhook if 'x-webhook-get-auth' in req.headers: # translate user's request header to sysmeta # Add callback URL (webhook) to container System Metadata req.headers[SYSMETA_WEBHOOK_GET] = req.headers['x-webhook-get'] self.logger.info( 'webhook.py: Adding webhook-get to container "%s": %s' % (container, req.headers['x-webhook-get'])) # Add auth credentials to system metadata req.headers[SYSMETA_WEBHOOK_GET_AUTH] = \ req.headers['x-webhook-get-auth'] self.logger.info( 'webhook.py: Adding webhook-get credentials saved successfully' ) else: # Do we fail since there is no authentication? self.logger.info('webhook.py: ' 'ERROR: no webhook-get credentials provided; ' 'Webhook-get not created') else: # Webhook NOT in header pass # Removes webhook metadata if 'x-remove-webhook-get' in req.headers: # empty value will tombstone sysmeta req.headers[SYSMETA_WEBHOOK_GET] = '' req.headers[SYSMETA_WEBHOOK_GET_AUTH] = '' self.logger.info( 'webhook.py: Removing webhook-get from container "%s"' % container) ###################################################################################### # Account and object storage will ignore x-container-sysmeta-* # Process request downstream... resp = req.get_response(self.app) # ...if successful, call webhook # Update hook so that hooks should also be called when we access an object # Need to check in handler if req.method != 'GET' before executing if obj and is_success(resp.status_int) and req.method != 'GET': # Get container info container_info = get_container_info(req.environ, self.app) # assign webook from sysmeta webhook_get = container_info['sysmeta'].get('webhook-get') if webhook_get and req.method == 'GET': self.logger.info( 'webhook.py: webhook-get found for container %s: %s' % (container, webhook_get)) # Webhook-get detected. This means the current version of the data # is not updated from the cache. Call the action which tries to # push the action to migrate this particular object. if resp.headers['r-data-migrate'] != 'ok': ## Call the webhook-get parameter and fire the post webhook_req = urllib2.Request(webhook_get) creds = base64.b64encode( container_info['sysmeta'].get('webhook-get-auth')) auth = "Basic %s" % creds webhook_req.add_header('Authorization', auth) webhook_req.add_header('Content-Type', 'application/json') # Forward the url back in order to get it by the wake_up_migration action # Add response data reqURL = urlparse.urlparse(req.url) path = reqURL.path[0:reqURL.path.index('/', 4)] swiftURL = reqURL.scheme + '://' + reqURL.netloc + path mesg = { "swiftObj": { "url": swiftURL, "token": req.headers['x-auth-token'], "container": container, "object": obj, "method": req.method } } self.logger.info('webhook.py: mesg body: %s' % mesg) webhook_req.add_data(json.dumps(mesg)) # TODO(): make request asynchronous with Timeout(20): try: # make request webhook_resp = urllib2.urlopen( webhook_req, context=ssl.SSLContext( ssl.PROTOCOL_SSLv23)).read() except (Exception, Timeout): self.logger.exception( 'failed POST to webhook-get %s' % webhook_get) else: self.logger.info( 'successfully called webhook-get: %s \n%s' % (webhook_get, webhook_resp)) #### Normal Webhook behaviour webhook = container_info['sysmeta'].get('webhook') if webhook: self.logger.info( 'webhook.py: webhook found for container %s: %s' % (container, webhook)) # create a POST request with obj name as body webhook_req = urllib2.Request(webhook) # Get credentials from sysmeta and base64 encode creds = base64.b64encode( container_info['sysmeta'].get('webhook-auth')) # Add credentials to headr auth = "Basic %s" % creds webhook_req.add_header('Authorization', auth) # Specify content type webhook_req.add_header('Content-Type', 'application/json') # Add response data reqURL = urlparse.urlparse(req.url) path = reqURL.path[0:reqURL.path.index('/', 4)] swiftURL = reqURL.scheme + '://' + reqURL.netloc + path mesg = { "swiftObj": { "url": swiftURL, "token": req.headers['x-auth-token'], "container": container, "object": obj, "method": req.method } } self.logger.info('webhook.py: mesg body: %s' % mesg) webhook_req.add_data(json.dumps(mesg)) # TODO(): make request asynchronous with Timeout(20): try: # make request webhook_resp = urllib2.urlopen( webhook_req, context=ssl.SSLContext( ssl.PROTOCOL_SSLv23)).read() except (Exception, Timeout): self.logger.exception('failed POST to webhook %s' % webhook) else: self.logger.info( 'successfully called webhook: %s \n%s' % (webhook, webhook_resp)) else: # No webhook associated with container self.logger.info( 'webhook.py: webhook not specified for container: %s' % container) # Response back to user if 'x-container-sysmeta-webhook' in resp.headers: # translate sysmeta from the backend resp to # user-visible client resp header resp.headers['x-webhook'] = resp.headers[SYSMETA_WEBHOOK] return resp
def account_update(self, req, account, container, broker): """ Update the account server(s) with latest container info. :param req: swob.Request object :param account: account name :param container: container name :param broker: container DB broker object :returns: if all the account requests return a 404 error code, HTTPNotFound response object, if the account cannot be updated due to a malformed header, an HTTPBadRequest response object, otherwise None. """ account_hosts = [ h.strip() for h in req.headers.get('X-Account-Host', '').split(',') ] account_devices = [ d.strip() for d in req.headers.get('X-Account-Device', '').split(',') ] account_partition = req.headers.get('X-Account-Partition', '') if len(account_hosts) != len(account_devices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error( _('ERROR Account update failed: different ' 'numbers of hosts and devices in request: ' '"%(hosts)s" vs "%(devices)s"') % { 'hosts': req.headers.get('X-Account-Host', ''), 'devices': req.headers.get('X-Account-Device', '') }) return HTTPBadRequest(req=req) if account_partition: # zip is lazy on py3, but we need a list, so force evaluation. # On py2 it's an extra list copy, but the list is so small # (one element per replica in account ring, usually 3) that it # doesn't matter. updates = list(zip(account_hosts, account_devices)) else: updates = [] account_404s = 0 for account_host, account_device in updates: account_ip, account_port = account_host.rsplit(':', 1) new_path = '/' + '/'.join([account, container]) info = broker.get_info() account_headers = HeaderKeyDict({ 'x-put-timestamp': info['put_timestamp'], 'x-delete-timestamp': info['delete_timestamp'], 'x-object-count': info['object_count'], 'x-bytes-used': info['bytes_used'], 'x-trans-id': req.headers.get('x-trans-id', '-'), 'X-Backend-Storage-Policy-Index': info['storage_policy_index'], 'user-agent': 'container-server %s' % os.getpid(), 'referer': req.as_referer() }) if req.headers.get('x-account-override-deleted', 'no').lower() == \ 'yes': account_headers['x-account-override-deleted'] = 'yes' try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(account_ip, account_port, account_device, account_partition, 'PUT', new_path, account_headers) with Timeout(self.node_timeout): account_response = conn.getresponse() account_response.read() if account_response.status == HTTP_NOT_FOUND: account_404s += 1 elif not is_success(account_response.status): self.logger.error( _('ERROR Account update failed ' 'with %(ip)s:%(port)s/%(device)s (will retry ' 'later): Response %(status)s %(reason)s'), { 'ip': account_ip, 'port': account_port, 'device': account_device, 'status': account_response.status, 'reason': account_response.reason }) except (Exception, Timeout): self.logger.exception( _('ERROR account update failed with ' '%(ip)s:%(port)s/%(device)s (will retry later)'), { 'ip': account_ip, 'port': account_port, 'device': account_device }) if updates and account_404s == len(updates): return HTTPNotFound(req=req) else: return None
def run_wsgi(conf_path, app_section, *args, **kwargs): """ Runs the server according to some strategy. The default strategy runs a specified number of workers in pre-fork model. The object-server (only) may use a servers-per-port strategy if its config has a servers_per_port setting with a value greater than zero. :param conf_path: Path to paste.deploy style configuration file/directory :param app_section: App name from conf file to load config from :returns: 0 if successful, nonzero otherwise """ # Load configuration, Set logger and Load request processor try: (conf, logger, log_name) = \ _initrp(conf_path, app_section, *args, **kwargs) except ConfigFileError as e: print(e) return 1 # optional nice/ionice priority scheduling utils.modify_priority(conf, logger) servers_per_port = int(conf.get('servers_per_port', '0') or 0) # NOTE: for now servers_per_port is object-server-only; future work could # be done to test and allow it to be used for account and container # servers, but that has not been done yet. if servers_per_port and app_section == 'object-server': strategy = ServersPerPortStrategy(conf, logger, servers_per_port=servers_per_port) else: strategy = WorkersStrategy(conf, logger) # patch event before loadapp utils.eventlet_monkey_patch() # Ensure the configuration and application can be loaded before proceeding. global_conf = {'log_name': log_name} if 'global_conf_callback' in kwargs: kwargs['global_conf_callback'](conf, global_conf) loadapp(conf_path, global_conf=global_conf) # set utils.FALLOCATE_RESERVE if desired utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ utils.config_fallocate_value(conf.get('fallocate_reserve', '1%')) # Start listening on bind_addr/port error_msg = strategy.do_bind_ports() if error_msg: logger.error(error_msg) print(error_msg) return 1 # Redirect errors to logger and close stdio. Do this *after* binding ports; # we use this to signal that the service is ready to accept connections. capture_stdio(logger) no_fork_sock = strategy.no_fork_sock() if no_fork_sock: run_server(conf, logger, no_fork_sock, global_conf=global_conf) return 0 def kill_children(*args): """Kills the entire process group.""" logger.error('SIGTERM received') signal.signal(signal.SIGTERM, signal.SIG_IGN) running[0] = False os.killpg(0, signal.SIGTERM) def hup(*args): """Shuts down the server, but allows running requests to complete""" logger.error('SIGHUP received') signal.signal(signal.SIGHUP, signal.SIG_IGN) running[0] = False running = [True] signal.signal(signal.SIGTERM, kill_children) signal.signal(signal.SIGHUP, hup) while running[0]: for sock, sock_info in strategy.new_worker_socks(): pid = os.fork() if pid == 0: signal.signal(signal.SIGHUP, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) strategy.post_fork_hook() run_server(conf, logger, sock) strategy.log_sock_exit(sock, sock_info) return 0 else: strategy.register_worker_start(sock, sock_info, pid) # The strategy may need to pay attention to something in addition to # child process exits (like new ports showing up in a ring). # # NOTE: a timeout value of None will just instantiate the Timeout # object and not actually schedule it, which is equivalent to no # timeout for the green_os.wait(). loop_timeout = strategy.loop_timeout() with Timeout(loop_timeout, exception=False): try: try: pid, status = green_os.wait() if os.WIFEXITED(status) or os.WIFSIGNALED(status): strategy.register_worker_exit(pid) except OSError as err: if err.errno not in (errno.EINTR, errno.ECHILD): raise if err.errno == errno.ECHILD: # If there are no children at all (ECHILD), then # there's nothing to actually wait on. We sleep # for a little bit to avoid a tight CPU spin # and still are able to catch any KeyboardInterrupt # events that happen. The value of 0.01 matches the # value in eventlet's waitpid(). sleep(0.01) except KeyboardInterrupt: logger.notice('User quit') running[0] = False break strategy.shutdown_sockets() logger.notice('Exited') return 0
def read(self, size): raise Timeout(1.0)
def account_info(self, account, autocreate=False): """ Get account information, and also verify that the account exists. :param account: name of the account to get the info for :returns: tuple of (account partition, account nodes, container_count) or (None, None, None) if it does not exist """ partition, nodes = self.app.account_ring.get_nodes(account) # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses if self.app.memcache: cache_key = get_account_memcache_key(account) cache_value = self.app.memcache.get(cache_key) if not isinstance(cache_value, dict): result_code = cache_value container_count = 0 else: result_code = cache_value['status'] container_count = cache_value['container_count'] if result_code == HTTP_OK: return partition, nodes, container_count elif result_code == HTTP_NOT_FOUND and not autocreate: return None, None, None result_code = 0 container_count = 0 attempts_left = len(nodes) path = '/%s' % account headers = {'x-trans-id': self.trans_id, 'Connection': 'close'} iternodes = self.iter_nodes(partition, nodes, self.app.account_ring) while attempts_left > 0: try: node = iternodes.next() except StopIteration: break attempts_left -= 1 try: with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], partition, 'HEAD', path, headers) with Timeout(self.app.node_timeout): resp = conn.getresponse() body = resp.read() if is_success(resp.status): result_code = HTTP_OK container_count = int( resp.getheader('x-account-container-count') or 0) break elif resp.status == HTTP_NOT_FOUND: if result_code == 0: result_code = HTTP_NOT_FOUND elif result_code != HTTP_NOT_FOUND: result_code = -1 elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) continue else: result_code = -1 except (Exception, Timeout): self.exception_occurred( node, _('Account'), _('Trying to get account info for %s') % path) if result_code == HTTP_NOT_FOUND and autocreate: if len(account) > MAX_ACCOUNT_NAME_LENGTH: return None, None, None headers = { 'X-Timestamp': normalize_timestamp(time.time()), 'X-Trans-Id': self.trans_id, 'Connection': 'close' } resp = self.make_requests(Request.blank('/v1' + path), self.app.account_ring, partition, 'PUT', path, [headers] * len(nodes)) if not is_success(resp.status_int): self.app.logger.warning('Could not autocreate account %r' % \ path) return None, None, None result_code = HTTP_OK if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND): if result_code == HTTP_OK: cache_timeout = self.app.recheck_account_existence else: cache_timeout = self.app.recheck_account_existence * 0.1 self.app.memcache.set(cache_key, { 'status': result_code, 'container_count': container_count }, timeout=cache_timeout) if result_code == HTTP_OK: return partition, nodes, container_count return None, None, None
def _get_suffixes_to_sync(self, job, node): """ For SYNC jobs we need to make a remote REPLICATE request to get the remote node's current suffix's hashes and then compare to our local suffix's hashes to decide which suffixes (if any) are out of sync. :param: the job dict, with the keys defined in ``_get_part_jobs`` :param node: the remote node dict :returns: a (possibly empty) list of strings, the suffixes to be synced with the remote node. """ # get hashes from the remote node remote_suffixes = None try: with Timeout(self.http_timeout): resp = http_connect(node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '', headers=self.headers).getresponse() if resp.status == HTTP_INSUFFICIENT_STORAGE: self.logger.error( _('%s responded as unmounted'), self._full_path(node, job['partition'], '', job['policy'])) elif resp.status != HTTP_OK: full_path = self._full_path(node, job['partition'], '', job['policy']) self.logger.error( _("Invalid response %(resp)s from %(full_path)s"), { 'resp': resp.status, 'full_path': full_path }) else: remote_suffixes = pickle.loads(resp.read()) except (Exception, Timeout): # all exceptions are logged here so that our caller can # safely catch our exception and continue to the next node # without logging self.logger.exception( 'Unable to get remote suffix hashes ' 'from %r' % self._full_path(node, job['partition'], '', job['policy'])) if remote_suffixes is None: raise SuffixSyncError('Unable to get remote suffix hashes') suffixes = self.get_suffix_delta(job['hashes'], job['frag_index'], remote_suffixes, node['index']) # now recalculate local hashes for suffixes that don't # match so we're comparing the latest local_suff = self._get_hashes(job['policy'], job['path'], recalculate=suffixes) suffixes = self.get_suffix_delta(local_suff, job['frag_index'], remote_suffixes, node['index']) self.suffix_count += len(suffixes) return suffixes
from . import constant from . import data import time from eventlet import Timeout timeout = Timeout(5) def getTeamInfo(team): from nba_api.stats.endpoints import teaminfocommon from nba_api.stats.static import teams team = team.replace('-', ' ').replace('LA', '').strip() with timeout: try: teamInfo = teams.find_teams_by_full_name(team)[0] except: teamInfo = 'timeout' return teamInfo def getTeamByOpponent(team1, team2): from nba_api.stats.endpoints import teamdashboardbyopponent with timeout: try: teamDash = teamdashboardbyopponent.TeamDashboardByOpponent( team_id=team1, opponent_team_id=team2, timeout=15) # eventlet.sleep(5) df = teamDash.get_data_frames()[0] df = df[constant.OPPCOLUMN] df = multiplyColumns(df, constant.PCT_COL, 100) except: timeout.cancel()