def test_single_increment(self): """ test that we can increment a key once """ partial_key = "get_request" queue_entry = redis_queue_entry_tuple(timestamp=datetime.utcnow(), collection_id=42, value=12345) # feed an entry to the queue self._redis_queue.put((partial_key, queue_entry), ) # give the greenlet some time to start self._halt_event.wait(1) # verify that the key got incremented expected_key = compute_key(_node_name, queue_entry.timestamp, partial_key) hash_dict = self._redis_connection.hgetall(expected_key) items = hash_dict.items() self.assertEqual(len(items), 1) collection_id_bytes, count_bytes = items[0] collection_id = int(collection_id_bytes) count = int(count_bytes) self.assertEqual(collection_id, queue_entry.collection_id) self.assertEqual(count, queue_entry.value)
def test_multiple_increment(self): """ test that we can increment a key multiple times in a time interval """ partial_key = "get_request" # create a base time, rounded off to the nearest minute current_time = datetime.utcnow() base_time = datetime(current_time.year, current_time.month, current_time.day, current_time.hour, current_time.minute) test_range = range(10) for index in test_range: timestamp = base_time + timedelta(seconds=index) queue_entry = redis_queue_entry_tuple(timestamp=timestamp, collection_id=42, value=index) self._redis_queue.put((partial_key, queue_entry), ) # give the greenlet and redis some time to operate self._halt_event.wait(1) # verify that the key got incremented expected_key = compute_key(_node_name, queue_entry.timestamp, partial_key) expected_value = sum([x for x in test_range]) hash_value = self._redis_connection.hget(expected_key, queue_entry.collection_id) self.assertEqual(int(hash_value), expected_value)
def test_multiple_increment(self): """ test that we can increment a key multiple times in a time interval """ partial_key = "get_request" # create a base time, rounded off to the nearest minute current_time = datetime.utcnow() base_time = datetime( current_time.year, current_time.month, current_time.day, current_time.hour, current_time.minute ) test_range = range(10) for index in test_range: timestamp = base_time + timedelta(seconds=index) queue_entry = redis_queue_entry_tuple(timestamp=timestamp, collection_id=42, value=index) self._redis_queue.put((partial_key, queue_entry)) # give the greenlet and redis some time to operate self._halt_event.wait(1) # verify that the key got incremented expected_key = compute_key(_node_name, queue_entry.timestamp, partial_key) expected_value = sum([x for x in test_range]) hash_value = self._redis_connection.hget(expected_key, queue_entry.collection_id) self.assertEqual(int(hash_value), expected_value)
def retrieve(self, response, timeout): try: return self._retrieve(response, timeout) except Exception, instance: self._log.error("request {0} _retrieve exception".format(self.user_request_id)) self._log.exception("request {0}".format(self.user_request_id)) queue_entry = redis_queue_entry_tuple( timestamp=create_timestamp(), collection_id=self._collection_id, value=1 ) self._redis_queue.put(("retrieve_error", queue_entry)) response.status_int = httplib.SERVICE_UNAVAILABLE response.retry_after = _retrieve_retry_interval raise RetrieveFailedError(instance)
def retrieve(self, response, timeout): try: return self._retrieve(response, timeout) except Exception, instance: self._log.error("request {0} _retrieve exception".format( self.user_request_id)) self._log.exception("request {0}".format(self.user_request_id)) queue_entry = \ redis_queue_entry_tuple(timestamp=create_timestamp(), collection_id=self._collection_id, value=1) self._redis_queue.put(( "retrieve_error", queue_entry, )) response.status_int = httplib.SERVICE_UNAVAILABLE response.retry_after = _retrieve_retry_interval raise RetrieveFailedError(instance)
def test_single_increment(self): """ test that we can increment a key once """ partial_key = "get_request" queue_entry = redis_queue_entry_tuple(timestamp=datetime.utcnow(), collection_id=42, value=12345) # feed an entry to the queue self._redis_queue.put((partial_key, queue_entry)) # give the greenlet some time to start self._halt_event.wait(1) # verify that the key got incremented expected_key = compute_key(_node_name, queue_entry.timestamp, partial_key) hash_dict = self._redis_connection.hgetall(expected_key) items = hash_dict.items() self.assertEqual(len(items), 1) collection_id_bytes, count_bytes = items[0] collection_id = int(collection_id_bytes) count = int(count_bytes) self.assertEqual(collection_id, queue_entry.collection_id) self.assertEqual(count, queue_entry.value)
# translate version id to the form we use internally if "version_id_marker" in kwargs: kwargs["version_id_marker"] = self._id_translator.internal_id( kwargs["version_id_marker"]) self._log.info("request {0}: " \ "_list_versions: collection = ({1}) {2} {3} {4}".format( user_request_id, collection_row["id"], collection_row["name"], collection_row["versioning"], kwargs)) queue_entry = \ redis_queue_entry_tuple(timestamp=create_timestamp(), collection_id=collection_row["id"], value=1) self._redis_queue.put(( "listmatch_request", queue_entry, )) try: result_dict = list_versions(self._interaction_pool, collection_row["id"], collection_row["versioning"], **kwargs) # segment_visibility raises ValueError if it is unhappy except ValueError, instance: self._log.error("request {0}: {1}".format(user_request_id, instance)) raise exc.HTTPBadRequest(instance)
def _retrieve(self, response, timeout): self._log.debug("request {0}: start _retrieve".format((self.user_request_id))) self._cache_key_rows_in_memcached(self._key_rows) self.total_file_size = sum([row["file_size"] for row in self._key_rows]) self._log.debug("total_file_size = {0}".format(self.total_file_size)) queue_entry = redis_queue_entry_tuple(timestamp=create_timestamp(), collection_id=self._collection_id, value=1) self._redis_queue.put(("retrieve_request", queue_entry)) retrieve_bytes = 0L self._log.debug("start key_rows loop") first_block = True for entry in self._generate_key_rows(self._key_rows): key_row, block_offset, block_count, offset_into_first_block, offset_into_last_block = entry self._log.debug( "request {0}: {1} {2}".format(self.user_request_id, key_row["unified_id"], key_row["conjoined_part"]) ) # if a cache port is defined, and this response isn't larger than # the configured maximum, send the request through the cache. target_port = _web_internal_reader_port if ( _web_internal_reader_cache_port is not None and key_row["file_size"] <= _web_internal_reader_max_cache_size ): target_port = _web_internal_reader_cache_port uri = "http://{0}:{1}/data/{2}/{3}".format( _web_internal_reader_host, target_port, key_row["unified_id"], key_row["conjoined_part"] ) self._log.info("request {0} internally requesting {1}".format(self.user_request_id, uri)) headers = {"x-nimbus-io-user-request-id": self.user_request_id} if block_offset > 0 and block_count is None: headers["range"] = "bytes={0}-".format(block_offset * block_size) headers["x-nimbus-io-expected-content-length"] = str(key_row["file_size"] - (block_offset * block_size)) expected_status = httplib.PARTIAL_CONTENT elif block_count is not None: headers["range"] = "bytes={0}-{1}".format( block_offset * block_size, (block_offset + block_count) * block_size - 1 ) headers["x-nimbus-io-expected-content-length"] = str(block_count * block_size) expected_status = httplib.PARTIAL_CONTENT else: headers["x-nimbus-io-expected-content-length"] = (str(key_row["file_size"]),) expected_status = httplib.OK request = urllib2.Request(uri, headers=headers) self._log.debug( "request {0} start internal; expected={1}; headers={2}".format( self.user_request_id, repr(expected_status), headers ) ) try: urllib_response = urllib2.urlopen(request, timeout=timeout) except urllib2.HTTPError, instance: if instance.code == httplib.NOT_FOUND: self._log.error("request {0}: got 404".format(self.user_request_id)) response.status_int = httplib.NOT_FOUND break if instance.code == httplib.PARTIAL_CONTENT and expected_status == httplib.PARTIAL_CONTENT: urllib_response = instance else: message = "urllib2.HTTPError '{0}' '{1}'".format(instance.code, instance) self._log.error("request {0}: exception {1}".format(self.user_request_id, message)) self._log.exception(message) response.status_int = httplib.SERVICE_UNAVAILABLE response.retry_after = _retrieve_retry_interval break except (httplib.HTTPException, socket.error) as instance: message = "{0}, '{1}'".format(instance.__class__.__name__, instance.message) self._log.error("request {0}: exception {1}".format(self.user_request_id, message)) self._log.exception(message) response.status_int = httplib.SERVICE_UNAVAILABLE response.retry_after = _retrieve_retry_interval break
break if prev_data is None: if first_block: assert len(data) > offset_into_first_block, (len(data), offset_into_first_block) prev_data = data[offset_into_first_block:] else: prev_data = data continue self._log.debug("request {0} yielding {1} bytes".format(self.user_request_id, len(prev_data))) yield prev_data retrieve_bytes += len(prev_data) prev_data = data first_block = False urllib_response.close() self._log.debug("request {0} internal request complete".format(self.user_request_id)) # end - for entry in self._generate_key_rows(self._key_rows): if response.status_int in [httplib.OK, httplib.PARTIAL_CONTENT]: redis_entries = [("retrieve_success", 1), ("success_bytes_out", retrieve_bytes)] else: redis_entries = [("retrieve_error", 1), ("error_bytes_out", retrieve_bytes)] timestamp = create_timestamp() for key, value in redis_entries: queue_entry = redis_queue_entry_tuple(timestamp=timestamp, collection_id=self._collection_id, value=value) self._redis_queue.put((key, queue_entry))
def _retrieve(self, response, timeout): self._log.debug("request {0}: start _retrieve".format( (self.user_request_id))) self._cache_key_rows_in_memcached(self._key_rows) self.total_file_size = sum( [row["file_size"] for row in self._key_rows]) self._log.debug("total_file_size = {0}".format(self.total_file_size)) queue_entry = \ redis_queue_entry_tuple(timestamp=create_timestamp(), collection_id=self._collection_id, value=1) self._redis_queue.put(( "retrieve_request", queue_entry, )) retrieve_bytes = 0L self._log.debug("start key_rows loop") first_block = True for entry in self._generate_key_rows(self._key_rows): key_row, \ block_offset, \ block_count, \ offset_into_first_block, \ offset_into_last_block = entry self._log.debug("request {0}: {1} {2}".format( self.user_request_id, key_row["unified_id"], key_row["conjoined_part"])) # if a cache port is defined, and this response isn't larger than # the configured maximum, send the request through the cache. target_port = _web_internal_reader_port if (_web_internal_reader_cache_port is not None and key_row["file_size"] <= _web_internal_reader_max_cache_size): target_port = _web_internal_reader_cache_port uri = "http://{0}:{1}/data/{2}/{3}".format( _web_internal_reader_host, target_port, key_row["unified_id"], key_row["conjoined_part"]) self._log.info("request {0} internally requesting {1}".format( self.user_request_id, uri)) headers = {"x-nimbus-io-user-request-id": self.user_request_id} if block_offset > 0 and block_count is None: headers["range"] = \ "bytes={0}-".format(block_offset * block_size) headers["x-nimbus-io-expected-content-length"] = \ str(key_row["file_size"] - (block_offset * block_size)) expected_status = httplib.PARTIAL_CONTENT elif block_count is not None: headers["range"] = \ "bytes={0}-{1}".format( block_offset * block_size, (block_offset + block_count) * block_size - 1) headers["x-nimbus-io-expected-content-length"] = \ str(block_count * block_size) expected_status = httplib.PARTIAL_CONTENT else: headers["x-nimbus-io-expected-content-length"] = \ str(key_row["file_size"]), expected_status = httplib.OK request = urllib2.Request(uri, headers=headers) self._log.debug( "request {0} start internal; expected={1}; headers={2}".format( self.user_request_id, repr(expected_status), headers)) try: urllib_response = urllib2.urlopen(request, timeout=timeout) except urllib2.HTTPError, instance: if instance.code == httplib.NOT_FOUND: self._log.error("request {0}: got 404".format( self.user_request_id)) response.status_int = httplib.NOT_FOUND break if instance.code == httplib.PARTIAL_CONTENT and \ expected_status == httplib.PARTIAL_CONTENT: urllib_response = instance else: message = "urllib2.HTTPError '{0}' '{1}'".format( instance.code, instance) self._log.error("request {0}: exception {1}".format( self.user_request_id, message)) self._log.exception(message) response.status_int = httplib.SERVICE_UNAVAILABLE response.retry_after = _retrieve_retry_interval break except ( httplib.HTTPException, socket.error, ) as instance: message = "{0}, '{1}'".format(instance.__class__.__name__, instance.message) self._log.error("request {0}: exception {1}".format( self.user_request_id, message)) self._log.exception(message) response.status_int = httplib.SERVICE_UNAVAILABLE response.retry_after = _retrieve_retry_interval break
first_block = False urllib_response.close() self._log.debug("request {0} internal request complete".format( self.user_request_id)) # end - for entry in self._generate_key_rows(self._key_rows): if response.status_int in [ httplib.OK, httplib.PARTIAL_CONTENT, ]: redis_entries = [("retrieve_success", 1), ("success_bytes_out", retrieve_bytes)] else: redis_entries = [("retrieve_error", 1), ("error_bytes_out", retrieve_bytes)] timestamp = create_timestamp() for key, value in redis_entries: queue_entry = \ redis_queue_entry_tuple(timestamp=timestamp, collection_id=self._collection_id, value=value) self._redis_queue.put(( key, queue_entry, ))
variable_value = urllib.unquote_plus(variable_value) variable_value = variable_value.decode("utf-8") kwargs[variable_name] = variable_value # translate version id to the form we use internally if "version_id_marker" in kwargs: kwargs["version_id_marker"] = self._id_translator.internal_id(kwargs["version_id_marker"]) self._log.info( "request {0}: " "_list_versions: collection = ({1}) {2} {3} {4}".format( user_request_id, collection_row["id"], collection_row["name"], collection_row["versioning"], kwargs ) ) queue_entry = redis_queue_entry_tuple(timestamp=create_timestamp(), collection_id=collection_row["id"], value=1) self._redis_queue.put(("listmatch_request", queue_entry)) try: result_dict = list_versions( self._interaction_pool, collection_row["id"], collection_row["versioning"], **kwargs ) # segment_visibility raises ValueError if it is unhappy except ValueError, instance: self._log.error("request {0}: {1}".format(user_request_id, instance)) raise exc.HTTPBadRequest(instance) except Exception: self._log.exception("request {0}".format(user_request_id)) queue_entry = redis_queue_entry_tuple( timestamp=create_timestamp(), collection_id=collection_row["id"], value=1 )
timestamp = create_timestamp() archiver = Archiver( data_writers, collection_row["id"], key, unified_id, timestamp, meta_dict, conjoined_part, user_request_id, ) if not conjoined_archive: queue_entry = \ redis_queue_entry_tuple(timestamp=timestamp, collection_id=collection_row["id"], value=1) self._redis_queue.put(("archive_request", queue_entry, )) data_queue = gevent.queue.Queue() reader = ReaderGreenlet(req.body_file, data_queue) reader.start() segmenter = ZfecSegmenter(_min_segments, len(data_writers)) actual_content_length = 0 file_adler32 = zlib.adler32('') file_md5 = hashlib.md5() file_size = 0 segments = None zfec_padding_size = None try:
timestamp = create_timestamp() archiver = Archiver( data_writers, collection_row["id"], key, unified_id, timestamp, meta_dict, conjoined_part, user_request_id, ) if not conjoined_archive: queue_entry = \ redis_queue_entry_tuple(timestamp=timestamp, collection_id=collection_row["id"], value=1) self._redis_queue.put(( "archive_request", queue_entry, )) data_queue = gevent.queue.Queue() reader = ReaderGreenlet(req.body_file, data_queue) reader.start() segmenter = ZfecSegmenter(_min_segments, len(data_writers)) actual_content_length = 0 file_adler32 = zlib.adler32('') file_md5 = hashlib.md5() file_size = 0