def test_single_increment(self):
"""
test that we can increment a key once
"""
partial_key = "get_request"
queue_entry = redis_queue_entry_tuple(timestamp=datetime.utcnow(),
collection_id=42,
value=12345)
# feed an entry to the queue
self._redis_queue.put((partial_key, queue_entry), )
# give the greenlet some time to start
self._halt_event.wait(1)
# verify that the key got incremented
expected_key = compute_key(_node_name, queue_entry.timestamp,
partial_key)
hash_dict = self._redis_connection.hgetall(expected_key)
items = hash_dict.items()
self.assertEqual(len(items), 1)
collection_id_bytes, count_bytes = items[0]
collection_id = int(collection_id_bytes)
count = int(count_bytes)
self.assertEqual(collection_id, queue_entry.collection_id)
self.assertEqual(count, queue_entry.value)
def test_multiple_increment(self):
"""
test that we can increment a key multiple times in a time interval
"""
partial_key = "get_request"
# create a base time, rounded off to the nearest minute
current_time = datetime.utcnow()
base_time = datetime(current_time.year, current_time.month,
current_time.day, current_time.hour,
current_time.minute)
test_range = range(10)
for index in test_range:
timestamp = base_time + timedelta(seconds=index)
queue_entry = redis_queue_entry_tuple(timestamp=timestamp,
collection_id=42,
value=index)
self._redis_queue.put((partial_key, queue_entry), )
# give the greenlet and redis some time to operate
self._halt_event.wait(1)
# verify that the key got incremented
expected_key = compute_key(_node_name, queue_entry.timestamp,
partial_key)
expected_value = sum([x for x in test_range])
hash_value = self._redis_connection.hget(expected_key,
queue_entry.collection_id)
self.assertEqual(int(hash_value), expected_value)
def test_multiple_increment(self):
"""
test that we can increment a key multiple times in a time interval
"""
partial_key = "get_request"
# create a base time, rounded off to the nearest minute
current_time = datetime.utcnow()
base_time = datetime(
current_time.year, current_time.month, current_time.day, current_time.hour, current_time.minute
)
test_range = range(10)
for index in test_range:
timestamp = base_time + timedelta(seconds=index)
queue_entry = redis_queue_entry_tuple(timestamp=timestamp, collection_id=42, value=index)
self._redis_queue.put((partial_key, queue_entry))
# give the greenlet and redis some time to operate
self._halt_event.wait(1)
# verify that the key got incremented
expected_key = compute_key(_node_name, queue_entry.timestamp, partial_key)
expected_value = sum([x for x in test_range])
hash_value = self._redis_connection.hget(expected_key, queue_entry.collection_id)
self.assertEqual(int(hash_value), expected_value)
def retrieve(self, response, timeout):
try:
return self._retrieve(response, timeout)
except Exception, instance:
self._log.error("request {0} _retrieve exception".format(self.user_request_id))
self._log.exception("request {0}".format(self.user_request_id))
queue_entry = redis_queue_entry_tuple(
timestamp=create_timestamp(), collection_id=self._collection_id, value=1
)
self._redis_queue.put(("retrieve_error", queue_entry))
response.status_int = httplib.SERVICE_UNAVAILABLE
response.retry_after = _retrieve_retry_interval
raise RetrieveFailedError(instance)
def retrieve(self, response, timeout):
try:
return self._retrieve(response, timeout)
except Exception, instance:
self._log.error("request {0} _retrieve exception".format(
self.user_request_id))
self._log.exception("request {0}".format(self.user_request_id))
queue_entry = \
redis_queue_entry_tuple(timestamp=create_timestamp(),
collection_id=self._collection_id,
value=1)
self._redis_queue.put((
"retrieve_error",
queue_entry,
))
response.status_int = httplib.SERVICE_UNAVAILABLE
response.retry_after = _retrieve_retry_interval
raise RetrieveFailedError(instance)
def test_single_increment(self):
"""
test that we can increment a key once
"""
partial_key = "get_request"
queue_entry = redis_queue_entry_tuple(timestamp=datetime.utcnow(), collection_id=42, value=12345)
# feed an entry to the queue
self._redis_queue.put((partial_key, queue_entry))
# give the greenlet some time to start
self._halt_event.wait(1)
# verify that the key got incremented
expected_key = compute_key(_node_name, queue_entry.timestamp, partial_key)
hash_dict = self._redis_connection.hgetall(expected_key)
items = hash_dict.items()
self.assertEqual(len(items), 1)
collection_id_bytes, count_bytes = items[0]
collection_id = int(collection_id_bytes)
count = int(count_bytes)
self.assertEqual(collection_id, queue_entry.collection_id)
self.assertEqual(count, queue_entry.value)
# translate version id to the form we use internally
if "version_id_marker" in kwargs:
kwargs["version_id_marker"] = self._id_translator.internal_id(
kwargs["version_id_marker"])
self._log.info("request {0}: " \
"_list_versions: collection = ({1}) {2} {3} {4}".format(
user_request_id,
collection_row["id"],
collection_row["name"],
collection_row["versioning"],
kwargs))
queue_entry = \
redis_queue_entry_tuple(timestamp=create_timestamp(),
collection_id=collection_row["id"],
value=1)
self._redis_queue.put((
"listmatch_request",
queue_entry,
))
try:
result_dict = list_versions(self._interaction_pool,
collection_row["id"],
collection_row["versioning"], **kwargs)
# segment_visibility raises ValueError if it is unhappy
except ValueError, instance:
self._log.error("request {0}: {1}".format(user_request_id,
instance))
raise exc.HTTPBadRequest(instance)
def _retrieve(self, response, timeout):
self._log.debug("request {0}: start _retrieve".format((self.user_request_id)))
self._cache_key_rows_in_memcached(self._key_rows)
self.total_file_size = sum([row["file_size"] for row in self._key_rows])
self._log.debug("total_file_size = {0}".format(self.total_file_size))
queue_entry = redis_queue_entry_tuple(timestamp=create_timestamp(), collection_id=self._collection_id, value=1)
self._redis_queue.put(("retrieve_request", queue_entry))
retrieve_bytes = 0L
self._log.debug("start key_rows loop")
first_block = True
for entry in self._generate_key_rows(self._key_rows):
key_row, block_offset, block_count, offset_into_first_block, offset_into_last_block = entry
self._log.debug(
"request {0}: {1} {2}".format(self.user_request_id, key_row["unified_id"], key_row["conjoined_part"])
)
# if a cache port is defined, and this response isn't larger than
# the configured maximum, send the request through the cache.
target_port = _web_internal_reader_port
if (
_web_internal_reader_cache_port is not None
and key_row["file_size"] <= _web_internal_reader_max_cache_size
):
target_port = _web_internal_reader_cache_port
uri = "http://{0}:{1}/data/{2}/{3}".format(
_web_internal_reader_host, target_port, key_row["unified_id"], key_row["conjoined_part"]
)
self._log.info("request {0} internally requesting {1}".format(self.user_request_id, uri))
headers = {"x-nimbus-io-user-request-id": self.user_request_id}
if block_offset > 0 and block_count is None:
headers["range"] = "bytes={0}-".format(block_offset * block_size)
headers["x-nimbus-io-expected-content-length"] = str(key_row["file_size"] - (block_offset * block_size))
expected_status = httplib.PARTIAL_CONTENT
elif block_count is not None:
headers["range"] = "bytes={0}-{1}".format(
block_offset * block_size, (block_offset + block_count) * block_size - 1
)
headers["x-nimbus-io-expected-content-length"] = str(block_count * block_size)
expected_status = httplib.PARTIAL_CONTENT
else:
headers["x-nimbus-io-expected-content-length"] = (str(key_row["file_size"]),)
expected_status = httplib.OK
request = urllib2.Request(uri, headers=headers)
self._log.debug(
"request {0} start internal; expected={1}; headers={2}".format(
self.user_request_id, repr(expected_status), headers
)
)
try:
urllib_response = urllib2.urlopen(request, timeout=timeout)
except urllib2.HTTPError, instance:
if instance.code == httplib.NOT_FOUND:
self._log.error("request {0}: got 404".format(self.user_request_id))
response.status_int = httplib.NOT_FOUND
break
if instance.code == httplib.PARTIAL_CONTENT and expected_status == httplib.PARTIAL_CONTENT:
urllib_response = instance
else:
message = "urllib2.HTTPError '{0}' '{1}'".format(instance.code, instance)
self._log.error("request {0}: exception {1}".format(self.user_request_id, message))
self._log.exception(message)
response.status_int = httplib.SERVICE_UNAVAILABLE
response.retry_after = _retrieve_retry_interval
break
except (httplib.HTTPException, socket.error) as instance:
message = "{0}, '{1}'".format(instance.__class__.__name__, instance.message)
self._log.error("request {0}: exception {1}".format(self.user_request_id, message))
self._log.exception(message)
response.status_int = httplib.SERVICE_UNAVAILABLE
response.retry_after = _retrieve_retry_interval
break
break
if prev_data is None:
if first_block:
assert len(data) > offset_into_first_block, (len(data), offset_into_first_block)
prev_data = data[offset_into_first_block:]
else:
prev_data = data
continue
self._log.debug("request {0} yielding {1} bytes".format(self.user_request_id, len(prev_data)))
yield prev_data
retrieve_bytes += len(prev_data)
prev_data = data
first_block = False
urllib_response.close()
self._log.debug("request {0} internal request complete".format(self.user_request_id))
# end - for entry in self._generate_key_rows(self._key_rows):
if response.status_int in [httplib.OK, httplib.PARTIAL_CONTENT]:
redis_entries = [("retrieve_success", 1), ("success_bytes_out", retrieve_bytes)]
else:
redis_entries = [("retrieve_error", 1), ("error_bytes_out", retrieve_bytes)]
timestamp = create_timestamp()
for key, value in redis_entries:
queue_entry = redis_queue_entry_tuple(timestamp=timestamp, collection_id=self._collection_id, value=value)
self._redis_queue.put((key, queue_entry))
def _retrieve(self, response, timeout):
self._log.debug("request {0}: start _retrieve".format(
(self.user_request_id)))
self._cache_key_rows_in_memcached(self._key_rows)
self.total_file_size = sum(
[row["file_size"] for row in self._key_rows])
self._log.debug("total_file_size = {0}".format(self.total_file_size))
queue_entry = \
redis_queue_entry_tuple(timestamp=create_timestamp(),
collection_id=self._collection_id,
value=1)
self._redis_queue.put((
"retrieve_request",
queue_entry,
))
retrieve_bytes = 0L
self._log.debug("start key_rows loop")
first_block = True
for entry in self._generate_key_rows(self._key_rows):
key_row, \
block_offset, \
block_count, \
offset_into_first_block, \
offset_into_last_block = entry
self._log.debug("request {0}: {1} {2}".format(
self.user_request_id, key_row["unified_id"],
key_row["conjoined_part"]))
# if a cache port is defined, and this response isn't larger than
# the configured maximum, send the request through the cache.
target_port = _web_internal_reader_port
if (_web_internal_reader_cache_port is not None
and key_row["file_size"] <=
_web_internal_reader_max_cache_size):
target_port = _web_internal_reader_cache_port
uri = "http://{0}:{1}/data/{2}/{3}".format(
_web_internal_reader_host, target_port, key_row["unified_id"],
key_row["conjoined_part"])
self._log.info("request {0} internally requesting {1}".format(
self.user_request_id, uri))
headers = {"x-nimbus-io-user-request-id": self.user_request_id}
if block_offset > 0 and block_count is None:
headers["range"] = \
"bytes={0}-".format(block_offset * block_size)
headers["x-nimbus-io-expected-content-length"] = \
str(key_row["file_size"] - (block_offset * block_size))
expected_status = httplib.PARTIAL_CONTENT
elif block_count is not None:
headers["range"] = \
"bytes={0}-{1}".format(
block_offset * block_size,
(block_offset + block_count) * block_size - 1)
headers["x-nimbus-io-expected-content-length"] = \
str(block_count * block_size)
expected_status = httplib.PARTIAL_CONTENT
else:
headers["x-nimbus-io-expected-content-length"] = \
str(key_row["file_size"]),
expected_status = httplib.OK
request = urllib2.Request(uri, headers=headers)
self._log.debug(
"request {0} start internal; expected={1}; headers={2}".format(
self.user_request_id, repr(expected_status), headers))
try:
urllib_response = urllib2.urlopen(request, timeout=timeout)
except urllib2.HTTPError, instance:
if instance.code == httplib.NOT_FOUND:
self._log.error("request {0}: got 404".format(
self.user_request_id))
response.status_int = httplib.NOT_FOUND
break
if instance.code == httplib.PARTIAL_CONTENT and \
expected_status == httplib.PARTIAL_CONTENT:
urllib_response = instance
else:
message = "urllib2.HTTPError '{0}' '{1}'".format(
instance.code, instance)
self._log.error("request {0}: exception {1}".format(
self.user_request_id, message))
self._log.exception(message)
response.status_int = httplib.SERVICE_UNAVAILABLE
response.retry_after = _retrieve_retry_interval
break
except (
httplib.HTTPException,
socket.error,
) as instance:
message = "{0}, '{1}'".format(instance.__class__.__name__,
instance.message)
self._log.error("request {0}: exception {1}".format(
self.user_request_id, message))
self._log.exception(message)
response.status_int = httplib.SERVICE_UNAVAILABLE
response.retry_after = _retrieve_retry_interval
break
first_block = False
urllib_response.close()
self._log.debug("request {0} internal request complete".format(
self.user_request_id))
# end - for entry in self._generate_key_rows(self._key_rows):
if response.status_int in [
httplib.OK,
httplib.PARTIAL_CONTENT,
]:
redis_entries = [("retrieve_success", 1),
("success_bytes_out", retrieve_bytes)]
else:
redis_entries = [("retrieve_error", 1),
("error_bytes_out", retrieve_bytes)]
timestamp = create_timestamp()
for key, value in redis_entries:
queue_entry = \
redis_queue_entry_tuple(timestamp=timestamp,
collection_id=self._collection_id,
value=value)
self._redis_queue.put((
key,
queue_entry,
))
variable_value = urllib.unquote_plus(variable_value)
variable_value = variable_value.decode("utf-8")
kwargs[variable_name] = variable_value
# translate version id to the form we use internally
if "version_id_marker" in kwargs:
kwargs["version_id_marker"] = self._id_translator.internal_id(kwargs["version_id_marker"])
self._log.info(
"request {0}: "
"_list_versions: collection = ({1}) {2} {3} {4}".format(
user_request_id, collection_row["id"], collection_row["name"], collection_row["versioning"], kwargs
)
)
queue_entry = redis_queue_entry_tuple(timestamp=create_timestamp(), collection_id=collection_row["id"], value=1)
self._redis_queue.put(("listmatch_request", queue_entry))
try:
result_dict = list_versions(
self._interaction_pool, collection_row["id"], collection_row["versioning"], **kwargs
)
# segment_visibility raises ValueError if it is unhappy
except ValueError, instance:
self._log.error("request {0}: {1}".format(user_request_id, instance))
raise exc.HTTPBadRequest(instance)
except Exception:
self._log.exception("request {0}".format(user_request_id))
queue_entry = redis_queue_entry_tuple(
timestamp=create_timestamp(), collection_id=collection_row["id"], value=1
)
timestamp = create_timestamp()
archiver = Archiver(
data_writers,
collection_row["id"],
key,
unified_id,
timestamp,
meta_dict,
conjoined_part,
user_request_id,
)
if not conjoined_archive:
queue_entry = \
redis_queue_entry_tuple(timestamp=timestamp,
collection_id=collection_row["id"],
value=1)
self._redis_queue.put(("archive_request", queue_entry, ))
data_queue = gevent.queue.Queue()
reader = ReaderGreenlet(req.body_file, data_queue)
reader.start()
segmenter = ZfecSegmenter(_min_segments, len(data_writers))
actual_content_length = 0
file_adler32 = zlib.adler32('')
file_md5 = hashlib.md5()
file_size = 0
segments = None
zfec_padding_size = None
try:
timestamp = create_timestamp()
archiver = Archiver(
data_writers,
collection_row["id"],
key,
unified_id,
timestamp,
meta_dict,
conjoined_part,
user_request_id,
)
if not conjoined_archive:
queue_entry = \
redis_queue_entry_tuple(timestamp=timestamp,
collection_id=collection_row["id"],
value=1)
self._redis_queue.put((
"archive_request",
queue_entry,
))
data_queue = gevent.queue.Queue()
reader = ReaderGreenlet(req.body_file, data_queue)
reader.start()
segmenter = ZfecSegmenter(_min_segments, len(data_writers))
actual_content_length = 0
file_adler32 = zlib.adler32('')
file_md5 = hashlib.md5()
file_size = 0
