def test3(self):
""" test wether will fallback only down 2 primary node """
self.skipTest("disable acorrding to current behavior")
return
proxy = BeansDBProxy([self.proxy_addr])
self.backend1.stop()
# self.backend2.stop()
key3 = 'key3'
i = 0
store4 = MCStore(self.backend4_addr)
ts_start = time.time()
fallbacked = False
while i < 2000:
i += 1
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
# time.sleep(0.1)
data3_ = store4.get(key3)
if data3_ is None:
print "store4 get nothing yet, round=", i
else:
print "fallbacked to store4 after %s tries" % (i)
fallbacked = True
self.assertEqual(data3_, data3)
break
ts_stop = time.time()
if not fallbacked:
self.fail("still not fallback to backend 4")
print "%s seconds passed" % (ts_stop - ts_start)
self.backend1.start()
self.assert_(proxy.exists("key3"))
store1 = MCStore(self.backend1_addr)
self.assert_(store1.get("key3") is None)
data3 = random_string(10)
ts_recover_start = time.time()
i = 0
recovered = False
while i < 2000:
#data3 = random_string(10)
i += 1
# time.sleep(0.1)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
data3_ = store1.get(key3)
if data3_ is None:
print "store1 get nothing yet, round=", i
else:
print "recover to store1 after %s tries, %s sec" % (
i, time.time() - ts_recover_start)
recovered = True
self.assertEqual(data3_, data3)
break
if not recovered:
self.fail("still not fallback to backend 1")
def test3(self):
""" test wether will fallback only down 2 primary node """
self.skipTest("disable acorrding to current behavior")
return
proxy = BeansDBProxy([self.proxy_addr])
self.backend1.stop()
# self.backend2.stop()
key3 = 'key3'
i = 0
store4 = MCStore(self.backend4_addr)
ts_start = time.time()
fallbacked = False
while i < 2000:
i += 1
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
# time.sleep(0.1)
data3_ = store4.get(key3)
if data3_ is None:
print "store4 get nothing yet, round=", i
else:
print "fallbacked to store4 after %s tries" % (i)
fallbacked = True
self.assertEqual(data3_, data3)
break
ts_stop = time.time()
if not fallbacked:
self.fail("still not fallback to backend 4")
print "%s seconds passed" % (ts_stop - ts_start)
self.backend1.start()
self.assert_(proxy.exists("key3"))
store1 = MCStore(self.backend1_addr)
self.assert_(store1.get("key3") is None)
data3 = random_string(10)
ts_recover_start = time.time()
i = 0
recovered = False
while i < 2000:
#data3 = random_string(10)
i += 1
# time.sleep(0.1)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
data3_ = store1.get(key3)
if data3_ is None:
print "store1 get nothing yet, round=", i
else:
print "recover to store1 after %s tries, %s sec" % (i, time.time() - ts_recover_start)
recovered = True
self.assertEqual(data3_, data3)
break
if not recovered:
self.fail("still not fallback to backend 1")
def test_head_object_if_match(self):
size = 1024 * 256
self.assert_head_bucket_result(result=self.s3.head_bucket(
Bucket=env.BUCKET))
key = KEY_PREFIX + random_string(16)
body = random_bytes(size)
expect_md5 = compute_md5(body)
self.assert_put_object_result(result=self.s3.put_object(
Bucket=env.BUCKET, Key=key, Body=body),
etag=expect_md5)
self.assert_head_object_result(result=self.s3.head_object(
Bucket=env.BUCKET, Key=key, IfMatch=expect_md5),
etag=expect_md5,
content_length=size)
try:
fake_etag = '1b2cf535f27731c974343645a3985328'
self.s3.head_object(Bucket=env.BUCKET, Key=key, IfMatch=fake_etag)
self.fail() # Non exception occurred is illegal.
except Exception as e:
# Error code 412 is legal.
self.assert_client_error(error=e, expect_status_code=412)
self.assert_delete_object_result(
result=self.s3.delete_object(Bucket=env.BUCKET, Key=key))
self.assert_delete_object_result(
result=self.s3.delete_object(Bucket=env.BUCKET, Key=KEY_PREFIX))
def test_head_object_if_modified_since(self):
size = 1024 * 256
self.assert_head_bucket_result(result=self.s3.head_bucket(
Bucket=env.BUCKET))
key = KEY_PREFIX + random_string(16)
body = random_bytes(size)
expect_md5 = compute_md5(body)
self.assert_put_object_result(result=self.s3.put_object(
Bucket=env.BUCKET, Key=key, Body=body),
etag=expect_md5)
self.assert_head_object_result(result=self.s3.head_object(
Bucket=env.BUCKET,
Key=key,
IfModifiedSince=datetime.datetime(1946, 2, 14)),
etag=expect_md5,
content_length=size)
try:
self.s3.head_object(Bucket=env.BUCKET,
Key=key,
IfModifiedSince=datetime.datetime.now())
self.fail() # Non exception occurred is illegal.
except Exception as e:
# Error code 304 is legal.
self.assert_client_error(error=e, expect_status_code=304)
self.assert_delete_object_result(
result=self.s3.delete_object(Bucket=env.BUCKET, Key=key))
self.assert_delete_object_result(
result=self.s3.delete_object(Bucket=env.BUCKET, Key=KEY_PREFIX))
def test_put_object_conflict_scene2(self):
"""
This test tests response when target key exists but file mode conflict with expect.
:return:
"""
key = KEY_PREFIX + random_string(16)
empty_content_etag = 'd41d8cd98f00b204e9800998ecf8427e'
self.assert_put_object_result(result=self.s3.put_object(Bucket=BUCKET,
Key=key),
etag=empty_content_etag)
self.assert_head_object_result(result=self.s3.head_object(
Bucket=BUCKET, Key=key),
etag=empty_content_etag)
try:
directory_mime = 'application/directory'
self.s3.put_object(Bucket=BUCKET,
Key=key,
ContentType=directory_mime)
self.fail()
except Exception as e:
self.assert_client_error(error=e, expect_status_code=409)
pass
finally:
self.assert_delete_objects_result(result=self.s3.delete_objects(
Bucket=BUCKET,
Delete={'Objects': [{
'Key': key
}, {
'Key': KEY_PREFIX
}]}))
def test1(self):
data1 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self.assert_(MCStore(self.backend2_addr).exists('key1'))
cmd = "ls -l /proc/%s/fd" % (self.proxy_p.pid)
print cmd
print subprocess.check_output(cmd, shell=True)
print "move log"
if os.path.exists(self.accesslog_bak):
os.remove(self.accesslog_bak)
if os.path.exists(self.errorlog_bak):
os.remove(self.errorlog_bak)
os.rename(self.accesslog, self.accesslog_bak)
os.rename(self.errorlog, self.errorlog_bak)
print "write more data to see if new log not exists"
data1 = random_string(10)
proxy.set('key1', data1)
self.assert_(not os.path.exists(self.accesslog))
self.assert_(not os.path.exists(self.errorlog))
time.sleep(5)
print "send SIGINT signal, should re-open log file"
os.kill(self.proxy_p.pid, signal.SIGINT)
cmd = "ls -l /proc/%s/fd" % (self.proxy_p.pid)
print subprocess.check_output(cmd, shell=True)
s = os.stat(self.accesslog)
self.assert_(os.path.exists(self.accesslog))
self.assert_(os.path.exists(self.errorlog))
print "see if write to new accesslog"
proxy.get('key1')
time.sleep(1)
s_new = os.stat(self.accesslog)
print s_new.st_size, s.st_size
self.assert_(s_new.st_size > s.st_size)
def add_node(self,
u_of_edge,
shape,
names,
initializer=tf.glorot_normal_initializer(),
regularizer=None,
shared=None,
collections=None):
""" Creates a node with dangling edges defined by shape
Internally, these creates dummy nodes on these dangling edges
:param u_of_edge: Name of the node e.g. "A"
:param shape: Dimensions of the exposed indices
:param names: Name of the open indices e.g. "W" for width
:param initializer: Initialization strategy
:param regularizer: If a regularization term, for example L2 norm, weight decay
:param shared: (boolean) If the weight is shared across layers
:param collections: Used if you want to group tensorflow variables
:param shared: When creating the tensorflow variable, ignore the Graph name scope
"""
if self._is_compiled:
raise Exception(
"Unable to add more edge/nodes once the graph is compiled")
assert len(shape) == len(names), "Must have a name for each open index"
if not self._graph.has_node(u_of_edge):
# TODO: How can we integrate shared property (share weights across layers)
self._graph.add_node(u_of_edge,
dummy_node=False,
initializer=initializer,
regularizer=regularizer,
shared=shared,
collections=collections
) # Make it possible to share (shared=shared)
# Create a dummy node for each of the exposed indices
dummy_node_names = []
for i in range(len(shape)):
dummy_node_names.append(random_string())
self._graph.add_node(dummy_node_names[i],
dummy_node=True,
initializer=None,
regularizer=None,
shared=None,
collections=None)
# Now connect to the dummy nodes
for i in range(len(shape)):
self._graph.add_edge(u_of_edge,
dummy_node_names[i],
weight=shape[i],
name=names[i])
# So can chain operations
return self
def run():
key = KEY_PREFIX + random_string(16)
metadata = {
random_string(8).lower(): random_string(16),
random_string(8).lower(): random_string(16)
}
self.assert_put_object_result(result=self.s3.put_object(
Bucket=BUCKET, Key=key, Metadata=metadata))
self.assert_head_object_result(result=self.s3.head_object(
Bucket=BUCKET, Key=key),
metadata=metadata)
self.assert_delete_objects_result(result=self.s3.delete_objects(
Bucket=BUCKET,
Delete={'Objects': [{
'Key': key
}, {
'Key': KEY_PREFIX
}]}))
def test_object_tagging(self):
key = KEY_PREFIX + random_string(16)
init_tag_set = generate_tag_set()
result = self.s3.put_object(Bucket=env.BUCKET, Key=key, Tagging=encode_tag_set(init_tag_set))
self.assert_put_object_result(result=result)
etag = result['ETag']
self.assert_head_object_result(
result=self.s3.head_object(Bucket=env.BUCKET, Key=key),
etag=etag,
content_length=0)
self.assert_get_tagging_result(
result=self.s3.get_object_tagging(Bucket=env.BUCKET, Key=key),
expect_tag_set=init_tag_set)
def run():
tag_set = generate_tag_set(size=4, key_length=8, value_length=16)
self.assert_put_tagging_result(
result=self.s3.put_object_tagging(
Bucket=env.BUCKET,
Key=key,
Tagging={'TagSet': tag_set}))
self.assert_get_tagging_result(
result=self.s3.get_object_tagging(Bucket=env.BUCKET, Key=key),
expect_tag_set=tag_set)
self.assert_delete_tagging_result(
result=self.s3.delete_object_tagging(Bucket=env.BUCKET, Key=key))
self.assert_get_tagging_result(
result=self.s3.get_object_tagging(Bucket=env.BUCKET, Key=key),
expect_tag_set=empty_tag_set)
test_count = 50
count = 0
while count < test_count:
run()
count += 1
# Clean up test data
self.assert_delete_objects_result(
result=self.s3.delete_objects(
Bucket=env.BUCKET,
Delete={
'Objects': [
{'Key': key},
{'Key': KEY_PREFIX}
]
}
)
)
def test2(self):
data1 = random_string(10)
data2 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self._assert_data(self.backend4_addr, 'key1', None, "temporary node should not have the key when all primary nodes is good")
proxy.delete('key2')
print "down backend1 and backend2, proxy.get should be ok"
self.backend1.stop()
self.backend2.stop()
proxy.set('key2', data2)
self.assertEqual(proxy.get('key2'), data2)
self._assert_data(self.proxy_addr, 'key2', data2)
with self.assertRaises(Exception) as exc:
MCStore(self.backend1_addr).get('key2')
MCStore(self.backend2_addr).get('key2')
self._assert_data(self.backend3_addr, 'key2', data2)
#"temporary node should have the key when primary nodes < 2"
self._assert_data(self.backend4_addr, 'key2', data2)
print "test delete under bad sistuation, will raise error according to current behavior"
with self.assertRaises(Exception) as exc:
proxy.delete('key2')
self._assert_data(self.backend3_addr, 'key2', None)
self._assert_data(self.backend4_addr, 'key2', None)
print "start backend2, (backend1 still down), test delete"
self.backend2.start()
time.sleep(10)
proxy.delete('key2')
self._assert_data(self.proxy_addr, 'key2', None)
self._assert_data(self.backend2_addr, 'key2', None)
self._assert_data(self.backend3_addr, 'key2', None)
self._assert_data(self.backend4_addr, 'key2', None)
def test1(self):
data1 = random_string(10)
data2 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self.assert_(MCStore(self.backend2_addr).exists('key1'))
print "down backend2, proxy.get should be ok"
self.backend2.stop()
proxy.delete('key2')
self.assert_(not proxy.exists('key2'))
self.assert_(not MCStore(self.backend1_addr).exists('key2'))
self.assert_(not MCStore(self.backend3_addr).exists('key2'))
proxy.set('key2', data2)
self.assertEqual(proxy.get('key2'), data2)
self.assert_(proxy.exists('key2'))
self.assert_(MCStore(self.backend3_addr).exists('key2'))
self.assert_(MCStore(self.backend1_addr).exists('key2'))
self._assert_data(self.proxy_addr, 'key2', data2)
self._assert_data(self.backend1_addr, 'key2', data2)
with self.assertRaises(Exception) as exc:
MCStore(self.backend2_addr).get('key2')
self._assert_data(self.backend3_addr, 'key2', data2)
print "down backend1, proxy.get/set should fail"
self.backend1.stop()
self.assertEqual(proxy.get('key1'), data1)
with self.assertRaises(Exception) as exc:
MCStore(self.backend1_addr).get('key2')
MCStore(self.backend2_addr).get('key2')
with self.assertRaises(Exception) as exc:
proxy.set('key2', data2)
def __do_test_put_objects_independent(self, file_size, file_num):
"""
Put multiple objects with different object keys.
Process:
:type file_size: int
:type file_num: int
:param file_size: numeric value, size of file (unit: byte)
:param file_num: numeric value, number of file to put
:return:
"""
file_names = []
file_name_prefix = random_string(16)
for i in range(file_num):
file_name = "%s_%d" % (file_name_prefix, i)
self.__do_test_put_objects(file_name=file_name,
file_size=file_size,
file_num=1)
file_names.append(file_name)
# check files
matches = 0
marker = ''
truncated = True
while truncated:
response = self.s3.list_objects(Bucket=BUCKET,
Prefix=file_name_prefix,
Marker=marker,
MaxKeys=100)
self.assertEqual(response['ResponseMetadata']['HTTPStatusCode'],
200)
if 'Contents' in response:
contents = response['Contents']
self.assertTrue(type(contents), list)
matches = matches + len(contents)
if 'NextMarker' in response:
next_marker = response['NextMarker']
if next_marker != '':
marker = next_marker
if 'IsTruncated' in response:
truncated = bool(response['IsTruncated'])
self.assertEqual(matches, file_num)
# batch delete objects
objects = []
for file_name in file_names:
objects.append({"Key": file_name})
delete = {"Objects": objects}
self.assert_delete_objects_result(
result=self.s3.delete_objects(Bucket=BUCKET, Delete=delete))
# check deletion result
response = self.s3.list_objects(Bucket=BUCKET, Prefix=file_name_prefix)
self.assertEqual(response['ResponseMetadata']['HTTPStatusCode'], 200)
self.assertFalse('Contents' in response)
def test_list_object_v2_etag(self):
file_num = 40
files = {} # key -> etag
for _ in range(file_num):
key = KEY_PREFIX + random_string(16)
result = self.s3.put_object(Bucket=env.BUCKET,
Key=key,
Body=random_bytes(16))
self.assert_put_object_result(result=result)
files[key] = result['ETag'].strip('"')
# validate list result
contents = []
continuation_token = ''
truncated = True
while truncated:
result = self.s3.list_objects_v2(
Bucket=env.BUCKET,
Prefix=KEY_PREFIX,
ContinuationToken=continuation_token,
MaxKeys=30)
self.assertEqual(result['ResponseMetadata']['HTTPStatusCode'], 200)
if 'Contents' in result:
result_contents = result['Contents']
self.assertTrue(type(result_contents), list)
contents = contents + result_contents
if 'NextContinuationToken' in result:
next_token = result['NextContinuationToken']
if next_token != '':
continuation_token = next_token
if 'IsTruncated' in result:
truncated = bool(result['IsTruncated'])
else:
truncated = False
for content in contents:
key = content['Key']
etag = content['ETag'].strip('"')
if not key.endswith('/'):
# validate etag with source
self.assertEqual(etag, files[key])
# validate etag with head result
self.assert_head_object_result(self.s3.head_object(
Bucket=env.BUCKET, Key=key),
etag=etag)
# clean up test data
objects = []
for content in contents:
objects.append({'Key': content['Key']})
self.s3.delete_objects(Bucket=env.BUCKET, Delete={'Objects': objects})
def test_iptable_silence(self):
""" test wether a node slow will affect response time """
proxy = BeansDBProxy([self.proxy_addr])
key3 = 'key3'
i = 0
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
i = 0
self._iptable_block(self.backend1_addr)
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
self._iptable_unblock(self.backend1_addr)
def test_iptable_silence(self):
""" test wether a node slow will affect response time """
proxy = BeansDBProxy([self.proxy_addr])
key3 = 'key3'
i = 0
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
i = 0
self._iptable_block(self.backend1_addr)
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
self._iptable_unblock(self.backend1_addr)
def test_big_value(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "largekey"
size = 10 * 1024 * 1024
rsize = (((size + len(key) + 24) >> 8) + 1) << 8
string_large = random_string(size)
assert(store.set(key, string_large))
assert(store.get(key) == string_large)
self.append(rsize)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
assert(store.set(key, 'aaa'))
self.append(256)
self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
def test_big_value(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "largekey"
size = 10 * 1024 * 1024
rsize = (((size + len(key) + 24) >> 8) + 1) << 8
string_large = random_string(size)
assert (store.set(key, string_large))
assert (store.get(key) == string_large)
self.append(rsize)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
assert (store.set(key, 'aaa'))
self.append(256)
self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
def __do_test_put_objects_override(self, file_size, file_num):
"""
Put and override the same file multiple times.
Process:
1. Put and override file.
2. Delete file created by this process.
:type file_size: int
:type file_num: int
:param file_size: numeric value, size of file (unit: byte)
:param file_num: numeric value, number of file to put
:return: None
"""
file_name = random_string(16)
self.__do_test_put_objects(file_name=file_name,
file_size=file_size,
file_num=file_num)
# delete object
self.assert_delete_object_result(
result=self.s3.delete_object(Bucket=BUCKET, Key=file_name))
def __test_transfer(self, size):
name = random_string(16)
key = KEY_PREFIX + name
local_filename = os.path.join('/tmp', name)
expect_md5 = generate_file(path=local_filename, size=size)
# Upload parallel
f = open(local_filename, 'rb')
future = self.tm.upload(fileobj=f, bucket=BUCKET, key=key)
result = wait_future_done(future, timeout=90)
self.assertTrue(result)
f.close()
# Checking remote file stat
self.assert_head_object_result(result=self.s3.head_object(
Bucket=BUCKET, Key=key),
content_length=size)
# Download parallel
download_filename = local_filename + "_dl"
f = open(download_filename, 'wb+')
future = self.tm.download(fileobj=f, bucket=BUCKET, key=key)
result = wait_future_done(future, timeout=90)
self.assertTrue(result)
f.flush()
# Checking download file
f.seek(0)
actual_md5 = compute_md5(f.read())
f.close()
self.assertEqual(actual_md5, expect_md5)
# Remove remote and local files
os.remove(local_filename)
os.remove(download_filename)
self.assert_delete_object_result(
result=self.s3.delete_object(Bucket=BUCKET, Key=key))
self.assert_delete_object_result(
result=self.s3.delete_object(Bucket=BUCKET, Key=KEY_PREFIX))
def test_head_object(self):
size = 1024 * 256
self.assert_head_bucket_result(self.s3.head_bucket(Bucket=env.BUCKET))
key = KEY_PREFIX + random_string(16)
body = random_bytes(size)
expect_md5 = compute_md5(body)
self.assert_put_object_result(result=self.s3.put_object(
Bucket=env.BUCKET, Key=key, Body=body),
etag=expect_md5)
self.assert_head_object_result(result=self.s3.head_object(
Bucket=env.BUCKET, Key=key),
etag=expect_md5,
content_length=size)
self.assert_delete_object_result(
result=self.s3.delete_object(Bucket=env.BUCKET, Key=key))
self.assert_delete_object_result(
result=self.s3.delete_object(Bucket=env.BUCKET, Key=KEY_PREFIX))
try:
self.s3.head_object(Bucket=env.BUCKET, Key=key)
self.fail() # Non exception occurred is illegal.
except Exception as e:
# Error code 404 is legal.
self.assert_client_error(e, expect_status_code=404)
def __init__(self, case):
super(ObjectGetRangeTest, self).__init__(case)
self.s3 = get_env_s3_client()
self.file_size = 10000
self.file_key = KEY_PREFIX + random_string(16)
self.test_cases = [
{ "range":"bytes=0-499", "status_code": 206, "content-range":"bytes 0-499/10000", "content-length": 500 },
{ "range":"bytes=500-999", "status_code": 206, "content-range":"bytes 500-999/10000", "content-length": 500 },
{ "range":"bytes=9500-", "status_code": 206, "content-range":"bytes 9500-9999/10000", "content-length": 500 },
{ "range":"bytes=0-", "status_code": 206, "content-range":"bytes 0-9999/10000", "content-length": 10000 },
{ "range":"bytes=0-0", "status_code": 206, "content-range":"bytes 0-0/10000", "content-length": 1 },
{ "range":"bytes=-500", "status_code": 206, "content-range":"bytes 9500-9999/10000", "content-length": 500 },
{ "range":"bytes=-1", "status_code": 206, "content-range":"bytes 9999-9999/10000", "content-length": 1 },
{ "range":"bytes=-0", "status_code": 206, "content-range":"bytes 0-9999/10000", "content-length": 10000 },
{ "range":"bytes=1-0", "status_code": 416 },
{ "range":"bytes=10", "status_code": 416 },
{ "range":"bytes=", "status_code": 416 },
{ "range":"bytes=abc", "status_code": 416 },
{ "range":"bytes=abc-123", "status_code": 416 },
{ "range":"1-0", "status_code": 416 },
]
self._init_object()
def test_put_directory(self):
key = random_string(16)
content_type = 'application/directory'
md5 = compute_md5(bytes())
# Put a directory object
self.assert_put_object_result(result=self.s3.put_object(
Bucket=BUCKET, Key=key, ContentType=content_type),
etag=md5)
# Get the directory info
self.assert_head_object_result(result=self.s3.head_object(
Bucket=BUCKET, Key=key + '/'),
etag=md5,
content_type=content_type,
content_length=0)
# Get directory object
self.assert_get_object_result(result=self.s3.get_object(Bucket=BUCKET,
Key=key + '/'),
etag=md5,
content_type=content_type,
content_length=0,
body_md5=md5)
# Delete the directory
self.assert_delete_object_result(
result=self.s3.delete_object(Bucket=BUCKET, Key=key + '/'))
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
# -*- coding: utf-8 -*-i
import env
from base import S3TestCase, get_env_s3_client
from base import random_string
KEY_PREFIX = 'test-tagging-%s/' % random_string(8)
empty_tag_set = []
def generate_tag_set(size=10, key_length=8, value_length=16):
tag_set = []
for _ in range(size):
tag_set.append({'Key': random_string(key_length), 'Value': random_string(value_length)})
return tag_set
def encode_tag_set(tag_set):
encoded = ''
for tag in tag_set:
if len(encoded) > 0:
def generate_tag_set(size=10, key_length=8, value_length=16):
tag_set = []
for _ in range(size):
tag_set.append({'Key': random_string(key_length), 'Value': random_string(value_length)})
return tag_set
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
# -*- coding: utf-8 -*-
import os
from boto3.s3.transfer import TransferManager, TransferConfig
from base import S3TestCase
from base import random_string, compute_md5, generate_file, wait_future_done, get_env_s3_client
from env import BUCKET
KEY_PREFIX = 'test-transfer-%s/' % random_string(8)
class TransferTest(S3TestCase):
def __init__(self, case):
super(TransferTest, self).__init__(case)
self.s3 = get_env_s3_client()
tc = TransferConfig(multipart_threshold=5 * 1024 * 1024,
max_concurrency=10,
multipart_chunksize=5 * 1024 * 1024,
num_download_attempts=5,
max_io_queue=100,
io_chunksize=262144,
use_threads=True)
self.tm = TransferManager(self.s3, tc)
#!/usr/bin/env python
# coding:utf-8
import os
import sys
import time
from base import BeansdbInstance, TestBeansdbBase, MCStore
from base import check_data_hint_integrity, delete_hint_and_htree, random_string
import unittest
string_large = random_string(10*1024*1024)
class TestGenerateData(TestBeansdbBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
def test_gen_data(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
self.assert_(store.set("largekey", string_large))
self.assert_(store.get("largekey") == string_large)
loop_num = 16 * 1024
for i in xrange(loop_num):
def test_list_object_page_v2(self):
self.clear_data()
time.sleep(5)
files = {} # key -> etag
file_keys = []
prefix_keys = {}
for test_file in self.test_files:
for _ in range(test_file.get('file_num', 1)):
#key = KEY_PREFIX + random_string(16)
prefix = test_file['prefix']
key = prefix + test_file['name'] + random_string(16)
if not prefix_keys.get(prefix):
prefix_keys[prefix] = []
prefix_keys.get(prefix).append(key)
test_file['key'] = key
file_keys.append({'Key': key})
result = self.s3.put_object(Bucket=env.BUCKET, Key=key, Body=random_bytes(16))
self.assert_put_object_result(result=result)
files[key] = result['ETag'].strip('"')
#print("put object key: {}".format(key))
file_keys = sorted(file_keys, key=lambda f: f['Key'])
last_key = file_keys[-1].get('Key')
prefixs = []
[ prefixs.append(f.get('prefix')) for f in self.test_files if f.get('prefix') not in prefixs ]
# validate list result
for prefix in prefixs:
prefix_keys = [ f['Key'] for f in file_keys if f['Key'].startswith(prefix) ]
contents = []
continuation_token = ''
last_marker = ''
truncated = True
while truncated:
result = self.s3.list_objects_v2(
Bucket=env.BUCKET,
Prefix=prefix,
ContinuationToken=continuation_token,
MaxKeys=self.max_page_size)
self.assertEqual(result['ResponseMetadata']['HTTPStatusCode'], 200)
if 'Contents' in result:
result_contents = result['Contents']
self.assertTrue(type(result_contents), list)
contents = contents + result_contents
if 'NextContinuationToken' in result:
next_token = result['NextContinuationToken']
if next_token != '':
last_marker = continuation_token
continuation_token = next_token
if 'IsTruncated' in result:
truncated = bool(result['IsTruncated'])
else:
truncated = False
if truncated and last_marker != '':
#print("list object truncated: last_marker {} page_first_key: {} ".format(last_marker, result_contents[0]['Key'] ))
self.assertEqual(last_marker, result_contents[0]['Key'])
# if prefix_keys[-1] != contents[-1]['Key']:
# print("{} {}", prefix_keys[-1], contents[-1]['Key'])
self.assertEqual(prefix_keys[-1], contents[-1]['Key'])
truncated = True
prefix=""
continuation_token=""
file_count = 0
while truncated:
result = self.s3.list_objects_v2(
Bucket=env.BUCKET,
Prefix=prefix,
ContinuationToken=continuation_token,
MaxKeys=self.max_page_size)
truncated = bool(result.get('IsTruncated', False))
continuation_token=result.get('NextContinuationToken', "")
for content in result.get('Contents', {}):
if not content.get('Key', "").endswith("/"):
file_count += 1
self.assertEqual(file_count, len(file_keys))
self.s3.delete_objects(
Bucket=env.BUCKET,
Delete={'Objects': file_keys}
)
def test_list_object_page_v1(self):
'''
test list object page
'''
files = {} # key -> etag
self.clear_data()
time.sleep(5)
file_keys = []
prefix_keys = {}
for test_file in self.test_files:
for _ in range(test_file.get('file_num', 1)):
#key = KEY_PREFIX + random_string(16)
prefix = test_file['prefix']
key = prefix + test_file['name'] + random_string(16)
if not prefix_keys.get(prefix):
prefix_keys[prefix] = []
prefix_keys.get(prefix).append(key)
test_file['key'] = key
file_keys.append({'Key': key})
result = self.s3.put_object(Bucket=env.BUCKET, Key=key, Body=random_bytes(16))
self.assert_put_object_result(result=result)
files[key] = result['ETag'].strip('"')
#print("put object key: {}".format(key))
file_keys = sorted(file_keys, key=lambda f: f['Key'])
last_key = file_keys[-1].get('Key')
# print("put object key count: {}, last_key: {} ".format(len(file_keys), last_key ))
prefixs = []
[ prefixs.append(f.get('prefix')) for f in self.test_files if f.get('prefix') not in prefixs ]
file_count = 0
for prefix in prefixs:
# print("prefix_keys: {} {}".format(prefix, prefix_keys.get(prefix, [])))
prefix_keys = [ f['Key'] for f in file_keys if f['Key'].startswith(prefix) ]
marker = ''
# validate list result
contents = []
truncated = True
last_marker = ''
while truncated:
result_contents = []
result = self.s3.list_objects(Bucket=env.BUCKET, Prefix=prefix, Marker=marker, MaxKeys=self.max_page_size)
self.assertEqual(result['ResponseMetadata']['HTTPStatusCode'], 200)
if 'Contents' in result:
result_contents = result['Contents']
#print("list response content type: {}".format(type(result_contents)))
# self.assertTrue(type(result_contents), list)
contents = contents + result_contents
if 'NextMarker' in result:
next_marker = result['NextMarker']
if next_marker != '':
last_marker = marker
marker = next_marker
if 'IsTruncated' in result:
truncated = bool(result['IsTruncated'])
else:
truncated = False
if not truncated:
end = True
# print("list object count: {}, maxKey: {}, prefix: {}, first_key: {}, last_key: {}, next_marker: {}, truncate: {} ".format( \
# len(result_contents), self.max_page_size, prefix, result_contents[0]['Key'], result_contents[-1]['Key'], marker, truncated))
if truncated and last_marker != '':
# print("list object truncated: last_marker {} page_first_key: {} ".format(last_marker, result_contents[0]['Key'] ))
self.assertEqual(last_marker, result_contents[0]['Key'])
# print("list object total: {}, maxKey: {}, prefix: {}, first_key: {}, last_key: {}, next_marker: {} ".format( \
# len(contents), self.max_page_size, prefix, contents[0]['Key'], contents[-1]['Key'], marker))
# if prefix_keys[-1] != contents[-1]['Key']:
# print("{} {}", prefix_keys[-1], contents[-1]['Key'])
self.assertEqual(prefix_keys[-1], contents[-1]['Key'])
truncated = True
prefix=""
marker=""
file_count = 0
while truncated:
# print("marker: {}".format(marker))
result = self.s3.list_objects(Bucket=env.BUCKET, Prefix=prefix, Marker=marker, MaxKeys=self.max_page_size)
truncated = bool(result.get('IsTruncated', False))
marker=result.get('NextMarker', "")
for content in result.get('Contents', {}):
# print("file: {}".format(content.get('Key')))
if not content.get('Key', "").endswith("/"):
file_count += 1
# print("next_marker: {} {}".format(truncated, marker))
self.assertEqual(file_count, len(file_keys))
self.s3.delete_objects(
Bucket=env.BUCKET,
Delete={'Objects': file_keys}
)
import os
import sys
import time
from base import BeansdbInstance, TestBeansdbBase, MCStore
from base import get_hash, check_data_hint_integrity
from base import random_string, delete_hint_and_htree, temper_with_key_value
import unittest
import telnetlib
import glob
import quicklz
import struct
import re
from gc_simple import TestGCBase
string_large = random_string(10 * 1024 * 1024)
class TestBrokenBase(TestGCBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
#
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901, db_depth=1)
# only generate keys in sector0
def _gen_data(self, data, prefix='', loop_num=10 * 1024, sector=0):
store = MCStore(self.backend1_addr)
