def run_quarantine_range_etag(self):
container = 'container-range-%s' % uuid4()
obj = 'object-range-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(
container, obj, 'RANGE')
with open(data_file) as fp:
metadata = read_metadata(fp)
metadata['ETag'] = 'badetag'
with open(data_file) as fp:
write_metadata(fp, metadata)
for header, result in [({
'Range': 'bytes=0-2'
}, 'RAN'), ({
'Range': 'bytes=1-11'
}, 'ANGE'), ({
'Range': 'bytes=0-11'
}, 'RANGE')]:
odata = direct_client.direct_get_object(onode,
opart,
self.account,
container,
obj,
headers=header)[-1]
self.assertEquals(odata, result)
try:
resp = direct_client.direct_get_object(onode, opart, self.account,
container, obj)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def run_quarantine(self):
container = "container-%s" % uuid4()
obj = "object-%s" % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, "VERIFY")
with open(data_file) as fp:
metadata = read_metadata(fp)
metadata["ETag"] = "badetag"
with open(data_file) as fp:
write_metadata(fp, metadata)
odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1]
self.assertEquals(odata, "VERIFY")
try:
resp = direct_client.direct_get_object(onode, opart, self.account, container, obj)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def _run(self, thread):
if time.time() - self.heartbeat >= 15:
self.heartbeat = time.time()
self._log_status('GETS')
device, partition, name, container_name = random.choice(self.names)
with self.connection() as conn:
try:
if self.use_proxy:
client.get_object(self.url, self.token,
container_name, name, http_conn=conn)
else:
node = {'ip': self.ip, 'port': self.port, 'device': device}
direct_client.direct_get_object(node, partition,
self.account, container_name, name)
except client.ClientException, e:
self.logger.debug(str(e))
self.failures += 1
def run_quarantine(self):
container = 'container-%s' % uuid4()
obj = 'object-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(
container, obj, 'VERIFY')
with open(data_file) as fp:
metadata = read_metadata(fp)
metadata['ETag'] = 'badetag'
with open(data_file) as fp:
write_metadata(fp, metadata)
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
self.assertEquals(odata, 'VERIFY')
try:
resp = direct_client.direct_get_object(onode, opart, self.account,
container, obj)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def _run(self, thread):
if time.time() - self.heartbeat >= 15:
self.heartbeat = time.time()
self._log_status('GETS')
device, partition, name, container_name = random.choice(self.names)
with self.connection() as conn:
try:
if self.use_proxy:
client.get_object(self.url,
self.token,
container_name,
name,
http_conn=conn)
else:
node = {'ip': self.ip, 'port': self.port, 'device': device}
direct_client.direct_get_object(node, partition,
self.account,
container_name, name)
except client.ClientException, e:
self.logger.debug(str(e))
self.failures += 1
def run_quarantine_range_etag(self):
container = "container-range-%s" % uuid4()
obj = "object-range-%s" % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, "RANGE")
with open(data_file) as fp:
metadata = read_metadata(fp)
metadata["ETag"] = "badetag"
with open(data_file) as fp:
write_metadata(fp, metadata)
for header, result in [
({"Range": "bytes=0-2"}, "RAN"),
({"Range": "bytes=1-11"}, "ANGE"),
({"Range": "bytes=0-11"}, "RANGE"),
]:
odata = direct_client.direct_get_object(onode, opart, self.account, container, obj, headers=header)[-1]
self.assertEquals(odata, result)
try:
resp = direct_client.direct_get_object(onode, opart, self.account, container, obj)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def run_quarantine_zero_byte_get(self):
container = "container-zbyte-%s" % uuid4()
obj = "object-zbyte-%s" % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, "DATA")
with open(data_file) as fp:
metadata = read_metadata(fp)
os.unlink(data_file)
with open(data_file, "w") as fp:
write_metadata(fp, metadata)
try:
resp = direct_client.direct_get_object(
onode, opart, self.account, container, obj, conn_timeout=1, response_timeout=1
)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def run_quarantine_zero_byte_get(self):
container = 'container-zbyte-%s' % uuid4()
obj = 'object-zbyte-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
with open(data_file) as fp:
metadata = read_metadata(fp)
os.unlink(data_file)
with open(data_file, 'w') as fp:
write_metadata(fp, metadata)
try:
resp = direct_client.direct_get_object(onode,
opart,
self.account,
container,
obj,
conn_timeout=1,
response_timeout=1)
raise "Did not quarantine object"
except client.ClientException, e:
self.assertEquals(e.http_status, 404)
def test_main(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
apart, anodes = self.account_ring.get_nodes(self.account)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill(self.pids[self.port2server[node['port']]], SIGTERM)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
self.pids[self.port2server[node['port']]] = Popen([
'chase-object-server',
'/etc/chase/object-server/%d.conf' %
((node['port'] - 6000) / 10)]).pid
sleep(2)
# We've indirectly verified the handoff node has the object, but let's
# directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(another_onode, opart,
self.account, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(cnode, cpart,
self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
self.pids[self.port2server[onode['port']]] = Popen([
'chase-object-server',
'/etc/chase/object-server/%d.conf' %
((onode['port'] - 6000) / 10)]).pid
sleep(2)
exc = False
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Previously downed object server had test object')
# Run the extra server last so it'll remove it's extra partition
ps = []
for n in onodes:
ps.append(Popen(['chase-object-replicator',
'/etc/chase/object-server/%d.conf' %
((n['port'] - 6000) / 10), 'once']))
for p in ps:
p.wait()
call(['chase-object-replicator',
'/etc/chase/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once'])
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = False
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Handoff object server still had test object')
# Because POST has changed to a COPY by default, POSTs will succeed on all up
# nodes now if at least one up node has the object.
# kill(self.pids[self.port2server[onode['port']]], SIGTERM)
# client.post_object(self.url, self.token, container, obj,
# headers={'x-object-meta-probe': 'value'})
# oheaders = client.head_object(self.url, self.token, container, obj)
# if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception('Metadata incorrect, was %s' % repr(oheaders))
# exc = False
# try:
# direct_client.direct_get_object(another_onode, opart, self.account,
# container, obj)
# except Exception:
# exc = True
# if not exc:
# raise Exception('Handoff server claimed it had the object when '
# 'it should not have it')
# self.pids[self.port2server[onode['port']]] = Popen([
# 'chase-object-server',
# '/etc/chase/object-server/%d.conf' %
# ((onode['port'] - 6000) / 10)]).pid
# sleep(2)
# oheaders = direct_client.direct_get_object(onode, opart, self.account,
# container, obj)[0]
# if oheaders.get('x-object-meta-probe') == 'value':
# raise Exception('Previously downed object server had the new '
# 'metadata when it should not have it')
# # Run the extra server last so it'll remove it's extra partition
# ps = []
# for n in onodes:
# ps.append(Popen(['chase-object-replicator',
# '/etc/chase/object-server/%d.conf' %
# ((n['port'] - 6000) / 10), 'once']))
# for p in ps:
# p.wait()
# call(['chase-object-replicator',
# '/etc/chase/object-server/%d.conf' %
# ((another_onode['port'] - 6000) / 10), 'once'])
# oheaders = direct_client.direct_get_object(onode, opart, self.account,
# container, obj)[0]
# if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception(
# 'Previously downed object server did not have the new metadata')
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, obj)
exc = False
try:
client.head_object(self.url, self.token, container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Regular object HEAD was still successful')
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
self.pids[self.port2server[onode['port']]] = Popen([
'chase-object-server',
'/etc/chase/object-server/%d.conf' %
((onode['port'] - 6000) / 10)]).pid
sleep(2)
direct_client.direct_get_object(onode, opart, self.account, container,
obj)
# Run the extra server last so it'll remove it's extra partition
ps = []
for n in onodes:
ps.append(Popen(['chase-object-replicator',
'/etc/chase/object-server/%d.conf' %
((n['port'] - 6000) / 10), 'once']))
for p in ps:
p.wait()
call(['chase-object-replicator',
'/etc/chase/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once'])
exc = False
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Handoff object server still had the object')
def test_main(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
apart, anodes = self.account_ring.get_nodes(self.account)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(self.account, container,
obj)
onode = onodes[0]
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill(self.pids[self.port2server[node['port']]], SIGTERM)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
self.pids[self.port2server[node['port']]] = Popen([
'chase-object-server',
'/etc/chase/object-server/%d.conf' %
((node['port'] - 6000) / 10)
]).pid
sleep(2)
# We've indirectly verified the handoff node has the object, but let's
# directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(another_onode, opart,
self.account, container,
obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
self.pids[self.port2server[onode['port']]] = Popen([
'chase-object-server',
'/etc/chase/object-server/%d.conf' % ((onode['port'] - 6000) / 10)
]).pid
sleep(2)
exc = False
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Previously downed object server had test object')
# Run the extra server last so it'll remove it's extra partition
ps = []
for n in onodes:
ps.append(
Popen([
'chase-object-replicator',
'/etc/chase/object-server/%d.conf' %
((n['port'] - 6000) / 10), 'once'
]))
for p in ps:
p.wait()
call([
'chase-object-replicator',
'/etc/chase/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once'
])
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = False
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Handoff object server still had test object')
# Because POST has changed to a COPY by default, POSTs will succeed on all up
# nodes now if at least one up node has the object.
# kill(self.pids[self.port2server[onode['port']]], SIGTERM)
# client.post_object(self.url, self.token, container, obj,
# headers={'x-object-meta-probe': 'value'})
# oheaders = client.head_object(self.url, self.token, container, obj)
# if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception('Metadata incorrect, was %s' % repr(oheaders))
# exc = False
# try:
# direct_client.direct_get_object(another_onode, opart, self.account,
# container, obj)
# except Exception:
# exc = True
# if not exc:
# raise Exception('Handoff server claimed it had the object when '
# 'it should not have it')
# self.pids[self.port2server[onode['port']]] = Popen([
# 'chase-object-server',
# '/etc/chase/object-server/%d.conf' %
# ((onode['port'] - 6000) / 10)]).pid
# sleep(2)
# oheaders = direct_client.direct_get_object(onode, opart, self.account,
# container, obj)[0]
# if oheaders.get('x-object-meta-probe') == 'value':
# raise Exception('Previously downed object server had the new '
# 'metadata when it should not have it')
# # Run the extra server last so it'll remove it's extra partition
# ps = []
# for n in onodes:
# ps.append(Popen(['chase-object-replicator',
# '/etc/chase/object-server/%d.conf' %
# ((n['port'] - 6000) / 10), 'once']))
# for p in ps:
# p.wait()
# call(['chase-object-replicator',
# '/etc/chase/object-server/%d.conf' %
# ((another_onode['port'] - 6000) / 10), 'once'])
# oheaders = direct_client.direct_get_object(onode, opart, self.account,
# container, obj)[0]
# if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception(
# 'Previously downed object server did not have the new metadata')
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, obj)
exc = False
try:
client.head_object(self.url, self.token, container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Regular object HEAD was still successful')
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
self.pids[self.port2server[onode['port']]] = Popen([
'chase-object-server',
'/etc/chase/object-server/%d.conf' % ((onode['port'] - 6000) / 10)
]).pid
sleep(2)
direct_client.direct_get_object(onode, opart, self.account, container,
obj)
# Run the extra server last so it'll remove it's extra partition
ps = []
for n in onodes:
ps.append(
Popen([
'chase-object-replicator',
'/etc/chase/object-server/%d.conf' %
((n['port'] - 6000) / 10), 'once'
]))
for p in ps:
p.wait()
call([
'chase-object-replicator',
'/etc/chase/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once'
])
exc = False
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Handoff object server still had the object')
raise
self.container_deletes += 1
else:
part, nodes = self.object_ring.get_nodes(
info['account'], info['container'], row['name'])
shuffle(nodes)
exc = None
looking_for_timestamp = float(row['created_at'])
timestamp = -1
headers = body = None
for node in nodes:
try:
these_headers, this_body = direct_get_object(
node,
part,
info['account'],
info['container'],
row['name'],
resp_chunk_size=65536)
this_timestamp = float(these_headers['x-timestamp'])
if this_timestamp > timestamp:
timestamp = this_timestamp
headers = these_headers
body = this_body
except ClientException, err:
# If any errors are not 404, make sure we report the
# non-404 one. We don't want to mistakenly assume the
# object no longer exists just because one says so and
# the others errored for some other reason.
if not exc or exc.http_status == 404:
exc = err
if err.http_status != 404:
raise
self.container_deletes += 1
else:
part, nodes = self.object_ring.get_nodes(
info['account'], info['container'],
row['name'])
shuffle(nodes)
exc = None
looking_for_timestamp = float(row['created_at'])
timestamp = -1
headers = body = None
for node in nodes:
try:
these_headers, this_body = direct_get_object(node,
part, info['account'], info['container'],
row['name'], resp_chunk_size=65536)
this_timestamp = float(these_headers['x-timestamp'])
if this_timestamp > timestamp:
timestamp = this_timestamp
headers = these_headers
body = this_body
except ClientException, err:
# If any errors are not 404, make sure we report the
# non-404 one. We don't want to mistakenly assume the
# object no longer exists just because one says so and
# the others errored for some other reason.
if not exc or exc.http_status == 404:
exc = err
if timestamp < looking_for_timestamp:
if exc:
