def run(self):
# try reading various ranges (these are (start, end) absolute ranges, not offset/length)
ranges = [
(0, 16384),
(1, 200),
(0, 4096),
(0, 8192),
(0, 1000),
(0, 6000),
(100, 4000),
(5000, 10000),
(4096, 10000),
(5000, 8192),
(4096, 16834),
(5000, 16384),
(4096, 16000),
(5000, 16000)
]
self.exitcode = 0
for (start, end) in ranges:
if self.errors:
break
# only clear reader's cache
testlib.clear_cache( self.config_dir, volume_id=self.volume_id, gateway_id=self.read_gateway_id )
for i in xrange(0, 2):
if self.errors:
break
exitcode, out = testlib.run( READ_PATH, '-d2', '-f', '-c', os.path.join(self.config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', self.volume_name, '-g', self.read_gateway_name,
self.output_path, start, end - start, valgrind=True )
out_name = "uncached"
if i > 0:
out_name = "cached"
testlib.save_output( self.output_dir, 'syndicate-read-thread-%s-%s-%s-%s' % (self.get_ident(), start, end, out_name), out )
if exitcode != 0:
self.exitcode = exitcode
self.errormsg = "syndicate-read exit code %s on %s-%s-%s" % (exitcode, start, end, out_name)
break
# correctness
if expected_data[start:end] not in out:
self.exitcode = -1
self.errormsg = "Thread %s missing data for %s-%s-%s" % (self.get_ident(), start, end, out_name)
break
if self.exitcode != 0:
break
return
for (start, end) in ranges:
# only clear reader's cache
testlib.clear_cache( config_dir, volume_id=volume_id, gateway_id=read_gateway_id )
# do each read twice--once uncached, and one cached
for i in xrange(0, 2):
exitcode, out = testlib.run( READ_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', read_gateway_name,
output_path, start, end - start, valgrind=True )
out_name = "uncached"
if i > 0:
out_name = "cached"
testlib.save_output( output_dir, 'syndicate-read-%s-%s-%s' % (start, end, out_name), out )
if exitcode != 0:
stop_and_save( output_dir, ag_proc, ag_out_path, "syndicate-ag")
raise Exception("%s exited %s" % (READ_PATH, exitcode))
# correctness
if expected_data[start:end] not in out:
stop_and_save( output_dir, ag_proc, ag_out_path, "syndicate-ag")
raise Exception("Missing data for %s-%s" % (start, end))
ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
testlib.save_output( output_dir, "syndicate-ag", ag_out )
if ag_exitcode != 0:
raise Exception("%s exited %s" % (AG_PATH, ag_exitcode))
exitcode, out = testlib.run(PUT_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
volume_name,
'-g',
gateway_name,
local_path,
output_path,
valgrind=True)
testlib.save_output(output_dir, "syndicate-put-%s" % i, out)
if exitcode != 0:
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
for i in xrange(0, NUM_FILES):
path = output_paths[i]
exitcode, out = testlib.run(CAT_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
# put a file to publish
local_paths = []
for i in xrange(0, 1):
local_path = testlib.make_random_file(16384, dir=inner_dir)
local_paths.append(local_path)
# start AG
AG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "AG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, AG_gateway_name, "port=31112", "driver=%s" % AG_DRIVER )
ag_proc, ag_out_path = testlib.start_gateway( config_dir, AG_PATH, testconf.SYNDICATE_ADMIN, volume_name, AG_gateway_name, valgrind=True )
time.sleep(20)
if ag_proc.poll() is not None:
ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
testlib.save_output(output_dir, "syndicate-ag", ag_out)
raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))
# should cause the AG to get updated that there's a new gateway
ug_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
# remove inner dir, and have the AG refresh
shutil.rmtree(rmdir)
exitcode, out = testlib.run( REFRESH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', ug_gateway_name, '/to_remove_1', valgrind=True )
testlib.save_output( output_dir, "syndicate-refresh", out )
if exitcode != 0:
ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
testlib.save_output(output_dir, "syndicate-ag", ag_out)
raise Exception("syndicate-refresh exit code %s" % exitcode)
if not testlib.gateway_ping( 31112, 15 ):
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# should cause the RG to get updated that there's a new gateway
gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
cat_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email="ANONYMOUS" )
random_part = hex(random.randint(0, 2**32-1))[2:]
output_paths = []
for i in xrange(0, NUM_FILES):
output_path = "/put-%s-%s" % (random_part, i)
output_paths.append(output_path)
exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, local_path, output_path, valgrind=True )
testlib.save_output( output_dir, "syndicate-put-%s" % i, out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
for i in xrange(0, NUM_FILES):
path = output_paths[i]
exitcode, out = testlib.run( CAT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', 'ANONYMOUS', '-v', volume_name, '-g', cat_gateway_name, path, valgrind=True )
testlib.save_output( output_dir, 'syndicate-cat-%s' % i, out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (CAT_PATH, exitcode))
# check for correctnes
read_gateway_info = testlib.read_gateway(config_dir, read_gateway_name)
read_gateway_id = read_gateway_info['g_id']
volume_info = testlib.read_volume(config_dir, volume_name)
volume_id = volume_info['volume_id']
random_part = hex(random.randint(0, 2**32 - 1))[2:]
output_path = "/put-%s" % random_part
exitcode, out = testlib.run(PUT_PATH, '-d3', '-f', '-c',
os.path.join(config_dir,
'syndicate.conf'), '-u',
testconf.SYNDICATE_ADMIN, '-v', volume_name,
'-g', gateway_name, local_path, output_path)
testlib.save_output(output_dir, "syndicate-put", out)
if exitcode != 0:
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
# try reading and writing various ranges (these are (start, end) absolute ranges, not offset/length)
ranges = [
(5000, 16000),
(0, 1), # 1 block, tail unaligned
(1, 200), # 1 block, unaligned head and tail
(0, 4096), # 4 block, aligned
(0, 8192), # 8 blocks, aligned
(0, 1000), # 1 block, tail unaligned
(0, 6000),
(100, 4000),
(5000, 10000),
os.makedirs(testdir)
local_path = testlib.make_random_file(16384, dir=testdir)
output_path = "/dir1/dir2/%s" % os.path.basename(local_path)
# list files that do not yet exist
# verify that we can't list them
descendent_files = ['/dir1', '/dir1/dir2', output_path]
for i in xrange(0, len(descendent_files)):
p = descendent_files[i]
exitcode, out = testlib.run( LS_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', read_gateway_name,
p, valgrind=True )
testlib.save_output(output_dir, "syndicate-ls-before-%s" % p.replace("/", "\\x2f" ), out )
if ("\nFailed to stat '%s': No such file or directory\n" % p) not in out:
raise Exception("Successfully listed '%s'" % p)
p_base = p
if p != '/':
p_base = os.path.basename(p)
if ("\nname: %s\n" % p_base) in out:
raise Exception("Found '%s' in listing" % p)
if i + 1 < len(descendent_files):
if ("name: %s" % os.path.basename(descendent_files[i+1])) in out:
raise Exception("Found '%s' in listing" % descendent_files[i+1])
# start up AG
rg_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", caps="NONE", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, rg_gateway_name, "port=31112", "driver=%s" % RG_DRIVER )
# start the RG
rg_proc, rg_out_path = testlib.start_gateway( config_dir, RG_PATH, testconf.SYNDICATE_ADMIN, volume_name, rg_gateway_name )
if not testlib.gateway_ping( 31112, 15 ):
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# put the file...
exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
random_file_path, path )
testlib.save_output( output_dir, "syndicate-put", out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("Failed to touch %s" % path)
# finish populating expected built-in xattrs
exitcode, out = testlib.run( STAT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, "syndicate.conf"),
"-u", testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', getxattr_gateway_name,
path )
testlib.save_output( output_dir, "syndicate-stat", out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("Failed to stat %s" % path)
"AG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
testlib.update_gateway(config_dir, AG_gateway_name, "port=31112",
"driver=%s" % AG_DRIVER)
ag_proc, ag_out_path = testlib.start_gateway(config_dir,
AG_PATH,
testconf.SYNDICATE_ADMIN,
volume_name,
AG_gateway_name,
valgrind=True)
time.sleep(30)
if not testlib.gateway_ping(31112, 15):
ag_exitcode, ag_out = testlib.stop_gateway(ag_proc, ag_out_path)
testlib.save_output(output_dir, "syndicate-ag", ag_out)
raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))
# should cause the AG to get updated that there's a new gateway
cat_gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"UG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
expected_data_error = False
for i in xrange(0, len(local_paths)):
# do each file twice
for j in xrange(0, 2):
local_path = local_paths[i]
expected_data = expected_datas[i]
cat_gateway_info = testlib.read_gateway( config_dir, cat_gateway_name )
cat_gateway_id = cat_gateway_info['g_id']
# look up RG
rg_gateway_info = testlib.read_gateway( config_dir, RG_gateway_name )
rg_gateway_id = rg_gateway_info['g_id']
volume_info = testlib.read_volume( config_dir, volume_name )
volume_id = volume_info['volume_id']
random_part = hex(random.randint(0, 2**32-1))[2:]
output_paths = []
# make target directory
exitcode, out = testlib.run( MKDIR_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, "/newdir", valgrind=True )
testlib.save_output( output_dir, "syndicate-mkdir", out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
# generate the commands to feed into the REPL
repl_cmd = ""
for i in xrange(0, NUM_FILES):
output_path = "/put-%s-%s" % (random_part, i)
output_paths.append(output_path)
repl_cmd += "create %s 0644\n" % output_path
repl_cmd += "write 0 0 %s %s\n" % (len(write_data), write_data)
# truncate while open
"AG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
testlib.update_gateway(config_dir, AG_gateway_name, "port=31112",
"driver=%s" % AG_DRIVER)
ag_proc, ag_out_path = testlib.start_gateway(config_dir,
AG_PATH,
testconf.SYNDICATE_ADMIN,
volume_name,
AG_gateway_name,
valgrind=True)
time.sleep(30)
if not testlib.gateway_ping(31112, 15):
ag_exitcode, ag_out = testlib.stop_gateway(ag_proc, ag_out_path)
testlib.save_output(output_dir, "syndicate-ag", ag_out)
raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))
# should cause the AG to get updated that there's a new gateway
cat_gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"UG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
exitcode, out = testlib.run(CAT_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
volume_name,
'-g',
read_gateway_name,
output_path,
start,
end - start,
valgrind=True)
out_name = "uncached"
if i > 0:
out_name = "cached"
testlib.save_output(
output_dir,
'syndicate-read-%s-%s-%s' % (start, end, out_name), out)
if exitcode != 0:
stop_and_save(output_dir, ag_proc, ag_out_path,
"syndicate-ag")
raise Exception("%s exited %s" % (READ_PATH, exitcode))
# correctness
if expected_data[start:end] not in out:
stop_and_save(output_dir, ag_proc, ag_out_path,
"syndicate-ag")
raise Exception("Missing data for %s-%s" % (start, end))
# finally, list them all
for p in [
'/', '/dir1', '/dir1/dir2', '/dir1/dir3', '/dir1/dir4', output_path
volume_name = testlib.add_test_volume(config_dir)
gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"UG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
random_part = hex(random.randint(0, 2**32 - 1))[2:]
exitcode, out = testlib.run(TOUCH_PATH, '-d2', '-f', '-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v',
volume_name, '-g', gateway_name, path)
testlib.save_output(output_dir, "syndicate-touch", out)
if exitcode != 0:
raise Exception("Failed to touch %s" % path)
# do setxattr a few times
# 1 attr
exitcode, out_1attr = testlib.run(SETXATTR_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir,
'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
for rep in xrange(0, NUM_REPS):
# touch $NUM_FILES files
random_part = hex(random.randint(0, 2**32 - 1))[2:]
expected_paths = []
for i in xrange(0, NUM_FILES):
output_path = "/touch-%s-%s" % (random_part, i)
expected_paths.append(output_path)
exitcode, out = testlib.run(TOUCH_PATH, '-d2', '-f', '-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v',
volume_name, '-g', gateway_name,
*expected_paths)
testlib.save_output(output_dir, "syndicate-touch-%s" % rep, out)
if exitcode != 0:
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (TOUCH_PATH, exitcode))
# list them
exitcode, out = testlib.run(LS_PATH, '-d2', '-f', '-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v',
volume_name, '-g', gateway_name, '/')
testlib.save_output(output_dir, "syndicate-ls-full-%s" % rep, out)
if exitcode != 0:
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (LS_PATH, exitcode))
# verify that they're all there
def run(self):
# try reading various ranges (these are (start, end) absolute ranges, not offset/length)
ranges = [(0, 16384), (1, 200), (0, 4096), (0, 8192), (0, 1000),
(0, 6000), (100, 4000), (5000, 10000), (4096, 10000),
(5000, 8192), (4096, 16834), (5000, 16384), (4096, 16000),
(5000, 16000)]
self.exitcode = 0
for (start, end) in ranges:
if self.errors:
break
# only clear reader's cache
testlib.clear_cache(self.config_dir,
volume_id=self.volume_id,
gateway_id=self.read_gateway_id)
for i in xrange(0, 2):
if self.errors:
break
exitcode, out = testlib.run(READ_PATH,
'-d2',
'-f',
'-c',
os.path.join(
self.config_dir,
'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
self.volume_name,
'-g',
self.read_gateway_name,
self.output_path,
start,
end - start,
valgrind=True)
out_name = "uncached"
if i > 0:
out_name = "cached"
testlib.save_output(
self.output_dir, 'syndicate-read-thread-%s-%s-%s-%s' %
(self.get_ident(), start, end, out_name), out)
if exitcode != 0:
self.exitcode = exitcode
self.errormsg = "syndicate-read exit code %s on %s-%s-%s" % (
exitcode, start, end, out_name)
break
# correctness
if expected_data[start:end] not in out:
self.exitcode = -1
self.errormsg = "Thread %s missing data for %s-%s-%s" % (
self.get_ident(), start, end, out_name)
break
if self.exitcode != 0:
break
return
path = '/setxattr-%s' % random_part
attr_name_base = 'foo-%s' % random_part
attr_value_base = 'bar-%s' % random_part
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume( config_dir )
gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
random_part = hex(random.randint(0, 2**32-1))[2:]
exitcode, out = testlib.run( TOUCH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
path )
testlib.save_output( output_dir, "syndicate-touch", out )
if exitcode != 0:
raise Exception("Failed to touch %s" % path)
# do setxattr a few times
# 1 attr
exitcode, out_1attr = testlib.run( SETXATTR_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
path, attr_name_base + "-1attr", attr_value_base + "-1attr", valgrind=True )
testlib.save_output( output_dir, "syndicate-setxattr-1attr", out_1attr )
if exitcode != 0:
raise Exception("%s exited %s" % (SETXATTR_PATH, exitcode))
# 5 attrs at once
if abort:
for i in xrange(0, num_threads):
threads[i].set_errors(abort)
for i in xrange(0, num_threads):
threads[i].join()
dead = True
for i in xrange(0, num_threads):
if threads[i].isAlive():
dead = False
if abort:
stop_and_save( output_dir, ag_proc, ag_out_path, "syndicate-ag")
msg = []
for i in xrange(0, num_threads):
if threads[i].get_exitcode() != 0:
msg.append("Thread %s (%s) exited %s, error = '%s'" % (threads[i].get_ident(), i, threads[i].get_exitcode(), threads[i].get_errormsg()))
raise Exception("\n" + "\n".join(msg))
ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
testlib.save_output( output_dir, "syndicate-ag", ag_out )
if ag_exitcode != 0:
raise Exception("%s exited %s" % (AG_PATH, ag_exitcode))
sys.exit(0)
for (start, end) in ranges:
# only clear reader's cache
testlib.clear_cache( config_dir, volume_id=volume_id, gateway_id=read_gateway_id )
# do each read twice--once uncached, and one cached
for i in xrange(0, 2):
exitcode, out = testlib.run( READ_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', read_gateway_name,
output_path, start, end - start, valgrind=True )
out_name = "uncached"
if i > 0:
out_name = "cached"
testlib.save_output( output_dir, 'syndicate-read-%s-%s-%s' % (start, end, out_name), out )
if exitcode != 0:
stop_and_save( output_dir, ag_proc, ag_out_path, "syndicate-ag")
raise Exception("%s exited %s" % (READ_PATH, exitcode))
# correctness
if expected_data[start:end] not in out:
stop_and_save( output_dir, ag_proc, ag_out_path, "syndicate-ag")
raise Exception("Missing data for %s-%s" % (start, end))
# finally, list it
for p in ['/', '/dir1', '/dir1/dir2', output_path]:
exitcode, out = testlib.run( LS_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', read_gateway_name,
p, valgrind=True )
# start the RG
rg_proc, rg_out_path = testlib.start_gateway(config_dir, RG_PATH,
testconf.SYNDICATE_ADMIN,
volume_name, rg_gateway_name)
if not testlib.gateway_ping(31112, 15):
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# put the file...
exitcode, out = testlib.run(PUT_PATH, '-d2', '-f', '-c',
os.path.join(config_dir,
'syndicate.conf'), '-u',
testconf.SYNDICATE_ADMIN, '-v', volume_name,
'-g', gateway_name, random_file_path, path)
testlib.save_output(output_dir, "syndicate-put", out)
if exitcode != 0:
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("Failed to touch %s" % path)
# finish populating expected built-in xattrs
exitcode, out = testlib.run(STAT_PATH, '-d2', '-f', '-c',
os.path.join(config_dir, "syndicate.conf"),
"-u", testconf.SYNDICATE_ADMIN, '-v',
volume_name, '-g', getxattr_gateway_name, path)
testlib.save_output(output_dir, "syndicate-stat", out)
if exitcode != 0:
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("Failed to stat %s" % path)
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# should cause the RG to get updated that there's a new gateway
gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
for rep in xrange(0, NUM_REPS):
# touch $NUM_FILES files
random_part = hex(random.randint(0, 2**32-1))[2:]
expected_paths = []
for i in xrange(0, NUM_FILES):
output_path = "/touch-%s-%s" % (random_part, i)
expected_paths.append(output_path)
exitcode, out = testlib.run( TOUCH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, *expected_paths )
testlib.save_output( output_dir, "syndicate-touch-%s" % rep, out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (TOUCH_PATH, exitcode))
# list them
exitcode, out = testlib.run( LS_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, '/' )
testlib.save_output( output_dir, "syndicate-ls-full-%s" % rep, out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (LS_PATH, exitcode))
# verify that they're all there
for path in expected_paths:
if path.strip("/") not in out:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
# make target directory
exitcode, out = testlib.run(MKDIR_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
volume_name,
'-g',
gateway_name,
"/newdir",
valgrind=True)
testlib.save_output(output_dir, "syndicate-mkdir", out)
if exitcode != 0:
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
# generate the commands to feed into the REPL
repl_cmd = ""
for i in xrange(0, NUM_FILES):
output_path = "/put-%s-%s" % (random_part, i)
output_paths.append(output_path)
repl_cmd += "create %s 0644\n" % output_path
repl_cmd += "write 0 0 %s %s\n" % (len(write_data), write_data)
repl_cmd += "close 0\n"
attr_value_base = 'bar-%s' % random_part
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume( config_dir )
gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
gateway_client_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, gateway_client_name, "port=31112" )
random_part = hex(random.randint(0, 2**32-1))[2:]
exitcode, out = testlib.run( TOUCH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
path )
testlib.save_output( output_dir, "syndicate-touch", out )
if exitcode != 0:
raise Exception("Failed to touch %s" % path)
# do setxattr a few times
# 1 attr
exitcode, out_1attr = testlib.run( SETXATTR_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
path, attr_name_base + "-1attr", attr_value_base + "-1attr", valgrind=True )
testlib.save_output( output_dir, "syndicate-setxattr-1attr", out_1attr )
if exitcode != 0:
raise Exception("%s exited %s" % (SETXATTR_PATH, exitcode))
# 5 attrs at once
# should cause the RG to get updated that there's a new gateway
gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
read_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
read_gateway_info = testlib.read_gateway( config_dir, read_gateway_name )
read_gateway_id = read_gateway_info['g_id']
volume_info = testlib.read_volume( config_dir, volume_name )
volume_id = volume_info['volume_id']
random_part = hex(random.randint(0, 2**32-1))[2:]
output_path = "/put-%s" % random_part
exitcode, out = testlib.run( PUT_PATH, '-d3', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, local_path, output_path )
testlib.save_output( output_dir, "syndicate-put", out )
if exitcode != 0:
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
# try reading and writing various ranges (these are (start, end) absolute ranges, not offset/length)
ranges = [
(5000, 16000),
(0, 1), # 1 block, tail unaligned
(1, 200), # 1 block, unaligned head and tail
(0, 4096), # 4 block, aligned
(0, 8192), # 8 blocks, aligned
(0, 1000), # 1 block, tail unaligned
(0, 6000),
(100, 4000),
(5000, 10000),
exitcode, out = testlib.run(PUT_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
volume_name,
'-g',
gateway_name,
local_path,
output_path,
valgrind=True)
testlib.save_output(output_dir, "syndicate-put-%s" % i, out)
if exitcode != 0:
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
# make target directory
exitcode, out = testlib.run(MKDIR_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
volume_name,
email=testconf.SYNDICATE_ADMIN)
# touch $NUM_FILES files
random_part = hex(random.randint(0, 2**32 - 1))[2:]
expected_paths = []
for i in xrange(0, NUM_FILES):
output_path = "/touch-%s-%s" % (random_part, i)
expected_paths.append(output_path)
exitcode, out = testlib.run(TOUCH_PATH, '-d2', '-f', '-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v',
volume_name, '-g', gateway_name,
output_path)
testlib.save_output(output_dir, "syndicate-touch-%s" % i, out)
if exitcode != 0:
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (TOUCH_PATH, exitcode))
# list them
exitcode, out = testlib.run(LS_PATH, '-d2', '-f', '-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v',
volume_name, '-g', gateway_name, '/')
testlib.save_output(output_dir, "syndicate-ls", out)
if exitcode != 0:
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (LS_PATH, exitcode))
# verify that they're all there
# look up RG
rg_gateway_info = testlib.read_gateway( config_dir, RG_gateway_name )
rg_gateway_id = rg_gateway_info['g_id']
volume_info = testlib.read_volume( config_dir, volume_name )
volume_id = volume_info['volume_id']
random_part = hex(random.randint(0, 2**32-1))[2:]
output_paths = []
for i in xrange(0, NUM_FILES):
output_path = "/put-%s-%s" % (random_part, i)
output_paths.append(output_path)
exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, local_path, output_path, valgrind=True )
testlib.save_output( output_dir, "syndicate-put-%s" % i, out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
# make target directory
exitcode, out = testlib.run( MKDIR_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, "/newdir", valgrind=True )
testlib.save_output( output_dir, "syndicate-mkdir", out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
# do several renames
for j in xrange(0, NUM_RENAMES):
exitcode, out = testlib.run(LS_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
volume_name,
'-g',
read_gateway_name,
p,
valgrind=True)
testlib.save_output(output_dir,
"syndicate-ls-before-%s" % p.replace("/", "\\x2f"),
out)
if ("\nFailed to stat '%s': No such file or directory\n" %
p) not in out:
raise Exception("Successfully listed '%s'" % p)
p_base = p
if p != '/':
p_base = os.path.basename(p)
if ("\nname: %s\n" % p_base) in out:
raise Exception("Found '%s' in listing" % p)
if i + 1 < len(descendent_files):
if ("name: %s" %
os.path.basename(descendent_files[i + 1])) in out:
# should cause the RG to get updated that there's a new gateway
gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"UG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
random_part = hex(random.randint(0, 2**32 - 1))[2:]
output_path = "/put-%s" % random_part
exitcode, out = testlib.run(PUT_PATH, '-d2', '-f', '-c',
os.path.join(config_dir,
'syndicate.conf'), '-u',
testconf.SYNDICATE_ADMIN, '-v', volume_name,
'-g', gateway_name, local_path, output_path)
testlib.save_output(output_dir, "syndicate-put", out)
if exitcode != 0:
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
# read the file, to populate the cache
exitcode, out = testlib.run(CAT_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
volume_name,
'-g',
local_fd = open(local_path, "r")
expected_data = local_fd.read()
local_fd.close()
local_paths.append(local_path)
expected_datas.append(expected_data)
# start AG
AG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "AG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, AG_gateway_name, "port=31112", "driver=%s" % AG_DRIVER )
ag_proc, ag_out_path = testlib.start_gateway( config_dir, AG_PATH, testconf.SYNDICATE_ADMIN, volume_name, AG_gateway_name, valgrind=True )
time.sleep(20)
if ag_proc.poll() is not None:
ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
testlib.save_output(output_dir, "syndicate-ag", ag_out)
raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))
# should cause the AG to get updated that there's a new gateway
cat_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
expected_data_error = False
for i in xrange(0,len(local_paths)):
# do each file twice
for j in xrange(0, 2):
local_path = local_paths[i]
expected_data = expected_datas[i]
output_path = "/" + os.path.basename(local_path)
exitcode, out = testlib.run( CAT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', cat_gateway_name, output_path, valgrind=True )
volume_name = testlib.add_test_volume(config_dir)
gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"UG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
random_part = hex(random.randint(0, 2**32 - 1))[2:]
exitcode, out = testlib.run(TOUCH_PATH, '-d2', '-f', '-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v',
volume_name, '-g', gateway_name, path)
testlib.save_output(output_dir, "syndicate-touch", out)
if exitcode != 0:
raise Exception("Failed to touch %s" % path)
# do setxattr a few times
# 1 attr
exitcode, out_1attr = testlib.run(SETXATTR_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir,
'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
def stop_and_save( output_dir, proc, out_path, save_name ):
exitcode, out = testlib.stop_gateway( proc, out_path )
testlib.save_output( output_dir, save_name, out )
return exitcode, out
if not testlib.gateway_ping( 31112, 15 ):
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# should cause the RG to get updated that there's a new gateway
gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
# touch $NUM_FILES files
random_part = hex(random.randint(0, 2**32-1))[2:]
expected_paths = []
for i in xrange(0, NUM_FILES):
output_path = "/touch-%s-%s" % (random_part, i)
expected_paths.append(output_path)
exitcode, out = testlib.run( TOUCH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, output_path )
testlib.save_output( output_dir, "syndicate-touch-%s" % i, out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (TOUCH_PATH, exitcode))
# list them
exitcode, out = testlib.run( LS_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, '/' )
testlib.save_output( output_dir, "syndicate-ls", out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (LS_PATH, exitcode))
# verify that they're all there
for path in expected_paths:
if path.strip("/") not in out:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
"AG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
testlib.update_gateway(config_dir, AG_gateway_name, "port=31112",
"driver=%s" % AG_DRIVER)
ag_proc, ag_out_path = testlib.start_gateway(config_dir,
AG_PATH,
testconf.SYNDICATE_ADMIN,
volume_name,
AG_gateway_name,
valgrind=True)
time.sleep(30)
if not testlib.gateway_ping(31112, 15):
ag_exitcode, ag_out = testlib.stop_gateway(ag_proc, ag_out_path)
testlib.save_output(output_dir, "syndicate-ag", ag_out)
raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))
# should cause the AG to get updated that there's a new gateway
ug_gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"UG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
# remove inner dir, and have the AG refresh
shutil.rmtree(rmdir)
exitcode, out = testlib.run(REFRESH_PATH,
'-d2',
'-f',
'-c',
expected_data = local_fd.read()
local_fd.close()
# start up RG
RG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", port=32110, email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, RG_gateway_name, "driver=%s" % RG_DRIVER )
rg_proc, rg_out_path = testlib.start_gateway( config_dir, RG_PATH, testconf.SYNDICATE_ADMIN, volume_name, RG_gateway_name, valgrind=True )
time.sleep(5)
if rg_proc.poll() is not None:
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# put the file
put_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", port=31110, email=testconf.SYNDICATE_ADMIN )
exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', put_gateway_name, local_path, output_path, valgrind=True )
testlib.save_output( output_dir, "syndicate-put", out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("syndicate-put exit code %s" % exitcode)
# create separate threads for UGs
for i in xrange(0, num_threads):
t = UGThread( config_dir, output_dir, output_path, volume_name, caps="ALL", port=(31111 + i + 1), email=testconf.SYNDICATE_ADMIN )
threads.append(t)
# run all threads
for i in xrange(0, num_threads):
threads[i].start()
# wait for them to finish
RG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", caps="NONE", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, RG_gateway_name, "port=31112", "driver=%s" % RG_DRIVER )
rg_proc, rg_out_path = testlib.start_gateway( config_dir, RG_PATH, testconf.SYNDICATE_ADMIN, volume_name, RG_gateway_name )
time.sleep(1)
if rg_proc.poll() is not None:
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# should cause the RG to get updated that there's a new gateway
gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
random_part = hex(random.randint(0, 2**32-1))[2:]
output_path = "/put-%s" % random_part
exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, local_path, output_path )
testlib.save_output( output_dir, "syndicate-put", out )
if exitcode != 0:
raise Exception("%s exited %s" % (PUT_PATH, exitcode))
# read the file, to populate the cache
exitcode, out = testlib.run( CAT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
output_path, valgrind=True )
if exitcode != 0:
rg_exitcode, rg_out = testlib.stop_gateway( rg_proc, rg_out_path )
testlib.save_output( output_dir, "syndicate-rg", rg_out )
raise Exception("Failed to read %s" % path)
# check cache
attr_name_base = 'foo-%s' % random_part
attr_value_base = 'bar-%s' % random_part
expected_listxattr_output = ""
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume( config_dir )
gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
random_part = hex(random.randint(0, 2**32-1))[2:]
exitcode, out = testlib.run( TOUCH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
path )
testlib.save_output( output_dir, "syndicate-touch", out )
if exitcode != 0:
raise Exception("Failed to touch %s" % path)
# do setxattr a few times
# 1 attr
exitcode, out_1attr = testlib.run( SETXATTR_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
path, attr_name_base + "-1attr", attr_value_base + "-1attr", valgrind=True )
testlib.save_output( output_dir, "syndicate-setxattr-1attr", out_1attr )
expected_listxattr_output += attr_name_base + "-1attr" + '\n'
if exitcode != 0:
raise Exception("%s exited %s" % (SETXATTR_PATH, exitcode))
def stop_and_save( output_dir, proc, out_path, save_name ):
exitcode, out = testlib.stop_gateway( proc, out_path )
testlib.save_output( output_dir, save_name, out )
return exitcode, out
email=testconf.SYNDICATE_ADMIN)
exitcode, out = testlib.run(PUT_PATH,
'-d2',
'-f',
'-c',
os.path.join(config_dir, 'syndicate.conf'),
'-u',
testconf.SYNDICATE_ADMIN,
'-v',
volume_name,
'-g',
put_gateway_name,
local_path,
output_path,
valgrind=True)
testlib.save_output(output_dir, "syndicate-put", out)
if exitcode != 0:
stop_and_save(output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("syndicate-put exit code %s" % exitcode)
# create separate threads for UGs
for i in xrange(0, num_threads):
t = UGThread(config_dir,
output_dir,
output_path,
volume_name,
caps="ALL",
port=(31111 + i + 1),
email=testconf.SYNDICATE_ADMIN)
threads.append(t)
