def __init__(self, config_dir, output_dir, output_path, volume_name, **kw ):
threading.Thread.__init__(self)
self.read_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", **kw )
read_gateway_info = testlib.read_gateway( config_dir, self.read_gateway_name )
self.read_gateway_id = read_gateway_info['g_id']
volume_info = testlib.read_volume( config_dir, volume_name )
self.volume_id = volume_info['volume_id']
self.config_dir = config_dir
self.volume_name = volume_name
self.output_path = output_path
self.output_dir = output_dir
self.exitcode = 0
self.errormsg = ""
self.errors = False
def __init__(self, config_dir, output_dir, output_path, volume_name, **kw):
threading.Thread.__init__(self)
self.read_gateway_name = testlib.add_test_gateway(
config_dir, volume_name, "UG", **kw)
read_gateway_info = testlib.read_gateway(config_dir,
self.read_gateway_name)
self.read_gateway_id = read_gateway_info['g_id']
volume_info = testlib.read_volume(config_dir, volume_name)
self.volume_id = volume_info['volume_id']
self.config_dir = config_dir
self.volume_name = volume_name
self.output_path = output_path
self.output_dir = output_dir
self.exitcode = 0
self.errormsg = ""
self.errors = False
builtin_xattr_expected_values = {
"user.syndicate_coordinator": None,
"user.syndicate_cached_blocks": "0000", # for the getxattr gateway
"user.syndicate_cached_file_path": None,
"user.syndicate_read_ttl": "5000",
"user.syndicate_write_ttl": "0"
}
# other attrs
attr_name_base = 'foo-%s' % random_part
attr_value_base = 'bar-%s' % random_part
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume( config_dir )
gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
getxattr_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", port=31113, email=testconf.SYNDICATE_ADMIN )
rg_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", caps="NONE", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, rg_gateway_name, "port=31112", "driver=%s" % RG_DRIVER )
# start the RG
rg_proc, rg_out_path = testlib.start_gateway( config_dir, RG_PATH, testconf.SYNDICATE_ADMIN, volume_name, rg_gateway_name )
if not testlib.gateway_ping( 31112, 15 ):
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# put the file...
exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
'-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
random_file_path, path )
# path to file...
assert 'DATASET_DIR' in ag_config
testdir = ag_config['DATASET_DIR']
if os.path.exists(testdir):
shutil.rmtree(testdir)
os.makedirs(testdir)
local_path = testlib.make_random_file(16384, dir=testdir)
output_path = "/%s" % os.path.basename(local_path)
local_fd = open(local_path, "r")
expected_data = local_fd.read()
local_fd.close()
# start up AG
AG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "AG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, AG_gateway_name, "driver=%s" % AG_DRIVER )
ag_proc, ag_out_path = testlib.start_gateway( config_dir, AG_PATH, testconf.SYNDICATE_ADMIN, volume_name, AG_gateway_name, valgrind=True )
time.sleep(30)
if not testlib.gateway_ping( 31111, 15 ):
raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))
# create separate threads for UGs
for i in xrange(0, num_threads):
t = UGThread( config_dir, output_dir, output_path, volume_name, caps="ALL", port=(31111 + i + 1), email=testconf.SYNDICATE_ADMIN )
threads.append(t)
# run all threads
for i in xrange(0, num_threads):
inner_dir = os.path.join(testdir, "to_remove_1", "to_remove_2",
"to_remove_3")
rmdir = os.path.join(testdir, "to_remove_1")
os.makedirs(inner_dir)
# put a file to publish
local_paths = []
for i in xrange(0, 1):
local_path = testlib.make_random_file(16384, dir=inner_dir)
local_paths.append(local_path)
# start AG
AG_gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"AG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
testlib.update_gateway(config_dir, AG_gateway_name, "port=31112",
"driver=%s" % AG_DRIVER)
ag_proc, ag_out_path = testlib.start_gateway(config_dir,
AG_PATH,
testconf.SYNDICATE_ADMIN,
volume_name,
AG_gateway_name,
valgrind=True)
time.sleep(30)
if not testlib.gateway_ping(31112, 15):
ag_exitcode, ag_out = testlib.stop_gateway(ag_proc, ag_out_path)
testlib.save_output(output_dir, "syndicate-ag", ag_out)
return exitcode, out
if __name__ == "__main__":
local_path = testlib.make_random_file(16384)
local_fd = open(local_path, "r")
expected_data = local_fd.read()
local_fd.close()
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume(config_dir)
RG_gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"RG",
caps="NONE",
email=testconf.SYNDICATE_ADMIN)
testlib.update_gateway(config_dir, RG_gateway_name, "port=31112",
"driver=%s" % RG_DRIVER)
rg_proc, rg_out_path = testlib.start_gateway(config_dir,
RG_PATH,
testconf.SYNDICATE_ADMIN,
volume_name,
RG_gateway_name,
valgrind=True)
if not testlib.gateway_ping(31112, 15):
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# should cause the RG to get updated that there's a new gateway
def stop_and_save( output_dir, proc, out_path, save_name ):
exitcode, out = testlib.stop_gateway( proc, out_path )
testlib.save_output( output_dir, save_name, out )
return exitcode, out
if __name__ == "__main__":
local_path = testlib.make_random_file(16384)
local_fd = open(local_path, "r")
expected_data = local_fd.read()
local_fd.close()
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume( config_dir, allow_anon=True, private=False )
RG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", caps="NONE", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, RG_gateway_name, "port=31112", "driver=%s" % RG_DRIVER )
rg_proc, rg_out_path = testlib.start_gateway( config_dir, RG_PATH, testconf.SYNDICATE_ADMIN, volume_name, RG_gateway_name, valgrind=True )
if not testlib.gateway_ping( 31112, 15 ):
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# should cause the RG to get updated that there's a new gateway
gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
cat_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email="ANONYMOUS" )
random_part = hex(random.randint(0, 2**32-1))[2:]
output_paths = []
for i in xrange(0, NUM_FILES):
output_path = "/put-%s-%s" % (random_part, i)
if __name__ == "__main__":
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume( config_dir )
num_threads = 8
threads = []
local_path = testlib.make_random_file(16384)
output_path = "/%s" % os.path.basename(local_path)
local_fd = open(local_path, "r")
expected_data = local_fd.read()
local_fd.close()
# start up RG
RG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", port=32110, email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, RG_gateway_name, "driver=%s" % RG_DRIVER )
rg_proc, rg_out_path = testlib.start_gateway( config_dir, RG_PATH, testconf.SYNDICATE_ADMIN, volume_name, RG_gateway_name, valgrind=True )
time.sleep(5)
if rg_proc.poll() is not None:
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# put the file
put_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", port=31110, email=testconf.SYNDICATE_ADMIN )
exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', put_gateway_name, local_path, output_path, valgrind=True )
testlib.save_output( output_dir, "syndicate-put", out )
if exitcode != 0:
stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
raise Exception("syndicate-put exit code %s" % exitcode)
"user.syndicate_cached_blocks": "0000", # for the getxattr gateway
"user.syndicate_cached_file_path": None,
"user.syndicate_read_ttl": "5000",
"user.syndicate_write_ttl": "0"
}
# other attrs
attr_name_base = 'foo-%s' % random_part
attr_value_base = 'bar-%s' % random_part
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume(config_dir)
gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"UG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
getxattr_gateway_name = testlib.add_test_gateway(
config_dir,
volume_name,
"UG",
caps="ALL",
port=31113,
email=testconf.SYNDICATE_ADMIN)
rg_gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"RG",
caps="NONE",
email=testconf.SYNDICATE_ADMIN)
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume(config_dir)
num_threads = 8
threads = []
local_path = testlib.make_random_file(16384)
output_path = "/%s" % os.path.basename(local_path)
local_fd = open(local_path, "r")
expected_data = local_fd.read()
local_fd.close()
# start up RG
RG_gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"RG",
port=32110,
email=testconf.SYNDICATE_ADMIN)
testlib.update_gateway(config_dir, RG_gateway_name,
"driver=%s" % RG_DRIVER)
rg_proc, rg_out_path = testlib.start_gateway(config_dir,
RG_PATH,
testconf.SYNDICATE_ADMIN,
volume_name,
RG_gateway_name,
valgrind=True)
if not testlib.gateway_ping(32110, 15):
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# put the file
