# create AG source dataset. extract config
ag_driver = syndicate.util.objects.load_driver( AG_DRIVER, None, include_secrets=False )
assert 'config' in ag_driver
ag_config_txt = base64.b64decode(ag_driver['config'])
ag_config = json.loads(ag_config_txt)
# path to file...
assert 'DATASET_DIR' in ag_config
testdir = ag_config['DATASET_DIR']
if os.path.exists(testdir):
shutil.rmtree(testdir)
os.makedirs(testdir)
local_path = testlib.make_random_file(16384, dir=testdir)
output_path = "/%s" % os.path.basename(local_path)
local_fd = open(local_path, "r")
expected_data = local_fd.read()
local_fd.close()
# start up AG
AG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "AG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, AG_gateway_name, "driver=%s" % AG_DRIVER )
ag_proc, ag_out_path = testlib.start_gateway( config_dir, AG_PATH, testconf.SYNDICATE_ADMIN, volume_name, AG_gateway_name, valgrind=True )
time.sleep(30)
if not testlib.gateway_ping( 31111, 15 ):
raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))
STAT_PATH = os.path.join( testconf.SYNDICATE_UG_ROOT, "syndicate-stat" )
SETXATTR_PATH = os.path.join(testconf.SYNDICATE_UG_ROOT, "syndicate-setxattr")
GETXATTR_PATH = os.path.join(testconf.SYNDICATE_UG_ROOT, "syndicate-getxattr")
COORD_PATH = os.path.join( testconf.SYNDICATE_UG_ROOT, "syndicate-coord" )
CAT_PATH = os.path.join( testconf.SYNDICATE_UG_ROOT, "syndicate-cat" )
RG_PATH = os.path.join( testconf.SYNDICATE_RG_ROOT, "syndicate-rg" )
RG_DRIVER = os.path.join(testconf.SYNDICATE_PYTHON_ROOT, "syndicate/rg/drivers/disk" )
def stop_and_save( output_dir, proc, out_path, save_name ):
exitcode, out = testlib.stop_gateway( proc, out_path )
testlib.save_output( output_dir, save_name, out )
return exitcode, out
if __name__ == "__main__":
random_part = hex(random.randint(0, 2**32-1))[2:]
random_file_path = testlib.make_random_file( 16384 )
path = '/setxattr-%s' % random_part
# test built-in attrs
builtin_xattrs = [
"user.syndicate_coordinator",
"user.syndicate_cached_blocks",
"user.syndicate_cached_file_path",
"user.syndicate_read_ttl",
"user.syndicate_write_ttl"
]
builtin_xattr_expected_values = {
"user.syndicate_coordinator": None,
"user.syndicate_cached_blocks": "0000", # for the getxattr gateway
"user.syndicate_cached_file_path": None,
# path to file...
assert 'DATASET_DIR' in ag_config
testdir = ag_config['DATASET_DIR']
if os.path.exists(testdir):
shutil.rmtree(testdir)
inner_dir = os.path.join(testdir, "to_remove_1", "to_remove_2",
"to_remove_3")
rmdir = os.path.join(testdir, "to_remove_1")
os.makedirs(inner_dir)
# put a file to publish
local_paths = []
for i in xrange(0, 1):
local_path = testlib.make_random_file(16384, dir=inner_dir)
local_paths.append(local_path)
# start AG
AG_gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"AG",
caps="ALL",
email=testconf.SYNDICATE_ADMIN)
testlib.update_gateway(config_dir, AG_gateway_name, "port=31112",
"driver=%s" % AG_DRIVER)
ag_proc, ag_out_path = testlib.start_gateway(config_dir,
AG_PATH,
testconf.SYNDICATE_ADMIN,
volume_name,
AG_gateway_name,
CAT_PATH = os.path.join(testconf.SYNDICATE_UG_ROOT, "syndicate-cat")
RG_PATH = os.path.join(testconf.SYNDICATE_RG_ROOT, "syndicate-rg")
RG_DRIVER = os.path.join(testconf.SYNDICATE_PYTHON_ROOT,
"syndicate/rg/drivers/disk")
NUM_FILES = 10
def stop_and_save(output_dir, proc, out_path, save_name):
exitcode, out = testlib.stop_gateway(proc, out_path)
testlib.save_output(output_dir, save_name, out)
return exitcode, out
if __name__ == "__main__":
local_path = testlib.make_random_file(16384)
local_fd = open(local_path, "r")
expected_data = local_fd.read()
local_fd.close()
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume(config_dir)
RG_gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"RG",
caps="NONE",
email=testconf.SYNDICATE_ADMIN)
testlib.update_gateway(config_dir, RG_gateway_name, "port=31112",
"driver=%s" % RG_DRIVER)
i = offset
for c in list(buf):
if i >= len(expected_data_list):
padlen = i - len(expected_data_list) + 1
for j in xrange(0, padlen):
expected_data_list.append('\0')
expected_data_list[i] = c
i += 1
return "".join(expected_data_list)
if __name__ == "__main__":
local_path = testlib.make_random_file(16384)
expected_data = None
with open(local_path, "r") as f:
expected_data = f.read()
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume( config_dir, blocksize=1024 )
RG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", caps="NONE", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, RG_gateway_name, "port=31112", "driver=%s" % RG_DRIVER )
rg_proc, rg_out_path = testlib.start_gateway( config_dir, RG_PATH, testconf.SYNDICATE_ADMIN, volume_name, RG_gateway_name )
if not testlib.gateway_ping( 31112, 15 ):
raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))
# should cause the RG to get updated that there's a new gateway
i = offset
for c in list(buf):
if i >= len(expected_data_list):
padlen = i - len(expected_data_list) + 1
for j in xrange(0, padlen):
expected_data_list.append('\0')
expected_data_list[i] = c
i += 1
return "".join(expected_data_list)
if __name__ == "__main__":
local_path = testlib.make_random_file(16384)
expected_data = None
with open(local_path, "r") as f:
expected_data = f.read()
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume(config_dir, blocksize=1024)
RG_gateway_name = testlib.add_test_gateway(config_dir,
volume_name,
"RG",
caps="NONE",
email=testconf.SYNDICATE_ADMIN)
testlib.update_gateway(config_dir, RG_gateway_name, "port=31112",
"driver=%s" % RG_DRIVER)
import shutil
REPL_PATH = os.path.join(testconf.SYNDICATE_UG_ROOT, "syndicate-repl")
MKDIR_PATH = os.path.join(testconf.SYNDICATE_UG_ROOT, "syndicate-mkdir")
RG_PATH = os.path.join(testconf.SYNDICATE_RG_ROOT, "syndicate-rg")
RG_DRIVER = os.path.join(testconf.SYNDICATE_PYTHON_ROOT, "syndicate/rg/drivers/disk" )
NUM_FILES = 2
def stop_and_save( output_dir, proc, out_path, save_name ):
exitcode, out = testlib.stop_gateway( proc, out_path )
testlib.save_output( output_dir, save_name, out )
return exitcode, out
if __name__ == "__main__":
local_path = testlib.make_random_file(16384)
local_fd = open(local_path, "r")
write_data = local_fd.read()
local_fd.close()
local_path_2 = testlib.make_random_file(16384)
local_fd_2 = open(local_path_2, "r")
write_data_2 = local_fd_2.read()
local_fd_2.close()
config_dir, output_dir = testlib.test_setup()
volume_name = testlib.add_test_volume( config_dir, blocksize=1024 )
RG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", caps="NONE", email=testconf.SYNDICATE_ADMIN )
testlib.update_gateway( config_dir, RG_gateway_name, "port=31112", "driver=%s" % RG_DRIVER )
